From 92faeea35bd10281d8308c2f6117357aa8f1d1a3 Mon Sep 17 00:00:00 2001 From: DavidbrOFS Date: Sat, 3 Feb 2024 15:23:30 +0000 Subject: [PATCH] Deployed 747897f0 to ofs-2023.3-2 with MkDocs 1.5.3 and mike 1.1.2 --- .../assets/.doxy/opae-code/xml/access_8h.xml | 60 ++-- .../assets/.doxy/opae-code/xml/buffer_8h.xml | 20 +- .../assets/.doxy/opae-code/xml/core_8h.xml | 256 +++++++++--------- .../xml/cxx_2core_2properties_8h.xml | 146 +++++----- .../opae-code/xml/cxx_2core_2sysobject_8h.xml | 170 ++++++------ .../opae-code/xml/cxx_2core_2version_8h.xml | 24 +- .../assets/.doxy/opae-code/xml/enum_8h.xml | 72 ++--- .../assets/.doxy/opae-code/xml/errors_8h.xml | 162 +++++------ .../assets/.doxy/opae-code/xml/event_8h.xml | 20 +- .../assets/.doxy/opae-code/xml/events_8h.xml | 170 ++++++------ .../assets/.doxy/opae-code/xml/except_8h.xml | 76 +++--- .../assets/.doxy/opae-code/xml/fpga_8h.xml | 144 +++++----- .../assets/.doxy/opae-code/xml/handle_8h.xml | 170 ++++++------ .../.doxy/opae-code/xml/hash__map_8h.xml | 8 +- .../.doxy/opae-code/xml/hello__events_8c.xml | 182 ++++++------- .../.doxy/opae-code/xml/hello__fpga_8c.xml | 188 ++++++------- .../assets/.doxy/opae-code/xml/log_8h.xml | 42 +-- .../assets/.doxy/opae-code/xml/manage_8h.xml | 8 +- .../.doxy/opae-code/xml/mem__alloc_8h.xml | 8 +- .../assets/.doxy/opae-code/xml/mmio_8h.xml | 20 +- .../.doxy/opae-code/xml/properties_8h.xml | 100 +++---- .../assets/.doxy/opae-code/xml/pvalue_8h.xml | 146 +++++----- .../.doxy/opae-code/xml/shared__buffer_8h.xml | 200 +++++++------- .../.doxy/opae-code/xml/sysobject_8h.xml | 20 +- .../assets/.doxy/opae-code/xml/token_8h.xml | 164 +++++------ .../assets/.doxy/opae-code/xml/types_8h.xml | 210 +++++++------- .../.doxy/opae-code/xml/types__enum_8h.xml | 250 ++++++++--------- .../assets/.doxy/opae-code/xml/uio_8h.xml | 6 +- .../assets/.doxy/opae-code/xml/umsg_8h.xml | 8 +- .../assets/.doxy/opae-code/xml/userclk_8h.xml | 20 +- .../assets/.doxy/opae-code/xml/utils_8h.xml | 106 ++++---- .../assets/.doxy/opae-code/xml/version_8h.xml | 12 +- .../assets/.doxy/opae-code/xml/vfio_8h.xml | 44 +-- .../user_guides/oneapi_asp/images/PF0.png | Bin 0 -> 397633 bytes .../user_guides/oneapi_asp/images/PF1.png | Bin 0 -> 387436 bytes .../oneapi_asp/ug_oneapi_asp/index.html | 235 ++++++++++++---- ofs-2023.3-2/search/search_index.json | 2 +- ofs-2023.3-2/sitemap.xml.gz | Bin 127 -> 127 bytes 38 files changed, 1804 insertions(+), 1665 deletions(-) create mode 100644 ofs-2023.3-2/hw/common/user_guides/oneapi_asp/images/PF0.png create mode 100644 ofs-2023.3-2/hw/common/user_guides/oneapi_asp/images/PF1.png diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/access_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/access_8h.xml index dd4430588..8e3efe18a 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/access_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/access_8h.xml @@ -9,15 +9,15 @@ + + + - - - @@ -39,6 +39,18 @@ + + + + + + + + + + + + @@ -59,33 +71,13 @@ - - - - - - - - - - - - - - - - - - - - - - - + + + @@ -103,16 +95,24 @@ + + + + + + + + + + + + - - - - diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/buffer_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/buffer_8h.xml index 5a36951ed..50d8a410c 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/buffer_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/buffer_8h.xml @@ -15,6 +15,12 @@ + + + + + + @@ -27,12 +33,6 @@ - - - - - - @@ -55,6 +55,10 @@ + + + + @@ -69,10 +73,6 @@ - - - - diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/core_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/core_8h.xml index f005f8f61..03819feeb 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/core_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/core_8h.xml @@ -15,58 +15,68 @@ - - - - + + + + - - - - - + - + - - - - - - - - + - + - + + + + + + + + + + - - - - - - - - - + + + - + + + + + + + + + + + + + + + + + + + @@ -76,42 +86,34 @@ - - - - + + + + - - - - - + - + - + - + - - - - - + - - - - - + - + + + - - - - + + + + + + @@ -122,44 +124,60 @@ - - - - - - - - + + + + - - - - - - - - - - - - - - + + + + - + + + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -179,25 +197,15 @@ - - - - - - - - + + + + - + - - - - - - @@ -205,9 +213,32 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -230,37 +261,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/cxx_2core_2properties_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/cxx_2core_2properties_8h.xml index 2972cf634..52902ef90 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/cxx_2core_2properties_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/cxx_2core_2properties_8h.xml @@ -14,31 +14,45 @@ - - - - - - - + + + + - + - + + + + + + + + + + + + + + + + - - - - - - + + + + + + + + @@ -48,46 +62,20 @@ - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - + + + + - + - + @@ -96,6 +84,18 @@ + + + + + + + + + + + + @@ -136,12 +136,6 @@ - - - - - - @@ -154,19 +148,9 @@ - - - - - - - - - - - - - + + + @@ -184,16 +168,32 @@ + + + + + + + + + + + + + + + + + + + + - - - - opae::fpga::types::properties opae diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/cxx_2core_2sysobject_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/cxx_2core_2sysobject_8h.xml index 773bad5cc..a46bc7635 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/cxx_2core_2sysobject_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/cxx_2core_2sysobject_8h.xml @@ -11,37 +11,65 @@ - - - - + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + - + - + + + + + - - - - - - + + + + + + + + + + + @@ -51,36 +79,26 @@ - - - - + + + + - - - - - + - + - + - + - - - - - - + + + + - - - - - + @@ -91,26 +109,26 @@ - - - - - - - - - - - - + + + + - + + + + + - + + + + + - + - + @@ -119,6 +137,18 @@ + + + + + + + + + + + + @@ -135,39 +165,9 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/cxx_2core_2version_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/cxx_2core_2version_8h.xml index 8247f0ceb..27a1ddbd3 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/cxx_2core_2version_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/cxx_2core_2version_8h.xml @@ -9,14 +9,6 @@ - - - - - - - - @@ -26,6 +18,14 @@ + + + + + + + + @@ -44,16 +44,16 @@ + + + + - - - - opae::fpga::types::version opae diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/enum_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/enum_8h.xml index ffbfd4792..8d556bd4b 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/enum_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/enum_8h.xml @@ -40,6 +40,18 @@ + + + + + + + + + + + + @@ -52,9 +64,27 @@ - - - + + + + + + + + + + + + + + + + + + + + + @@ -68,41 +98,15 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + @@ -112,10 +116,6 @@ - - - - diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/errors_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/errors_8h.xml index 23a42b6d3..3e4add623 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/errors_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/errors_8h.xml @@ -10,47 +10,51 @@ - - - - + + + + - - - - - - - - + - + - + + + + + + + + + + + + + + - - - + + + - - - - - - - - - - + + + + + + + + @@ -60,52 +64,42 @@ - - - - + + + + - - - - - + - + - + - + + + - - - + + + - - - - - - - - - - - - - - - - + + + + - + + + + + - + - + @@ -114,6 +108,18 @@ + + + + + + + + + + + + @@ -133,22 +139,6 @@ - - - - - - - - - - - - - - - - @@ -162,6 +152,16 @@ + + + + + + + + + + @@ -170,16 +170,16 @@ + + + + - - - - opae::fpga::types::error opae diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/event_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/event_8h.xml index ee50eef59..0d8322a30 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/event_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/event_8h.xml @@ -14,6 +14,12 @@ + + + + + + @@ -30,12 +36,6 @@ - - - - - - @@ -50,16 +50,16 @@ - - - - + + + + diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/events_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/events_8h.xml index 5b08d2b3c..2b18d1ae8 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/events_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/events_8h.xml @@ -10,37 +10,65 @@ - - - - + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + - + - + + + + + - - - - - - + + + + + + + + + + + @@ -50,36 +78,12 @@ - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - + @@ -90,36 +94,26 @@ - - - - - - - - + + + + - - - - - - - - - - - - - - + + + + - + + + + + - + - + @@ -128,6 +122,18 @@ + + + + + + + + + + + + @@ -147,21 +153,15 @@ - - - - - - - - + + + + - + - - @@ -184,16 +184,16 @@ + + + + - - - - opae::fpga::types::event opae::fpga::types::event::type_t diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/except_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/except_8h.xml index 819f60ee2..8a26e2b9b 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/except_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/except_8h.xml @@ -12,6 +12,9 @@ + + + @@ -22,28 +25,17 @@ - - - - - - - - - - - + + + + - - - - @@ -59,16 +51,32 @@ - - - + + + + + + + + + + + + + + + + + - - - - + + + + + + @@ -81,19 +89,15 @@ - - - - - - - - - - - - + + + + + + + + @@ -103,10 +107,6 @@ - - - - opae::fpga::types::src_location opae::fpga::types::except diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/fpga_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/fpga_8h.xml index c69d5d301..9dd1de442 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/fpga_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/fpga_8h.xml @@ -22,6 +22,18 @@ docs/sw/samples/hello_events/hello_events.c docs/sw/samples/hello_fpga/hello_fpga.c + + + + + + + + + + + + @@ -60,8 +72,11 @@ - - + + + + + @@ -69,55 +84,49 @@ - - - - - - - - - + + + - - - - - - - - - - - + + + + + - - + + - - - + + + - - - + + + + + + + + + - - - + + + - - + + @@ -125,9 +134,15 @@ - - - + + + + + + + + + @@ -143,22 +158,11 @@ - - - - - - - - - + + + - - - - - @@ -166,40 +170,36 @@ - - - - + + + + - - - - + + + + - - - - - + + + + + - - - - - - - + + + diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/handle_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/handle_8h.xml index 30f803363..2c0fa969a 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/handle_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/handle_8h.xml @@ -15,37 +15,65 @@ - - - - + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + - + - + + + + + - - - - - - + + + + + + + + + + + @@ -55,36 +83,12 @@ - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - + @@ -95,26 +99,26 @@ - - - - - - - - - - - - + + + + - + + + + + - + + + + + - + - + @@ -123,6 +127,18 @@ + + + + + + + + + + + + @@ -142,22 +158,6 @@ - - - - - - - - - - - - - - - - @@ -191,26 +191,26 @@ + + + + + + + + - - - - - - - - opae::fpga::types::handle opae diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/hash__map_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/hash__map_8h.xml index ff24391d0..8b89de90e 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/hash__map_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/hash__map_8h.xml @@ -29,16 +29,16 @@ - - - - + + + + _opae_hash_map_item _opae_hash_map diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/hello__events_8c.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/hello__events_8c.xml index b365c330c..a6dbcfd05 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/hello__events_8c.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/hello__events_8c.xml @@ -15,6 +15,24 @@ argsfilter.h mock/opae_std.h + + + + + + + + + + + + + + + + + + @@ -53,11 +71,11 @@ - - - - - + + + + + @@ -93,58 +111,55 @@ - - - - - - - - - + + + - - - - - - - - + + + + - + + + + + + + + - - + + + + + - - + + - - - - - + + - - - + + + - - - + + + - - + + @@ -152,9 +167,18 @@ - - - + + + + + + + + + + + + @@ -170,28 +194,11 @@ - - - - - - - - - - - - - - - + + + - - - - - @@ -199,49 +206,42 @@ - - - - - - - - - - - - - - - - - - - - - - + + + - - - + + + + + + + + + + + + + + + - - - + + + diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/hello__fpga_8c.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/hello__fpga_8c.xml index 1ea5c1ba0..5fa429a68 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/hello__fpga_8c.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/hello__fpga_8c.xml @@ -13,6 +13,18 @@ argsfilter.h mock/opae_std.h + + + + + + + + + + + + @@ -51,8 +63,11 @@ - - + + + + + @@ -60,68 +75,29 @@ - - - - - - - - - + + + - - - - - - - - - - - + + + - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - + + + @@ -149,87 +125,111 @@ - - - - - - - - - + + + - - - - - - - - - + + + - + + + + + - - - + + + - - - - + + + + - - + + + + + - - - + + + + + + + + + + + - - - + + + - - - + + + + + + + + + + + + - - - + + + + + + + + + + + + + + + - - - + + + diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/log_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/log_8h.xml index cbb37d482..7db17703f 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/log_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/log_8h.xml @@ -18,26 +18,15 @@ - - - - - - - - - - - - - - + + + @@ -50,13 +39,24 @@ - - - + + + + + + + + + + + + + + @@ -71,16 +71,16 @@ + + + + - - - - diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/manage_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/manage_8h.xml index 341b51c0d..36a8d7bde 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/manage_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/manage_8h.xml @@ -46,16 +46,16 @@ - - - - + + + + diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/mem__alloc_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/mem__alloc_8h.xml index 5d9a0d282..ef4e9eb45 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/mem__alloc_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/mem__alloc_8h.xml @@ -16,16 +16,16 @@ - - - - + + + + mem_link mem_alloc diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/mmio_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/mmio_8h.xml index 8b0234924..0dfef58ec 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/mmio_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/mmio_8h.xml @@ -11,6 +11,12 @@ + + + + + + @@ -30,12 +36,6 @@ - - - - - - @@ -50,16 +50,16 @@ - - - - + + + + diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/properties_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/properties_8h.xml index 66f243b43..6f9955e43 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/properties_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/properties_8h.xml @@ -13,15 +13,15 @@ + + + - - - @@ -40,6 +40,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -52,50 +72,38 @@ - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + - + - - - - - - - - - + + + - + + + @@ -106,19 +114,15 @@ - - - - - - - - - - - - + + + + + + + + @@ -128,10 +132,6 @@ - - - - diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/pvalue_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/pvalue_8h.xml index 211b68f5a..8ce557dee 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/pvalue_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/pvalue_8h.xml @@ -17,27 +17,41 @@ - - - - - - - + + + + - + - + + + + + + + + + + + + + + + + - - - + + + + + @@ -48,6 +62,25 @@ + + + + + + + + + + + + + + + + + + + @@ -60,39 +93,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -114,9 +114,11 @@ - - - + + + + + @@ -132,27 +134,9 @@ - - - - - - - - - - - - - - - - - - - - - + + + @@ -170,16 +154,32 @@ + + + + + + + + + + + + + + + + + + + + - - - - opae::fpga::types::guid_t opae::fpga::types::pvalue diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/shared__buffer_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/shared__buffer_8h.xml index 4185c32e8..31e8d204e 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/shared__buffer_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/shared__buffer_8h.xml @@ -16,70 +16,57 @@ - - - - + + + + - - - - - - - - + - + - + - - - - - - - - + + + + + + + + + - - - - - - - - + + - - + + - - - - + + + + - - - - - + - + - + - + - - - + + + + + + @@ -88,10 +75,21 @@ - - - - + + + + + + + + + + + + + + + @@ -102,34 +100,52 @@ - - - - - - - - - - - - + + + + - + + + + + - + + + + + - + - + + + + + + + + + + + + + + + + + + + @@ -149,35 +165,28 @@ - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + @@ -200,15 +209,6 @@ - - - - - - - - - @@ -217,16 +217,16 @@ + + + + - - - - opae::fpga::types::shared_buffer opae diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/sysobject_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/sysobject_8h.xml index bdf48ff6d..fc4837329 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/sysobject_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/sysobject_8h.xml @@ -11,12 +11,6 @@ - - - - - - @@ -32,6 +26,12 @@ + + + + + + @@ -50,16 +50,16 @@ + + + + - - - - diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/token_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/token_8h.xml index fcd3c655f..dfc24d21c 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/token_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/token_8h.xml @@ -16,37 +16,51 @@ - - - - + + + + - - - - - - - - + - + - + + + + + + + + + + + + + + - - - + + + + + + + + + + + @@ -56,52 +70,42 @@ - - - - + + + + - - - - - + - + - + - + + + - - - + + + - - - - - - - - - - - - - - - - + + + + - + + + + + - + - + @@ -110,6 +114,18 @@ + + + + + + + + + + + + @@ -129,22 +145,6 @@ - - - - - - - - - - - - - - - - @@ -166,12 +166,6 @@ - - - - - - @@ -184,11 +178,9 @@ - - - - - + + + @@ -206,16 +198,24 @@ + + + + + + + + + + + + - - - - opae::fpga::types::token opae diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/types_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/types_8h.xml index dc052f13d..7a7398d42 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/types_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/types_8h.xml @@ -54,6 +54,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -62,6 +82,12 @@ + + + + + + @@ -70,53 +96,71 @@ - - - - - - - - - - + + + + + + + + - - - + + + - - - + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + - - - - - - - + + + + + + + @@ -126,19 +170,25 @@ - - - - + + + + - + + + + + - - - + + + + + @@ -184,30 +234,16 @@ - - - - - - - - - - - - - + + + - - - - - - - - + + + + @@ -224,54 +260,22 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - + + + - - - - + + + + @@ -280,13 +284,9 @@ - - - - - - - + + + diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/types__enum_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/types__enum_8h.xml index 6046eed0a..522f9a4fe 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/types__enum_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/types__enum_8h.xml @@ -9,6 +9,26 @@ docs/sw/include/opae/init.h docs/sw/include/opae/types.h + + + + + + + + + + + + + + + + + + + + @@ -17,9 +37,11 @@ - - - + + + + + @@ -29,22 +51,62 @@ - - - - - - - - - + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -59,61 +121,51 @@ - - - - + + + + - - - - - + - - - - - - - + + + - - - + + + - - - - + + + + - - - - - + - - - - - + - + + + + + - - - + + + + + @@ -159,30 +211,16 @@ - - - - - - - - - - - - - + + + - - - - - - - - + + + + @@ -199,44 +237,12 @@ - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -253,22 +259,20 @@ - - - + + + - - - - - + + + - - - - + + + + @@ -277,13 +281,9 @@ - - - - - - - + + + diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/uio_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/uio_8h.xml index fb184beed..9508f0dd3 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/uio_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/uio_8h.xml @@ -5,6 +5,9 @@ stdio.h stdint.h + + + @@ -13,9 +16,6 @@ - - - diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/umsg_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/umsg_8h.xml index 7fe298d67..aff12a25e 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/umsg_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/umsg_8h.xml @@ -50,16 +50,16 @@ + + + + - - - - diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/userclk_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/userclk_8h.xml index 892b435d4..34ac1ece4 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/userclk_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/userclk_8h.xml @@ -11,12 +11,6 @@ - - - - - - @@ -32,6 +26,12 @@ + + + + + + @@ -50,16 +50,16 @@ + + + + - - - - diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/utils_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/utils_8h.xml index c952b85f7..7a957667f 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/utils_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/utils_8h.xml @@ -13,6 +13,14 @@ + + + + + + + + @@ -28,14 +36,6 @@ - - - - - - - - @@ -45,6 +45,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -57,16 +77,6 @@ - - - - - - - - - - @@ -75,32 +85,30 @@ - - - - + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - @@ -109,19 +117,15 @@ - - - - - - - - - - - - + + + + + + + + @@ -131,10 +135,6 @@ - - - - diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/version_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/version_8h.xml index 9eb008bc1..7043ba651 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/version_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/version_8h.xml @@ -14,6 +14,12 @@ + + + + + + @@ -30,12 +36,6 @@ - - - - - - diff --git a/ofs-2023.3-2/assets/.doxy/opae-code/xml/vfio_8h.xml b/ofs-2023.3-2/assets/.doxy/opae-code/xml/vfio_8h.xml index fd7337ce3..628d3b407 100644 --- a/ofs-2023.3-2/assets/.doxy/opae-code/xml/vfio_8h.xml +++ b/ofs-2023.3-2/assets/.doxy/opae-code/xml/vfio_8h.xml @@ -12,25 +12,15 @@ - - - - - - - - - - - - - - - - + + + + + + @@ -41,12 +31,6 @@ - - - - - - @@ -57,6 +41,22 @@ + + + + + + + + + + + + + + + + diff --git a/ofs-2023.3-2/hw/common/user_guides/oneapi_asp/images/PF0.png b/ofs-2023.3-2/hw/common/user_guides/oneapi_asp/images/PF0.png new file mode 100644 index 0000000000000000000000000000000000000000..573e7bed4f37808d6570a89ee3ed990fd3c636c6 GIT binary patch literal 397633 zcmZs?bySpJ`#nsD#1Kly3?T*~AkEMcf~0~-cSuMxbV%3GQbQw1NjL}$As`GOjdah@ zJ(Tpz6W{0KZ>{%_nR}hH&V8S_PV9YM`$TD~E0Nt}xQB&>MW&*xppAuvw~vK|>q10u zmxJKRNxgeuyJ;)QVO0$>ZQdpDY-C@_Vqw)JkX)JJ-=&E^DC@gnVNtpN>xaEy&FX!Z zX<_&Jt-Fqs-G^6J?q-%|?q+v+SXjPn77lhFG@b37+}+s34SdcG|9_EwKHFN*35huas zvXqLgXXhT)B>#WyfQJ(kL!Ip9@QP&RKSTbHjYMrC!4ZGk?C#BWs-MI|2!pHYBpy=L z+iiPQM%C-=>QlDBO2?Wu^@j1D{&L5FUwVszVn%Yt8n@&bW5&N1vN#)2LGRSN^5Vd4a zc89FYkQ90&H5MuyV2QX4apB|kLK3Ji4Gmt3B82B(et#*J9fHAF?7qFD*%$Yfkq%jO z13wrJ6Hw`I*@Gfx)WTuqSoNow@ZLOyY^Qo##Ij$O0EwJbcID|9jnF)f@0i1F{v zJpFD|#&<0r=sF}?AXOhcWKnIfmGHcEeQXki$c3D~KV2C6g}*;+mPH-UgNqN>m;py zkrK-g)^!e22G9=Xz|k8`rV)D@o^BvE?rdq7DZuM_A3^-p^`~_n4$0}l2)f|$(tdmaW%JtK( zYB_!9xSc{ifA5L&D&~Pe$u6Otatq-%KA?zEe0mi3)<@X!x{d~z^Y&c&)<*xE{cp8v zB4ggoQq*#1#Ok)mZ4>GHtl`#1CJy!@vmbqr(ru5@!dhDcGP1L4->aY!1^M|O%ggs4 z!#u;^K2d3zU2_ZH>nFI@-g-Ry{k*yDQi4%K&(P&X^rZNRE*?z!&ksOfJhMcF2@nGd zxUehdIw=_(?~Rh4i4t`doEhn%tSx1Aq0sC)ml#6%8;v6_OwWQ4EKEh5mMC?+QCU|l z!nJ7U+6z;6miD{Cv<@&!xyo&&Qu`LlRqqEp1$RxVy4qh_ll| zn4ytgNdFSdMK`d}~zIvc|v|b6x-kiNF8mH@9#b=Zs9h-Rl=H%1Kk@ZBeLx zQ&zfjIKqFveS6)us*`(DUoqNPXTExmy`1DpDF5Ek%*hJ9|E8e3qI5MaFy43cvZcY9 z7uo#R>d&^y9(E*_f8a_gumPWSNj=}|cGZumyE74{TclMhJR*W;w-X@E zogj5l3h5&~2GCJ{wHabzf1+FQmfzx$#gyBAw+J?y?P_$Y!0;p=%EBUT?=^71P=LfD z6GH@*-)K=g34JZ*eXta3rv9#v(`m8QU-7<7eEzIn+r^vniH26IaET&ZJJ4hA?mQ(u z((C~;AMmB?e&=fq=+-Hd^+=Rtq6y0#M7By5A$hUc9*nD~h#`e)7|1io3CPyEFf%g?(epC4WPKS@l`u|P zw(5_C=I3~8ybFSjQ|7T-;r0~-P^2P@Oo--m2+=xxdyg6A*1~9wHT}zVZH_;~+{G3& zyYc}i@_=eaj)icK?O%e)UdWSHW9}95XBhF__!DLiQ_`_hx67cKqh05~*)6iizb^R# z4r3;0jBQx=7uJ!u$e`E)u!^d`cgddAL@DjbLq30GT>>Ee6*5GsKRgV}i0<;Ze_i?( zQoe8GHP6ZVr=1W@7h4UJjt=V3-s zXLM=NMrt1`(xBq~A`+M;*3$-PR~m7SWj#M9chg+Ar@Hhd+@tVm_S{XHY}xauHR}vl zoAt2V&HQ$%28Jq`l8){zOdtx|Mq+F(65CrEBs@ON68?RPWs_y6AlOaY7k#!XG2q1` zuxme019G=vk=JX%;F7`lbF}iVeT~j|d4W^>yw1F%u2lv;f`-{##hw(v&39rKi8ISD%GD3Tg+tZ)jXYl_0!~sGH(+V*w3tY=Fqh2<$mpR%A;l2Z{3WIY)<* z2A3{H`@zr1$VuP6vt;X3!QJbP`Nf^6AcbV@`=}5GJ@!ipb%=a8L!jNaDyr3;>h(H5 zzxGhGfK(Eg&c_I^g_JN&;?5iL$prK%@g6+2^PV)=Og-72==GR1vf(K4bp<86)H-~K z#I~{cxgEZjBF`{$uLF=Y(qH2~?@9x(lh`PEl~w;L;+MfNu~^?xK>j{GE~AK3N(^t4 z@pPq^Tw{LvAo13=2RrT0EE^$L*4u{-m8~`5X(Vo~TBo)$cXfTqjXKu2y-j})e z(4PlD-yg4+WH||Fvka-_ZvP_ed&skFdHo~x`8f8l6FTe{JF5rRpE(8KlE zI+&0ijoXE%*B2q$`Sktn%r`~NJY0{DcpXQi;K)$CJRaaJw9}UeABig&nps)m$iGUj zP1FKn%zlf7U?04UJoOSt@>~aVP#V9;y*d7Db+k4hsL++;wP2?}y{gT1@zKX6fm=Dg zc8yh2pWYYtv)BHw33DRARtzDV#mB?op8GE9D@j#)W1#di=o4?7y_rT;nD33) z>^&d!0VQNFY~5yNd^?5(YhJnEcdv~0L`Lbtf&Xy|Jl=lh!}=ie1TSx7b#d+*JtVg%xaabqV{m@h0lWv@nGNDQ60-pd7pinK1r33HKf_Nkx9(XMfAo^5z5^f>A{ z*NI){I0yEO4vTC!Tj=+P&{f+>w|qx&X7wYz01{l}QH6dN$&ehxWs)B7f^s;=C2j$K z#v<6po!}qH13Yl$=YvEXCLH=K`k&GNz{&^-j_n}}e-_RgF0JU*qMIyzJti&G$)v$S zx(o2%kXIZv^cwiKmik;w$g1d#r_UIje8SRbM`KL1TsOlq~?3*)2ANjgY)0QC9zdB4d`?c9e>vJyUyUm|Viuhy05kUlVD4>>O~E z6;}~t7+M4ZaE#kXUDJqDxFwIZP&?Xf4=yqTf{+4 z=8As;XfQ&V1FEx4NC-GC^n|n8Z3K3|GAXV#t=)PZ6dtTx(qeZvw=wjru^Kqnw_(j~ zrl{~61;AGKbZEVVzYG({@VwWeqXkULjFb{UAH&p5eNtR4Rt^s(;hMh8Sdi*_^iH$z}9Ei|0r z2a1|{4g4|Rc(0^0FHEK*Z?HH|Ly~H7sZT*Y%w06=c8O`Y7=9SY9ysv~9&~JcT#|#PTQXcy&)pLfZzE+N)T$wQUX%!gav&edbb&%EJ)oL%(ikMWE>|QysTYz{ zGW^fd&*sHzC&=RE0#~*F?)!llGC6A8TdqJ1SDb0k_n_?d@_3uAkK_~r-lnqmuZ98F z&_tgN<(uv5wgqK^Na0BA(b0)E%^WHVbMLrfIUx*r&b?>M=GT(kroWuHJY8pWR`nIhzm6 zJ)!m3u@$*G%)N?Y67_C}YqakAE~X()(*XU7yL?p?es|ODx;CKGr|~*o0)67_-QuuN ztFNf26x+@=AD#Mib?JvZW~qTCMZ&4A6G?Mi;wYw(=r~=imjl!~Kwb@cwOZh@TL1*` zF%+0cc*!*=#O{6h+0XQ;=AUxqvxD9yvOw9j0uodR=3piy@25esu(`oQO8}a>i9Zea zh0X*TbGws!Ey!-F5;_>`(oL&})r54mOYb|+UHJQYK0P`uC8ndK^9YCIJn0P7F{efFxh;xU z^1r^7f1k1+()jl?yy$xA>b+5>L4la$+>~34$xY1FY~ZEys`ORFEimvrJ{2W7oA!pBYr(i?|0Y4=~egbSnV!!%O$P&uZ zft>**_j_?t5W@>UhT2zg6;TdR_)xg*^@hVX_Gb;ELfwNjWlWG*ICxkJB;82Z>IsL~z2k=c=5l6feHRm(}=(X}{6J|2v~!%%&XkO`S%r%+IK8a0 z#4wAVkxOxom5qO;kBmSPtMl_M#aTIVEv^|nK4o`nwGllE5QPb%Q-T`&D8l!GfEid% zL{UcJmz~i=SI}b1ex_^a@Qh%Xa9_(kpAQtY;nq#GyXW66iCxcnhUZ>la^Pth3#Gw{ zX%gF1UQEsvmg5yTM539GaED&>alnKvenD7qWkS5Y?pJLjAc0ELes0S#AK*@>Te8Y%Dcw7|7?O^lOH)Phn(n%;ccBls6X@OHwT&g z`Hs!{wq3NlofAC!*zE!!&xoRt|8xeGp`i8N_1m#v6&zxUwIiYc1^i-Ey#ApOyjzbmX7>kH7%?m*Ppi?4b@7VGv+Ys!!Z<+S_$Q3%2YJ-uj zm0`i+alSZ;%0N)!|hzl+F=@<~U1C6ZXZv)E?0)tSs>W^Xbb zXLnctl0>@hJy`%YJWXK_%cmg7NGMOuF|h#+=&>>`v3#zqj2we(h1SFNqsZO*-&@CO zMxf^JqrCMGHnCmC@pE2zRZ^c6T&pOxdc2>^P$QB?QqE8@WBJH*e+AP~WmI2{8haZt z>&`>Jy)``^8Qh3=qEOzzKg3N$t)GvuKsCqX>hWfh#76?WlLaQG?y23@`rmh!TcO*o z$F-oUjcNtLTtSIYY)F_|9~)1Rpx&%;|93?_o}uNVprs*_;I(H-Gyr|cWvAA2YxdPC zhUha^RLJh?F~3hp@(H0u7(~(M__KPf6%l{cF&_%rYQ?R^H~Kl=7qAUoYEKc#7XoJWqvPf^O_ zRwmjCGo~O!hkaURX3;%c@U5m4cNh`G$c)M8=H*Vtlu!1ezOUbFRCw37&MbaF8C02W z;)Evrh`aF4o4kO;jtI>KfN6NYakZxZNMN%siVVoNGZj8|>4R~6r?pi?Q}x_jD>LKrFmMeaqx z-{$#k;Vl!Hbt6Z5L;|)LeiD^%8+!i`bv1IIL7nf z;xAGL`~c^_h3gB&o@bUgMseRCXpZMevQ{o%ez#dhXBUC(Z;3AThtRkZP8Nqo0H$fV z`5g}7E(O?y7cKk#)mPolSK(I|)wfNFL2cKwZ9HwK@7t_x+T!98GC)$SAQv`Km1W2< zpQ@);0_!2w#m{g$$Mh z!nUv_G&0mQIC7a@fG{aUbj<9G_vm})Sp07B?Go!1?s-P;IlV+E$qJVL@Q4bbG8+-@ z3bscW638-*7r1th*mVH@-a&#_VT5tEIN90~>@-98pE#y77t; z5{QM-02%oqMgnjXM+joFX`TvfDdv$W8~* zHWh!bv0p4U_Ay%iNj~E$W$o8P7%@pmv}ySx3!=RAWV3yr$>!Li^$dWsW`oX2kH-7J z;ahY*D?QgSyBl^+B)c@~N3Z(Us?`z9YojyOX7BNA57~KQn8Qr8uvs?BC|d8CfqO=R z-^Z|o1ZcXX@r3kE&+U{u=U1BR3Fqsys};VRf)l|mj=bGNxTS+YIB`laci-wulVyCd z2G+ke5sJ4C=p*}eR@(6|)us7{yrnK*hDNs{7gX|-lhq43nIlPrV%-Md^gk&Cp$v3w zbIqW=aKUgPMNtmC1xdPg3U+k{DEu||bUGWYn&hLo4<$?(Et;UW{2irlpJ6DqNoAl* zj^QF?A7MQb!-U%S{=jFzr-n*fKl9I_%+S=aZ(kh0G14*AC&)mB^^gV(M6Y_+i>4JL z3Q~Ofv@0hq&FuyJf4o^d)QZS*YMmyMlk_Q#Nspj~CObkw6wKp8Dva(Q)m)QeWU}O* zjG8pe`;=deUk1JmK!c^ZTfh(A>s9%Y78zGg)A|FLB{?`98KwSwSY6daZl#WU$@LZ2 zp86-Ge3K?1OlC<5Bd-_SjaD~i5tmOpAa5CFHSdgiv~A!hcwyZnt{OC|RJ3|;I+S$g zlj{|;v2)6cKz`5cpu@A+jGqrR{C0q>EmJR}*R!3gi&zh03k{F+0az!T@iiJujRr-a2y;%y)MrT#@dJv3_0*A))xN2uX;wyv9Xmcf(pCOTY_} zGZrJ|Kux~f&YHt(%frAke&jR zJ?!6IJZY)j{Zek#Zaor3-%|5I6h#_*bxDH^7Z*U%WM`|E>8P|3gsK1~rmr6DZh`ebvMcPYe+c=8S?P^6h~9OBQAsxs^PXA(69hIzCO-WYA;24ac<)~x z=zu!kK+8Ws059FCl+s#ESl-)dMZ~;=L;oKaA;d}&O8*?*13sQM&88sP< zOpwf!*>g8*nr^9X2}^RT6%`bECL(@AUF#mV4u{ewvBZ(XiRfs?ti5~p1HXxD; zbD%|b=O`1G^GWTqQ+;Gf`jHXc_XC&Eva{(#`w@Cz>N6=0nxl(s7rz%PKge+9z_5TB zNaVSzwktb(_(pq%)aY?1I25P=;>+&(fFbI@n+LrCre#lk&=i5$*&GMcnKkoT5*RDj zlZflkPDXh_Y5kBvyCd14ZeN^}kSW>8)Tg?1o7jo+^BXqqpSg+Dw%V2OfM8(N6wXEF ztdpR2&rq<1FQ3ntVl3^btdHv{LoKkPRN5z>PuzPJdc#)PkkBuP@7O~KIDmcZa?2@Y zzbGIvQ7v)Yz#WnYS2&4R&GpCJ$m(6BLb@6u>aGH5bT%7gVv7Dm4oX(O4^K*Rf%SI| z8=#4jk(8vvb>k;nVFN{DNVB%a%wj{_=;v6KNX28YexMAVoRq7T6Lkb3h@}cG%VL)} zlx8>+bjl7pu3)^I75AsnHWFgHvf?v?J>XsWX2JBg1_!Yy@aCsUba0C0PK{?`_!-g$ zp(8!GEzf$96i^tCbqAiWZ?+OZaqxXMDY zCROSzvNSF~|L~Ucx4BKK3kLADpsDnQB^}WCMHB?Q>wU6=GC{JhDH04x{cv1CF~H|Q zHDEhUe-=LTVT!y4saGUm3yc4w@tnK^q}{ZKG1hAw8;7cZ1P7PCUgcJ1hK6_}f?A@{@?{4qxJ-rV~byw+wyx#uVHX z&|@zD0i~zj0AfO|m5+~#?s))NRz6}L@0Tz-3nYDnm(1I;&ZO6?3ycnmEV?KE)Z%iN zV_D6^b<|*-X$*Tm@viAbFr3;Ta|G9)c)YlwdU7Vw^DbDCwJD!PMTgpmS@0VVYPDoK z2(#WIg7ipW3I!o{0Oe;hiLC`DrMT!^=T$}CqmRePoI}LeVXf`PM7rk-=_(5Ilo%zB zldyX8ISkBGOh0fq0i6$~;~OV#0o^t#5!jXSeFR#Boex1WF9Ah-hoidn#1>uPjZzWE z0q=aHN9n#75l=Dnw?)e&Q)nx0dBw{=ayET^6V%!?(;L60JOmP-<4LV`%oUq#=%J5T zS~i>Cr6$aLeTN_R9~H=d1PFBzbFKcD@fNtxG(kK-)@M#2zsOGS9`Ek2LhSGc^99F3 zd*s>KX@*um;nQcX{7d}aQK4T23*^cE81hy_l7eHGcUQ61Ok?+dJ`X$k8FE}ou1;$9 zT^?rYSBQIP2#FX;;N2jfmU*|;?$-QUSfeiSQ0r2Y&}9SAi`}avg;c66N$sIQMfIwT z1$$9YLn-IvO`)vJ2hSAblSaR=#E$2WYUyJ(dVY@$n`HZ1%h6(w`)Yb*WoOn4VC|wVviBx#&+kSh%ScYVgERDac7!n10%E60@g>+8=d9DGyVSb|@awp+Sh>jOw zTR1;*Dq{xT_ zIM|0->e+pu3m!#keR2=qlR#`O7LpU{H^^j_#QQi{S0P#K=}Z2Z`1^-4R~t&w_OOr6 zzO%F3tghcLB#U+gtO)+A@%gWHBa@T7oE$7D}jig{7@`DqJ@?OY;aw8M`#6OecvKHnFL=}?X`P4d@z}{;t z1<@B2N=EN86-{2|Zjspn7PLexFVi)4Lm#}uFCSaV^T#u{bQmA?o&V0o>I$-d6e0Z3 z>8LnsCzMVpo<*qT;9ddwhC%=l)d~pIqG1SdKNI+Vw*VeW+m-U^Z7E9Ui#TFa${H_d z#h(cVw{{gi%T={xb5v$<2lAbDxp@?juhdoybx}sjOJjWA^%WRibQhZuIH&4*<6(WJyS*KdsV~KfIsEn1BICu^YAM<<>=)Shl z(hrrrJR01NcgWIv_i&Xmf*jo$<6!P!_$L};4*V=|Nl2*X+?Q{d3mA{XD%B~UPWlqx zd*>E#Ed`uoyA-h}G$gN|$QZ~>RWE)GuvU{Bl#Nef2^pRxXp{=bw&h~U0fU>~`jEY! ztVmU?Ghl|q``kVKf-j^63<6CK#SSP{Zs(U`FPNu|;-_ytbAd+gr9(8LFG55f)nJWgg%J2y>UD4r)7cM=FrtX&TnHY+@k#MV}Lp zs`G|6=^BzA3R1zo7`Al3dTfQqyP;Ly%3_rG2wpTNtJMdMX{4)B%YY=<75n3WjC1a0 z?VTe>$l%pefYESHQy##_d*1G8B-g%#I~Um@8pLBDbl8Vx_K<}ofHBbUb}%sR4BrlN z>iLiiVh~r7s2D?#!>U}U)~D!oT+Ho$;N54U9QZ84ZwCsiuV-C?$*{0$q$v3U6MQYT zsv7F3KcR8a@~jw~6%`Fem(`Z-Vk&EdAMpmR7?RGNK1G#YYQZ}kahY&?C_O{7_OJLk z++(sIh=oaNrT#y~+rP2Xpa4~j0N~CY^N~NB-<|(8e+R!)BUNO75`!f5vk$%U2*xce z?HKP!?AGu;iBX?`J$^VXiL(8dfRqH|l(=PeW*~7b9+rzfib>|-ftv9`A-l1c!|j{Z zxY({vuFv?Ck75LTURpi2g}v0DRW8*VfP2a99R7Mh6)xJR2$&2LUhe;33V93T zsTZ323$SR%{UWE|Wp=(wzKQTJ*SS1dt{a4Vb*M05C0chm5db38HuyO^j{@mo3~{S^ zY`{kHu5UNag=>P7cZAqxm-oZ?Q#mEVbBG2QVN4;9T%m{XrkAI`@L3WEW_cD2W{xcI z5I_}}AF=vB%5C!YuN9rYSz`XN`e8NW;?A{z%5;e)?Ng#%aPerYCw5eQH~L|J6;}hb z1bZedWi6nrM30qV(zgnBnHwWOTB?I$9Dij;L^N<3*QC6-)zs_r(npj1VF%i`t5 zOcMJya*65(X{j^7M#4mwE~u4^Ks7JJ3h?(pWx(B+<6x7of;$cdj?L^~3Ot4(@JHr- z5FFS9tR>ouQalhC3HB^7M6wXBQfnLtpGpb|nv@kQ(}P$+WfM?hEQ& z$t+^J;l?A)TIH%O?*u!N#C5okVnmW#I4VjxcIga7yiZ?ThAql(c$;P0W3Wn|9**FYS>Vebo%PG z2Gb@Y5TYbQ*s(Gg_gb&7m>b7Ou3gp~yN{rYnr4)>$R#7_Vk}_%_d3;nqm4uIkd?1N zVjBDoJ=&_?4Y;d6g1zSmN!I00Z7ztMC8X|mz>P5Z$^Ei~ zE6UACqyGpw|FcoGyNE8|#W#kSBonk^4hIAYY8XecP(B~LGw7SqpoKVRQUjduD#np% z?>%e<>~m;b`mX|3tZD+0rW(fwMnJl$1*Ht9NQ)Ga`uJ;H+_W$sEgPwwpc-i@VS;8` zn;~W?Z{(nB{{*oB)@Z0Njz>sduw|Zcmlp33QeN0f&&Sv5OQw{uY*rq%&F5w2F0jjn zdX^gtoevw~Fe~zqFP$x2bd-jq3@~+LcRiw+b8*Q{zB17FyMEO*;rO*r1=A$0Cbz^oti!1Ik4Tm-9r9X)dRHQCk|zF)?mG=YDI<$I+JgHIbtFbSPm!Is z4O}sT3CAZE#}LEy3q?+f_SDE{>`kBff5*EnWfXTMXYw#}sMMXmQ}x>K_WDK+xH#${ z1TQZxZMsRW)qVLD5xf+4bkD>Ie>?*Y1k@@kHP%a7mY)4EZ*OSPv6+nUoyqp@PEJ_X zxpPGS?kV3^_TTL!e-a~#J@XOHGljyZy$Y;@Il)9FScaJ!pvI|80DU!U1 z@+|i2Tl!bzu5$kSI#^u5WU5asps!@Nx93V$FS%HtG(Ew2GD{p9kM#%Jd_)@f|Cp31 z`{0*^NC>f0ibNzXjh-_4Q|DOjJy>@?R%_*P1LV{pRrypqrB z8P$W5fb#xolcDdPcJ6qOLUQZV*&~Qg zOr7^DN0Yu+c1%UsDcDzHBfi!;Mnlkr^~IdWgw58r2Uc43)6VT5;2%W2dv%>-`Yis&JLieNhx%Q<8?)GmTaLmU z3`Lcty?DQ?rHhX7>uyFe?KEUkw838w3B zu3pBr{Y9G*O^LH@fw%p_(!En+6Rlv2sdJb4OA_X4_-V`qOpK8(cw~h>tCjMWPf>%_WbCNTAW*1)n&ap)T9({T&@P2-fIk#pk zE_1n&M$X$qy8v;0=3OLS^n7mp2+~4j-LJX?D#jfb`y53S6MU!GZtu}qpZY7B*h zoh_lY4XZ-yi{+>p@6~c4AONbcns~s=A#)P9w8tH`+)du!6yqCEz^)LZb1D zx(%ocU$&h&22Nx6=zeSk+x-}PyKkANRdYI{0yD1Y4J|SEjlAbsJaM~kJ4fL;{T*%cH+~CyzXSD=z-4ezKym;6<^*JKZLajOXu~_|Bjr zn?E1OjA}K=wE`hzKh3q$_)MIzmrr~KMjzaJQE8wvhUNyJDaFSkmzMI5Pfs(Qx(>-P z37%|&vyQUJVt7{A7FZ`iXIWv~a>*eyz=m{eKk5%SZTQ4@H0NFsPwGoyPa>&!BHo^A z5*9sLqSCXesZZbd-QpWxGXC*n_ns?j=YGG@^n1edcehoE-Eg3ITcX}GjFippg8jE0 zaX#y2eOh?H2tOjPN-C*Tdr?N^E{<{OIzOp<9lrf={IxzYF8t?5iTyL@q{u4Kf00|!#; zb7J2t+}uSPS~OzI(Ydgf<@)EqjGp1$Lw|q&hrwqyiVFksd!;udopNLNG>lOdVxCO$ ze}?y^$o&i(hss%+1y0nzth(TFs|tX9m8bgY^m24kVoWGUdpb zrLK8yXf>aEoW9Iu(sLsPo2DBt;e_51NEtYtt!b$z2F0H{$p86&i=h2H8s*5uwf~Dv zd_ql|B9Sd&y>TgkN+cAQW^<+zj{b`K+o|)o!>r@`Cv~tD$Y?UnX+2uknr$Ch4KGae zJDRr6Jc$MG*AwVt_x%1w60`qMQi}naqV5I5=4;X~ntM%wGY}PN`CR0rv6yTmv22rg zx-GDce!SQ8)klJ}fEQ^(^+wIi6P&DT#?<1YDd^sgz}HEuKNW=Ej#dpqy$po{pD^n` z{)D}QHzkSwG+9uGMc{r!WN47C*HlzlVl`pv0U0K)IY`KD;?8yVQHoSQ(VJ1t(&o;?vIS# za%|c~Si08Re@iaZr%EX*EBB>O%l*IYFu0fm!UYT)G4w?D~j`OcyjcG1w!4oGODgCQO&X2*#;Z2(4uBB&lO`(v2 zL2S(q`6a_2Jg0*gAct|jKeJ|#pgGO}7Ojx~XJKSPAc6w=F z=hHI$SCzIwACav8L>JWKTw6ADX5eRtJ0_D#u(92ZapVFDZE8rPkw9=%g1y&9kw*N)^l0^;Idh|JO$5 ziuTpYiUX1~+wmDWJ++h~oitvz&$o9u#CDP}aTylshg@8{;$X0&!D!;ZA4~ENYm9cd ze1MtJrtl(G2JVd&<9b%Qc+Njj>x5V{H$|X2si}+K1J2tzRn4%?rZ;7;3l{$gWG&SX zgJb3#s=5|U-CPii!nCJ!2%T>;^8)YZ?g@(KfOZc?1Not#JUn!$= zw(%cv(NB(3;yk4E;%hlhGW0wAc{*}$!A6B%aN|v&C9@+{q>ScL_*8ZW*~r^RhlGtJxfBj4qNocl|A-}@h``^Fv+&oN47(IOwK}MsRGb# z4yK`F#h&zJG1IuLY<0ZKYHZ+1VihihcT+f3AjZYCxM-HdXYU`_phs9Mtd8UPyM{li zQH=hx)vWaZi|YD>8LyYPL-Ov7E8O=BJ2dZh~f{)U*(qbXCA8aq04mIE3 zI_ZvGR`qjs&Sf~n_0-IVoaryPI|?ONl=P{9D4nB-vW2RQOaS_Y^JAKN)z$334=aRx zmLx-X?$6h3qteS0>;O_X>NIp-NDDvy8(`@{J*rRe5ZP1C_km#8vtrVIBF%9(>v=9U z|4Y_k+Iq3cMwXN#{Lx_z?}=qIB`Hg6cwa4V;=>D`|KS$I7C&pM>ph>JJ5RsO&fTo_ z(OL~y>q=BqSDW|<$*dKh9npsDrth>-qbhtBf(^OoV@PmLCT=;x{_u4@IIH_rr24*L z%3N2N{k}F^x`y*JO%-qV%ja~S%7-pGbuCy|7L#4$Z=We0=e!m;Qe7Ge)=x_)&l2=g=Rr4r}D z-J};O;u4KQTh_8m1elQJ?a~Y#sZR#!w#;_hkw~303>-&b>8Dt@O=pUI`&Px`nL}c* z?e1T5KXdZ3+jll#^YFVxczX>B^i;Ys_a9*O;b!G6BiH;;H&YD#Wz?-DHDw~|JDZ%b zcw^Th`rZToRH^1n$%A*J4!?h`Jt;jm91k*@yfzG*wiJ8SGqyu$n9!sDaKA!$(=+(f z>P6C4dz1j%n^)pj_WALeU5ZcXz+U-ex^Unr0()$@4)P2hcDC%cZB$-Ow@q z(MI4Ny|m%_U>x<2ZdVcJGjwQaUjQ=1yE`imrAKU={ButDX`?%yOxfGDA2Z37)jHx0 zp!H1=%QT;+_3_P?+nNlLQK5HrZCE2VEiDh|HNn15A6ysazt8Bdi?3PX81CtbkJuBi z{G{LU*KQ{HvqpHZ!{OVxLw?B(ZVnS`;iJq+qjICy5{N2c!#9qNc6FP&6-)xnUtd#@ ziAhW%@Y2I~V{O7`9H*R_-C94fa&hb}+6z?OA%7u3x?+Y8b zM$vcnv`d}2a8EsXKWq1f1&log1?cN96NyIlp}Y?S1>FoEXo&IXm=x^n<56;9ZUNoW#fC85#Pj32;05D>kl*E6lfCdpHTTTfCAii)8<~_+n%|^e z>pbhlX10)B=%StJhDE9S0%93b405wa_-}chYjrO0F5q3L#a6aGiP){-u({l+&9}Of z-hZvouj^Kp#IRQzo=Q^@sf1Y1yt)1CRbujg*JJN&xa3D@J+PVNHJy`(MbD$6{w9g9 zIFF9Gx(|sgWs>rF9nZwf&v?$x zr){U^Sv3VFRxW~^S)jU#T`8U}pI@ZVrBP$jabg?A~&m z-L@hh^?nw^rR(o<)9KCfo{ zfJa~IwK*eqj&_z)-v%=B`1M6mgR{)GG>?2JZb@LJ*6-j^;r(>aqwj0EVR0pBNP|TW zdE?#fAct9QD_K&R?1Ok4E_HI|mCy$<>yXKMf|48^ANIg{7b;M@zTp!+s{+CyiqzJy zj9Q`~F9L!RHiOnxo$HGIrcKGkr*YQlvElqmT$8vswBCq@r+CCg;SG=V5E-HuIk$f?g{PgX zR5a8nc8*_M=@;uI4#eVMQnXl`k?dH!-$fs%KmdS^Or0lZ$VlEl zxwjo)*WGWEeJ^N@*utu=tjjI79W^reOOFgAKDP=8R8;}I=cZ}x-z3~!p-mnFP*F z&WTkg_?w_J(|tB62x4iNGHv=n`NjEQr$*+wk+Vte{eDh*R@TXNyVPlDWtK?q+N*Ts zH@mGc9EXs!Vqqus)z0PTCoiX+R682l`R%m0xc29OijPVCkP>)%i)|+4ERwcG?ZXFO z>D?7|ZD$r1jQdISHMM^xCQ3~_Ga$t<%^f`~$2*UCm+0gFj+|+Vk5Qx|IrXXDCj{C* zVr0Qlh8<7rw#r=*ZPc8sOx*8$QMRMrDS0Q74HlQ}TR9{&OYfQ2lBj$AHU7iB{Rg?nQkoJ7ZIUXRh8jh3GVJ#*%ZvjqA-M;YsU;+)jE&RCFc6u zv~Z}xZ0FJFAQk{fkkT-D$CZm^_3lne9L&3EVac&N6CGbxMz1R6kob7!$<7~j+bTIk z_3zw+*DpJT(`lVcIi$J1mW3 zGtOR%ImqTs`&Tf(EjxCrlghLoSEDC7R+FHyBS-d$=>?5|nTX073iqoG*OT9Ry(g0< zd}njlVvW|b!n@5vUz|rKqIKCJbUF`RN77xOj+Z;10ROm%NN{^*?WbIv3@+Tt`{%z` zsu;Ey3z3tqRF#>FHeX51=B8|Je)0c`gCS=rn^sfOcWm6Dns)Dfky-r^Z2xYWUva>l zl+m;C9n?Ceaem8c`nnfB&{PvYN>^fGVR7m-S!JYX@3>I=g5_eb(|%lN;A_QiMnN=r zbWQR7Pb=UDE&U8*#@uLTMWN`L+lErBlisy~bbDBVL*O5?Mocu%8t3Tjm_-E#_d{b& zbVe>BSz!TuD>ox7QRggba8on<@TFx8rEBkzWs|s>M^8IG-R}pES2w`{Lf%icb+3Ol zzZjdWKkVt|vQn6}_4)F|AKd>O%=%5nx)(v$Kf~g#P(B8YXP7|`eA6%|^%_2#OY-_s z^sQ|D1-ago1-qbx3V?unU(%A0eiNKBMYs^R{*q~F%Zg(7&@eA~Ol4Z#hN+3AzHKtX zhRecG4*=_)ZHF?Q8oT5PasVuXd6IgwgeVg>HZE`tIO<#Ra@%4(wq!089&(zGMzwH5 zdS;-Gp+9jjQw1^VDRB<_w9fB?oEDU->KfX%SItUHXoyo9?@knfXz*&ikS^^9$+Rq= z>khDq!02w{x=BjgWuKno-U%7$;5hC1xW1_G|3}XbeD7tozmS5-8IzEDfrCqdhYG|djAFY+j((rbJjYKW37Gv>>69o zcJx~I$ zZvI($wWBbHH+fyedChBENiQZWj$5f7luxQ|e;>9f#r+Cte-c|V(Q&H#u%e;b!C-E_ z#SjPLJO*EyZ_HKd(^Jfq_nIVd#gO`Z(nEQrnDZ*Ox{EUTL}r^Rf^&0Or@t1v217dY z*wvu2V*}$Z`r89yv{prfB0`RupVEcQ-tKtbF?FCtb3Y>Jsb7q-8sx3BdOn9NI3FNG z4KzOTC!k}88WmQZ?+m4d+&c=rm(@Rz3X<}@{eAiK)|TzJ+>RMo!h{2Id9Ll$OO|xP zvi&ih9Vd&E&8j6n4!TgYiSS>pP1$fOT)oT6IdU#OP5`x+Xzz}E(qP1|p`;UCj~NYL z-pZT_#_mNGwz1%fF#@8!s8(|c52y0j=70ez3JU1Ggx*K$eYByFVjeC@$uys)>_m>y zs1FqZUJK9utgycLuu?xdwK{`oXk*SC`b?X-u{e!BnyOtUT?|!{yh6}`WE^DzZCqHS zVtx_(g>2fA!Iufb-o`J2wq$BZO`T9yQgLiK<~`pxkEH~=zSt?MU{uvXcV#w``#Ivt2MG+oIil5>Aei=_x zsk9g@x{@5)r7gc|b22MS8;ePkn4eQV_CoV{C5MNL*^I{)m3fdC`YyV5`${rvCwS$uFNP~J!CdPN=WP+F11SHyKe9iQ;$SUyEVbziIP48`D* z8jNgi$f*95VngVVbMSU6YV`ACaM#a5T!G3MX^j^EMaTPKiI%DHsF!2?dG-Mh^$l+j zt;hej$0)Jh+W&q;N@Rb;)3M-U?_*iK-!|DSyVgUv6P-zEmuWQx(L*%-8qx-AwonI1 zuwSV8RME6+yYLG(%)VqHr)LT=>+T{;VMvN(jQ$Yj_PUE@X%`RwiQ{%{<87bY4EK)u&TskRM`0D7CHdTx0vEdKKD?M7hr#^$_D zm0u<8p_cT!O~ct4>xx}g`XmJ-57wpnHLcnlIP~?KzA&MA3Q++bo3ana5i;Vea#_$% z&_6_Ua9QK^m2R?Ng7+n>{XP?u96gL2{T8=qs~2O79K3T%iy^dfEyXyOmYrOr?;+Qp zzgw#D+0WCmYO%w(A{W=bl4X%543>Nz)@E@#q1%?~s%O{}j==HAcm}U&w)F7(6qmX* zB*%QPn;BF0UPuxoDA^c+|O?$wsq|81_IkI8lR3w)e?5g{Z@Oa2(mbw0ZlzEA$K`%?{m+z?B2evEk}c*J4P(a^h2tYQPOCNcw~@~_9s z$)CStrTDDIMdlP-3#t91Pg@M(SulSu(F1gwcDe7_N%lGP6_Fc!l$L8 z2z2;F@@01d22(Hrs+9_WD$xhjO?Qt*>U zJRH@I5AF{Xi#%R(pRWhw55d~6dG8Li+s*XL?h5+biOakCKRO(5G5&X-*+C|5Qe*J1 z4mmv$BmM(%vDoK^n78b=G+RQcxU~&h6?%*$#_ndh;z;8Qfla^6`GkKDu+Z{f9yL!pEfF3>Be^&semQ92Gst5f!O? zI7|MN~ zwR0w5h7k;juHavGZZmM{-SNb4>KF1louGTj>uT-uOFKTk038=wS@ugKT1oM}sMegF1*=%CC9z~q5g-{u8qIp^1%ZEDA={b%VN zmyFC`pAFZPm6V3(=fCsFcay2irUDBZQ7UJvK15tNDb5sXbwf->onQ%QTd&!tu|&dy z%s%`0K9t#Gq11z1Pbe5xCh0M^C%W2@(?#l8mr6>3Dn%s2gfTBK4Emw4&WY}g09u8u zzIAl#A)iJs&$P0ZOe zD4Q6$F&`Gt2Pi94;?-0Sw(fR=Z)i-byYzxH+nrs;Zv~4ZDfZ3@=ljN+{L*1Jn~@xg zVYc(wtzHNt{Xe930njb`{YLL>j4SNO z!+xpgJWR*=1GU)tIPfh8U@Mr{W z`MB{2LQsVXUHl14W|@!_E`x*F_kfwA0<8>9eTUzh+_8I_4e_&hp>ev1DJ8cs0DpSc_|O= zUp86Rpz4Ss%GI~~D#fn5E0ct}?e-~f$B1U&tD2iy+4mswJJT>l z0HnEj>{8uN$$geR_xT=`Cr&2|4n8b`HeCxo_5#C2gvq{(UEcKEe&?~R)u)PsqZ5v! zpS;+^t~%V8dz`Rdm;kGAUFSlg&`a#H#rBmmeB2!`-5~lO>;?7&c?UaWM}0c@Y;*u7p6fj(u;7?W@nQh{VjH!$dlyM)%<5xw z*Z{n+diaFoEE(4bmFu}yRS&zyI|_laZaLW~G%|`T@F4sb)z>z6= zJlUZZWRk>guk!b-=~RXIr1`B1G<- zUuCGqg2r_8-8q=3do)7irgMudug=R^p6LKLrEIpKgtr{J=<04SRd}fUfa9d`NvU`f zH)%F|O79}Hu)1_;gPz0zi$yt2W1(%>r(m(-S6T8lHW0bIgp~YB&R{KeSuWkVk|r<`fbHGfsG zY)j7XoVVyZ_=_#HxX(M4FxV{(7FbOA zP)kro;9)mc78gyyzv!tDPAM;b&*)IpGhA=V1^#3#sMagmLvC6vd6%fTPMt9kS9vXA zB>7WPU0wCkB#z<3nf>!n(DniPZhY8ST5<*8E3p^u2SXSVYLUOJ5kG6EHB__VpHdN{ zSo;-%l-*v_EOQ8ZA>zxmjp(!uee;&jn)!#*)}B;=-TG_n4XbBDcdLlMV!#XiH8%cb z*IEO&i%}#l>qe#NDS65A;wubJuAiKN8Kc0xM$elu9kMF6VI2Z6%=7WUI~=ECTf0g<0N)K##gX5T>!~HoF&w5{T3*EG zQ(}yrL&ePM?RsAJv)e-t#-a zq$(3*>mrumvbT|2RnDm4=LSa2a#T0-8)aqbRS8G?@4^T@^DDollcm%g?S?uecrvACIV?gRPe zx|NtG7KLw49c2dX>g+8?T=W_?+VUhiwte{H=oTBCA!XXs5&N5m?4z>N@~q*qShA(o z5Q)vm{*N2Wc^nARi+o0W7o#j56=UFp+=4`w_ut#rw|MF=D}_-zUZ0LoGG}){!;C~b zhObL8J4toyX{U!JNaO`CEzZn`nQ6=eyz^rx;y!e$ELID|RENy#dBlu$Px7|yF}_SUB}P+|#i``?p+Yu0 zbuop$KUsJbuBx2AqN$XZcrLjn=e9U7(XOCTS7`_oWQS{rsTN zJ?YAenGAu-$|5=rM3br?$T5I=!86}+Kk-uGhvXd4`5X!2vPOs_nAh(pjY1n1ke{?& zk!%_Z`g|&*An#3-#Vmo$7xtJ;NI$S|#fc&z+v>9nzjQTiEQiaz;I~MEWC7tCL2qVE zQ?Is-hx*WIjpRbJM*_0DrG=}#2#Mjd*KsmF&P=D|k59P-$%K_cW0TVsnWw{6c7-S& z$^Vm9?!Bghx?$nfubZO7XK#zNhRgEQt4;R8@bPisZgP6jsdZ_*fVomzIbPn##joVB ziB4Zij;hJ3q8Ia3S3cdSo{*&CUgtibB#nSjMH#<=-RygKd7`C_vjmH%<%9gYHKXi5 zMDbpvx3A?T9gi+>8XE*~lhloV*k9TzDVqC89Z>!*dd?@9UMU1{QEsa*kuS~3ag;G? zH{^fC5eNLG#Q!DGGB{X$=J2v^$i`Oa>uW9?ig?%2`59k3y^gUaag8rd61x8Qx*F^S z7k^W2FJ0>?Fo|Eq?!D>baY;x|fT)J@1LESgDHRLZVhXKkD!yG)ONF^etoVsoyl6Dz z#MMOo&Pwem>~)-&A?&FqEGfUv8)E(1!6LtGh^~!@eGhvEx%TZJ6vm>L=pYKqBmmAV z9ezGNgWv37@1JyS-0b(yUp`4`S0)g}J(UI&VP=Z65}mGgFFbR7pK!fB>Pg%u5lD;Dv@@g@&>^wWRlb7^Y9N|ejFlCt=%quz86=W8Fk40KTXK6LzbQkqpo23(<4m5^g^zuiy#0v~CILvJlkvJLOBo^@rJ zzIl@d&t$e1DkU$(J z8O>gY7IwXq6Jl;od|TDCKVJ-uPVlM~NZ$Sur$|G_2teb%g=0Q@vF<0eE;h>nrT94) zIHWC^1v>wHJWfa-GC$>wq&hAx*gl&|h$VCm!73J@|cg0 zg}7qonTD@*`(DwgoR!zK(sRHE`yZ>_T;PF18!jWw8xBe|ALVqcsr@e1?iOL~OkQh` zz~d}x!i4kS8HgnIQq=A$<10Z_;2y4MeL2_!HNMlDsRs- z&qa@=I>TfJ((WTTlDXKYXrJbg+%V}d{2}~5o%mQr@i^+MH;3y5f4akBhc3oNt&ODY zKkQQ0WU9wrYE8{4nb^BG;04r6^WRS|9VemFtYtglQwo@v`yI}eb362DE73`?cl>cx zd_cC8k=|KYzN9r4@qnm2Ltj5?8gNolc(c4$r?WfGQ&tZ3SaHd9oHD0!g6fOv37syy zRwlpbNs{StcPpk+rM3#Z)>jjquyEE!dU_`(!M8&&F?}fEM_c)dEU6W%7Gc0&F;dWr z&ZSGyxT&mgw}0T1hj!2A-VWfi(;s`a!`e3-1{qdM8gHClToM}I{M(RqAJuyC&|!?E z^B{(-#U6B<_BSI7rl#$TS(~q$?JS3P^esX5N=V>(?h&loppV@w;bl#VmC%_^>3Yz(2`5W-ocX5XtUm}0=M2IncKw33$9i>hwfKhk6@F`q z$@u%R)g`Dl2Cx5U>TJDonNcZ#o>IYx(bh&v>q>N zf{vWpXP<82>C|&Hjo&r$8C+w4S?cGtD~Is**8Ar#zDL#0b`+srWs9k}Pa>A{Dt*^b zPF~Ect#M+2qT(Ues5gozB?vt&aeR^KlNl65nY7RvX@@uP&bMMF@JT=|>;HUl9EQBN z4zk56ND#E-x)kY+Zs0KMsd(%6Xwb>E1Mv95LdsSG(H!PX+|}2JHz#VPiWZn)wO!=1r6ElHPoVT5 zai+3ZbmB)e%xgGJV8;2ShAI-vMUz~V36i=#UZ9tYE)#x-9Ctv1o2hA*JB<5e<&N-s zsvf_@_t|;c+YG0OE#7m}0oVVd$#;gJHvp}=fSdM7qjv74gL8Agc;*>XyFZsQ9&0I2 z&lHSQyEvow#=BRk&0789*H^cz%U|7>Z%axD#Zaj`wCT*N)H#U*GzExg~lA_OQ>w z*vF$wmand9H`Wo(!p$bydR|#)-|~u7_>vA;d$yt|F0IB02M%R;;Z5gCJ_n;h5NT^w2Fx@kd(=;P*#btXj6{Y^I# zHtO6Y_e?eZ5@;5c*7!Zp|8IV-LlYNv@#!o#ILbU)uiC6>rP8ogPwUX9w_n7&LkZS> zWd{Q++b-0>UoA9SMwU%@?8M1+u7wk%VNwxHP2eK6)+VEbM$o;1|V-qFqDqQcV zO*Zuz>SLtNpN9^v`yE)tl6~l1QqkBtRM(`4*y^R-VR=Neh2RdjQ;Fp~|6K5wwUhyx zFdG%Leuy~aZz;>Y_SCgBuif~GqFapmHD(c{NBo_-Kgs3VD6em<^EzU@wVSqwz$2Kf zYBY|!D%1Bw5V##D7e(oP(quj2?1_Y2zBZZI{1ntC=f91@-~QS7osGE;SqY_{?JqG0 zhBqfcno%5b?RU6NN922h7kE?OL($W5XD2}fSUqVQB>&QW7^MGhzx+cMvhEjwO?Dps0X8RQL@H>}zM+0&$VurZkRe1u~zJVJ$xP8b)|2eK(BcPU5l+$+lNkIaiCRw zrOOvlcwUU{hcpK(nv`PYB32{!0&2Hb?|Z@YR6M}1^hB3-b=_$qb5DciW@xG9O6u2o z-&#&lOPcm<8BYT`q}c4-V{!Xq5+$IJnd#U^>fBMxM9J*6*_S*K9Qsv$Z?{uCdg9*! z<-r6XWKKre(b~68?6<01=tOG4Im5|ASBEbSEmxYMHLkNPgC!aA+=aDi%JxZ9Jj%vEGbo$M$GHzT zhz7ov-BgENOSv>U@d8B#0tD5aBSk}kP=GR?ETy^?{nJ4(6_ z&n@^&Lf*|T`J*Ms;!0H45XF`54_Mh~w>tXb9K*p)^>~*uIdd2;Ad^?GKL;?k6zWS=}z{$aiGL$lzag+Nc-N~Hi zP~Q%QhE<-UN4-!SVlYv3J2o#Z^F+@C3ALX3q6| zESoPya_g98yuSya0xhX36~4>Wr1&7{CJg0?#0BlFU&e>KRC^R*bQ#NB0iTjbHSovAX(3QPMWCK9`~8F zhBU}vMfk z(3|NdebTKNw70>R`A`)NT?41a4Gel1EdpCs=74sPKzm=l$X#W3P^z~sVdB^z&MVhX z@o~U9_I@Pb+&|xe7WMQ*>ZAnBh)8B4o@amA$7*a$GY%FZ7#DeH^1MvDVK#po-JK9z zw{6M}&h)*f1hNT@m}V`#Bts*41uEZYM&2CaC(J%cB*S-Ut)ACQ3AO;Y9o_W9%{AC% zGB+{Fu3Y-oJXbfp_!B11mR}1kc|aSSs4+6Et$^TC%-IzPadna8`eL#fwvCbW-~Hy# z?1>Z=|7XXU)CGw1a?9SGap~OU5^mB-_EEF|nMqw!kwN0()RyQ9-Z~sQaKiX}U3zOI zanY_f=F}xUCefX3YuAVc+Ch8WD&D&*QflwsIQY&Xt-m`9dszuo!d|JEi^#sN!_Dv8 z!{ftQg%Zg~)7Z1h9~;>6KhsmEuw(R#OVoVMK9K+z>&{tx`6wSOI9V_!NS9kCe@G5- zBd<~@l~HGkSH3E4k_GCCC=EJk#&3_c_D5NDv5OII;>P}msEez2G)Py-P?Bs-3VX)g zm5TQ72+Xj(uG27D0TS!g9S;dk|0$R&*+@hbTWyiP2igAC%i7?3_Sspxq~Cx3CX2MRf5L8XIPhmGK(TO#?6& zQ}!U-Yp-@=vUu!7swyt8m^l>YG~1Wke6Cm?%_pp_ou{lpJwf!Cp%@R*?pxugQ*Ktj zJ3F%HqMgjFuZZQ_ow$v~W-g17x!x4PuA8b}KaaBnT zj7;adz9DsU)zX#&ohNS!RaAN6Vdk4vQTFKLnx}%rufveR_4NRp>l+290=XZIW6^SE zHPGzncu~>VHcQLH9VJo3sZ96757m|PUS%<_NX}SP(m&6F!4951En#?wk4zZ`n^YJ( zc8*QGQKRNAB6queoOdV|U&~XXY=V4O9hxmntyh~U*SWBI%+t5JzB5l}`aM|eRb=dj z^e3CTciQx$z1V`p^8C+>#x-%M|9ZBwUL~DTtTk%pEAyE}4kol{|Hy3zxW_=?*Ef5? zoHFwsuEVXBlCzDHp3Z}tPW7>VpI=_+KCAuuX)mQ6T{j4!X@bMmKd^NXPbD-o^n)l85vA}rntb<9HS(bCb{cWpCZo9vz5wjaXblzB z_7&*~y8SZaXQ5P=KCX_o(^hmUc$<_NuQ_z|UBSq!mfFjZpBXxAb(d)F(zi`zooiaW zd8W4alG0W_6_(N`2n={-j|j4Nm0pYfR^jdOM)6a*eC$kN(a~eI03i?)o*XsE1`MFW{(&QYK7++v+lB@8e+8V3@SoHaT`_Q_?TtZn55rllo3=1)B84gyH;ZN}2V#YPk<$#-*7 zvwgzGe(xfQRmn{k>c#cWY~Y5efQpLJ2xUHg#9kJ-_o-Vb5n@w13~3ple{l)~>GoH#2U0O#odF zA*HdODi*BoXZx<+2kp_dg*5@*Mcz>hJeUaSwRk-wnqvr@CSEtRPkM*Tj18pEK_hs7 z0AK3Dmh?HNvfSG-=+Nfdc0W-AbmaZU&~9A+jm4`OU7llaS}8jd(3lTuQzAoM1)VVgKp_9ZC_21s#(X? zc2QU{>HImXnqgrzs<*14;RSqoB9{Fj24Pc4IBT{F<~L-xJ!^AKHi zZu7{MD;)1{g-z+(fJ&x?#ZgLp6Bo*%Y?rcc|M(78H($%d9fL%fMmBP}P$Yg!_CfV= zUsAP&HL~sstB*hnCIAW*Uty(vb^{)3=5a1TsR~n4RZZ0mvsQ9!H7#^W2~uVzW6x^) z9?Ix+*pf$He&>$IvWJ;H5Hn;_+~&KX2Kd&m5Pq?vn>g@56caJ+u}!5dkH|4=mZyLR z=R)?Bn)72--!;Sc`l=1<4OkNNgqdC_?v!pAnCRTqyp0fTr~iXQ~z8iKgHc)2$evZ^DAax^BBfGnDtTi zbWh{?6i|-_4Lm1C3=(NwTsIdV?cUm3S$Jkyl{`gKPYs|s4MypIRn>kzGWR5D7$!gx zEAX&chH)~OHJj#tC4lNil^eqyeR z_X|6*&izjSW_I9md8mnpL0wD}}3FG1Di|KFX`< zR2=J$8$1SysP*ar5(n!DvZ#|p2oFIq2oR4FFF$>`GcH42$RpBH-Tu+LS@$sxg_xb* zIft%K;+L>p)Oab|ApCes;Nf`tP??3_P=iuCt#sis4}Y{%?h<75 zEqmx-w?gdJw#@j@Ti?LFkqN9fLuP%f4CZX+w9<23KCZnKJIJWEh|a@J8Cjk)>$OfX z;{79(^kR^FAg|pVDOrD;ucn&QdJpAQ&jS;opD6b>)fxH}IIMNmAL+5&3B2neM*~q? z!}8dS;Lv}RsHh}(W4`BOxK^05T!A+#R)S4Ch{pxApLX^g>e(QUG`o7SCnXb#^aNCY zMsKQOB9zTnu455m62hl=V95IT|BfAehXE3^+H)jsAN#MBEHUS3qrS|T&-dv!qxq?30~|R1X!)0+tzdE zx3`r4^lBy6-Cb6Z5XN`b(VJx`qe)(rM`Io*Eo|PNu_@X}Tv);+FdvjyLS!|?@ow?l4zag^M#HB2NCl8w41U;JVyE+cP2knF!Q7uX5kL>_4#PZ%aK zMn32$@7tQ7O>1W%yexd2)F&nSG;t^6aoG$c2e-Yd0dzIJWkm~o`ND1QE3w<2*?$i` zIw}X|-UlFrv8|@>ECR=pzXH-MDGEU!5VQ6e>l+`f1vh_~y~jk|qX*%Q@f(|0{MX@> zSXXjJULD=^s8*pkPElC)^>KbB&+bKlo9Kj#ScZz$+L0(+ChNXWGF0>)x+3rq*e)IM5e=nN@r`p z!+ZIQDPY$owmSpe+5GY-qy zUywEuOd4$$@X$#AQ!jAl7oAOxJ>?vZ3l}z=o#*F~K+OOd(o+it&YSF_BI{kMZ9{tT z4^W@4`5MV)Ipa9DX6^4rWi|AU_1^9>Fh7r$CLCx*&uj*R47M-Kp{{iIAnTDK?lOJE)q@IN0Q ztbB-o1h3A|f>RQk&?RL>YZ;J{efhx7rQt}!%U73ThsS0?YdU6i|J5aJ;i?9g)b0ul zfgu9e(vltQe>#t2u!U@6NsbsVFC!5V5ffH0Ri#L)lMm((a`6hmq0|btgj7dw?Vq1&Z?!)MB&)&l*be_mdm>*;zI z+1z;a!)j)0yOUzhcR3>!Sy@@jg>zHxZDI8U1G+e`Sva6rW;VN6JIKnDKZbILC%+uQ z{7adPl5*y}h1&Z#hTOZ|u;Klc>}U^COZk~v?0eI9lpe!hS``T*m&kE2`FhTt)pw_z)$CL)($oIoS%%0%Tv0gw1?8--3&tcbDMNhfaN~5_m1T*=l_rOrU z&Aj-(n-S2}BoJU2eX%?-@n!MRv&Nwo?8(4n3e`(O+V=;LPH_fgK4xuWeY?1~Ndpdt zF%sToyt#ELZn$K$s1C6zvUO$oBy!=crk;mu$+N+^ZOHX>wl1xbR{+$<9kde^4f&BY z3%4OpQey)%m3nHAjlB6L@=Z0p9}^Wtu3Rh0Nz6H`%%}|+Q=BsWDGdSkhU%^CvW~Nn z%jO>@2{0q+E5Cr-Z^N2ws|2Yoj-}99f0h4=oNZ%all=!OTMW#<=EHMWSw*4TphYxh zf?I>r6~iWrj$IQH63&tHu_0u@<;Z+Ptmd{`{qg%s4F8rSh)OAah^vb2sKxdhY%1mH z_|T`}Jy5XS=kpbAM2+kc8(Gm^0b{wRv};DPAE2J!cv45b?gRe&G>1nHr43H?sW#xY zAg4`7iiQkWQTw*%3z*XESgdlBQ2qnD9M zhoGJNw@_oTQO++N1zd3N0{f_q+d)l+s`9HStq#iCt{gnYin8^UOMoT{KCE{0i=OlP+unu%BXh?SxVOZ`|eW<{jru$ah+rHZ^3QF4p@n%!b@+Z zpr#fR1j>2H^Pv{ z6Q@L|9W0f0b7&_Bbj)fTXJ)B$4=GRR&+<0-f%fRBWmug^(oUjQ&R;MsOUQfFRPV~7 zpPzFB+NhZ=QPfA{IxKHa5*GB4U$j=QI!9} zGoFCX{W-mmS-q2E3r}zO5!hKu4)$6Dt@|%Dx&-^tEs7#&&(d4zKaT6N)l@YdSg*oo z2Zb#xbTYZa{CIr`35~l&x!F9wvGL`8`@hdyClI^w<1K&1{PO&UeVXY1Ag3FxJxgO` z;`O6at1HNm+}#psO2m0cE9k&UoV{Skl`P|6!0EB&8wdT@@Rpjy2Nu~?r2l08#zgIH zEUS?)b9p^W-e2rb(X7STA;eJu85v)vY9cd!*|+?8b@%Y?uiyE~L;G9AgRl5oZA1Tp z*P13x2ArY^oA3_V`3&^^m&XO$p^N#i_x!JMg5X8N}Wo!tzgvX4T|#o;aWA9oD*3A>5Ar;2VOzd zb%4Q_<=>)l3|%4i%^NGOvvz+ojn){^D2!P{zAjmKPup2ek2)ndc(v)*uiUXW9eaqg z@poJs)_|_waIL&k%xN@8y-I&A?6b!5xh3b`{aCJkbuLvIPnN+Q5fW+aeCqx1G{zASfN>C6W+!V}zkSj<{KHH=vX9#;<`t z1IQz|P_$vfr7zd1nj6Ka_W(7PKqJ(f?Hu+%ZOot3BNsC_01z^tt{RzI8Y_72EoMdK zvH&%!?o9bPV1>ec-l4diU5W{dF74hDprlV(`>nWZtx&gixahO#`7hc8cB?cdl z|KayDqw-A}m~arP*c3ENCOyZ{oD2WGq&y0@7=Wk`yqO_ozuNzb`(*H$!8jRIVn*|G zfBT1en6q6mCrqom;!Cnw)7ua;e!uqOLMZt+7U*oKv)yOu=q)%v#8)%RKrE^9?-n7h zGS%gQ&uWQo7LL>yY+A_VkNG<@f?Q?{28_4HgFT&>p%N7Z2y(EB6(M5#IrVoDbNBdA zYi7Z6T%Kph_R8SLAz92R7e;)aXZ%M^zY9?za(c{@@>}PBGFiJ0mPVyTfCiWlRX@(E z$q;Lil13T|iiAA5*s=X_;c>aUKy7?dX@J(9ACh?$7oF85q81r~{9mKqRwDNzLqXQC zxYAgc3UrB7PyGviBFVijODj$%sy!4-f}d{GU|wx8yIv_vc87c#NgrQdhsRT)^Dd$> z)9i|}h%%~ne?lAjO&z(m(N^O|%uHf1q%N^5>0h0VHrj42RfPV}$iWu@B#MTq>Q4Lx zleUVr2??^y9K7#8f0b7D$%*($S8q!X$#U=4slvOO zd~Y0^kWp(ps(&w4cRHS6e_6gg?R{gou^Q5@27E z_#7OKXAszA;yF#Y=R?OzAOLwstSH3y(o@UoC5K3xE0ZxDzGRPxSvuE=?$FP)W}bN- zD?F=B-k1yCi@SxU5rLbgem74e_Sc@*0b@j2+$vx0KwAfHfB#_G87<6hmB4=~Y1?^@ zL%T2|Y}}6V=imxXS)vztk^Llf91b5EVqbgYYwgLsGAwc}i(q_TVlP!Y8L#(7Rm?%- zcyAl`SFe}5sL8`BwZIyZF@@YrSD(e&;r&#diTSOS9xYr9l)> zVhRewdD=9`S(l{OVE`JDNX_?IfV_QTcb<29nAGe=O!aU*A5P6`6fb2MiOgBRRd(h~vde zgp=NgA4wNS*X!pi;0f)YB4As3?`t6Y-_;`S2p+St&t3i^E?xvc>&P2)AQNP5q7ih_ z9b_7~GZCm0h!YIb6KPoIp0PR>KNwdPk zsn#BuX=Sau@v*V}Sx>^9(clCBI(aq(96215(ic%&Zu10A)TF#1F zk_hK`j*xKTjaEi}dh6jP$GQgh==+;pvg)_-Y^7|UbxGV5J=YU@_~Fd1!3rJB2{iY+ zZ`3o!w&49*1OYo=!HWA=B3J?`8wvwxJuGl;0K#4~4CK$n#&`39l_vXUs{{^M4Q_*g zBmo?B@Ewm^S?h`IJ&ytQ<;2!k0y#uh82daXwsxiXjO2GuBv6uqW9g(h5q`Pl2#M8< zcjf{i^~<+duvF__;C?=1595iCAzxGr4-NirjlMl26qj1#kSCJ2Y%Q)N3vMC=)TrPY z=qD9BK!Zo$%ARDtZ7@hl{PA4YZT&N+lKbHNoc`eqFU7R zIsHTrh|$RaUzfX5S{i7kAvLNZf_=E!PKT~FLNV=tf6sMIhIk3#RC+}UV=PGSu3cB}Jx9k7PhLQwxpuZI6#8CU=%d}0(7wyU) zAwg#&xV?4;4u=}AEdvRq`xxc3Kk&?ndD}EIyVI-f!OQ-E>Gr(cf;96az@V4;=UT=d zCK=NEH*m36rg@)Ep6qhV;;O;_Z{)F-Xw!>SVG}#`-P#s{OR`2i_ti;vT>33#GAQN zNq1Nvy8eKl!vt844ZO?6ZwP%3-d`XL$}hMJYjW`YdLSX5&CzYp-AQ6A7D#+nLGeYx zYFE*Ly5A*qV>o<^t`eWGsQMPE%=+*f!O7dT;OL*0vQ1i6T$=cqz|N~XW$Bl|JenYqFMz!W)l6j zdAGr3_BPL7_gmGa=HE{7fA0Iz@-f|&UUNa{mL6)X!1Tp(@odt=z^6|;9ST2=J{YOf zYZeHPr_eD>`u6c1SISokw?^=9gZ7%%qT4jt^k_S;IQ`|xYt>71laqFKPMrB>yt^L5 zj-ECiC3k`QrOx!-Z>hihu}}L{cf!F@Xtu}%A4WgGl~DC_W%Bhk@bT$jkh!{sMx(7~ z?CAOInv(rS0_h)hcFiU&k@^nJ2e^0Yb7~BVHV!-^lV`pFo`mfQg;5QFN~6i=r{`Ly z`h{-ti%V~}>`Z@rEYotT!#5vg9fLJ&#@#qiAn)K^-0$X=(Tn%E%}h8EqwM&ad@Wy) z?kou3IA1NdA1~)J-#oXub&G$4_{{V+NG;@mxnI6Z(PVyqX!@r;9LPu-=@NbyuEH4t zqj~bk*D_{?lLY=lTfvazUB4m*MJYRQw(!mp%)|XIyNi?7QJ!c z`p&0(_asxzZ#cfBa5VXyoRR%WVu)8bx8#X?uvEfPj8sC+7sUsIWr$(xfV1lI03D^H zLWq&^xX`$6;QNJ)gQpv5|4|=QFU8KOUgMH#^@v6awS6wMl9md56c3sfio|0R@+f^! zs6TV!9yXD&@!KE&L8D+Xfo;69Y z-z@(j`7*(ewcKimzwex(Q){-QtJav3tX^F}arANfWjn}&&Q&5fal=`_kQRcHL@Mvy z>oB2%)Y|93x@g^Lbx;zqZnT42WsL(=%1Th;9}%@66W6OAzfEe*y+quez6W??3=GOl9Zsp&qiAC%SuOmZ1Jo+% z)6?85*y-QM)2RxpFc|g2UMtN5dOgy1)_S(4QSu~`P+~o}w2DZVl!pTcviuG6(x2?( z&pETebpM|O$9|ozqcXF`?ag=OUn#}mLJsK~v91fZ%#L$pmliVaLwr}k7GB=}Bvf00 z-56AGc??cugZ+xD99zU&QGP4ggP(PZ06YV?+QN_3p;e;L8{%+i;HYKxSM)_4fNG5l zIk6)B8egTGhzH(y-Bs4@X_=~ zOg?|Ib>s>v#LVN(pq9rQd<^h_%2;y~=R}p9Cb$Nb`9&4a-y;ualPeE85w?Tz_BPmf zds|@a-1jG}u|UrK)iwX1nimPCjbl=E61Rr$ljX3l%msa|hZ=kuYIb~8Zg?~;smAyZ zJxSl-#{*#t=aA0ewFR$OA5Dpxi6yVUcuc-s9vUD&=BN2V=0v(XyqmpC_C05%*rTb< zYo~Z1b>-EkXDks4YdxydP0L)^gt1~n6NQz}lsR|VTG=WXot(>?PL^WqYd(M@&-7Vr zlPF1;9)`)OBCTGW8%$vdXl%7T6<~6qlKrsd1z6yHN_`fA;sO`}!Kf>_xFzVx>;= zmFQugDrGY6FWM#po+?cMR8o!U z)0(Pxj#TXe~|C>zH6Z5OHi9Y3v$g5{c7mExWMpQiu5wa`n z8=qCA$v_lhpb#i_QHsnx3LN83o49`Vg<~JO=VoiN1HVAQQ6}?e1_wjuGFPkf>Gm`c z67Px6T&JsghF`zR+aloST}_<9ATY~cFC5>4$<#gxFMczY_4p7jsw|fDPchf0fjKSN zmlqpYeKs>vGnifg+9OnJCOlyplxVtZg{j^h z+^|T)vV^|!20dwu=9fkO&rL>HPwMDTCAr9~%-1B3h8{mLZty-nG|<+t(J`8$i&Lzj zOVv}63A|npNOZSvhZYtV0q6*ekt6p=h5e4?2DCz!y>cfbt%+Qgt%S1YrMlfD zJDDTi!f#AgKTOwLC{~Pj2>Asz@jZ%S1V4OF#l#DISwvC^w%rV<;AK)EKObHIfYq?$ z3+N)%Ja)TVX86=sJF2KNN2CJp3&8+HkU z-1`dw(KUO$4zl#Q86inThi|Kp*_s_9vPxoN3~r?c2;2T#mi<3hQY2CKy1ZeqwS$oQ zbrWwRQX>v3zC1o@_-rGq>?JAj&bz9u(+_JrTLN+t*!_p?65c*|TIDFe%^$<#h-A^p zwSPRXUR2xdJY9PxGYv$AOf!&I(J5=O8UU4k{&}Kg)b+w+p?rIMR6Rfc?gHM7;)p)m16z~+XYIGusi~@jKh>gE&IY1Y zYEsNS_W3;s)iS6RlUjCIXH79jSxcLO!rP$VJJHbgmQZ2C`nR?XSF5DzoLviB=6E>? zB5?O?g*t1j=CEbHTGC<$#h=G{32@x~lKqDEp{~z@^jPVdP^)K^k68#jtLURLAERKZ zMnKYDVvfm}+C+T5{Md6tjfFQ}{K0wrjvX65>yw7k@N*U$@@(}C)cenk2LtkPv3Uwn z4|jVgHDmA~TNTbFvus_MXY9`zyN?y5}UXSXZ0n%IS*m^;Da*EP!z?N_bKL5ZgOvM~^7ZRyJ)p~DZ zO+|E7e2d{M0g!3n_&)jdDz$mkcz%|vq<29hrxd&kS!b|D-LEtzLkHPA8y>ej=;@Vk z;&(!dhCr#t3}aVK(*g8Oe(RqI6KrI(+|&-lKK;^z3@6r(jfPQp9Iq1vR< zA6oC+==%v~da=hqs6Y~2T+?cscKqT-edo-|CBo<#L}O*55Vgo7;hG(0*ZatOCn%Cz zGc)KjX&~SgeFqyj(YT_hd;)?^;9L5Z%8^S?H+HsC)V1giy!D@kSJ5562!4NliM)7W73ood6A$`T}-QP`& z(YOY?FK{2XKddU^&bH2$ncxn+&I>OWTbf$NusewP_P|Yd!76vSr`Bnnw8{(ei`l>c zO}q^-H!4`J@QHP%wWd;|l1FLNrwYyiv_;!9q88q;-~m(MDTUIqN~EP*m5BSKbMy;E z@ePYH_+N+{rVI8np!leiR=GCPWyrc7bkz6-s@_VZADqG@zWQ-QRGD+y%|Nd!{+y-q z4vpRiug*SRhO_4^4gb9227t*(`{qUsF&s~Rs<@v9>ob{6%oeMm#r&Qdy1}b>EN;QU zL2x>h+x87)Tz}vM6Un5hJkCEIS|iA{(S#fYIsedcVp@T&S2fy*TOg9cJcd|xs|*(xH_Z^~-js;X&kiALPRvx7=7cFL^L zr)L?WDA4)vPwWr%CM$K*`WH3D%=*wdcQYz69@!XrxUA>Nh%!N+uT%^K#wh_g?mf8| zJ-M%pYZ_nsKlJ?~WWxn=&IMYd*jLu_F5_&Snk%z%HTe&T&K7r^5i1|=W7QQRWQm^{ z21PE$yoLVykqs9I^YKuA7mZM|A=-&BXUv9f`(yaHe~3>RsMjYXI4idEk>{~AC4~1g z1f8_k2&XkzI;)W^t!S92=q;#{^011P+(}bW%=V`u8z5(c#j0{gDzW^$6<)JF@7Uf&cex2PYj0Qh(p zI=}TVk@pfJpfd&~I~#k=6je8DVHOpjR9X3F$c{trz1?X`!&%Uv?`iEgAU=Bi5G{EtE!Fj#+LKO z<2K4N=lP}Wy7|-W`5!xZY|(EoR4EpYBHN-ub}XPPii^iHuC>6d_HRktbB@LNuC#5mi4IY>z73Yl}nb~n1^iB1+{|G<*5zl-o>eARD_1u35lGEgNPLlloby5#W>RxVogY=A*I0j~c536>)+vxdVY=~VSgVL*EF;v%^Ic-pCo&UYnR5R|+#!(CGy z&3)s{E5LC9rfK5%1bq#iW1VL$@#vn*)LZ+gY2>#>K9=bTG{b+YZZH_3jzQA0MKDE( zw!ZrHc_m9yp>uL|;o=G@N7gMrt_P~xI8Xj_ z8eCC%bP1{;TAxjgnT3!gb9q#^Ndn&4#S0FuS$`<^P-V!jih1QZ14-o*$|<6r;E4si z{~$hHp;vtZ`0uq%voA5ic|F^1um07Wf!!8Ef9xHdH@{wf`)cjRdn)UY@-T5w@F2~C z4Yhl0qHmMqfrGE_(=|mh9dP2Em?#55`wp&m@Aqj7e@pM(%ZMy1xdZ*8Cy1=u%UY%+ ztFaY_@C;=H1s`G_v!GprJuilWO#>}0mBbmgsv9`v{88idI3@_YO;XMU<4V8&1QuAD zOJ*aPId2Bz*$(1m`!62(On>r-OVP@wAi80gYpE za{DqQ3>JYlpBdOhb1yK0D>N1-p{0>m7r9sOwzzY!O?dN-vFa^hSAzM$ZFI5IG%L2g zdwd4pazzk0(6TuD`mJdOnKggKtnA>j6O_CjQ*+7nS#*eUC{>Pe=RXI3K&Hy1hr(xo z>O7K5*K}~t>F1dUZ0i1Ke_^X->m+LiMct!)y#y{B zFMDE<^gftZIiLGh+RYgPsond7#Mqmm8@DiAzI>EC9CP=1!Am^bdqkegT2isn7% zMuLV}f|@TCt0pOl?N_0lMO>k%SI1BCVIs3c?2d|(Xrwp{lA9OtHtY)8ZWZ? zVD^{*KtVgMUt+NL!-`P9HPp@W*_QI_AilebH~3Tv8OT--29@pIWzf*DOW{}XjHUMQ zfc*6cj<+AJTm}I^ZeNUOGY=3MvVHlzqBkdZ;-X3h{i{l9PkIDhdu-Eq-}=kgPz#M# z)z1HFAC-j{trXska#O1@N7Y7j&WYs7%*mA|bnEsxf$DUPJFENQiLn!GkC;rX!AVq% zX3#yRs7Ri$g&{+E4}M%Mg6yn-&ziJuDRf?}oEv|?OK-zu1^g8um_S=yFweNGN#hHfuF zRUq43aQCc2AO>>idHVW}JyZ?sc--D5))Y#BZk98iGQMWCbq-%K)j%%oy|uk~cv!J} z0=2)iTO)N>kl3Z3JvZf5wpHm+rGHKz;=MT_A@n}HYl&(k*>dN*utcKa!vgWG?G7`as!*9ZP&Js zHC!*)ySc8vi8P+e5glnEE^pit>1zy}lVjm@A&t@>kg?gxEBE9Ysq+8$+R>vI8Fy;Z z`kanjq4k9JaJPK+^@$`l6R_M(VCd;~DlO%eXg) zFvNH=409A8mn~GO7lU5DJu~6~k6#~DtTeSJ@r5cDO5_utCFVjy(-?s0+vKz=qz^23 ztu>sw>yO?sGH=pUiOCX*UL-^e#9HJS!Fk-M@G>v~FL`Pt5;07XD-c zwKHSD_VfrT{|={%!>u#JdnE>yTn3$=kh3*lzE}>+Q-|-XFS7YK^S4{ta(s8S=33{L z4ErQpgX~ThNzv+I+ungt93H;l-M6H_@y1>J)#Kl{b-TFmXdZ=g#R+GH23&stO9!(l z$l<$%G<}@{WSN+8&DSQZFP#Pr%W-C3YI-XY<~Kb}XAD`x23v%z{cb)u1b&vZvXP|D zBk#$=P96$Ii6D0?ZDgIog1AzvBYwcknj>WY#FZteC1^8_C1^5E8Z|v>&s=`bfyo36-u`^Yf5yYU9hxiqcdGDu6vw60}WII~~aHa&DLW?nRp8*1sp9wdae#aBRTl zaY?d|{YBe{!s&c}U^kE$=J>+uy?Nm2jp3qNn+9oH`w`;UNtd0461mDdu-x2ltT2J z>~@S4kz$>F6OnFu{lIA0MS}b6d4nqZI7m~0geTi?CXq!{4kd4+3es!tL*!K5=m6u_ zJ0-BFFzuziyVeBW+%1|nT4?4$x%Ee{i!++*8m|9wtX^paE1`ePUrO&4I0jmNY4Yca zd=-vd&f#O?!R}SAx{`MD52@>>Kg=K2J>=SX@56PA>pf5Y?Lo|uG-SD0n66p6%tLe8 zKGv6e#gq2mtiIOMsN{f(rN5KT!uGt#mFDy2x9A(erA`4UfraDW3SQmKbdR_fD;oP! z#v_E|=k3+rCxVDBQ}J{##pKg))e|(n*|4kucb5;VsTIztiY6yPzz;%@%gbM~{JNC+ z+&zAR@@wqY**k7POg-0}CKQ-wC0G-3uTuPmZ@$Y)RyW5?0_5(a_jNTa#;&6~dDlv; zP)qk@&{LTO9*J$4Z=7!Z4uk4YYO3)lo)%Sa_8V)UO`99dZE>$SX0C>(>t3`?1eLl} zi=gLAHh4HR0{lyBrN_@C&~HRFpD!8BfaQqyF@ki4D3var<)YdorR~I8(O4%~X++a< zT|~}PcxuLFjoUZ#!tcvcm!N+Iw+D=ZeAofEToc`E^`VD{`9DYYS||61bzzI0tlP1T z08K7)t>Z7%s?mElX|B<`53RUCd#g+VJNrY++c782eAa4%VO#l){enB3l|p^~%xlYD z!SbX?xGi=caD23pm>2)%PxIYQ@8D%*3oKu%SJO*Hx5*&c zuE}a}<#i7(ao(%y>^rCD!}yg#gQMzZi?1p&D5)qZ=q;;YaB&F00N7PnTj!QS;=+NQ2V8QJb^YD91M| z|Ee)t!y_bcNC{V_&J#?!r-w`7;fs9J1^?SpZ+6TyHsdn2b#;387AkBCA0qiYSiVqH z+%$vseNW60>b3D2R@m`s4N`1D?!mKHUy9T5Rha3FvD%?$Fm&wv|N@CPJ`>Dy?)3A~H+5zss#Js~^XSJ3Iwa zhYaiE1U!N#9r-C zsH@X+IGUgR5;gy^vc^Oy?JVX2s#QJD4pwwlOAJvQK<07(N6GN89Wea@oVcPNFP&bD zebzzWF|vH4imKnIqn5$vb{1P@$J$CINL?dkZwHEd3jwk$<;Of`Jd98&qRMaXv{n>6 zZ=eDotW0RKwCP;Tz=7?PCT9@IAQ`@_&zyj^J&p73>n+Cgog@i{$FKQz5-Qev>y&2<_vXz4Cy3(`4k> z%FoThH8(r%Dd;TRB56`Y>xu|tuzjy_$|5U4!}nld{Fu+;Z1Qnu8Roaw)qgx*e~(9{ zH_yD-*U4+)?iBmu1N;+8^tP^_?5VqMU+G+NPT6BH@5tquuc^V+=8|VUiR8#EWfo(> zTvTM{TyB*SUf~N)q`{N!p}(U+zf5s)M?KUM7HY3d5-3l@e7MA6lutUE*7k8tvr{Ki zr48m3ShM&vx(hDbvoTU^Fv6@ixPq)q!prkD-B3KEuW8feXkty+;jMx5MEkQSWXU;n zK#F5STOJ3s@KYXs%lSox)1h)_#ka#E@_(Z-H}F#IAkuoquo?(r+05705roq;Uqq?e zMaFEkyf&w+*wqk#cK!ObmwCVW;_JkI5B0{*Mzv=`kiSlXM9O5Zo%*NC<*fvx^nT!- zHh#c8fILvV`lz9zB=-;U-l3XISw3+NJrubn=!V@lS01*H=53V^VnCY02UqL|JNOu! zu3Hig9vu1*R19*qmfdGX9X#!3Xv_k%i8kpSGy+@*6AM^Wd-@VrkYQA}8pNZa3xB9d zhi$jxb8#BSXW1HJun<6X@ZmB{_jz}EZ{Do)CtkS3{>hBkOv8Vs50?uz_ULRd4599b z05^JLlK@g;UkDHqu(!x3Uh+#&P_RFS_hz>v#(n+48mEH;%DVApFkza?2u`(Qh4XB+^u_A&3R0-3!fO?I+F~a0u5Z2 zCfa`=aaTU(^zfCYEX|#E+uXMe$twpQ5dN8w-<{D*TnJz82LW01Fpcw{0#HtFQ%j-S z0AwIu6@7oA>57#URW!kFpk)45_dDz+6!1@*3>R2o;}#afD9#3{K{bArZ+Q?ebUPUL zQQ&;3YvKb*&xx*n*6>99RCtoSN$H-BBSy)$Ak6%~*f!l#9$dHaUH?f5!FuSQZm6-v z(2UU$zGsDcKbiKxULBNF&`G z%iQR=(sOBZcCm9R|1m9cSXvvs@{@Dp4*1D`q?kYBbmey*{u*yHZyN|fw#Uoy;pinZ z^>KU0kp<{GeM7x*cEK-G=`vKuzlrHce11JmmuRfBvMGf)28{De&37Ki&V7tN`?y+O zRR1yAm=vD#O;V5{`UU(cV(eoRM3EPJ*~Xpl`;$mmeX?o2xV16=w5X!&m$a=n?2ymT z*^=10f=i8|K6PKT!|L|8DZIruhf^(wN1$jH`r(8eOS-8PhJ-e0F`!nu`vmK2(|3Sx z?IYiyEqN5hL%Gb#HPks$*9=5{UF5&RvZiUUmj%L2%kud|c}fw3-E2{MaqeCh=tT$3FFw;b`|F{1c@IMjR)NKa~T$T>8eU zu+30+Vh*Qvn#(pErY4eyU&kTWtWw~%uomB-EW^3N22vaFaJ<_ z{~m?=D4fe*Y8Xin9)H?1C$)AHXrf4-2^iqzu-`Z|9$a25_4WRs_XhF9#L+=+sY=)* zDc*7+>}Kqk2khBI5!l4yn`i%slCaK*b23YPciH)WPM>tJj{dz{@f4q;h zNtu`YRG`EkJ*kJs$4^|}8jF~ZiPp|}{M?3ORAKY*%R1+J=bwqN&Zp&`xah-BBwz3^ zTA0Qzf-`uxK*_|Gse>rKW8qp#PXr&AAxeN{jWyxr%&d!&_!LbY-F6b&R3 zBe6b@Q{B`AHlPqW5}tu4atm^aIlk_LeoNV!p0Z_WYwrDq48cmH-3RtF@9lf@7=Big z&~!iSk;huJHYrayj!4Ydama_iHK(WuE zl*NbYK%JXAnfzF)4i@?T-;yk;>_bK7+Cg+U7gS?ac+IHejFe~t#j2w(0-s5>RR3&! z!pkm$&{VY9Fw#E%-e2E1GxhjQZ9L)Azgava_N3{valT8QoRbsn?d{&e{}!vQ!>^Jtij_g0#+QddL8jhuACR7wrpNK4wu5if za^@uNp2v^dB2@s&~Mk_;MH3nZx%b|yJN9Dq2EQ_ z8QJpp?FISz)3=fg-z^hkiLh@(%jlzu?=k(lAsg(UzoLk}{mI9u@&g{pCk!@0+(W4a zrcs-UVg|?BRE={SL?KxBDG!*sk|CN|LD2;zK}wSVe}MLzLRx4lUS#+LdrnoFik3qb z4pl9%v!_Y-Pj67UXKNPfj2LLBeDX6P-PN}khVORvcuyNKjJN(1a{RdLS(2T}1m7Q} z-)RMTTh3+^6VhuDELy|E&CWyBk~-pK-c1_PIUey#kM^1Z zDUBTHV7+8n0uj_WMW#L}ehayJAMeKg z=&_IOe>%P?waV&6{ROYOwM}~QZgBwwwKIEc>%O{Lel4$}kOAtEM%=H(xM8E3=hN+J zn)aQxdavHqe-vpjzMEz)6!57m1^3wS$=e;4JEc?i!qS|R$4p#5dKsecZb>=UpV`$z z52W-qIa2Rb3;0nS8oO%djpd6tX5ToxR)``Ex%zgV<@iYK_plHjm}fZn<%3+AK~=%} z*}lG>Yns<~sV{*E-ajxt53EB-9|dSIC8Dc+U{O4=-On?4a(U{&4SsKzI@mw@_7jJT zr=Oi4^1d8p!zTHl!t*g`%3uT@nZL&65G^RY($&BIN~1<*c4gJ$iD3WSs+fS`;i20{ znoX}#<+xck0ebnckXLr&k-si`!+R+p&1uC`shH^D^Yez$w+_LyBQTVPP^u-W_6QqZ z#`<2I``L!%z4h->dX;w`7siagR1-$*Bw%Z~vyE3mCADA9*53;U&Uytb~<*&3+HB0|XPv{m`M zWd*BrEQpg(K1%jzmzpTNU-*>ug{UycXMZ3esAfteQN3Eq%vaMw$de>`CvUb9&u0D} z3fhnFawnk=U0?r`$CAr`cha#l)ucJFPonaTX@X8)>ZAF)T1M+CvSl=nD6v!CRP`G5 z9r_|0W)Pe{i%0e?cdk3Rl}M$0)zsV(jhh`Xxm{HQi`MgdNcxTuENWjoT0?_ddsg32 z^Wjn*9@hi>DasQ^F?X8MNW^0b_k2W?g7V7Asb9!Q_lEPQqd<&e$ho6eZ;k(i<5&iR z!I0?RUquI(dmAsY9~zpM!Vv^-&)2aI0@8+4*aj@5}S6#*zpGEB86 zLASVTIIN6UZ@WU+z^TM~$nlFIlD^Ok5w>fFwKFq$)ks0I1(+@!Pf?-L`jr|GJebTiNN!i2_z1cl1WEtmf;x_K*zU?0SLil!4EXHN>k%eM%s znatm+e`q8yl`}RX*cTaDP^+hxq2r)ySfC_T`cx{v{DtI0#IVUnf2N0@UEW&+T?KCk z?y4G1872pv#aumV#{E!Ez+a;LFyEZYlPfdR)_+MW^98(s18r^dlNr2%fNGp!fd5__ z<%;XovFplT8MFeBN9{p4n&^|7Fy*VA4{QWr*cZV)q5ZAUyhI=^s1j{>%3GJjWQ&NCSxYDr5~u!j)geo=p82v}G9zeLWzf;&G-2N}tZNQL3? z7g(xkmZ>V4kb@JYlQxFlze>qIe3?&^R^YNsU^|Xj5Q56(e+m_NMv2j5z<@bIoosA6 zhDXpwP^E=Rl^O+LuAbKjOzGVi5?62K_VMkkDPO^ie!LE!1^kZE3!I1|N*qt@cghW( z=q#Q?@$cLY3a$%>ZgLp28Vh3lk?&YnHQ;YQm#nz{*-0lY*olSGra?b3@y8RGKSW=> z3vpNvVGjqN_gEJd5{eD812To@R=ix+t`RyT^f6wkhcEh`e2Gqz@ZDrDR&^~k=RpL) z@4eA3*VZgR+)67gMy(#**cdza@GAd!O;(<-e9~F#3)NJb0{%Z^U1yTU#uwUI)W;9} zDRp)Bo|z6NAls9i`Vd!x7Fvx+b$V?BOl{B_KHqDh1}^qMw{9QX#}+RGR}p~^n=cEMQ}I_F^Jz1#%`Yi> zd$dU4ObPw{o<8?rl+TFc;A1oZ1!oJ< zfK`FXGP$Ku62$chw^hy`DxLN~dIgmLQ6){i<`l3z%JVW$ts^;P!XJjU@Wr<(x?}R@ z9PCjNy`(vVSmMFiJXY~K_V(ia(ZmP4iJeW%3YgPFgt9OYz?cfii*?^N9T{k;1zgRw zocmM;5`1BUerRh9KL(6t(PMzLD~5RU(cxR83tIcyfvQO5iZth@+941Xpg)rAo2x{| z7*Ofiy6sq_XCZklO#IK?zAt=vxE!L1`fW;l_wVvaxL z?rBsCab~?{#l0(^2t~!?d=m+}&5cT4UIW!$E`M5{4F9RlhrKvEzw5EmN$?osCy0N> zH;pOkjq-Ei%E({{>v|XQVYwUsb}9DEvfgrqX7pDS*hNfs!1n{~7HN}m9hANDk-G!F zBrPmi5!cv)KNok~FspR#WG`Ha5&x#n*mW^I)~=qj=#^$@3rw7BDQr=CDcve_ec5w;1zhoXii{N4SS~PuFr~;K*}qi$&meccAuPIp_?fEC{+ld~23ZzaFNYlstWuD2gdPNw^B>gQH^rdb5mOG)$Obm6_j z8Qa=p?SG9xl!b#%B7<(v)ViiwTpd}+m?dgWtz&H;$UgLw-BTPO(?vy6nC>;U3RPNWli;8{-c1TCmOMZ#mvnfoCGj7$#Wz zODWwTN}@B=d^s?U+5Sj<5>Xn&GHfm25jRx1!o}O)-&8vHSZS6@p)68qPnJFzg#V~uK&2J0AZT+d&}4MAVTQ?xmCC;gDaaHJ}R9W zvb&?nH=kFi-}vtkxz#8vlj{MmrK@!sY2C-iyjVGLMXwjYk%4O=DxcFs@I677ge!PS z)23em!u6T)3aGGnf6#lXx%q2Vrp|^W5P4($Lt{SWfIr4b61;nnM|t;U+;26%+P7)Pp?3MeXW^vfO;txp^5awXB~XjZwOx?b%tXuB#* z?Olex_+=7>WPHeI7j&o*WJ5MFNSGZoA~f&ExuUQa16b5w?aE)lnbo80W5(zRMiKt*ddrT;&;@-(b>F&y}cFrOk&Y*KdEgnTTq&WE5D-`1Ed*6Vg(BTu*08 zmc*ml>yMlAacnrxlDpx(GyT#p+w3Kd{dytSp|gfYq|`SY;kdRXE?eHm*e@nO>8P;)|#HIYD<@#$WWAJadQvd@8?bp4&PT2bDJ5$<{~cF8)nM z5R;XSZjWS`8Gc;=hN}`MK7IH10%P98t8eZPsWD{O=V|W0;1<4`t8w|XL;T&h$(OnU zGS3~LykMpkqPc3~Av!1z^XlCiNc~+YA%1P6`v&@SK88&1@$5f0^hwK@H53^i|1fQP zRTHmQ@WD7(PWC&gFv<6NT*E<*<8wDNiJG#@x?1NqrNMHae;x3QYA&RpTNTZR%ik~e zI6=BpgVY5>C>hH{c&^ie*9}5^Giud1TN+-i88$EM^#}q%DgXO8TTI@5%YblPgq)S# zi2b!1NY!Vzb@Iz1fF({-lqJUf0v&Wo3IMZN<5V-B;>5M=ONh!$Y9L&%m{LP-`Lr67 zRu`rA-<4Ew8ypiKB>C1IZ>45I)pE8EUn~~MB0s%6RXO@_teCZ> znHKH9d^}Yz>%Y!@@fB7P0K`651Wde0W8nesJ=BsXcLrWV0_Wud!zB?m0PD98I1?8B z+QmYANdAzq_0VV!Hi4-78KdoaL##>qF^c$jA0#P6VV~@8c|poeSl6JthxAza0E1Jj z5?HU6EZ7!~4o4u%f6dHv{v(R}uZS{pu1e?6gh`RWa8s@wOK*hx= z&)qR*Cr&NLP8832&mk@+ouYw%a1)gVFcQc5J)kBMp8soqNyTku5=kf`2t=KtV-{YQ z{q{=MeK1JY1FP0N75O=4O~Ii9_|*w~A8_rmE#y^IX&ZR(5*=D|pQpoS0{}x>m6K~e zhy+NDS_3tTSdEFV?(e%V_jD^?y3cw|m{BO4<@mV9H*@dlj*{#eTjmfk%)uZ1qACs| zQw;dC$xyncFkGEB0$Wkc4U~M8Bcr%SA3~Mc_PuHR)nd@uNAzfsadoU!S^3wagqOwD z$W+2UziPtW;xX6K(b9$u!(!c}QSrwLJux>m+)W?+R^W4gCdsfX&TP4@LfLY@kPW0Y z05!q?_i>mfFbHy;>OE@kNhs8Wny%~To{eHB%MBB0w^?2lU`f7Vqy!eek!RliLqYc` zG8aZ10_9lf*c)?I_;4BWTeEwairmrdnW~?7?5oeQZpm$>V2~p(v(JKo)0-DsZlT0o zbf2>ZhtqrcUiIZ6B|q%iFuT?-iVYz5*y1^7Bfo@z9}pWEL#zYc<+2E(x2&N8sY*PDHs#ObZPEu0!!r)ew zzG@XHYY3kzlL^SPi7Niu3u~@EAs5J?4yWItKKMDGW!At&))EEWD?b1q;=--8!z>7q z5H7ueDp0%oxBBT^y2G&3CMc$RzCh>;Iu$L|6>UEG8e)fA&}SzZzFY&;P88k_J!QcX zFy?EwMwNVvVu<8J{J77CBAx3i%4nV?M}ApyovQ2%RuE=rUP(3S*M}VaSkV@~0G29D@>xI~|o_w=!IV=#p2^D-{hsee`%1w;54jc z7D1uLd`DgP$M5S&PTbz!CkDmxWT9@YGunrB(2 zyv1+v`nJIqYPI- z^^Yz_ebCjpB-CWmqa0ppx2tFMc*4qxmmZ3~ge4MDglLltRNZ@lcEE8myB%+}Ty9ZB z5vFnpH!uHIr6+*fCxEp25Q$7B6OXPJu2t_#-nh%%`MjaG)(Zl1!I4~lfGDT{PDeDt zzxw(MEuED`!C~rBk{&Y4o@5A5`Vt_0!^$jZ^%<;ieTR||uXKKEpz;J9Cmgg2%sjWt z@9wLdI&Bo;=Bu63U}ZghIu6uBjfCSFx;LY^*IY?p_#@(IUSa&$Kjy@tOL0p@StQ$s zclI=F4M9sWCtv1^{G6hTQ-57!Bm{1P0jT7|F*Q(_uddcRK>>k#47r!P-^r2D#`8ya zrVQpsk{%tYa5?QfK!@BqEB&lgyk%}WXaX5LIm{CrJl7VLM5eLxVn3`wzI_HSj^&1y zBL}a7Xzt38+8&JI&C6z{eZ&`2UQgm?4(gTz@c+AE{_n0iYXn=}t;|rgXT@e_b)qGq zf+It!H=45Pm%b>-E%Ey=m}%@K<~FdjoLoBXfAPV#4>ku-Ube~mEURH>!{8SfV)I4{ z?m9nK^i@sv$)4ZIxgzC$o6q*Ib94d3O&!3v7C>OyKTFc7!_`H zF70rKTiNz~b61J6&zGrDI3tNu-wE=AUuy_aB+G+Ql4&xsjm+a`PMtR@3c^_b8Wl2R z(lyOJa8!QBQ}%SDBkz38Y2OmxZn)?aV55j zh4@Uvb%|0HmajupTgXw=S~m(YVhFTBq+A8Y zxmnx#U`5DH_ZD1BIHL)`ZS3&Cl~0qEf8cIg5c@y1P|3zQt{~Q4+J~h7BTYW0W*#_S z3=&?v{<)}W53^BNNv-lClZ51`@Z{`pqdVMqAs^3OWd3D)C#?K}BG^;rZ&*>{+F z)|ldMOCt-9U@4njt^0WBarDoo1=9!tFL-Lqbkg%c`yq#_($C(6eL5V$^WK}5hP2?i z>KUB;by18cNIch_ViV4@ReILKN%r{Nh1lEw+zNm12f|!}vt8+X6IXefMm=bYD~Zl3ZX=M%?QlCJ79b(KN&xRx8E!KMBz5ED1uy$2t+P-sUnZ2m1bde zls;SMM+9hJ8~tpRa(FH0W(SwdcH5UZzR4zABD0L!s{*2Ve97uB&apjI)`>eOrUA9{ z<^1Kpm>%)vtfeV5lhshS{zEPH&pKbQrYcVgavek9hsew^=aJbEFp$f9Ve%A2dg`bn zq=Ay#e~8|V(URXW#OHNMXfsWRF~ckjtqy{I-{K7kCn@mMiTiSX*Mx>hUxD}~3CdGv zf93bYhvF)7H|kkEg7L<+tdWC;`Lj8aabtyNQ(h|X1BDo#7iV4TRhuCHRXFhfaP^Mi znRHS2XKdTHbH`3P?ATVv>Dakr+qT)UJGO0G9ow2b^Ugch%>Q$Jsj92$oPB$~v?yBe&rg4drfBnTn}V6dkst1TZi(7XF0#DfnkH5}g5a zeZDbo_yMeSVd~SUN~jH?Y@w&mvl8^DxnVN2Ca&V5Gpbn&s=GM|Q8ztAIWL3}q;7NK z_%D&(wRnG<<7&v_mwf^VB~FB%JN{#cS7M8y00W+yFn-n?Q1UHz+ z!EUj@jq(m{TSY1!AQ!3VugU-|vLY-BiC zfwr&;{0N#mWzUZ%h#XowV9scT>`n>@L$XLu7M}RYKV1JXqQAQw8B&&~JjKPlCn~@) zYeo>I>Bf$242@};oVi(!4|Tz@2FR%s6;oA$WQ7~;Cp9yFr!{kV&xMn_|Kw}gbZ@QG z-cu zvF0bbKYOwAnhkZ3QnYNmQ4|SXsXz(?wH! z^%@m&HWTbB02G>V6mg~XM!Vy#`f0bY`&sFIKGqwo`#wH*xchyf^qSb`NoeQmdHD;8 z_~knEruXR!!RLm&J4oR@QDKiu@W!`)*!NK5=3DOy{@#!59X98hcBM;DjJ{>Q(CuRX1d^d5us*aMjC#AC}6^< zF26HC?DZn`NKqytW>jETs$g~2vux_2g-rgrlhZ}t)n#SCU0h08 zN|!o^@vn^i5SH3cML&RuX77-6&ii6_S-b7eyR(4{npD~Gy^f*gnO3lRE2gt@^=WB8 zz4KUwAE!XYGo3N~fhrN$fp$;kWw#QJwhb{1w4 zl4qUAth2yGbMmDQ%eOJN3I%8QVE=GkHNl7~uA_1bm5@SQ1X+@jFdLl`wa)PnkZWqJ zAH>|CAQW2qzuW&ffOG+f>iIv|u#AA#m3y3-I1k=$5k=1){usO??FMk z)O}$HZW-z6K)riW!=K~Ee;@_*I^1WGbY^bF<5?;~&VaMuX9glTnZH|c$D-ge<4$Qg z={>h&WYNzNplspp_`2GXM1E@Gby9__ilW?T!UN$(Pu2bJa^wsYiWB1H36g*m2^t)h zo$6gUoZl}iCqftg%WKdz!Qx}>Q{dB7LlQT+pYy%|6d7oqf3=Vo0@NVVp35gm^YGYA zEBUZsuZ>Bnp;Z42A%UwWGZ4T%SZn3??YGWtwmgN>qEiTUP;{^)2qV}T^qNMyOeCM@ z18SjgGyBw-NAp|4SH=&o20R z6PxLli6@hnEMKj^5(GMDe-jhYBncHw8k zf1u6w{gwLbAQxw+eBLEF1w?RBtH|RtNSiTy5w^uc9oj$aYrBf5t;BWC@i6!EB)1uy z4Fc;_bxpzhRAp%0_v70Qa#6%ZrPcfO^QYt6Gx|2CVpC^T;QPIwxxdw3OBtkc1#?pu zeasK;b4Gt#;EzAVoTs<{5OluuwYAypb^UCM3q)1;6%_;)ty-FL1wW*uYFHcFGilv# z+bm1pPAk?#+&U?n$S%H0uo^ori-caro183MI7Y4(hFym}&+hhrGWhmXWU8%zeH}j^ z#9w;yTkvFCWnFhQYkt)>aagbneG+wN=5 z$!~hu9rK@_280_F1uPzd6J|zb6GGPVXW3&rI;_~+f1*3^YzS=_S!I4;_~^b&PnsVG zV;!bD=~km3Vmy-5-|ur50?m95a5#pLZRZU68$#%V!MJ2pJjS6*m7a}#O{B9ue@m1_ z$0cvnm<}1QFY`l*>N}5BkqDVc=SC}jNAg{Dp#R4Zip!^CeeMh0x8JtR4Ri7Irr{|T zdNIIeolmPyA0)qln>uIHiQVUsopvU@H)ztB!g!F&_27RW)~Y+@v>0$}a(a4wmvHRq zKU|P30#fWJ37ae*tcL5d6MFe{f4}qJuk@Q_wx#QV(etr=QM9K-SQs>AuPH@P#sc#8 zW9sTXX7KGBa#W=Xljr)dyefTE@RNEymLf{ziNoHGEq@u<`P> z^RGvolqS%M72pPCvj9#wL)iNoD^x9}Wh-da-i}`O<8`4y{e%3e=oTZA6+D{>1w0l& z*!*?Dwn9QfCq3m2<4*tZ{@Gfyx|)?F9_;G_lh5IQQTED4qOak*5q8n=y7}?0L3dR%a7K;yg_XM9VDj}$CIF_hHa?t8@ zi~y0`^3BLLrqea@ec(q_1i|bY&9|QZ40pe9eGPS7C8Y|h#@{AM7Ckd>ec#C433MSb zqWcrzx~`VR?$-XIY57L52Fe8|A|=pMrri5g3BAdr;oKWe6%j@g6E?}SSV#+gOk06b z(aO7n`0cwDx+q5E%8a?x{uuGvbze9|yyhW--66o)i7iC*Jumv!wsG@+9Qb~zfTS|% zu~drUBolN5wRZrg`93+U5dX5DTd}{JGC~G`%1+2->^==7#_rcq69sc3*DN18h+2&m zD;ATcE3IBd$AR;2$kiw0ZwAH5!JwKLpda`gz&ZH&tFwee&b=oH*>v9yeGSiKZhgQYZ%DRxCfGUnz5B1%L zy{0j`JXwdN+_L0S3-1?g&*>`Y301p64g6EwVp<+8VuiP@aAZ2siV3jSE)4-^y%Rd* z;!SCVZcGhJI{h=A+%8|h_2rG7alaINN8iGoyAZjUdeNwuP}BtZx8t>Ld8AjHADCU$U2WI4*jE4f zMjyt-0J~)^K<+_%*#@7|;{DW9tDdDHM6S2kwhec-A1V9z^5U4@iBl71ECQ_VejRhx zCj;hlR76#6EV|}sP;kT!YS2k5Y&X)Vznn?X`)W+rP@WI6!CYe=2+Y$>Ynvp1XwN@w zhbQe+%=x^Wq>k=~U3NZR${gys)o2;WX!aE#FAkm-zLv{d38@Kw;M%u9ZKYF_2^rjO zn2D*0AfLWzc)`FRj1yf247<84q!mCjHZ0A z@-j$4JnNR0dFrUa604!9@wr(|Juqk7OeIU~c-}&khdt7$ylxj8vp(Knc3z8a%{DJs=Jnzb@{#Z12r{qQ;Ild%L&ySE73&Z%yttjSDw z0J0%H5EAEc{^)4`He_~Uf)oyehozvG4iq~El zS5feM>fqN|?&7MdbGhhV{=`~sx{6;p8@?I)WixzL)%@r~-}Thv=JD9%qT^|M;d`}Q z;VxZ&b~e|TRJOAqQ8O9()ERW1v)rvuXvx@DaclA6A^xJHBrZMORTiZmlzMjUsLl9$ zw3q(QAQ#=YY0P}|TFA%l&~6{C29h$;)n?NPZZbqqhj}<9N(wnnU)oa-s~|P;_oA?i z_A>F(NM?HF6?N3Kxj{|W|8n0C6%_NSED1n53%?k>R1=xX%4p_O$G)(3Rgy*AeBQAk zonpyPasz#<2MNSO$5H-fqqb5R@Xg789?+Q>mQ z2Bfcf%FW#ZqI2`ZKJ98eSl{4X!`Jr^q$_7hp;F(MY=02gc~(=xg?1HPLWw4{G;HH- zr>EgwX!@!Da@zH;eGt{V)V(G(x6H-n<4S$ws+6cke!7&H2)MIp++3-*75^JM7;A>^ z^>Rn=IXTB|0n*mSW{*Hyhu8Or!nSJ{Yv?9rd<|{FEPA^beu2v+y57dEqr9f%h@(W! z*-8@uNYrkFNJ737h)Sj^^N7)HAKD6hACfu1u|@I$O-!D!1I03#BwkhrFOU`qOJ?65au|Qwtq>yCrY%WFAu%DVi8cGZ`^@HK3b7*QXf?;jWuf@0O-#Eu=@Ai zwyDDEmC%2ln6s@kI{c_dLHk;wGi>G1GKW%QAc)Z6Nwy0+OsT}AhfM4>jH_B%7*B+X zh!!tF7b_eJXh)?m_PEK%Lcrr&>BuX~3sY+b{o0@b;x}R|lEx@8nKJE2tJ9j0nh})c zE78?&lAEX@G#L@jI^(#+09}P8*p0kK8(auWu$3Y+Wlt%`w6)T69+%)@QL?Th_FWZ2 zcr#oLlFY+@yhkpPm0_HgnL9%5i2sIdb^&9ca%Aufc&}4R-(Hm9H4&}HGRlbQy6&n@ z&SXGd=RZ`reze)O%4EsGw^@GNQ93;Qm$aX0EXOz`4dC=m(*Xh6OsUVwbi!8}` zbN+EwrJ2wQYk(X2c3qg}Nr}BhkDItFPbIn~YH&+NnZDc~COIEf zHin^S13(@CW-l)(O)}PYtBt6X7^Ehc24G?ai2pnZUZZm{N-&yd_r`S7Bo@?AT4%YS zd7P+!LL)*6NxglG{+m#jrco-r-Ff%3=oy)R?8IS)TEw#cRr9$ z^DdMKhgI6kGD`p|>IYc+3Su3pLB>B`crjXD84uY=G4t5>>4R&mLd{vl>`x+dEZ;Y~ zD6UI&YccP;obZ+OeFM|M26HfL>WwO#*owM17{^k>W#$PM7qna}7Bi z|Jnmm-ICwOSz~eferqhX4BfN)@Fa!j6<0hmD29wkfvO+C7snWNE194~x6&lThe;Sw4M!X+(}osZNv$Qe{nQ&Y>GYX;h1Lev*%Sq4 znvS)}Kg@dNE>&257Prv7!~c#ldQnp+kA2XjWl+uyEze|oFXmq*w~;lq&|(D_*8hhD>|VI!5*<8q6c1vr;ndeaV$~%eGQ&%#>6pR*Zk@Mo`4N zM{pd!7-c;Y5=N0am^(#ck(i>x+KPur^`ba-+^263KXx^D75kLH$*4Cf8dhJcGoFQb zgp^218^2tE)8d#tA&GfRF76%P;!1P$0iWc?fdAot%DB>pBF1r(FGoF_Y_prLpHc)=Mz~MksNG%k>8fUvaHd{w2`Z;xOajTvmOfv z=F8Qa|E}uT7{!$3be<+Us4L+#I zZF%jaIwhlpy)o$4*8D)oV3`F(BIuMacG6UlUw+XH==6aI zFnGR=f*7J29HiJlm$>}>0_jat5+dq5>FHApL%rw9TUL~a5Q`Mkgk$BX>7{YmcshlX zQ7m=60Dzwb1MM$M*+cW(M#_pt%RlV4@cB?V9=kNv`fYG)ZX=CN^VU`v44)d4egr>6aw1y26^F<6Mo zt)d-j%{B`CrN!UWa*D2uzigT(!!-J&8102xIq*JFR;;vKrn!Yk=ss|-2E95t=+V0- z1!d+ZQGGCU;CBcQ2Re_Y8_FoVbqcc<7iFnVhx7Ps1|?uxO4c9U+=al^s}JUsWETfu z-h#tW6lJdfoe=u#w^7W0`U`JdV2?1`x7peSu>)@W8C_EpSPp#(L6;wd1qHEFc8S{_9yBC(p?zc!__fi7uKn`HtGTCo`QN_m7F)s7!&4yU=R-x2r9 za->3kFEYyARVxu5)4-d;ti{q^dGAsma zo+e|TX+X>a^x=R|N?@@%7L^^Nku!Pv&2Q|%uxn;1E#q+7W6%RQ(;yo7Q*g4y;L%m| zQ)sf}Dv7Q?-DNdEC713WpKw9@PU^lszt$ei!lxT1z{Dv<_hQ9oggOV<8Hc#fBhM?q zJghiQa~vU2PdVrHBFLu|z3GtEC48-=#~9;k*pP3w-SY%(1^i6u*kVAcX);V7?|SH7 zrkg8*6dBkUsYrloH@~9+;yEAs4~&U{2Nlkw$8W3&tZ2asi!ctR9;YV?A z3n(_{C$w^r7Rc>aB~FRe{LSB0^eDB+YL;q>cwJi7#)oJ)U}aBJQNKp9?+A6c;`kH zS{oC{Q1Y*#ft!B3k4~S9 zmZwtA_0v_{CgvDFs{1Y%H)84PCh|ptZ)GqVrP#|NUeP-u z6%0o{9ZB#`eMbmh#MX>3Tl7=M%9_OG>Qdz~Q1)yeRH z?LVKL(ZO08b*?gr^Yu7Ei3sb)LxzZQvMz{0fG%WELoppwO(vBVph8h*?gT?DQOMRg zYT$VkIn2k_|8*ktKnd!&)a)g3qP_0U%u#X9_1lR$&QHe-_j&UbOlXmavc+&a@V9w1 z?WyNM`zWZXep^@mZ}-d+kM~!k`_0S$>G$1tGh0q1M^`MyuFX}|)Z`jN4G%TF387eO zCYTRM`#?IuHz2TSYHRiIBR(D;ip$Ew7U7F`#r5CLk)>c^Keq-gCY$0t@*w>Y_tb~h z9Y9mUbNGq0L|0^fJ$h6*$1<~HV>d^ZvEz#JN-T%X3v4hHF;_QOADOGOB z6onfwk>492=OE~-Ykt^%d-d3!w%rd69n)U8c#xUFE_ukVsc%#;7t_9v-&seb zXFXje)!yqKj7o-{<~9c*o(>U{_K@lf--w4m{#g`Zw}k)9q9a~1IOB!&AE>@La04X^ zDw(Srejr~xsNZU@iAC8#%VtIq^uAxppo0Xwzk=tCmU1g;mm)B12pZlh4;LM+2b0|{ ziUb-3l-Vo53#ErHRuZ=wPpmvxR|1cqz{dCWIlS( z?L`aouUl|OXKu)yhDB0#Y~Jk!Kw1G)iP+Sm!fT1K5UsM{QW5LfB74wGPBD5wDKYe~ z=pBC_;9F-mN27B)F>g>V5FRJ{!s*5rFZBMZdRN+(HpA zO_E3?q_qhVeEmnx4M$3O`J^!}ZC~R7Vus<1D1)7zO_6Kd9V#*Q6=d16z9EmG&bJ8} zN$)-=Vq+O*{GPKudlLCH_MVcr&tt zWvF~)5{yV<*NF0UT4^2)3vtG9B|337o)lxQ`N<|{a-}1FL4bG%`RI03{At)@=I4#e z@dyrjl_gV4uI0oUq&Z#t9rkI@8}Xa6k2kcoO|`GCblBnLmi?G|jb1}MWq1LP1v9DT zFhc|Pj8V@tuksZH9V^Ye2MSpEsg!(1H%=T+t;06eDx2t(69Kvl=86w?aEN;{gkzB7 z)gj;NxMBtdJm5IIZ76tE89EHt$tNh!mAWLkYM^q_=dgSINn)r%q^t%=vMIB;)raFM zq+v{AecfO$A*L_zhz9_pZr>THZU`dN%ncoHn-QQB4~SkwUKA5KAjm@R6GV-YOj-+| z$RJm*4zg`Lk`zY3TNLSPb>~RIShTu%P5s5|K6`H|%XBc758{*+8I`66`dH2+1~uX( zNnX%QJcDV#{VlNvu{F1b%M_*+QF4pUzieRLvpx#@k^zH$}$>;F=O?;J+Xv^?ak1D6o0>&WpS8$ao zy1=N#%IPxw1&#&KIXAFE3b?9Q9ur9a#2|2due% z!hE0xUOo2(9xZdbUkByT!X9mNzkKAyYQS#2Qd~@Yz{(F-wVWF415IP{y<%QPLkAZ( zvu^tD8|F;D(8|v0k+I-8vIOhG23yhIDxCr2gPI=cYJafG%>N|G-Dbts)7Ql;u*eQ~ z;1_%9jg9_>X}zn^^ocP(v`n!a6904>huKgtiwZ7J926a^0_`MHEO#wa-VGwS4Thyx z;Hva0qqh^Wi6L6TGdz(A`S%aPm+4_)j)Kp_b3rBRf6a*01C;C2BpFBIx<`!f;Po(s`e(%MfkXH#E_8Qqi;vx~i&JTAqNoG)@~m z2)cT-3@G6e3x?8c9J;{-Pl$o|FQ{)k%V(N2jKm9LB+FBz)uz;VYo{WsZzCK2!itTe{4=a~!cnK>`$%Ir+0EQpbU2>`FNEK@iR?LwSTxG$K zLdIlR91p-mgd9htWb$!uKZ?2)e&*RMUX>vfP+jbEtNp9%*MN0-$r?3*`D!1Br+*2h z@1ikK^%D@fgU%`z0jhJGk!LN|f4-cwZ6vn?V?}XYHt=W<&JfBEz1shxcTr|~A%}re zpY#F%M-PwN?!bP$NtDYk5ewpr|+r{m#tkhGm zEOfwn$SD)2QOck5=Rjd4UKXAu@tlncU1PgJVECDuMDoVEoZ<}i$6BIOa}9sNgwuF{ zCG61C*q*BY$3DkuiJIXq#CFhcmLh9%3&uji8OrTL~Iop*p$q4Ry!ab@RkgihZI!=)DlLA zlr%NXzfTa~lKi8h_K{7?LR({ddQIW#OR;!$#?Ww2{@}}yJe1#FE&8^E^vw)RYp;}; z;VaEXD?c_*c;SZ2SiAN7J!2?=86Tf^GBEs8>=j84V}X8}RvHg{;FkmFpDeUH0#;c~ z+HrO{vb8pWd%v=IgZRT>9^-9|a@5_M{QWe2d|{sxr&t@qb0dmevE2vEal4Bnon?5Z zo`s2MtA)8wX#DFqITN6jk8ezpj1x`B2Yq^)OPG?+EnusKh3`H^!{4!_==5lWwf2)= z8{%AIbs67MtMiaFyjrwa^el*GWgAu*8m3J3jdP2T(Bl}17B=|DDLVyLvu|-L+drn_W8Hh7QPn@yR zOsBsX-acAJ0+p6RV=Z)JLdpy(Je2GbBpw^|AScua!KbW5wxS?yR(-QYIQ{wa3CEh( z*n2pm>-L8!Y6!ZUwmDV@d`L$!+oE4}*Acl9fr^o#`rVB~3*%Tv`KC9Btz)L(G`SUI zPtLHx$K@K-0f%pF>-Y-lM6Y&3I2EY)x(Yj#6A)1>)>R`onYl|Jv8- z|Ciruqq$RuayX?}tU_mHS$N!nCx(!BQg*q27PQ`3zHClq^Eqth_d^YBI=m77y^G_+I9zJkS<9!{rrynn~zf~>)J^ybm&^_Vk87>Ev%nk$>41l0J&Aq1wUg<73zF`rbCk-xK zs)&7OY635g@kece2i46;si<7#PgdNtc|+Q0^{o$Rt2828D?!Sg*1D%`;w2jOHY)X) zvt>}Kl_pIxYobA~a$Oifw?XXng<&X7~|5!w7)_+R*7{`x~WbsRg^)WHbCGBo3K ztrExdx5OP!lW4dYacabh7>Co>(ZvR(vR%>gZ18Yq`XSjd)TSLU&G7 zRpyBq7c+raO|BH-1{dh`CWo`$McJo!nM3^)9k-7g4(al&GS&xql~yJkKcq`Z9Y27) zGhiSYzgjc`83Y>_@k!ETVl-b6{T-Q>B0UesfO!^GcxZrm;PgJVeY=L1n=CobraYw1 zB)O@_Fe_CdOr4lIE*D9UY|Xenr(BG7C1Xa{kvhIu+onBm3b)Qzddy!~rEaKeXOuwSdEt2c{mSxgJ0D9E@R9;zCzGmGyq1S^?%EWY5; zjf+1fGel^sYpZpj01bLi-vJv1Ts@wXGF@?!(fo)zg8+ghT3ONUq8|YbX`y^-!oT<< z7sDCGDvb+QNFs4Wd!S6~^vS=DSEg5nT$-5IyKx`#ScO=lT0R_Lv|NJ@PQrGd+wCzDQr zt_t=c+=vQs`IYJfI#<9(MC59Ezy&dt0&h7BBrxswV9%ao4q(mDQ|1bi$Bwq%u6A3?R3g$ zWa|r%8}cz5bL_*Tqe%4Ju5W0E$}N*KP;OTtJ{IMigfH2aAZAk?webuv&Fm;o zXT~&5npTqdvoNT(RNB$=InmYVax75<@CpH_m2Ob$4jc8pgF47?p;29+R|?n{`(9_tPZ zFhU$T%egAF@vKNYpk91`jDuk1-PoLC+yiH*}lLx{F2S#6@#* zDxKJjww&Pa2&UDGru)m1e=O9UyHS3>!cE8o$W>yU*Yv}xI${~Tz5>-6O)DBCHigjXx;gfO_FRIy|Tpp7Slq9G3X7zU>u`t*u9^KC0Kz3M0Kz{1M! zxb>49fwy6=dfV6|>$EYZ4V0%hAblKD^egg2C^FTM{ZisIzVG8!F}*h}bkXzGJRtKH zBc2HynBC8^!TZ+L1MUj~+V$hrQa-nWkbS28YMC4uWg`Q*8i~=}|GOgyjQH_F#UJ+^ zMHAcI_P`B^;(i@}J^1bol&f&MpYe1yE$OVw(nYKC_lAp4Lb*pDdz_VV8D-#D@I>v^ zac6mXgop%!JMt#ibvCj~Ev4D<~}x;Ga@rCTM*z%N!W<6xIjtuz^ZLSi{8=l418c=ya%K zkB-z}E)0d*d&m~WA;Rso6MEvtvV4A^UPmoKeS3#z&cpd>bQVMSyt>Kfpx|2rT;eu z{o^tdZ;)%aW^QF{|DLsZcm~Dw5qW=PhiaqCNvLgPe)fcCW8CvNGSYoo;HpPw{=;l1 zNfcC8P2J)C{AU(%caW!T7rbXEC&^DAdYk%GUaiT3;>sh;P*=!t9hYjD8KJsn>f>DrwYLW8%YpppeYd_@`A(l4wkdC35^P% z0G#{L1X80$mQ*LrNK4}xq1NRM<&k(LqD#4ZX^|;mfSnbqDV>oj5$=BR` zP)|T4y@GtE2$3`=ie2fAG%n27J`UO@tXPD@!W4AvwL4MdDgo!#OkguOt7`k$h{Eyg zCo!+qBMO|`e3-8lX1Oxju5WJuQ!ST0jGlwK!^yq)z!|DFe+OkPZPsz#vzQTwX^A6!TBTI!u0;C(T&K_Gs70gGTqC_yeFnsdA21w-Rg z7{v2fKF$XJ&xH5{U#h@K65~dP#y|&OGlSXaff;MZt0Ly}(}y9NxxnTx_QMpfdPXFB zScL^&Bc=@(FU>$yN@mL*VYEg+#TjDsLl-E1>kK>}*sw-B`8QlEH?$H)f>%A_a>oyM}lWCr=r^~(G zb2nXS_}ha3znOxCk1zOL8XU^aYm8(11N&9r`PX{&WOm#%f=OSfgHQdHn!=;Gz zT4Ok`EeteL(F_|3fLX^%m@CN6dmoHRPm?`P=@r>F)PGIKRl;F1o)ZDVS(%D|$1dgL zb0SGDl;WH(R_C{XL7-?*eb5S3Jh%DOdDzjIIruuXZXlqTh$xWN||!h=g*;_B$Q(BPwVeelct>-|~$k@V^C|L()tQq3(xaM{Q= z3>RbHTJp@vj}qj{8L^7ecH*~3{lNZEr2Uaza&E5+)D8E|=#_3eA?m*My6<3SpKhqP zr>hqGe>!ZHCAJmgI1k{xt>W3-P}m%%3GD^&!9YM3kIBT-Bsa8@qbo;Y&!Mq}1=;f!#cq z@#iwL`C@C_$++LYJ9u5}zFjY@VjpA#Zi3_2R=DBV#zBq$6W2}8YE-f2Tjyi_>wumB z-v4*PidB0{2aUDSbj`+~tnTC|zmE$>swIuxdJ}wqXST)FxtOH`?%<2K^W#H4U$P%T z4O6P4yU{7#_h2%V_j>az_g3VEPIEzf4fd!BTBzq+XR%JDB17^(&kU4F^VQJ79Km&w zUL^>~*Zog15!JaH4BM4f;;YtAUV^pG$TmxIiMIBcDc++BaentB_G)>B-cgfU%Vk^Z z)i!Vq_1nL^Ji(v#Akc4~b>(hh^KT^8va-AD*i1VIV_<9TZT1$^g7-+sB-3qj4Kdoi zoO4^^pW+Ub?S}8t%3qgP=Yh=#^Y~w*hro@eg?)R=3A>fQr~)UPc*xf1pW)lTF)rYX zdQ#Zzpn=J0TtrriXH4^V^eRx|xflyIl z+3#*JI9AA6Z*}1Iiwa1Uw$+XS%p1o4o#6i9mHU{eJT|vj$Qf;58eeH}qu!4tLA~nR z^TO4r(HHr$N8jAM;4HzQK`)h;-yItyJ4@>u8L~yN`&XirJGAm_N^ci%!Kl@4?}HZ0 z?+i7~?-T>Rrp5oY!4o3mem*)U5=6Qrm7q1A7XO`N@&J-&^~f58xrFz_PGO%lb@4dp_PY$hLzs>Z;& z;8Pa-@DR?(tv-e}Wfsv0*cQg75JjqjQ1B&8yoYZ*6LK4QjLq_K0*hDYK*3oOzqS%` zZ%>U^4$`)IDNhA=!Im5&KC^2w(_IJsG3n#1?r9-R9S^jF^7d?-X{8RlKf>s{sRe55 z335domTI8nAQXC&M90|RI=7UL#scWZDT@N$1yRK6R3$GoaZ#IpdN3bmdKB^NvnbLu zwIa&%M;uRQiLMJ1q>E_pIbg3Qs|_d&&N6WA-t$>F;wh)o7JBR8qY2OkDQDHdR2FOi zVfs9j?Pk0rxD2R(TP|+Jwcc+iynKKx617^>J0xA-I}{{>t)8GMIR7MLwZ+)0*Qfm3-H+P3-c^1M) z_}_YzVOX29+iQM#!7r^aRelXbvcU+bi*@7z+{y^E`BuBLk)pBKn7AIsrsIu(C-rtgnTaj&_W#{HyHoK8|Yt0r4al*q57{6IpJTjH6Io{HCx66*Bp z)$6oaA8kHcTImC%)AJ`=_BuLSEjtcm|W~ll= zNw@-ST0>TQfiuMX$HA7^&&E)Lq;#66pB#F{?y_0Jc)m{nt&tKrlA!V79jFZ{P1xt~ z2TRhk1Y4*1-pFJg%GuR5T|-Au$9&yI33{3(7Num!ird#Uy}mAju${}WKSTuK0hx3q zN@;nLZ1Yl?Ii$S5?@`&q%oWss2&5UXX`=&8Lk5|32(VFlHg?UC;% zgN?-vHO$J+9kl!F^h<)Tgz6YIB58&TM?7Rd+T`Y3E5>R%#>nhXkiWmk`TDZL=2on7 zdDv(7z6*yHL5m2vLe?OO32y=F$bXtd1=8^BMQBsQZ%k6DpP-oEk=*X+Y35yU)d z(o{0*Ll4H3fM?A6reDmZ?R@?%nJ%G$cvp>|eF85Fnppb&`@xID#0RcRsHol#@AW47 zN!Dh>Nm+>Oq#LPH@YL5V6}(4(5b1Mpf93hd)v2Pj8TaY{?l(&(bChVrwXf-3>6O8S zcZMm0oPl(i{j{8H$v8lD{|_{IkY5+^n)6+O+S(t|*$p{e>jM9yTKaq{;lHq0U3V-? z&&~B61zyVeyOA*(U!4DI{y1s#^sxB*u;#cNpO1=gt+EUa4b7v4(;Pkk(v56?%gVw% zF*-)_`ic`O;X*7bsyVb~^uv#L~@V#2--q*?JQ$=^%!=#H5d@NX@;&~=AIQf1DPPAMMy6`m{dwSx(T9m>4oxe1z zI0Sqi!%ve-iW)8NfY`2c2k0%B@}$_=IudO5g47@vw3HGz%H14k^4Js2YBiM=a6ZQXSh?v~~P*G7i>KiswM3#*-9kUZUJ`8~tornnojdy`6GgM#)j%_$4Gw{P zK4Uh7R**935Ab9twf4Y8Ya&db={$-&g)#dAKRIoV^z6TA()iy6o&CSOJy7LFhl~Bgjpow{}7Wbx!7wd~nKe`S;lgSPTXI2!xAY#k2N6l~`f${yYz)E`rKX+qC z+SGFix`ZY5_J*p9$0@0gOw-e>EjAJm{jkQ*`>QxKcBi#~MgOm(8vW*MFU;0lPePX+ zIuqte0#yoH5fRV{#6BG7kd2-yJkH*nIePFvB#}0!)AaH}wc#qBarjFZ9|Lpxg6F8A z5${U?e_5BzgG`+k4>u}OFe9<_^8gK)n_b6xaID(%&gy!DVU{XZ#Bp+K8dpZRN zKOyAXAJ@ntOKJCGnaS;W!eW{c5|*jr)`*NVYZ)pQu;)Psq%rdP5A0e4_R8ThUQl># ze}^VU7}^>UMNhg6Y@>;Hc81W$Hq_XZ{OV$jHXPIOA(R_JCHS?y@gsX)MqnQEhodJ{ z{3T6*LkZ(F+n7J~=c__S8K$Z<3uWyd{>ltDglsUJgCs4 zpD@b!b1*Q@=ME^VGC({Ew=9%^MHIRQ(}~HE9QK zfrT%8y`Tf}51QI{oOMS1KqnoXS$tahH}o)TlUIDaBqfr*uHxdL-x-?x5evfjdwLEN z+gzdkh@Db)3S=u+{?y8vk7FV^rmr~%Y~T(Qp=d|H@J=-h9A|ct6rm{_wbkV|7tHcY zZaT2BUGpMSnD10%bRvuC%#N`fP6teNJwMF8i_tpMT8-f$`(d3Yd-*kS0a3k{gOj$h zEZ5cl3%5W>zXuZW1iHJ@oWn8hja%_pC(i6Uu*K&p6;T&P+yLu(b=6=hB$Nj5_M}Eb!r^xwgAed z1|Is>809~PE!!v?*R*43C_~we;^L*V2t|w3s~wy}x0!xE@%$VnCvr$LpKA^bWBblF zj*&@}iUBOnlg~_N`>qW0qP$eO&1fJkqMx{>XiV&B~!M}Yme z*{(KcYqS3pibc2+RU{VT=osojlROQUL&zU1pnQoo#_mpp_E0XOeC{aQ^VBHIYc>T^Ow(;G25I)x|cdT6-A?1M(N@TYTtH{A&=8HvfR|8$qi9Ny8}oJP(Npa zlpX5qYQy$CtwyQ{cR5H6CK2nVz9Fl^d#GQwF#oood9BCEuZ;GZ4aN|whtSZA%Ip#* zsKYPiS8=&n!!pOD9?GIlh1b~Bnj@{J*SqGEPd@qNlTV`MWzPLgStTsOvKwumE z@_F6Uz^1pRmJcUH0ubdfViU4;q~XsW0)6u7e=V`svhQ*Rt@S4)EnBGt*r)lu&JI0a zF5>x_34HD3F?{ptDLlF`ifpBfeO-h2;LUg8V>i7MKe1sSHpR3=M>}Itw?yIy(U95J z-GkeDdT@wF>fUq`+i9?Bapz2>0v(pPN_}#Eevy;tBKA_(9OT6Lx$w|AhQ&rK&| zI66OrN`rdDVHsmoeX+3Z6<1x#AwN5Z;KB+9$^q<7b|R>~yJ#qcf>e_ADpJg&i=l@T z_knO6-Qk2ed8DbUI;rzwrn3UnQ9+!ZUqCU_hT*Pm?5EMQiw4y#n}#tErS2ksYAm_7 zqRR96lVej@nw>{?xrF$_A|}sYz~seIE{Y-a`gha7Zt2C2`> zXzz&7u0ORChKOVmr*JgXj}%-)6a4Z(Ei`lKp&0l0Ad&)n@5%UU*O&H zat^D@CG_?7qqk3a)X_I6&kqNQq{DF`MMIW`7u&B;DB$wttNhF(Us%S>>;y)~M(w2X zP_q|vlc3RDMU3q>G~9_BZ`p=B@7{;Ez5NK@diQm>{jP(!=iP^J_q(pat+(&R?tNRB zUlMLmo3KaG+z>|nW*UsQ1>tT0GBWy8icUP@AaA_5?8 z<1_ntdmrU@R~co>j&n8v%=4PkaHAYk>&O*QLps;#XZ!JEmWyUE-AK^oHz+HdbdjK+JQGX%2BRujso_W&Hpc`NXSbq3}gXfV_* z=tS%WD=Ga{w=F$NNVQieF9N%p2-)c{n%l5FLIDg6#xS%oh;nU;vY|c7Dz*>a;;igT;k9=4P`@o4hDjaP<<~w$Ki# z(52NeT)KQ2IqH(?QVsd}Dhf*_IVN-R1{^rB0e9TFA9qvs?|J)mc*~u8@V0jz!aeV~ z4tKoc01n=?1>Kvv4bXJuf!7ts(19UzP!608WS2T~c@WR(4hPqT%1js42$rM}pgLXu_1rXT_b z*+1XLG9O`_h;l3()jC5Et4j^UBJxm9GJVc* zL+n@a40UxJ8MbpI z_i^6Oal&~=2ZC>nqCq{X16VXiXtK#tdy?o*cA@=327$8yL~C)l3oPIG3RWsByiXa> z{tx8_f)fFxE~PP29zk|y84Gi>SejWxVXg=_%le<1M`gYOP0Mz7*8E?&YHM@O+zESfFUdKv1IPd@qNlTUAy zv|p^YxRSl5<+7S92@BIK=LaSwmQLzzp(bD69$2Y3CtDmSE7ghx+XJsqG`Dw~ZE60j zK+^N_+KGfYQ3%Hv?rkVeS2B51J>;Yn$0aQ;OzM`c;Wfy715T&-+B8j zxar7#^!BtNs8dZ7AsRmsL_=ZP>@^xM4Ku^z(YSdBnjyy2<8dvarxC3UwUs_YxmvZQ zLmLJLY}X^t6okc%#-h-H586OmC#iNP+tDBIkbQyECr{$Rr;p;HXP?9A%NK2<>2NxR zU^IkMtz?H9u(z7B%x1IZQ78*rhP0i^$h6%+kh;#bWq7Mg%gCfs=W|NQfL<68d5H15L{I=2ZzHS?k<i1o#gohV@P@mXBGe+uJ|%^*KYUY1z|Az z(Sa}8OH(>VZLIM6X0jOXed@3ETg#VY?7yjW7{N%5I%35GSlGte4^1Ab)q0JSC#K81 zJ31m58cI>0gmK}*Wh^aaF*9AoF@1jf3Xd_tS*_1Jhf>2bDqOlf(PnRSOw(H z&d%5=`pI~bd<@|D@fY#jbEoj=qtD@QzWfmW@-OelXa4+oJoC(PEGb-vs-0JLgs**ABa| zQ#{uM)!|%gc;hhQU6ef?j#Ji2tJK38z4^@{m*qrKvMt5JN2E8>@eF!9`j80QW@BKd zGr4aDDR4SLxLRlV!xrqhH84$W@3K6`Z+-I5YeW5J88t6-)JI*^*$;HEhPLB;2^aTL zvEWgwZ8ZjI1mH8wPy4&2sngp#I!xCIKvOZ)7TS`aT=7!ViLP2+O3}0DR#$VldS#yN zGL8oxIElag%27P_*fE?xe*x1ovzVLDd1)y})0I)QwUJS4rIStwP{zHN(&R1sh(KW- zOft8S#p%LQI2IR|IK~vI>!csS@{oPhqit;g>Ims-%0Z}( z?%pQ4x|4{-IWCf&RrcHb5->3~jfsg-T)21%U-|0e`13#e7C!yy2l3=n&trVzB1Xq9 zGR>-$ty)9i;a3xF|452+bk?y}mq#GSV#-#8V_+fA zbXZ2IxESMyCGXm^j>;_@;e4LDO*~Q`l4{XDwbe$|##DKQwhgCkf5&o``eC3%KyrGdTS}j^o0goWsO}lUQslqNo#hC73$U zi25or(1mKehNX+k7<+gem%eoglTS~gGE+fwku{?ZS98fjRV^8pH(gLKqIB>?nR>_b z9Mw2s=W#F;`J=`*b=dEE``fYqz%IP)=9_Ti_1B}PqZ6}N$8hwK$5Eo;{Mc65Awh!e&LuW1%z zQ9R;UW~WbbG}nYp1G+|cO_Vk1)WE6bcV>ucQb2=8L1ht8sPfn;iki4H4gG#;S)Y>9 zfBa^LW$4;kg}t&eZ_Rc{t-7_I{a&QFt%2C`;`7R5eR^WFjNMjxir;!p*A(AE-R0L(4c@zAGi=!9E;M80k>FLHk23mh8P&b@Bc?$JH5hFdl*fcc62|IA++!-2- z0SxyJFmxcH{>JMsE-&G^g?Sdu#e24HMwDf3Z-#In(}TOWA3&l}SGP+@*^&I$aF(bRokaZ@1@9UBFOpH$M2bx8mpD_7iyDzMJvkn{L6+ zU3(*bdjIwKslC^uHiz{wY;FnVPX2M=yW zS3mW-!$F7ET5hF+ix(F#G~CBAr5&kc7^}+#T)eQ1n{U1zZ-3`@-1+tmxb?1$xb-dD zu>Z(bmS+pL?dU=_U%(45oWa-r<|H2c)(d#@(Mve`)FhsM;wm1${|p}b#t9rhdIs6~ zWwa-HXt2lGejx+F1XhHaL4Eggukj)-N^EY`97*%d@6L+$|w&#|lbV^HKrD(L9_Pfx`_b`J0|+PN z-B`xTsVNjG%R_G&wgWU)j!hvrR7dREg!RYe&s|2>=03#sw!<9^8z$6bU(=`N8QQZ; zMzts7l4nV^C)c8u?6*jB5HE({Z@m%-QvP%7pJlr%`MY%aD*F3-F*w+cM2dW6asn;& zB&R0!I~b~>+Q{L|i+OC_G=RnVJVviBW9P2z*s`5^)Y1eq4Q7%iC*da2In&|;2EGWF^A^v zD#Ayo_gSyNgOtnZCY);`yr#Fp99OjUTqJ-7WkMQ9CsPMFcdL{+DsZkA;&<&oclNXA z(7l&(aUhLgFUuySuQ{D@TdellQqZjStkchifH3yqNPu~g(`x@3LxVxjiT4V8!aj8> zw-a33_pe;FJ>pKC7{$ROThQLtj%oIfix;UsZ{A{n?Bz>yw!ho8*KI+eQp2(5o~KTl zqWrf}hwaC~Yq#K<16!!u+EAymNTwXNUjk9qcY3mj$?+`vLKEBfw6UM5J~|X7kMn28 z$;SvbZR$fuM-nrWEBNNuj$?jq89R3D#-SrStW5j%?#9xryka}i(;r87A9=+w<*COm zU}R)~*V++_hwWvyz`$eAo#OpE<+Tqz-EAn7_a}~DMi2YT-S6Iocf9jD+;Hm_+JPjgBT z`!0JOn^AKEZQ~%%Q~`Vf&2q&YoKh6wVjL4!rqQ-1gOzVCq8u$E_KqmJcJxrk2|TNz z^i&za@euldhP?bY{b>JC54zvSapb){==*sgG6d9@2TUf_7F zZ9tY;mr*kP=;T#=`KhPz?W0Fgn_R+miC(;G_aR)rbt95-??vNFb0pIG?>_nDlTSYR z^m??guwZzSyzaCWOACMVkgOk&YSkKE;N;<%r(VFLk350<@4FxO-Twd{eE4BJ^2j3= z9^&`=dF=sRzyBdlFdybb^V@jvfrs$mg9;Dgfd?2KO9kAU^-4d+~LCp1gPtg+dN#E#*w5 zEuBWSLSv~2w^~G)eKsI4k$LC^fi!`9T~hM;D&#RSGXpIoYU-6f;|NSwpO?3oyrIIO z2x2s1lMKD7cI+P+#M=+-#qHN0!N`tnC^Z|HnVUt9`LUBC9F1FAS*EE2BwY62Ci``i zDn(v&Rc(3|6u2pnvx&}*4pd5I6tgQ*lJ=|sM=A!S(kXPa9DSiMjhFz9zm$}a>5_D& z4TGH-^tZR6Cze51IK%Q~5Q~tPdYpCah$paNaL7)_Jv}>(@s(xsTHd>DAC2NLqO^{p zG=|Jr@ZKl9M9sComOhk9w3jDqFF$lmYBj?AWE?7#wRAB zlV=?n=d8PQoBkVfBTmXyX^C%Zfh$UkPB*@b!$B-)dNgg_FG4vg1$Iv~Su;tp_l%IYC z=b83Z0VCw5JmA>YVH%#9-Zb)ydDKqVP#fh~hccX@P-~U1S0;U7`9d85#I~f-wZ9#k zuj|2{BO8&MUO{!9so;8PgHKAo-`<`;RS z6`6moSn%{oi_}iyu!RC!}2t|47! z0d#kFVs#~tOBZvPoD^VP$MA5H`he}hDk;85eM|@QP^3|>2F*)N`vfm6%%iijgEHR6 zQs`uV2SY=H{7mtg2z5>Z>9!Qwx{_$?h_PQd!$U|>o+#z0G$;vm>Otx%d9ksY22Tj22(W+HDYYzf zfOXT0iVEX4iqf$WgrX5NbCkvDWhB@}2F!?2N+j8&Y}7G7zl!OZC2JD0)IQa>wS1lm z5Dn@H9qJIGZfS<7Z|ZCFE)U%iDg(F1&L{xyRirC+mH%m(v-RE9HMVf8jpb?ga%=_i zi-jpwSa$L}!0|-h!SccmxM~aa2evKi%b|nggywYA=hY^%tLz(5hxM8@J-B&GKX&Zq z__ZMcojP3ANzD12={U8wH;-cp_#u5`eUfSGx<2&++w;J5Y>&k3N*R+AGqy+cwjG_= zv9lLLgT1^@c`uZhE@gsk5s&Mz2bC{MohhEOpRiooqJhJj$(O3C=Aeq^#2)DD?`9dJ zEF&KXHPG3WvD3Wci4f(dNu3-+T>6=MNZ#6mg9G*=UFA3Z+?0MUv+i@`H}j`b?C*(U z_nu+gc;hX&{q}2d$L)u4=N;GJt~>Vf^B``!^({Dba4)uO9x}bnAwx{DHc=5@{`Az3 zz|??CT}U0LHkCo5eGqHOy0`b`Gt}6VTCST!=SZh{=1Shw+R1bRo$+=ACqk&TH<0X2 zA~TpmY?ytyC(Q9Oj!>OCk9`deM_HD|NjYdg)Q0ZcyD)fTza36hm@S~TQb(iG;P}_% zI2L0T`eVECQBOm7=4vGzJ9`R$^^LFK)A#-ro_OXNL?~M~-+C+F{}b=P zPu+DhZrQul4vaCqBEj&*-{8Hx%k49i$8EMYajh+-v2 z!zGV=Ig4yz6*(I66H^mdSX{74jR)RoD$jvc11bkbn+UcBNKIxG_*E0Hma(czw}MCp zi0aV#{k=$O2#{+%(|S(Vbxo6g?c4MU2J=(*m6vA>d*$`Xr@xP+$!(B^g@QbrqFO$+ zW*EdN_r0_*wzYTS?w-x~$>Cji-UCaRIT?TwdajIAB*h(=-%+dl*%uFK7z8h`NAR%xs5Hb%9Fr7LLj2eGy;ci{-KBR%R9t6@!Tnbfi zo@L51&IU6bGZn%ELJvj?++d zJaEo&XzPkhGScM%=WU& zt~61~huFSBEX)UNvI`Mhxmdv1l^Rx- zYSgPu3=DK(-+=)fxqc(V2o7F1fP>ffVfPKg*ni7L?7emqMmF~w*ynOg3O1_-entf@ zi$}2vnvoJ3p(+QZ5#2%P0k^w#Qni z%|X07iQpnrpJTey{5%(j`(yz5B`rzwmOO{F%&Jr~KsC3tipq&9B9Af6<52`&V1K!& z0~csE?q-B;ve^xxI$uHkEX#c=h{Rk3sYnLxp$_x1D|8gljB^a8yj57Ydaw?sErgDZ zX$(h)ffG%v&S#Myp>e!R?asXQ;;Tx&vmNSVZ0{FYuJd6;mm|moATAV$k~^#?+b0#Tqd2{S;3UgBMqyZGT^k*!mfFzl3&K63(`?yWAEhij283S> zAo5}m&anW(M+1nSX0PTgx4?3gDT^jlB=R!v3S5=E#VKYntf>EmsYfCL!~``-w|Idf zz)vm8YX$MS^_`-(F0{y#euZ^_f`K?eRH&zFb?T@@7?olQ<-7x`4^-qu#>k4_nn!49 z_~3v}E8SpnIXkO|79*I}_NtdjR{7jNrhvL)d$; zAJ-h|#o-$UaovprICO0{cI--{yEA1#zBd=*d__9SnwI?J_b_=KZjf^nO0~1~{~F^J zw>vD$tCnA@fxavgNlKJ1_TDtTC(D^rK%JpiBMk-%N?SZYNbyjG{aq0l%IQuaBeVP&PR(S5R+ z#QT(OdDz7{CWa89yoJqsd1ZAD!#SwuDGz^w+6T>{20H*noTGl8bUiM4w?w&oj^G*z(VCbQ@!PW;yni zICf`4C{9o#oopgPUd4)0s?P-4V{ORx6j1G{0SQmOB>%Q=g5=9&&dV>x_M_TkUD%ev z8po`Py||j=JSAAEVubqO$nXg6+P(vK?ciKy%SLSM??hXIb&*W?@AwUC)2#2(%u*mJF%*r>FA`iSR#t{OuLMJL5& zabc(i{n+9PPR_65{MB*vH0wCf(_!9ojvGLl-G2&IJXbDYqL8COn8%f+ zIh>vy$K~8SD$x*zBN^NKQ+dc6QQz;KIgQc6A|fo<-rivx+`Pl~`P1HG!tUfGL@Sa; zk4##%hV+lxhI)@iiKi|Dlz}Gel*fzDjbZ4SUr zZ8hUi2ClXN%r4Vltc5XlIghDv9b>*i{SI7v{T5_&lAx3`joVVGg0azQ)6p_|=Vt1* zPuR-J5(YQbF+9>|9u`7-WlYbE2T{z0u{d9&5n0C4d=-~3uhMXiay)>Y&Y6fu5z(gh zWPlT;0mm^Xni7~x9UGU|49iJXtG!m;K!NGfKnzlE2NcMU5Up^B`dR=L+gMsS+N69} z$gMH*@ni-0GfOCq6;U25pmCK(-eMiWl#AM(Y==IziCpi<$T}tJae>n@xVZ-ESL+Cz z3Ltl}f@DV;&NX2)w$Sj))e(KjL2fQbgOEn#Y7+Sid6dqS;ViH|d)dafg^=t{8i-Pw zE&yj-G*8vBI=X_=L>BeYBJ!mI>O(ZVuWd)8-$kijLX(zpD9r&q96;yOT_~N+BiEUQ z^Ogv@5BH!Mt(iwvpu{$ttD$(gfWWf>)UH%f8Lyx`Sw(5KgkrFco;_WNZ>2uylr2mC zhUAgL@od6HXta#dN)BZju!So*+V(D7j@!XN3QiB}oT?%`9zw?p9LwjT$g|9~i_GIf z%}%4u)Z=!zMm@_qom{4IT}S@`R;B`6IW~pRhNzX-Erf9K8*}K})q~8=41%5HVS{yK z+o`Q2U6Nsa4{8aXUGwzL`p@r7(qqE6wVM~~*$6=1pxSG!lIu6QSz2C2E}uu7!sLXt zzovXNC0jA_B!T7G6vnO!d@f?g-Zt#o-A5jKsmnuZd9loXw1ky;o&MZuo?Q#?-r_1QR zoI*#b4fWM3a%{um`4X0Dt7zNXiTJ)4(5?N<*jG|PEQe;0oJpYTB<1c(35B^T+q{8F znR1!-yr-1c>f{Ov$MdK@Q$pp%GQt-lh&ge}QW2{;_O;al5(8Rp$=0++Tx&@+kUps| z1>mvdXv)=;H`E@|=OzcNtDaDF-4U6WG@ldrtD$L*GJf{d9QN!M_?pJt+$v6;9>a~d z?BKjM%J!P1929W<4Lj_4-LF*=aBDFXaz)O8%b1_zcrjDp^$>b`I|4@&4xk;{;7fZNub((WrV0gm-QshUqT1PfpL8YkCki1zfv7HN;r@k8-8%GD{ z)5Dv{(_Yq&V^<{Df$=fv;ToR_V``#+naLnVsTbQi9Bkg!W^?8Uo2FXNBDX^MnT_+4 zb5G7o=VvK*SBsdRVVUX<`iHfrG#S#2aLfr%j%O^N#f2i~rd_5DtSncsaZ?O~16}69 z8Ks;h6HO#ib%f&;%54MDWD_yAajGqdSdzLTtZ_ywTlA@MM*UP{zgme+CIGIcwqtuy zSB3Qwp6%0=*CnsH4rQ<$Lgm{t=t#Gr?ba?hhgd0gR=3JNz_B?P3!xAxVr>2@+OJYq zT&DWC$g%ib8THFGlv727+rs8CS}!$`9V;P3y<4U}E{!wZ4CRx$d_Fyo-XncTY~t9% ze-gJ~LoWzbQc98Ji}^b2n>x1!_V&JhT(@}}4s6?op{`Dw185%N zxU4^AFUrBvn;ZG$lTSYR0lF#DrmRw$k&>fp;|=KYoG+Uz(G@1`6dI z-XP0#Y<$(@Yrvh#Dbllcou7JML7U7gDD74mbX@@38znts%X(A*l9;OH zZc0bY)IGhG*LQ7>uj|T7p#>W18P!W={eJTH$)_JS{jkA1UDGlmk-+tGsf5+VMGS1( zs0%M$x96jt@hmv0Az&r!AlcD@zP?@zclDq*&XhR1fMb)>Ign zSXe}ilj**W1iCqC?nDf2nG9BGERECXothZK`O7C!r6D#*gZH{kTXDmd?VMP%49qXr zs9}0}8CNe~#_2QXaQ@tREaz6yJJ5^UZ@dZDbaW%edPS&YB2+cn$4yG5(sVICKZn!j z&f(J8bC|n0jzXb?E*dw39ce_IB$^#%joPKLW(@z)kPz}w+mg} zG=gcQ4lr)BJwpRHfW_GvoO=8bEKE9a*+n_-Miq&No=gyzT<;$Zel^Qgr#n~zi)CdL!X+S072FUgG^rBQq z(h#1>w%zJWb8PQl1ZS5&L(YB+L0?-VZr<9hj*%HSPYegBe6I?h zwoB5Il_O1!eIOJ=_)-AH=^|Dp=TRwSkz!f9QUkWZdvzOi{+1?^L5=}029eQ=sqPpG z`7Fj?xI#Xz*vX)MclIE;H;G2RW-?zd(NLT!Vd>mFE}fu!UYg*wBI4Z{bnfp(?79d- z3HGxpjaIg;P8F?{>xf+B_;Yy$i5+qD-O&%HgNAyNJX52v%(ZL{x%m~im-soejMd33 zmM5~vFBP!;=AB6G;`pwUWNDb|&!G&6r%v8MFz29JDC5GJ6PP%C1^HkD;o%Ir*e+TQ z+0m6oSG@!A)i`Q}BF>+=gxqWn`Dzh$8u{&;Qov?;;*jrie6|rm+r~~S*Xl$8k&y(F z+tW1En^=9JfZk1=h>k>2NpY;@wI~gKHWl028|#hu+arBx{qR0ty7e0Jo|j(JM*UjS zskurb-hr_zSzNj_YlrJBF3+H|vmaeuECc(NbXG8+`3(73Oi(_j(9@s9!NZ%Ve`4mr zC6G~55|=-+t1c=<&VihurFHT08Qbgb$h8OApSsv*)E|ME4v84aFkYH+QOA`l<2Zie zBKxMiH>(E9wReQk-kzY&rS9aPRI2uZlU}aXs|Lzxdjj(WV*w=7lsuM0bE6>d@d3$z z?px)VO4UWx(Y~F0uFQ01>d0O#U??+ycsPdn#c7O$SuJB}Igf+v*P6FYu>8e*1zWakwdM2aG_|T5!{o#= z$BE0x=9f+OB9S=KsW!BCCebsfvbc8Ya6U)bjx@1zPnUhFEIPe5n=RXNRV|ZEC3Ucm z=8By-b>b4Hr#V(I-{qAhlR<4+FgVnYfuRKDgZjnc7}w!Yw}-I2T&K*78&a9MmOyr&E*TU zfhw$fm}N|bqDWAlY(HF|eDcXBpM3fONcOeH7vWa|@3du*IgicE%xH%38WO;y2qH~z zwP)1Dix;tP-#$(by6w7vt8YB;HB0!!x#w{7_*1CX%BTdF5u){^joOQq0`fF&V>D_t zqtbpFA@hMJaYE?<0p43&4dl&=26;_*IHBa;h+9R^LW|2^BP=#_y%u4<)3U* zjg6X*Djk7|s;GixrZhR>)1a(6C{7E2P+XD9ZWE91gO+LZi#MrM%F=C7>$}p__f|gQ zr9eHAev5CSbUJOHKKb;&f+UBJKmIuT*aq{SmCPy#rjbHyxY^Hu7VFv5v6ZU2TXiIgW$p;kv->(-iDz8*l7#<`Z~1jEcn zrQoDor{9*f&ZxnM#W9ffVc8#@`OhB7TThCGv8yT)lVX-X4NXhvg3^Dl|kAAX`k=#^O=(sBC+aiM>^p z(GDGAnFPve?qnl?%0s7Sf<|%}m1==`)R0cd zL&?QzHf!aRcY|c20qS-EEQ}jj8<$ij&!FTF8cu?Oyy-awM9)z`s+6f*5QWtWVreE+rXd?= z8Qa+Z+2*VH634kHqU1|}9bC)hYYiIQY7ojlxr0VsOO=a=B{Ha%qP7oQri0@pd5|lz z{>&>83mJGfzf?kdhUuxTw7GqdeNqWqYpFIKiBBH!Q^vQ*GJPefKexUq4Uv}NmspRA z<{M<8PP#4^z4=J4u)?;eqrb0@{8$4eK-qUg$mJ^-J(IwrkDSET-BH|r&j{jc2XhtK zmjxCr&6n`VBWF=A1aZqP+t5Fx#r>W~QY1pXK)EWi4T>cfQR>jPbc!-feMFvDY9*$V zXTDyaiA3s*BQMA>dxC(R3d^B`yx3mUCt>Y7$vl;Rt8OOq)QnpG-gG18Fi#otSlF?@19-g*qcEk#dOG4^*^1!U1ypuc@c@m;j--7 zmr$OfI`>1mw7@>26NRPoV^Qq|EMSl_JiEaBsfUv(+qX1C-mfl`AC!j#^9aULh_Jl{ z(5U2O1xa=B#v)Elydf44A8TcYK}T8gK&s?k>Viz+1{=f!sI1owp`YW zS(jsibRz4>>ow}krPW2I&}k=PYtK5BM|&Yw%1VHK)CSZJyj+B)kdGCaD zc!KtERhQICk~;Nsh4bqYdV7;rUd@puB~^|w$p+)5IciF;OHpxb60aNDl7W3pUXto7 z+DlP&3UTlbS=*3#0cIkioJpr?ejf@`KT?A1MGD#45FpNRT~>j>dah9UA!%dwYtol%wQO0cSs>*>O!2YLs zk>rCqjK4KDTLKbC@+(YUD-EA~^2sNkeELCYYHA8cj~>Of*Iw&6&Ab`UJ9*h$xN^ZJ zD;KYx$LVv&QQ$blNur(~oZV@Rxm06Py#nFCw+5@X~K!qk7 zTK3wC*Q%GmgYSm~T<964q37k!sP78OL-$u#S6>lF`RcdIrcVJM`tFlY|EoxV+zTKxm$!(3yc+v>O>fsZ@l_uYP@;np zw11j3PT0KqXp~T=#A)m%XvjvGo|e%I>Znk^Xm+oA@kki$?E6XfvpB6*hj|Iaaw;V= z4rLJQ8jFD@haJ43`kB?BvhrCOcf6Tq-IK{E;yO{33a(nG zLCwD^`I=;&Dc(;-V+;xYc95pUl$4>ZOvMWtVcC@x8Xhj%;z_i{5;WG5Ub?JvT=i2K z$sh4sWF9qEM+cdBoJMHE04+H*$cB)iT&Dpg6DeYcLr7oIm|{JGGVnc(Y3-$N_MLeo z6A>CkY8x7nP8H!84HFq2wmeNnn&&+veJnk#MvBmommwOLw+O%MpbHuDGMKbX)x3FF zc5T+4PQ^XDk7bKQ6gO=7M)Kk-6f zh=7$^gT|ZyS&khtJZm%(8Qo;k1|S)D@&pl%MV;*NLDjSW&|td=bdlG1ko6Cy*bi6^ zC+USyl5)U$3dArFQsa+sGbSBKtB=$TxND{Yh;ob!rWs#(MOfkp4NA(TywDXl)K1<| zsAVKZ(n)@2dNG=RY(zUrRc)tyXnlutP(gySppz2g97AMyGd)H`BbQ}72aOU9-`NVX zlUc+crO{rCBYY@|;C`m%FfGQFFfeIrBfVS;GF=O-S1`#wz|UfYa!(d%c`9?V&k&7P zEd|tmib2YW!*~&mVJeS2`j}pW{Xy-fes1s(+c^+tzmCxWj#C~4@=;C%{%C2mqfO=c z+u?O>F;$^#M78WwUX)s%DmiFl+AN<;;*k0_Kb5Z66IlJ1J|*QO198^At4&`;t>?UN z(qWo2csfmchDBL6=@PdQqD<>>1L_XSpg^c%W!c(Xr!%i)bC{oB#>LYDX~WpKHG`oM zfjwj#v(*Xt>a)3Q2`5inp*U8tdv8Cwdoye@a+Umw#N-VseG;)bioiSVJ!c?`qh+sw zH4m-0^%ns&Zm?`|f&m?}!8Txr($Z}$Nvu`ubqJc&DHKDByY#gLla*wHvK3AA)dd7F zh&xQx>aIRyyeY8jOq4NqVHO4Ivc4O8k=~dx`BJ$gVAnydBmavJDHBGM{VBrq6hFogic)7aD}2UV*`U!!W%WqEJXqf3 znAZguiR6u^KabR>y|z`or86rvwFU1}kJULJZ7`&gDVAMQ%2M;5PQ}(s3wb!IgQ%{W z(+OO)IUMC$@}YPMwnvOQAtXn@2pBlUm8zT}i9pu1vVEq(V zpL&TSy_QH&-?1<0d5Qw%K`$vdwg&ivHP7dP@rh}sr~@dpbQ(B)asubiU%}9DJNE3` zOu1#b*ggipsV%fuB=tlySYqFjOl!G!*nskuUi9>H2b!AB$db|2CG`N8Ss3r=`M-w-Vy=&uRz2zOm-}yooXV0C% zUw-;Cc<_M-@#v$E;<3jb!!yr3gR57s8aSx9t!1unl*9*-p4G%hywSj}^j>`NMSSy{ z-^4=?J!D>r^32l-mzn_TxfV}bAf8V?{jVhTmmfBGr%!ofH8|1F7jl@LoyEot?6VSg z^;OdtGYDWIq4UOV0MM>u?$rjM4@2V>{7GS)SviQ zp+3^J2+PCN?N0~Xn6;vRs)NcWV}yo=;@Gks7C{@h3n)`DD-{~hG`_T4P)4uxOk=IS zO4BS7#uor7ZzXxW2z->WDZRugy$8BA_^r+-SO;CN+Qi;lhG^wx{6^LD4m2Z<^^iw_ zw65+66p=wBBT2xJF3JYd%Z{>^E7HJIakQ_K=|k3EUUyCMEy!zOCu-%D7nlqwYd?Nt zb?j3?|9gNSzwtliFFwiJLnm-H8q#rmmi0BGkfBDyUSt3(%OoEySKe#NhH<^Nq@lxS ze?jVIIu8Xc|?TReoTkYiKnV>z2WI?Ge*^piqCjDXhq&pT>VpYq;0eX3dbFG z>~w6aV|LiFZL4G3wr$(CZQE8=>_6Wg{~2fRb5ZYjFRE^8tY@zE%(d2>NdiGo22p2B zx;WtobR+iWc?)t0pjO$g7)mH8BaDW=^Q!9FSu&G$vG%0u@p%+yjU&$J}= z@QD`*6YuGVI7gzvbO?5wwUlrPl*tj_tJo*zo$+G!1q#|PUH$G=nGAY>GhLHn>KaIiw7(nj|hsb>=zp{Aj@Y-Lb@&&Z5 z@x~AAVI&B;|8{kpB95uZ8UEU6so=t50iF*ecD48xjIzvx;iO7gd9Y#EfacSnsd}O2 zwa8;E!%f~~*eGTPNzz`7x!yJFYTPV8e+Q7xu4}!c0fhd|!_v)bGgO}8yd5_oR1ChB zZ@Tx1d&5BNy$d_dPKbHycM+#z=?<;2B&6c$_HVxPh_X59FD$betv3nF9OVkvxV<0S z8*6p4ns~XqMGH2%6jp*z7yEI?|5a>wm;I-^@>{mMNh|{dOj25^QUv4KPpeW=)Wc*i!jC;R?o?)hA+N;|H z91Z?lyxeetRy~-+iX~$4gycLobwu`(AgqS9rs0br&PA!LMP|f#kP}>|JVBg{N*ly< z8ADpM0T=zmnpG`-9UZS%TX9S7GZ0C&!yFF%%(OYsmnu4BKM90>Nie$U#q-y>>KKi7!Oz;u%79C7V@h?~47`d` z-AzSc%)XOT$V$+ znE?N^8StG;AuP+%QTf;`)jgE7MLjS8*0{1pe*8bM?Ol50N&cfYEe#tU9^O_QAbLiS z%Bo&!w|@;qgSWtO=TIJMcCdm)L6`?t-iL>?2bPIOVF!ajC<;!gA-YACe~gAzA}*vc zVJ2ZKWL&IRuV^yykbg<<(NNa1iuZ#P+Fs45HtZ^W?%87ANFoI{llBC*g2Uk${`0oy zQD$DZlpc6@&?qp$#B1G;C0%vC zf`WqPn=Ep%^i3GT#Rg*R0y?DG|K_dZNIM^jd%bHjQe%V^L7=OUylo)0F~vy&w(^ik zY{1=q1e%FM$0NIz43OaA$YvT07!#C#uPy9RQk-ViRv@MaQ6KzqCVC?dU`zQ4;Peaq zW@lt3)8at6o5bkm&?i+Kcso$u zT6P&;$~mW;!FTgCbJHuJzv0EsKCDX;d|QfBUG%U(1vJggDiEhLF+-%a_(Do187E>k zfx1@82VPk~F}fA;kmx5_SIJ3L{LN+`WzY<-=p8f?u&7Z?nyRJ&=QR{f=J_@8KJ?n~ zOURGIO;kq|6y2fR$aFcvAwEV`gB zM9FDRQsjoK&0PeVEf0ioi}Mo6%lZfq#I-SJx4+!L;W!ekEMaPV+vMc3`3^aKEYCUx zAnzIvniwx=Yx9%^#$Xckd?Fikf$Gyy5W#Db3KyI321Q^De>Kl@@i@t;KtcWe3E(|f z6t=q=*Ik_R@6y?(k062>?c8A2%{kx=&=-_-;~`+A;zBBZv7xGDgT=xoW>EWAstUjx zf@Zjf5&s&vA1v?{pk8hx|L9W)>gTGYoMJP@#_tn(?Owu=%0J*H0I;}|whDeBBb59sK%+{#Tv*xD3%1(UOVzMJCN4n4q zup5YH^UgT7Ws=GG*A{c2()TiHda9|~ACHm%@DMhAv(^wh2)z0nIUO9Z-WN@hl<`s_ zR6L-_Zi26qO&Y~U$Jv$UYB&MRk1peB6dj-n^U;}j9fR3HmW47&jRkeh`d?`S!7rw4 zFb0ng!NCioa7G4RT-&mY46ed}v-)a;=rB+qOXFnn^b%6E3R5dx*e$|4?`U%2<`Bf_ zU!QYXd(pM!Zu|YBbw#f^ooz-TxJfaWMQar-D4Q7J7z&+kvT>-5n7ZE{-@^<1aHGpj zAV*|cC###Y`q(I*=TPM6v&aY5gRDU%bd8L>X7(qyI%sx}isM<&g5Cd*zZ9ZGf9bh9 z@;H;oYRa%h!^`$$wo8#cvd>-B3}HX$*T&3dNJ!2L#yqwR5VZoY#jJ6T&@LaeP{BMA zRzpWaq-wA%U#FefOL~+wthwLz58)lhrr7~RklV#!clqOW>3)mP#TN`+##X}tj+V!* zJFf6=P-pY2)eN^O@w8xgL+m+sIQOG-5zmJT-GP%&Al!4-&Pz+{S#d@r%$gO3?sGq; z>rF-LK{WSmU5wY{wvx0_%nYt`?AVK5u3y-4uu9( zQluRF>?sALycx=cj?KAVHQgFz)B=UqhsjVd6O@bRVJ@7_y*Q}x{hHJ@32Dv zY46HL$jm{)`L;&3vcsLqHqrOJy!&3fO(IW?1CuKrRrjk?TBL1Sq@5UWsfKWDR z9L`HYK2Q`6qI_H^lynAr$fi|fJn+7`l@Oe1uzA;YFaziwr(eTmxn?s~=ntU&t!Z&J zNY)|{H(zdLU}W7OFSuPMhYu-dL6!0fj+DZCgT8bDuYsOsu$x*Q0Y*B(I%sJ{;N~W( zN@*$CSrnaQbV~ds6SpUSBXcbcJH~8*?5ejn(Zj8NRcGX znGf&ovCs$&KrMld#4rx09!S}vB9IyJfj$ z)6{b9UN=T^4(p}>gBMngyvB^6N|7MJcHx{Q0Xl>tS!5hKk8G5{C?TWB@f3Fn8w>1> zk>H(x&qz$+cx(410frG5y2d*l5apoGlC7a*Rj`GpN-;?T#oHAfaJZlSy~I77#^1(h z*x&TG7Fw2FG?Gv+7OzNIX=4QVZNe%X6UaR;5iRX-FIOKst56eJUGI?HOoAlP;vsd4 z-zmB2me5NvVdYCdrD9&t*({67 z-QpO>uRv`7)lF9+X=tR#5m#o?QZ9olB>C+&voRBGY;&7_-4N9nE>u+d-7zS)p$b)E z`^9SDVK3prjObRFBvBPG+{t|Rh=X{)AyO7EL_McJ6Snw!wF_a=)Naf}w!5M!e(FK^ z*l+9hxKdj6N2;)aBMVHDrw0-ai%j;nIvf6k3`DjLX1_ z3XVxKNS)y0*N~_uH|O)BqXHq4-QQKzlS-cVqsYi(J)`ZXGx}|Y1|ZgsYNr~}bnha~ zIuZQPRU}?3icW&n8OFMcP>Z(&)FqjClmBWd*5W0VHN~y${If#k80QBKeEl-6$C_;M zhdto~82xaoc`URtt3C0uA2NV)@^=+_kuL~Tr#z!;F7q#jhl~!%CMX!OiUQ3eE{L<~ zV|b-meO8pBRsz4hfM4({m|#?t05Qy|f2UZ9UM zn!kH2_w(4iQDt@;d*IrO~H5KHbP{CtbKPN>nsGDSDMz zS6YpAY`xJO)%$A28{vB~w)g$LlP+&5u(}&+)LbDSCa7291EY5iY~#Mnl9;Im(j>GI zUmbq=FX6zzKr>ohP&{?CFIQ2K<3r~7yoBV;xekp_+;{=ufm)&L*2>+G!Q6G2hMmSt zHDcBMNv92_8*_@q_Qpa z*~ULZ z(vh#)j9$)yq*p=kHY7t|#Qe=U{Lp%65Q~rnxZ(wP!~+sl(+UA%#Z9~5X_fmys_wFf zXoqA(AX3`~Rk(Bq3^@C(L%Bx{VC;C&RJ7;;yHr45yPy{}E?QmMVmr~c+A!>Z`SH4u zyTwHjflo+wfA87r(>j!LyGR;I~Es~hPhH^Bml2c_O7ka_k4r87f++e*`T*ImE>># z+n+}TCsoynL`_|xA^`vHnjj3ff8S0<+G zm;?&xXvMu|w%0u8zZYVHEr;;vlct1^Y5ESSgo;4 zT=MGih8=}6{X~^__{kLd>+g2aX7C`c2s-5?<+y}>RWgPYjgNr2W1OR-Sb5Hao@V`x z|IyR4v;mr1dtJz_`O(BiWoe!Adv2zj2JPt#)<63*UQ1t+p4CR>6(ytxqKbl)@)ds_ zJ5A-sW?g+BW1VdUg=GHYe`Ca6q`U_Y|J=fW3H8nRt*Bu;ckCwVC`QrK_nNpD$esti^uJ0?^Y= z0>p@Y9|duk!Z^sMP5#+NvnH6KsSDr?_dJB{h?dC>e|Eju-Z`PjM4qc0LGGSCh48=YZ#r-Mg9`OJlho~p zW9ZbI&{lr>x6!=Ed*c@4Mjl2G5fSlxwe$I}^DgU#HF(_e?@CbM_=U_x+)zdoF8lhn zZ_jf?kJ)#Ncoi$#{hUgx%^uRh!J&C3`}ujfr5l}>mp6H3=Y48t_vIvKWMli0?ORe} zpjmabWBY-ABlGsj`{wBul^=lGJgKTz;47H)((4BvJebYnh5`o%2ZdCg&A#nQ^7{Hp z+cV>N%#xa&-TVf|=yJVjZs%wa7Pv>F`x2h?@9ptQ#ru~Ct)M(llI}&t5Cv$829D>~ z#}jT0DbM&px@C3WRm+a`!?J@$^7|-B%xb)GlEc&WR_nvEW}VO1XUewR+x>wcKkQ7$ za*6nEtI=Jurgfcni3hX02A~_03LX3g3$5c}P*U;VcCF=j%#mT9o`-8$CY0z|GUi_$ zU4-ElkJv9YPV!YjtkD!W&}}*8+U~`()zKLRq*5Sf!_u7Pe%*MnSZF z6)ZcBKqgeGXAjN;RCp4KLR=s=mO;ThhHJ?R7Bf*%lXkOi*oS<{@p)peC@BOxImw1T zE!{*_`Z1a>(X6;+$t0oi2ZZCO;+)Jn`kRqkT+VudYH+#Q^NP z^N;IPz>;2pL^2!mZNnO8EMlUU`vZ~y^4sq(*i2)h%7jji>po0HLMjQPP;Rn62#a3| z*tsLtknY!m?!kPKB3U(sJc-A}mb9DW;+@Z?k2 zTt(i+Rj_%8+EVXvMZG+}Cd?h(_aWWRz{77hlxbhFB*hmP*F~o!+1~8)Dqt3`$MSf| zMS00SOeX5z()-B(L_V{3`SlLxtNQwEKCks^xJguy@Bi8+aj~Guu^hdTaaa@Yt~@vN z7@u%Ay4{yRStq3=(_e>l6)53A1)Y~Cn`dMlz*Q{qPch%x;+F4#6}J@a4oYuqL*h+F zd5h3OI;4kpm%jHN&GFP%#(-57vY258`Kqj#%(B_DY1voN@(+z!{%=hAt1A*D`43$5A0L)(w;FPL4aUfF$N1WkGZlow&j9vY?JNcGGUKz|8GVROal_=c`0Phf%Dh8POT;m^j+kWD^ zqes!BsA-Y)1|~d)IiQ<)(FcUdgLm3;4Kj z_X82X@g1U^>a-oG$OZOxxJNE=a9%3L7osd%XbKbIW!qOTH2UW{^(pfT=~GsY^a zhQ*vT%FHpyNEn)YBh>ntYPj#}6AX@Wd^vtC656nb1VP{JLFa@m@*>GRz}1 zzp?Cp_7pQT2(vOv0_RRW;A)V&sBUFEL`Nm1CQO>n?AsKC>k@^ie~Enw#9V#EX;&F| zcbjJ{6)OnB&8b1xwaOe;=-HNHW@TN?@{xeeD=}V)#mkgjD(1@Z<*>+AXjH7}Sgu>C zWfb*D)i51>yPZo*OCRumj(mmw*J>D|-~+&LOZ~~#150hZSBg~E(P4{Wr30X?k}}E) zWPdL%Hp9d!@E$dl5XN3kbKkJea9@>B#MU$+W_8ZqYfE8Zm7E+QKW#sCulUl_1$6BN z<6gQAVRDonl}o5TdGBF*_jMj67V~UEtiyyZuxM zyIKCN!O_XK3$Nj_z^(?Onw8d%ImiJHy(o9DZ}$dNbhj`|sdnYJ2CH9<{7}(EL~Kj3mCMp>y^>=rz%Ut9Sopc*CDd$6b9V@Z7<vn2D-HqbPbi>3N+wTiJPs6Zs046~B`4p1HQ#%z)#DWc%t)jDjp0B_FiT6oqXdY9 z2wQ_hiGpW2xNm*f%dm$y%zX6JO!IEF;3D|45=k^TK3<$BP+zV$S*l)nxhS{oLJQBe|?n8%U zcD+Fm9itm^+WCO^iPM19IS)srzn;cI{{qwRg?7N+PE}=v)XgM9V}fAh*Q@oB*XuR$ zN!hEPjIB|tjCiam*jy4sp{*ZY)!?b{`uN^e%i=q&j6d!=FJZn%4R6R_eP{6`*U3(yKR2; zXr|(5`$ijRXiyUi54kQp^6|V6ioW;8T=pS)yAwEXEdH*;P=qLMH z$a1@d@P5P0b6U;|wu4zvq&742SnY$&*HzDNkBPB?s}P^(5P)URqvdezvS({xqzZ9i zWGHhvRaWoAc+@5Ax{oUqG&E&)9sWB@rW!ItcLlX z_S;)=)&(0|RYgULlev}(ab>rJL!#gYh@P7eiD@jRLprr^FA-}hW$F@|DZ6;U1U!Q z#&^BV@@*~JuPKf)3z0HVTZeIIzyLmc}MV)q+uWps?X5QLm&PZzG#bVo|Geat$Ovhn@YS9MWCO zzw$m@N)E0k-Z3qYAtxXY0#J@EDzz~WvuZPX+@fJIr5k99fXqz|`N#P-lVrhmT#O+% zsZp83Rvc5um&p5yWqT+&__J5HQGS#;N=6HwTrWi9ZwSX7LT)iaYXH$caxQ#n0e)E? zPHkGx&tbi;f!~FHdHyNlm#WvQc{4Qxx`(==99c94Yt;pU3C)%h4Z#+x{m$~nA;yn3 zdicL_ce~pK!6`K=M=`>*W}$`16LhoYO7XB>z9d(z6gP#z#KspfV3XZTQ-e3WH!x!R zdJkP(91bn-H)T{m?4K$=95;3h0MeA$?jLgExqpyG=cM)?lA^0n&-3zbl`4&6;o|l^ zT&!TRoAT(u26-MRshHkyiZ%I|GM+d7!)0K;>MPD2U^!%>I7ow6Wab$|BEP|=3;3AY zd6zb9Oz4x8H2?Z9!I}K63?(~K=NR2B@xX2q7RN45)Pdp|Qp8(Q1{f(Ra)#3cq~FSG zVNW^Mcq$QeDe^HVo@GTES{qWy9u>K^t8)2mpXms`oTP=ox?iUsuSGAZR)Szpq{|*7$k@5@0WRh+a zYpF&e#XFpEJ~TyqFgdl{+ouSL{vcL9s2iRIL9b|=@x?U4p5G6A*WLg3(f0&jpEkNp z+!66c`xa|O{~u~`6RIzvu$X}6ky;dS9Ft45I@)89oUn{e%WqV!m!etGPSKNKOOv2q z3+hvaTVi4`x^qSV_VDZ@&=;wd3RT0Q71lCyvwNYIQ@k{={lKHEpBf?+@6yPp#A00vP_%qt(07tU%liY-Ey z81371oSBEs|CVKIA0$Vby%xIw=M_JhZOwCd{Ki%A+byUnBHioDVcC<2W?gvU?q%Fs zon{-lC&WuC1f1gcq=xzY&dFl<+$PT2eZIUEDYD`#j_SsR>;em_$0w$AXRWl9vdc4o zDXQ5r)k_{j%I?|^v;n2SS>*{A{`WmZTpYe{ow;L2&7=#zA3JoVrtN)>&&9jlDSI*{ zm9JJ~B##Ia0OE#crWty>qWFOb2a@NBi4vyn)vlQf_`QH*J|R==ujPTv=YCsvNf z6;d~FIKx%P!_fk@9rn#5>7$Q8Ans##Y6#o&;S%P3EHB{s1N_Hr2m?WHUz@sC3mqD^ z*xOK2h4Hs0PJdpUWovt5DDj&ZA6{=^PvnYe^j{S47#&_; zn=I}CK$~$Qbu>k7t}wusS+|d!L3hB1-xJNp@s_dM@f}_VA9til0TNpmWFpeX+vMj? zdb@v0ctdMAvqYt3Qr;h&V{zY3Z9%eLPp9+B{~K<8KYYjt-@+}w?E0F(v9q3ia7um8 z2M3Yvq96=IAvfZ}AN!_<~=Cro?Ca?!CPA!Ya6B{ znqZe|fhR}muG`2^)SN3RV^^c;dhCteNdX|LnIwa0iyZFa0Mc%hK3xk90>QLyA5<~m zN{>w=nZWFf3>J#j@!H~#bE8Al=y4$(>uS3xI@KF(l7f9I=uMaYzsu??nld-a#XK*K zknDT`x>}iHh&TxkD8?@#`RSp~eR1`Z-82WMS0SG%!`0eVF?SQMF2A^boF3boV@bYtG~J!GjTQJlfnM@X5o*A3-sbH4sr?=xXc^6;fObSYL(EF(jZ$`3G^u z7y(D3SZ3hr4!hH83nr_>8G|`2di%bL)$RNV)@^@I8cj>5muG@2)u{B)ze&5D2(Y$8 z7HP;_iQ&{m4=JhH+r5R<%FO%|&AE~~Opbq$2*V&PyDz)L5o)8`7OHA840zpy_$eK; zx0EIiZ<-QYrl|(AXxEfDM5D^EH#42x%Pv~8+j~#>Q#yL@XbxE`EmbhD@YnfjP1xP@ zJ6N|n0Hxc_7=j^2`e1jEHXAofFN@*ebdtDuLfL3-CL|>pumpg620n9o_XHjcevH4v zCy$FHS|r->;i~kCk`;P^cJW>Qzv%Yg+pQ|-H^Oz3aL2$^f1Lb_a)N29o&sF=(=a#OtP( z%t%{inXv`i8vo*kNDuk1ZArh^e37;RbSnad} z=L5`9+>+!xbKra53R+=l1mvOzG7NTO{?rTR`8cm5X|;W&eO>1U)FJYO=R6-G9>o11 z=-p%A84R-=P@^nokROFApj4tx)zlEKj%9x~7yRe%lVp(ORqd-$6ka+LBS#!X2>>+Q z7_DN8iH_Jl%!^d&cH!`U*2(S@c%D;y4+n3DbXQv!3K1n%kYy5dg_q=bV7vrDZnjt= zdY_B&NAht60WcJ3v znKH4lPuSRf+fnxF)-!tBVbw5yuIRc#&EWG&y+#(uQTM49cvdghS)q4+ekYFqop*0l zA%>He3LV;-$@(^MZj37JQK{?UMXLd$kl2P6%D94AxucoqN${|e4Yw(Dq+ZC+_C#o= zV4b47Gh|Gjo)~u|G|(HMt3lKTp~M<+ByypI{OG--XYl7mq*lT6_DF3(IOA&|{htt* z7QV2g6R}k>PIL1dZfnnFeG}j%5}QgQd)m}!a7|M5B40eQD8=C#zaXK1Q!VNmU=Y3|Eiw+fzH6O2!3 zGoHgdU-z*ZbE-f%z;0@nDzwv-0kNa>~F%eU2lZ=00FXl>;!LU-ufpb zYUnZ~!Y>I_rcBD&B1CGv$=%LYp`x{6US>Y3nE2W~_(K{Z~kh<@L2{ucP%aZHCCnP4r$G3jcUotM{3Ocj>Q9zjHc_8^j{ z`p}QehNVk)jP@Yr>%Kv}rO@v1eS*VvpL6hlDG`9;4l%%;yxK=ebjdSL13 zgs6%kIR*{-Pc!7K8LV7j8||@)38SYM#CQL?)vw_~y|}_sNnj^n*^9 zdlQrM*o3qw6?TOqN=2mM_()OQ_j*3E?6$K8EDbSDSM~axOtuBEeW3x~S;ORejc4VS zjfcl-Tt&mJi%J%IwNZ7uOtGZaa-|yAV`j{@^toTGr03h@%(|xQ5oftl-7xvGeCqI{ z*fU8*(9Xy+Q7jkU`L&vPvASOsio0lms)I{VCB!L|tzR?S9ayNy3lUCpBZy6&)UqE6 zwLT#$=^vC68J6F5T4dZbJ`vD^{cQ^c1VfLBI9c?DrALGw4A_TzpnBSlq7bTf6$d=% z@SN&EYT$bxPkvini>}^g(5O0oS?6j+`bG5pUW9o^2rGgUfeQZ(#~_7pFgr)m+Q|{2 zXUcuO*8amIfNF4keGWIY0P)h4>6p;8mC?32dt{|Ldsw&oIix`fDH&tq4JyA&>bzP6bk8Q9-i z2~u(Ucg`?DNn||Qy2BYKGI^FwMC%8zRH8@;!a7t$x&gL!Y`7gWJb$3M&kH=pXc}(Q?3F+ zPOTgyiSg1N+LYKl=4RC zJ%3BNSp7f2(AdmINE@~K0c|OLe}`dK2=mCAnbpQ@_vXb_yB50bnG44Rj&Z0UnDxfg zO1e&rG85)xIH+`oP$|2%FS2l?Uf?LOhZ1Eb^pt-j6}d2nmeFk<)xDA%?&}qFkBiMP zrYeMm2<2%)`p?R0*L?(b8GDvv3=sBY(6L&Rq{K#16iYwyL{P}8kzt5>uj-lf6Wbn@h zaeh$qnUE#=N(4=7W}2YB9X5aafetCj zV1`e;o8a)0`4%%8{i~IhdS+kHooUzt(&K&2AD?)y6xW)Y)4}Fy9jlwuzU=KKwXNl! z{4hxfudx);P3$|uzXQTv%%!F#^BFsQw>6+QxZLaP%s|Y?DsOVH{ud?rKMdvbwSap| zW`$Okri^lZpZLjVwiaa}D-=@DOmuFC6B0Mu^++_rTb;*1z-r5p{-v4=e{*FwZpSjO zuQu-WNOe@FwI@;s8}HyP-{tmuEsO`&BlG9>)Xhlt)sw-~az~v9?!cqh>NhDP<6T+7 z>i0A1w(mym9R3uA5w?iB^r-Q-;U=6|StC34nvp=PokXsS!Y=ye%`16+>oDUMoG>eI z2Nc3n`He)4=}Kyz>540bZP<13To;@*PErjkb8)L%+s?zLR(3|se5=y}chIq8%TC7& zbvFEOca=12o-H^R_U;Hpmev>O`z-{plxS-_cIf-=mo;$2ZpGM?EDyb)0dt5D<180= zB3%$ot~Wq$vUaP>!p};H7DF8mNu93H75eNx+!co1!AVMb`e~8_n*2c6_(T$yBJSIs zYYzy1`x_Ehvjuv`r=NO-8iT4aUH)Z7oC^dGEFvMwLbNGDQ9GCIBA(~aWjVLJ;omMA z8GpRL=hA(~W5sFx1ybu)m7O?5hsraGDwAJBbU&0!4(fHhAK3X=WJ*m)M&F4OO9UO9 z=}{Z3oyPgD*iQ;NBSh2K7ZU5;l;F@0j7vW5|2U09!cjK(EIFdId@_+zeerR$;^(Y+ zv46IN@Q3=`Ka1TW^51fNzL3c75?uQ5Ut{A46Do-j>Grm@nd9ZOhqH@Kf8X4&JNdjJ zQ!@vmjWDlwS`oO}tpsGb1T$?K{r^vK`km!(h(?bK&|@^nD7eEdZQ_5OcszD;dt7${ zI?pw?qW*O>?|#ehk9>T83J6@$T&}v{xSMYVKR=#!uYcSyz4dIl)ZBe^d)y2Jyb<(! zJ#EE!-0Z}75VT?<^4RicyY4f2?t&BO@geH%Tzd@<(BLh4TeGciaL>)Hi42xeDuxxl zTHVIFgdY@uEn58w*8}JT=X23qx8jX6p5Sz7fhTy>@NR1jq@p?VUUXap9eXRf`&n+< zFTmYzd7a?Y-%oqhRKT^X;yq<`cLa00O_XV|XGP>hgy5<+s(N-UutWMqRAi{+x>~zG zsn2RfYpS}fovGVpOepg}Zqan@L*Gx+Y2KtCw4FgPLr|Y)Wh@8u=pYe%NI!O>a^Ojs zh0_OQ!L|R+dCtl4TwUIYc+hOoy9GOn0GMr&a8kTh6S3Nx%gE5HnJ&^ME zYzCz`A_d@bg&doVX)>ez19d(+KbQM!oE~mPjz*a=t-bwQ>tjkkTO^$GJRx=*Vo`&SR|1U}10eUtDY)-f)*Vbg? zsH$Xje|j?LOt3pyZ-N^&#$;sY53*dTIk+riX|>%)5e*h&@&v#RSL=*y-ow*t^Tu4L zhA_DsLy*7@DB0FTD)w~-Uy!IW+J7)t%M0>-9>ZwyxL~i;XpJl$q9tP{GjekxbXero zwBUXyO&|FwghUEr)I<|a*(gb1$AI0w8EqyA_Hkw{|E3?^B@76HSE`nmfFi4+Qu(;A zBqYhIuHO4=uEoIG_00)^R;>{h@!adEuBa8&-^=VCuSG^Y4md;^S$ykWM}@pryH@Lq zTOtGYV_cLwny2Gr)+6D=Vbve&LuE@P^r>aR_d{Bh5yw>QcR>6P+VxWKd1J#I`F

6XLN_J>)C1_H&U*ciPa+01qIH zQr7K_CSE~NlQV32BHN!bV`%8{gY9RvV7mBQ6^^KbBnK~3{T@ zKZ(&{h$o+kyZ*nh%e$KA(rpTI^vP{mzq2^x*cF%-J!|z_u{)o{rEE(b(#`> z#S!!PkiR<6n5^7Hm&=D2zUNmyiull6_{1g@x19butYi}Tbx7x>8i>Ci;T=ptCJnaG z{CS=rJgba^@}h=zh#r#H_76``2uBAvju~cBm_!-Z38X!oGDn7ir1Mi8qD}XU4gotY zuF>7S;RO%vE!X_G?XX(FddhEQJ{J$1A*YG^U7MfpJ$t>d4_ews z8)wfh?Oo{~If7cn?-J<4MuUtKbDz%fM9fz|K0whbsZAWPX^raDcGy8OoZ9L$`_js= zBO_xB#|XZirv%8%NK&H>Ya8JBJG|lA4VWN=ybmReGt*LsdwSlfB@&J6?4kWKVVjv& z(ffWlUk`m&d&5Kg8C)68>xa$u+*3DdfGk9%t1U-QR4roSasOrcAO!DQ9+*g z%j3-Jni{id;>2U_W*4AzZ^|CZmda|)k6rMDD3S4>I=YYl7mr+P1lzBUCQ_mXGVlNW z(kWjm)wp`)#lE?x&zQ#iVFAU)LD_px&j~w_#{aik{akrnao7j`+0K45y-Q7w~QO z-LrzvOYoSF4foZ_E|SvJBt;M2RrI=2CX^>@jF?zB!yt#tlq^eMtP0mP>)bxvt-?hU z%hZE$iYwKrlL1#_6H+NZfQ2GqM}b0WJ?%d(xZ30LJK+}T0#{v9)LJ42RBIa>!78X{ zmZ;~2v_}sk#1)~c%`J!YZb0VQ%>R&z(LiRZrw(l{vhp>#AR6RaAyvUoy%*!JKPt|b z|7lAdJF%l>2^o@tT9W#yzro39N|`>W=R$!3z>=@gr^*=+XARz)lc$xI?H_q1U7!;W zdOx+c8AyMmcN|2a1bPL!nltdHi5eFs;9=5Lyptbfh2s4@?|M9Vn_i#(KW+TmZ>1Q# z!jHZi+hcl$D@p=U;l|j_J;AyBfwAb!AyR>Vm|t(V>*b*j z^%WJ1qRj+g^;W}EE3RtpGWg{Fo_sOGQ|7}vuWxFH$S;;PzLs)G8?p-k&e^M}i*0!-HFt+ax1VdPYm<(T$}y>KTd)?%V+Yei6uH=0T2r*Z zKV|G+=oq)Vj>|LiJI*%z3&pfr5^HB9_ajTdeq>O)Iw4w$zL8oRc(TihH$MI)U z6W_-kp-_#Pxf=hFWI^c)x7f2e$=`_&BFxLk8g>=Cx78V}c`Yx!`_-4WM?`gCMEZDe;Yf zPH6qDU8D>0zH*t`u}7>UTzvHQ--iFq%YK)>KEDaO7MyZ5)YX}WJ9s@Fv3T3_e)gaj z>UBu8!nM$By8jgO52YXxCrc0?SrGr;jvX$bjU1;*;%&1T1;IoRK}H-wwhVM7wU#l4 z9~lT09tahC$r6O>fv%}D1Dm5n*QYM@@rXUFT0k zoi!Xw+JQe|QymbgP>q=#nm0g%rX|PKrHT}4TP-NNqeT*QZJa7uvOwOo-wN2w^%^wV zl*7N?wgD?rmBZ`m;Pq*NKcWf9f+uI*Jw+PL=Yk0IYs0sWugTI^MBfH_5Sl$Q&cb_A@`hK{TmKok$iFb< zg`YBqC9=FKDx`^Bw~?UimHahm*O*^8hI%rMdL#g=E;(AFoQ+W4u)dL4W7#ngX1UQY zp|81hfcFl)mDb7mPC0OdyzzHC{_2b7N&nq= zeQSOys-%uRFq$J?LXeBUje3wr-f;oZQA=hob%-YvjSH{0Q6 z{ATL9!TrDa%}uy9miyJ*tAHDY=z0elm`DBR1_y3=)AY%$jr(*1Oq{0Yw}E$$(F}KL zH&aj}IB+XZ-}>#X&n~>(dx87y=J&UDt=rQ!r@1}c3b&F;4~=@754Xm5zux-xc6hf~ zH^1xdW^n%4fz6AX?;UWz>)GMSn|bK2k*pEiyJm3GIO+JLc51|NuiptY$8~bJnMZS; zdUG?}4(@vgm$!#I<#DHX|1+TU4h>y!aBs@DaqIB(9pdTVjn|ELf2H7-(Hw4rx4de^ zGl<^$QBSv0;7;&vv2K3X-OX5yrzU=Ob8&+6CvdKA9^_He!JT*pn&|I*b8|AE8uxB^ zE4=*|4?OVTy#U!y&o%-$D9$2A|GVI9kqR29O$s4S@yZeS0nBG^cinEuU zjB3;lr7S9qKSWKH|Kx4wr`B4bLK~z4?HZQCfrwEiEnCZKT?3Zm1(q-5O*|>v7Qe|@ z=MQKpu9m{p8K25-FlwHqjJqHOwxL$v0HdG_pGJ^DP1ojcF6h~X+tW3}JH@*_Pu({g zux^j%8sM7q6u94-!8M4vdAXoxy?67_vm0D+pPS238n^s+i|f8?#@GFR`}IzsZ_QJW=f+&AY}aa+gkFf zd*SEsd6@Uwa!x%kz~@NVSpsij@mgbfbv;uS>Bk8@YiX{@J3!$oSr(+6QYMs-LBHa0 z3dQfFq)J7Rc~$jnypE|Zimp{Hs}B+3ATgE4T>v5B>9gRYtPHXp^aF>y^$NZkN_Bb6 z*_5YYgHKr-)z!5O)XFzE^p5I7l^hCxvc&fs>1q<~F$TNpaoawyi$8`MFm& z*sy99(}oJ7OU>54o_eQxigy`Hyv*+gnu}K!f?H4$&Tjth2i5iV$Zou5oSN(Flqa0Z z;$sRDLl@u8o`@kA56u4#R;*_hyT%6MMzhibP@PBdzU)B&L zFVuN05e?(XUSk4fqwjteTz3jrjVcgcL9C!`_BafZw?#LrN_sLnS>Of=W`A406c*wd z(bXq(`Mt{YWq*Nrl3%q`r7Fe*dC5d>dUL@ZaP^z+ySIN^M&O@M$FC|fYtTFo)pMw0~=>ZC= ztiZjBzIg`HoYJMLxfAK4d5@w~92nt+6H!ZY8#3nIY`cPYe#PYqh;XfO3$d z0n?NB`8CTn&9hHZDjM2pcT(}H_)|HWX9LPdX&uH=sm3$7@z%uB994wVkHw5vt7Z9} zd_|ciY3(FZ3i2x(yUtV^2~-)2uhlQ))f^-#jJV-{6<-ON`U;}Hn2E2_3?DF}tz*W? zH@A#mO|+2=*H$im=5vz|6m)4s(xb}Kf6eWtzG6P!+V{l=#e({(#vWCu%yP;s!wfFw z-g$1cnE{7dH``LcZw3{q^6K~QcfMr(a#=GxaQxyM$xdxuL3Src4x!OPW{No#PL=pcjQKqx2myzP}BHk##d!YFwqPysy}0iuI42(SJApP7vUor zQDt5s{@>s$-qqM<{w1cPye+eCC8jY68Kcbc!l^->{hyMvRxm4lDwHuf+ov5eOw}Yd z4G!d6$qfoXWkPvXVhQ583R@~-_SY2;XsoXiAI{htRJjv4UnIMw2T6`fg6rO$Z&it( zf@ok?8kruXx)N*6SwxE)AX#4||A`lzA0*u}45*?4BNx;rYJc&Z;@s(-Lm3Ae`RJ7! zZt#S@Iq_?D&hVn{HLp-Ki?UPm58+xTpNWh<3g&V}eB>O))f}%zV%wyzGY&Gc%FwO) zprC=uyTk4H5pKLKn?fMqaIHzR7&5);#fW=1!IsK&f1#j!mBQiT(u6Lro%sohYE$pK^9yt9b+>t3WG5 zoS06~Ypr%M(71Tufd}sl7RO7;2L(BhXbEj!U!P-}Ys7gA7)6z#d2mohuHx=`QAu&v zWZJx=pmIB=ma$qK{7s;c3+mKH=A{8hMkBo|lnU(B&cPHa%TYjlh(_dr=L|l|SH`@2 zF^@7QyBgeNAaw9@rbpimG}q_dKykm5`0o_&-M+mY-09of<@I*&-woa=-P@Jpfd_vh z;ZD4H+dDnokQYtg*bB`6F6h~PW9E4LlYDZ%HU4LM?Mp`iW%`BYCCb}KFg@RlX288F zz5`AicVgoPKV_-3g=&=(u@Wb4X4WUAlV9EZIC+!Gpb4ft_++@05iid)if4?LjOMFy zGNw!k=E9c-=f<=1$;51WW_=hmPj#M~;|d%p?f(DPa1)$2&F^lH*Zl1cU}$xF+~zdy zcPE&#n5NFL#j43jLxqa%rU|RYCX16*jVEqf8febWg&XhOo7*Gl*$sm4G@jbe;U!&b z#IIOd;#-Ypj^eD~L~rH{LDRLtL(N~B=`1dAIKlZM2sX+zpjXlZIF4#>*>bIj0+p_! zRiYZIF_z;rl|LDgbz(b*XDT2jr9q3ycya#Ra!t+-E17pvZ}cY+&CRX!0O>5!>}5HT zrBaA6!%&Fx0^%oAbCk+sMCJ1@$ns@CsIfhZg+;~}T}&YckGeUETFcZUMqw2;s(m;t z`pR3JUzu+prum0{y<~qOEu?^(751u^sgro#kEAcb@)L{|F=e}bS&F|-zSb5WW*ZNt zk#SO>9wU%R+yVD`JLtXecY<)xp1cBmsZx=uR7f={il$bk^ET2n+WcnTyxw>xc!zj5 z;P(5s3-1Ou<4HD(4z>)j#8+)zp<=2w7cXx zRI&?IK<%l#m8n2Tv)pn4)j|cbd+3!|#I%@b%C@PIex)MYufiOd z)bw8pA)h};79y`n4^fs(s~`9Ead386M?#BSRUCSGxF${|x;^JA(c<<*G06N%_AsG%(~vx|NJ}@|c%yWquIl zpSB^Aek3_T>XLVkIE24T2el$9c6q8_#!4-2O4EG!!2#;!rQ%TzTOUR{jT|o zjPOqXa4905&_3qmHGhSVgh8%y()NZHJ^W+dMXaHwI_t_B#hRa#NxkuWtXQ zYamv?EFK60dAi?iWebjc6VVXzTHu;Ker?FIH zlfDVm%bdF=yl&jwa%6M$nwYgr<0!M(bC3S;+#NAnCVe`Ky`0$gK5^ zKa-I0q+qH<`H%!S!O~<;Q+36aVkS?VitXbhjw)|Z6L=+aCSqoSMcQi;7-p)lfbGw7R>R&PyQ;_ke6erZ>5Ca9X^J6F>ZW)evtEu8lP$$KYH2i$yg zb%Vj4w+_FmfIz@8Ksk!4V{FpI*;GssOqS-qrgzQd@|$TlJvWrx8~7?+GhT}8(7*}q zy)KHc{MqVk8`q;nidu+mr14F!4Cw{#FWx49afm?a35`h3WLz|3e&P-CjPf)F493;8 zTy^Ixop;JdApTPw8DzaMZK8tN{`w=DVNTTjGJFv?89HjxVuhcw_mhqRDi;R*MtPLy`S;{;q<*X<9!U5M1LcZV}Bvhh; zoDVc`cjTIyPV!4lE?m^dRJOcaL^WS6$ZG3O@r08=nKpv^N;hg`Nr}2zhjrugOGRQ;!58bL1_7sXM%^qyeoBQS*Lb5nc+x^LuvBPfo7(kh6E zq^oc-`3WZlx971fn73Lf!25veMOuhXj&YKa2sP&6NLle1XmZpINp6u-KxY{23dKG8`?4G&v@cA@H6z+RmJf?wTZfm zguja!dkxVcuPCho`=CT6H3W`zWz%0BUrCm4@YZ}^)RR%F%>0C>mL`u9({xZ)tD!C=p ztIh#(`XR{VrwW=M4pdi3PV$}P0CAV`SaF>{(dk`4*$^KiWB4Hck-<*)s-K4Ox)_)h zRuuIW(_fgo=3#cd#GwAg@7i+1l@%hsL9x*aqLaCbCVHpxZs!-_tM3loZU)6~1f>_> z8r~J?>g4H+H_TW3VemCA%0Kak`k2Ef{-An=WU=zrHXYg{o!`m7ra27{stjJ7{|J01 zo>2Zb-nroLtTRSy?9{XB;>L6OtUSU@f8=Y2r}bSycse*Mopbmh-!mAQWIj20sJ7-u zQ2tXYbMqeYGs$4e74N#T!Czy)Vf(YC4L|b>k!O01YvgB@5oXj7`O(UgpR3H#@Uxs^ z*uIUvs(aJ_I8FwbcfiPJ$CKIoAZ4+iGp_on`f!jFNvDaKhiO;kDFpRT^-}S=bHLD4 zR|ZK0y>AXK7`l4kfd}sj>MKTC-(uJ(QR1<8NoYlR~Zt?!5*Hx$J4#tPXfE zLYX~Wo^^t$sVSU3a~fyQp2gh!9EE^{j}J>rOJgLVR6A6d24Zk{^Cn78Y7>A z40r0?%H!1OQ<#{Tz)EI?N=*p8J-rBr!p4{=F(J|ExR2iy0G=rDz=Q7|tiNT(JAs*8 z8;QX>5eH2%%)HonMZ9jkD!AWn#A|>iDK$n^J~G^eIdOChcbe-NHG&fymDb2Wcbt6b zS5QrEf~27__AygU@)g74pNtD;X=}ib7$;E5i;czP%u&DCDt{s)Rmh~`dpUuG`Aw5B z8FOUp(=u8qvzkgNn2DRNngcVwVw%acN@sHmZ#_8^c;2xL{pp>|D`wdPvrzd?YA2q| zkgSDy=Yr^}CzHNGKpmpt#CK>XS?)KJZN1TC+nIn%(Q_KWeJ^MzBu&AY9y)otL9{bU zg8o!D(Zl2=z9mhBt>HyOfCjiXEYIAk+y<6r38L~A{pB*?;OGRs*LQ)73$HH2G;ng1 z9i>x^^_~9{H+i=Csc4zy+r-fv{G>ry)>or|2t-%63{kBC%HHHALlL7r zmBF-}h#S6fqVt~bjS@jx$bcn9&<-F-#WU;=Qwg9F#m`#uW09#uF)aZnvL;Qa25CT? zjd4QXnMPi9em`Z6j6ev|;n?e0H)8;3fbtO^7=st2&kh-0lV^)nStc)#enH3JAS0}{3vl|cayPw&dGQ@fsQJ{!1>ReTSTQ44lqdgV zka9|i$hI}6Yqh`dfR^HsZ~48hpF2b+z9*X-o-p1elEd2OMRhhxuaSt1PsQ5w7x6yX zjJ&Pw1H@;dtx=|#hvLduryNwG`fmesN(;wW;82geZE&G1AYFBpif?Ex1zY?sx`|=k z_sWLp>udwtYyrR*b#37w9@4n1k{sSKWN;EV`P|4u_l@_BTk)I0%J7Kt5Dgfg-yCl_ z@*nRj#8CzV(ZcX8BdC8H8EuRLdQT3JL0jKcoL(_sc{?km+E&KuD%+P*h1sxwX+BSR zQmt3aa_bNig~S8=t8=Wg{1gU$d6~(bN%BTWN7+^(ej_ia98-+Slss@`mD)B${t9a8 zCGob{p2C^rVR`z_d&UVfKe2YDsJhDYm+8cZq^Y*i(1BCpP05H-VTI+8tN7h7{vyBl zW#mw|VB1Tch$bMqj^U2@upE6eq<T^$ZKdAZ>R=w9E2uXxz> z0^;WML-DifyNfTu}K= zJ33bve|PSq?+(85p4N}8nzQMfN>rH!UxRleh+%X!2s1V7tJPvo-*@CkRbF%Ci}#eD z#ee+f81B^elrB}$h)Am+TBMQr0Zj1~hwa9%#*10IV3bSFk0jz}vsXOZL$gfHGbBf4 z2oB3=$~el~7!FyIenhTY>aUE;x`dft25)H3X2z~_EY$Wceg=al%axQi)Ky&wqPP)H z44Mb)n~B>9M7VLIOXJ1^4?K8B(3uI+x8(@u+Tir8{`D5n2N~|9u$y1{6wwSS6bMw* z-M!Mx;G)5;TrysB%gf6qo#M;OPF{5~+UdkedF;tJH+*#%6H^noe*HQ+J3CGOIz>Zq z^{%6%!+7V-&(C9QdC$DTxt2OfCf!S@54X;d>fZ7H*xz6;IK?vUihyLb9_Ypmw*Zs1~cXK5YQJ_Eyk1DucHJZr*Qqb@A6&$a}pu6MlhtFn=yI z=GO$Nc~S-)K2wUC3@K7#El&HXNYuj-grjjJV+_0|H0Cp=c&bW`wZfphrRFGGv8(WR zZp=St5^rV~4iD=fmoRhDVp#K*k!~z?6E0NFsLLP{k02UJu&s2$xx6vzM)6kuT2dx` zOjzp&KrJ)Xy&dr((w;&rm0~&U7s{6kmtV$0gCOq%A+)ylA(`$(Fcf8-sR(IVI`c65 z?h5Zf48Bww674YrQ_MiRqbeiqMzfBZv)_=aDJ94#)tr+U7eBq!dL+d+aD(7Y;J&;0 zP4zPU+T1%qylWIIDOBdm?Vz^|H`==i%)16p7+_g98~+`kk>`zc0;PM0z^L+CGAZ~U zNrNhNQ)APSY$84EYSk#OS~_NoIi_uFgj0Ufxeb9tn$J|m8Q(0Yz9S9}I2`TtYr_Xh zBll!|Q<-|tHj&4j#d3l{XC7;4${^bf*$E*k*^wYaSf}~N5sR?h*#~Nz9P8jCl@UUW zaYJMQF@cupR*k-=LA!Deu9)?f3z`=q|mprr}!rGao$ z`-)FwP}ba7hDM1GRku=6zi{Tqj(~22TiU#l-z}h}a59oOd&0SOlI%3g4nPIUepTWKJEKgCNsGDfQ+2QauL~57G=`WF?xm(4JOULtQ8bdA#Jrxu~I5x zDL02qIb)2cp-`Oljj#l@t@@r*ADLdSBjGyY z9mJu9f+y&Vr>41Od}SIv5;>$g`w(yKf}e7uN_v&p=C!hImP(7ARhDF_rDcW0SN&FH zI7<(i$Gd=Nr+gL6c+Nys_L)+dV}+lxEgWFr_(cY;uxtfO%Q{sJ<}Yu%0O@0lq{N5W zm^{qKTNOz6a>w&To4|nFAynnPr{a~&I&P?pEX|R-eZ2| zP5^Y>iPYebd{473e;^#!W!_Rff3E?kU=z_}8UUqfL5q>oMS=Lg2b$u2P z2y4s@nez2q*AHUU<7?$I$9?u^GMvFhncV^IM8{@uYUzY`%Eto_Ja9m=Mjm&r0Z;N% zW2za`-tNE(fjg;mJ!@oq>+S7zyyrByaI&NQ>8_7n$I+ularyFP6EB<1nq{&w+__6= z_wL<`S6_YAER~fv&iME^wrtsAmf4Pujv5ajeN&k_7(;oe+<}1sbF!=1E001)N{ngcM}>Oc;LZXL4Ejdo9CS)EgA+4gTu|>-U&2iGLyE(pY9wVGX~Y2lR2}Ll?gQI zqB_MtGxKNAgw$x0Zq{a*p`%-=9QS)uyarS`bCQxa%1P5z(k~Q?A;HO2P`}7h29nr} z<>YG=B<8n3!G3*l5wU0tW@#8Fi1J9$zGXMTsf{K~##@E>RDGO)`Z(G4ncrKTHj%<; ze%)3?ryPs#N>q4t)vv95#UKjGFyl-*6`c3ZcTGOc1XQ?~U#ktpO>km4zuP*raA~F# z`nB4ni95a1d-t7YnvKb-Qv!p)iL0EKmMR>ZRIW2#INnT5!6-^4ql*+F=Sk_^Jj~Zd z5a_*IF3U0-#5V``o1l@VL9<(f@Rc_J>EzR--NCz5v@thdMkqo$GAk`v3x$JbtW#Uv ztha)|t*^>bo>G#E49we;i?g%R>tO~4J)lfkjdQ)MoRgkfl*!09l{A~qME!2>+a#YTlMtm=8VAYRGD9wc;yA!Nb9~f!1um0t0jOXI)W zKyg(@b8yR6-&8)5C3@!uzE}SujxwgnlTs`v)(encKGLcfkjJHr4^ilVu_EPEF`tKB zEgQPS&wOR!t&zU!oTRA^LQz}FvqN6XMre{A0?|Fd0s{3g+lT!pK;@VUX)qH+eV!3A zM2T6ILM4>K@`|K7&GzxL9Ge~>PNbOhQpXwS8O72xi;wS4wrd1}{7>_?g*sP9qU^Dp4zd{Z)oIFyP$%+2vBBEFNnXCDiaPgp;PI>JF^ zijP&k`i$!3)=xq2#M`>M^2){i=A1z-t{w+Uplljofr;Ur(*(RkMkQr;&gC(U|WEt}PzbTYObI(!JHPM*U2!o11! z?z`{C(9jTCTU*Vk7;Dz7G0zI>%(`yau)%y=y?V8oEVs3_8CO6V>ZItmx3`<};*aWtiL zzB@}n^-j~Kn=zbl6Wp?#FU@fp9|*eBB)7te_^FZWI6X3oAnDsj1t=8cq*QJOq#H@^ z#OHmL6SIlMMI1SK!tA>hkHrv=Xwzc08UGxdREI_)f-n{6cT^d3^d`*No59P>V$ElQ z6OZaCC$Hw#OxSemhFjt7-`)y0q2a6PsYM(* zbOhZ!DWid9{^BE}EIi&2d7AokD(+eJ%W1Tu=rh|!pWllC&g z$N*iHa>+-1kuo8LL@P?gyV_h^Mlda1Gn6M@GCBy4x{MUnAc9oBgJELH$6A$jVBTi? zMk-W=dck-R1a0T=3E5giw}51l%JRohEK3Pw8U{lT8Hb3B6iY29lJSVQ$kZ}sxo|8P zGt1^QuQ1E1geb{umVC=y%P(V4h#w&3Nso zZlbGrvQ(_%(#7i-AD>_yEu>o$%u7mWGr0Ll{ZTH=;1=Exh}RSpx1n6$0=MI&eB9uI z;wYW2?;K2BnY2`{qC~!s@uO5*G|R*sr??;@R5A+{@pQ;BxE*)CSF!w7@yTGuK4UTO`7*Aaoy7d?3S#Mq@tUj!Whh_+mK;NZPwmNnK~pvr zWRfjFe6*ZQOljAtPOAS&i&ghxU4*`t&uOWurS*2=Ve*sqPSj@nrXK1OjH!H#5u0V1 zJXN0tI8bKaU@Dx_eD)wl`)T~&W7%%w7(e(DvRI&~38j|@}3%_5icA(7_z8`an# zUROPdgDTc2;UnH;hJG%K&4?A&9{I}+;7tT8w6_k?` z*Sui@%k&kFG&Olb24I!Tj)95&$XDVRRUqx7W%knu6fu?4#xc}kX(@A-7un4iJ*j4 zvCJEm5hgD(j_k@IzYKq@9|ce-ME)Y*lt~}$Yi9Xs$W%+n7IH{MqNo;&tgn_FOP;@# zf8P!atquL$G)_B|O2F0ODV#oa8NYH*qt@I`O9J zXZnZ649;_`#yB{+%yIBIx_gtPae#DFx&X>-c1?TK&dxX|J{Cx>xrn=UY_y{RUELtK zIbMp+<_r6>^G7g)Vgt;r^DHA7F?C~^XW?tHO*9?_h=x^p5BXVTh@aWEHf6ls;AgAB z&stfb?H%+dJ}D6q!_RWTborS)XJ)628dzXu+C_h4}_|J!D~bAKIlg@!IBus)ps+GRSjYA;ilWNYJXFBw@{%a9 z%qw%VIQZt9h)0s>=xRqg9y5EFaRyD4oN1KtfMcQ?<*PXJPQONTvehu&sd^@uduOtx z37)2e%E{nh^3WGe!Ay)sP;UCFI|5%hd$+jCsPj#E6EkyvyK0~}E`1!(y@I23IutWh zBk-M>EjHrwUTpiu8>cWwMLwNwL0^BX^NB(B8?~R50|%7uPLqE39fKh_Tho+rJGS~J zr9${L=jnnV#a4^by)lcaf<=)6z zc?w7?DGg={9A@?9v8OhWH>E>ihfkG)UzE9&1zvq+=J^Vhs>@WWR)Q$cTUb1{jD+o{ zvKlpZI4dY)4EbW2O6nyWV&RQh)J}&Go}*F~V_#|` zov4{>sbd6Dgs+ETG2f+^mry*$_B=<9Ut~`XR}g5WmZqgiHPX2dL}iW>s%r(57?=|N zi>&vX#3jRgdx)c&*(HX1Yp&uGwTo#D_I1+4Aqk548qS=Wz|2$z$z&WoJ?)gMH#Ap* zuI_z9`Ez=%UJx2(nXC9k?;U=-nNHv2JE8Q#MZt~l##7L(f)Y6wh_78Gy+IVLhM`TXi+?9fOE*1xEmuP#y^$avjKoMn__Y)2N6 zpNWo?S(PQlQJ#5vr>>`^*xG9_Ke>XjW0z5%veDMjf>>MB@rtU6%8WyvaB8JL_LoVf zxs-!Vey$w^<_|6)o68wxn;j2USQTR$la48`5TnB^`!f4W%#W3`i>S<0$a5i6C$qOG zajg<5{U9P^w#sv8>y&N&Gl6Z-;O?8}G_TusE;#AbS9M>n*NERTf)N`l%QlwhgGj{N zFtj#-Sd7#aU8ulUtMa@f?Tpu@GIrZlpnLIzp0!j?zOS~12-Miw%j{z(k4?g=u;0n6 zmF;0DCL@q!Ad`uj0%6CurXAF-278vNRz{WOR~VCW#Gm&eGm=H_N*VTC08xvy`^$NR zRwAeq?_#>9#f2m}`)@4@|26j8*KJe|7ujEn_aRXLH_g#N>c?x?Qlr^ zB-$`EXm=}!at{4uvGM~gNN-J>V-JV8yxz8>%@@PsU<6*hz$Axn< zSX?5%vL8jFan^~vLA;!HqHNGu!T7@4X8SD6=Q$1zW8>CNL}JePtU6T4&yE4mEkosL ztaJI;4SKI2aN`OZbrYkSaa8ieF>WzaJ~#DEg8HTN3d9BgTxf+i`J{XTje$>{7qD6!J9(dqE z6S#73adFY{wq`%o_Hwd=ndi4N-nq)Tlx~5Pce(7zAScf|?MZjyoG9Tq9Bc6Lwt(1QKIwM1%?p;JJ%QD6~83(m=R__%@d31Jlnjn!Y zCD#S7^LyaIcMs}of7^_Anx=B@U_cT?w^|14-U}_~v!smk$J+5b@0ig1-7u)|EhjwA zkC9BpI_g%d4K;0aUyG8?M8?TmdFfub$bhHWjxloaUP@1Z6SPC)*YWz&4BGm7u)VVj zQ7u2>L{h7%%;w-G9gt;Qyvz<=3c{QycPxxl>HitU)VNOJe|29H(5FFXR&_ zaC3A1cY^cmf@0i^)u5Vl+rZgCqv7=Q=9_1b%hu4_+lPTwEmRU@#B}h{k3I%dlT809 zxcR&H&1IH&QJK0|dCj1iy=w?*_6BgFfv3q!N{7mJ_5%`E$(u)BWoy@Wvi~tX+ear# zs-1#?IIj&Q#bwwV*^8C}wuteJ3A4?D431)A`AWQJo!QP>(qr02hd{_H$HZZ}0#kiC z0{sjP;lqbSEo zq23ztcP9L%uI3;K#^F=IW@)0mA-GzAc?75^^U*i%NA*n$i?8HSe2H`%4I(s54!T@L zb;d$@D;4Afb73d3s6<%HeE&PF*O4IV*Qw%+RAHZIAE@Na(w|_GIK;`Jj31n+WAWe| z@~3O9bQOPyuk@{%*YsXN_bxOFgO-cB z#y^3}yNc@ux2$&oCiN`>3r8nWxR6627eX?Sfc04$q3IYxom4sp9EDIG#Dz=&r2|z| zzFJ5AXc_gX0)iKTmH85~p$b~M67a>!4g@w^|m9@6Gbpi1+&I+jO?QyIh>_&i+x8~;J4XxWmI14$NU>}C`=X+jD`?r``4+2OKH}=c$K;g z)11%Vpf73U2pit;9dBj0Q~z{+6lrcFf#NFd%`~#%u3FuW{?)ChvcnZJan7Y%uzpj5 zb8pGg8t2j#j^|Z`L-K|d!IZCfL!f){gOjInw;TI@%BxV`TBV|e7oIuG;*vX0T!IoF@eXR3#Il6|u*;~k?ZZpMe%le1Vp%@J`agmf&7#owJl zw3I+~l^@wb_UDL=q#Z|PzJls2#OWyuwbK>~*D8o!3Zu3{ehF0(Nl?(V$_t43S1FhD zqW}q0(;*^Swp=5=wHSGVN)7qMjKq|!wF0?))<)^GCDf1cc@!v*@zVt#mhDAE;*=XH z)+bKf$y)WPFy;;}Bl~I&)l(&uMk!k;Z~POKuZ(B+QKlMN%2>$}f!`ueg%KSI!@rvK zePtS@=`2F&AY$z);FfJ?b#vo^pq60ZKa2A#?Rh6Ym+?d(896bY#qi}>XuHq5?%s|) zcdf?Inl7}q#!cI+4Xah{l}A3|{6pT2%FuCcP-rf%AtUt71eLFL(|g5Je1V%*@7=Nc zCjU6sMtt*%|7lLaxl4sj{pR7Tn4OzJoYn8_>@dmYxmdHStY_K$NTnQ=PJCijDa$RJ z>Suy%${+-16P`x;BIRtivIQzi`tMUtVo9M|0Acd>Ksk492K# zxVh<1qLO?hFH4Ee=~I))oLPW>xsGHk?eKH)z2Rpq+2`bsc(6?*-|UoccOqH*}Pvz=aOM~5@crsXO`K~+xUC^_F z^1K~nkaUd}ZklF1n&Iu+Y8i!U#q@o3 zY5tFFsffiK<&9*H0_9tlakTNT^0)zfC{wmAR!W>GtzdpBgJr7#ixcxO<(_1yb{NwP zM|<#ODPvcx66W$NSjcCvSSlDC5TODQi&54ZwZ$k`bxy)EH46*GWw}x|-7f|3kQ+^|! z6ulDDXE|0l`Q{BRmP8NA^E=BxlY?`68z{Py#KA|WtL1V70OhHAc%t?Q~YRWATPg4eQ&SlF??~J`}-pwStX<%jGJC&pm%2n`W|mb%e@`2 zspLs6IE%51Q}FfH(f&vaQun1%2o!MQ@L4RMUx9x`#s^lk4nMwf|G)0m%JU|%7B5XGrWV;DPojeJtVs{2=? z|3^BI+8)Bf(j2b6ejPZ)-k~*hauc8qD38QUmVIw&(I{dWi~V~=d-gTR{uXF7OcQ8s zC3zu!)7F{t`U!_4oa<0wV>u=D!+G*3c}e^z9(DO%eX&g5(0lQ}(mUVS&gB~Uq{_Bb ze7$snmKlrJ#n%}skh@%eO6tmpl3=MEV2HjY6-P)4YM=T zDCMbTa=wRD+-#mMqr@2LeEsa0*_&Jqofj}_`d!dHv9F^o{3#Em!;T?Hd8t)`| zg(iV`R~*sRQN)HKR7%^;kI=ptWoMZEv(A23E1JVFw8BzepYqVOWfWuDTq$SfjfG0V z7-zJ6&M2V-kwnt^oxo)g(Yjo&v%SVD1n0!FON^Um z+#-sU=gyK>hyE_O_^VA7^kXjXpSo(Kpx!&k_NOl|+PekUUxD^L!P4)>B^Nei>*; z%PM)hM84J`JT{-pD;%FEbGUwK0s+d6o=4iyeqRfU{sK-OI*H}8EAXvQ!sw*piXYZ% zbzFI682LgDYaUpG9e-~pdUmWrelm-h6Vq53&%hTU{o-s3@{BeTtLG?H$z$sOi>&L? z%Cb3iRJ?MNSZ@!K1!2mUL|mSf5#mDrt#FK5%41@D3Jb>bqK5v}ezt{$GRL1X+eQ2= zS?{k$;j>9>TL!Wk+t}d==imn&a6#Nm`*ECSI*lhrMsQ3Mzd1Ztq3jUPIpD|xUfI67 zYW#6{)CrOu&Vd}PJI6H1qg*b}F|0&>%TSkE!s0yp2*;~}@-}^t*pfemvko??l!ycI zi<6dn_im-A=t3mW!aQtq5KKIhFticenV(fxgRBAJQe~el6`Vd*D%OZ6$25+Yd5)vn z15W+eBrtVz(g}EMdI z#o2)7y}XxL*6@FWoN6*Y$cU%f+yIfbaczF>fd?MEJ;*NR+TNONaBu5*C#BJu{5hRk z1H24hTJkF6oD6b0T3(Y_of0bpn>_Diq?1w0qEuBJ1>GwTO;%lx9=&tDc@*?r>16nl zAx~Es+cZhiQrxXux0)rmGWf{|sMDB)U9)g#Ha046m8ZOQf9ljJeCbPH!XNy>AK$mV1fAJT1>Zzy9ff>qI2X6f0Fa9F_!+-b>`2FAief;4c{vp2bg)iW1U;COlP($8H z+M`c+{?ae~5)K|bXuJ)TR-m#3@<>#^KmYSTk4>948Jg&u=pb<8x%3cz&3Jp@!S@T) z2mjVBo2Aq+a)bYw6+w;=F`%AVAzsej9K7nA^gT{G^{izCWzJM)8D}cM30l2|uVqK^ z=T}Z)ac%+q^)SNmIIjDv_|q4@h5zy5v-pcQ-@vzyAHh@S&R{+oMk<*`CzW&o=i|mZ z?$wnU{9k8Y!>@h)i+JVSahzD3z~wv@s6Z4Q={7vt)`he`fEg-eFlGa@)V9;zDy;if}z!HZz=-J60^kn2o=L`oI8upGQY2!IDseM zID*$soxwQe`_5GZW+QVofkR~hCxldf#DV6w+9qD9QVG`yYC$TWdVk=>OL+c;V|eA& zLpb>AX;|4Un79_?*c3+Jswn#US~-ai8vdIct>cYXm+`{0XYu^AM{x1nJQn84{1!xa zx0d$+Q&Wq0@tJFQ>FHT4Ovmu_lW$;UF^=&O3vV2{h8LeZO(npO_O=+6jtDZG;a)sH zgJ-^V1qTmY!fUTy!ttY{7`vXw;zAxR={S`#9a6!GIF+i*at(j}m2cvkUpcC=j?d%z)f`S78^)o77kC||Qet7_ram#W z2~s>X;bt$R?xDQG2x#B85@{#?)gBqHw*^D6#MdupQDGbSR#WdyTA02t3431(E$dnk zYGHm<24$xF6xI0kmvy!R4__n4eleB9K7)0JVf=;NV}ph^42N z(LU0HXdC%F#BndnzVlKZg)c6mbUcgbzBB@1w)ufNqR$4=+uDQBzvT0tGBCt3G*Q5* z(J}PLl1Rr}5h6`0BY7-5H-cm{^j+7al8z#gFDMIFfg z{W?0Py3qbuH~c>yL3My*aIYWJBUH>;r)aDlt*r@Qi3<0#vnY@cf;*#V`4|gY zD%4Xf|HU4ptTb>h=@=#LsMr-UB}|RYz@pOIy{(&yeI3~`3*B2wO3x;awKmKGBKVPUi7I5aG!LT3>2I(uyH} zFEfsYPHrGeVUqbhmR{PB9)8dHzv)FxZ$F zsbYHEk7Oc^p8g1Bv%IaCc|E}SB+-!2MG#}de!TYLGM;(z3=TX`nQ>$a)ZdYa#gR@0 z&Hh`(Vhu09bOK-b(sPtUS1~uQ>ATv3S3g_^#qz^4aZMgpOWu$uA(6TL!WG{PmhU3YI zA4>XeEq90u$`~q>Ls1D9LM0Agu$$0mCa58bvtC_ z!joSc!As9w!YeOa#Ty5waQQ+89o=E_Vc3jCCr(_$pa1!@c>d`*${HEjEDT?n#VZHS z;oDEYMm|ZQqn$k68e_lkV`Zt#_=|)u|pLcJ~)P1%9@v6IDuntOyc^mg}Iq3x&~_; zU&)^}>7|bLD6d?rz)+#C$al6CV84@0X3t|k(IF?sBA+8aPb{LkST-xj(yLo>{gwBY zp933+6KU#OaQOKu`T5AD3zT)OXc0dj%VCWCY_(3H?XeE{9*n}?Uq|Y$1d{(fX874& z1(Y^cSuf6mvH`3NbKG67Ae11_bhl#ZFy|#bDMYujoK>W?h|I>;vR(CQoy1-b`!ULD zfAiQO#0NElJa_J#Ia&Ad z#~;VqwQFzSB*UKWMNik5p{oZT{B?kM_-~W(PSVDeK$@`J=y?rPIM*oWwO>8T%+-_HEjNDwV!V6VsT_m9T#G zAl{d1MTFYk`K5V0bKx|Oo;ra?A9)b{gR4+U#4x(NjBA%JVP|U}Hd0=7B;p4D0u{`Y zb`>ulJ%SuHx8BYU46R#-ts6F@SBC?2_h2C1Y9{s1T)l>Kqoc;-uW!w2tlhYQvTcYH z&?V%@rjaaHG0@j#40lsgGkE6gDV!ocyYIRS+cs@S@6aH6Q&cG9NlxlzY@~eV*f2+p z|M>I-4jg(7L)*4v-KNdhux1^4JG#sdM0*DMs2EB))sHbUG^iCM$HgO3X7t03JX9>` zWKr>r4xEsez{@XQ!TEFI-xPN3-ir0>H(+J91y?RlARMV+&H6NYd#I4JPV;jsIC*Ro zBUhKPnlgL$o_>x?2~14Nvta?N2G$@-CA`cs&Yhmc%wz%Ew+$K3mW2gsy_~&ww0EmufK5x$M}4j3j5h3v*fW+yn5sXDr9xU$S*QF2C1m{ zjs>6%vIDC*K2y)EOjl4_=SORI8c`|`HW{?IoJHnySww>g1n-H$x0mCW_7h|KjvpUI zDN{wNvjy#|nF0GpB9$bAQ0YBiMY2kTv5!13yMXCK<7O|Y&iAcCU=#CJPdvdsc!oH% za(?j%AIgE8Y3pi%%Fjpvwek{L209Ss_`E#2fD135LQ8iCI_~O5q@TFxFbyWiePbC( zJAv@FF#J^J?L`~&pIQ?F~Q(;XpzmByXh^!B1uUuTsq-k@Taj zRdX*N#>c108#5-4oqH%>+1Ip})V1LyjE&^b-rj~a>mn#sbI2^$F)>nNUm3x=H5;*U zV?XjXAaBq3I4@3NU@%A?mweRt%(7UH_By+E?HZ20If~iID)Swp z?C7H0YB4KI<`!nq(b-R0vrQ-$cyuGx zlBB$`apuezo__ijJpbG|9B1D@&9*sC9yxsYBHlcF9M`YUqE@US$@#wK1@cHLFXS~4 zAx62`+S!iqs1HfXi_l;Qe(E5;5DAy@VfyHd`BA2ILmOJw#Zl@gm?eOL^Xz*k$sLQ&n(R3FhRae_E3J` z)eHaXC@hL7c?#CQSVq*MjM%{pb`aNW1=&AdA#Eb?Q*M;Dv!8RokFwn}!zC0JGVpU; zX#7tYNxpo`ve{-Ja}mtE$RFgGQJ@^q z7{U2LXB!4MpNK>>b|f*lx(^?GY#Vm%8l*ndhwk1KLQy~4rHl)w3n-DseFHJ98cdJ} z9DVJ~nM=rJ>t@Ut=$FHV4;L?vWAyqm>}nK4Lj%~guhlpuEH70tJ()pUOD{OxAnkk{ z+x%?PNj&}33s_l*V*R>)Y}q`(v9g@#6wJP(I-$co?8ar1^Dv?etUu5iwgZj6JJwWt={FjlBFCUORBz{NQ%_%mmJyo4~Q7!#H&KP1bz|)ne5g02UKJ7bwrj&pxtr zf^&y7$MNvQd&kc+~1=fSSM1C&5>G1QKj|`zx{LFUEk)J2Y&#mO=w!OUw3`Sw6 zH1q~g{Zq~fYMlG+2&2w%s8XvSyc|M(k$g%a72`Z4-H}3cB#JoYvpvK)Ri_SP@jD*t z6e12U`cNv@aJHPm!Q)3U(AJKJ_T0sO5!DcnoK?bjW)bJdMsVuv8C)h^Cl?k_XPT5R zY`mD6CG%lq2K}H>;oQd@0>wUH=G793w|55*Jn-NxASW-`jI@$dD?(+rGJUr3{VgEn zycyJ~l|UdxUWPSI2BxU|NJ*F1oZjgwLx2Wv#dDXADy@`!Q|Q|Sfjs;Ku0cSE%djRR zLq12r$#;sQes2VgiS2;OQxNX@CgbDb!-tKRobK>C|;~&Ql{@@Sd!yo>zF|IxE zzyrp6P6j-MjT<-OgCG1LKJ=jv;bR~B7=GXfe!vh*<;zP?1`n4Oy1L+&>w@CBMh<-w z$cU*%5RFs^x6U4T;K5%#sJF^Yq@evxstlG{Kj?srX(8!$RZcd_I#j^M3KLzfmvL}u z9A7y8I)4AvXYl;-BUoNoL0?A?eqdlDexz?3_OuUBk&04o`Oy(hV%N|X+`oAz?%uQ+ zyEbmdz1wzDt+R1WOT-E}EV8_5cK?G@qnMbS#%5m%pWL_?A6UHwj|^_bP`n*>*)lk3 zG9_FuPmkmL%mjMZ4&jNlyYP|09e7{gCTwZ#LmMIpa{{LQ>NIQ8vHtSJiE2$()kaIj zhwafeJUX-yAKtzbAKJ1F+hrUe&J?mZK5`jzl^hkjPV8R08T%RD$2hH&i8Gl63@^?g zLsAv28fKSQusAc16zTN-joa|yb(`?`2G)1OR&3~7g*b7LA)%BnV_|6tGYivLSzf?h zgKMyV{YKnRnmn{^EAC#m77@xrwSXpeGHz+2rdIwc+=~)EHu^73{mWlS<1V_S`){ zWwisD#UL)7&zPmOIz=)Vwy3aLShGHiox4_{P+TBxE7-856+3qKU}ky-xlED#L4FB^ z(9zL`ZQHxBZAUM5?Cht4)sGDux~U+=F?L-8ZkBlH1Uj|{74pF~eb};XHC7L{u&K(Z z*;=%0lXsLK>q^B*RFF56zQ>a`)@?{()0Pfw+u4V%?zCAlG&M;DQrndR^C41(3Iz595Zn+#`q6fDZ|FcU zCa-&r)$-n}#84%F*e#^{k{^|)%NYOT8PrB<=%tny=VZMwSCZv|8V%5~z6ae~dk`Sb z6Q?;|9-)HpR0&q82H!&*|MpUW3$bh}Q!B+h7VRaooKBm*7rs?R`Iv=pF+{#hVQJn% zMb>h$bjgQMCW78o{lKyx%dgBM^KG{Obt*C12bVb3s(IK!&PD8;;XUEwpZCLlhD!L? zs;EzKEY@;9($rU=1}05}c-Ko*Y=TK}27v6z99F(Ch1zQ+L~8+ry0j#kiixcE(m29C zL_0(1+un-qdpi*6rLwXYh<>aFYY*vgi{Z`%xowCGc_m$)XGtR`m@X@MM5V~8m61pW zv37k2_T94vyY@(~=&-OkOpGmvP0Z@upg#ufgH^5+4bM2I6LSv2Hmh4~tQsrN3fffK z@;NGov-6mqy@m}NTd{Lb4|eYDVjmDcwHSWWkD<|60O^(>mG=;mZDF)_#3>Kj$#1>r z>Fq&hPa5fV_OCcq#XyMtkwwJiWt2we-Zmt*#NlJ}M1FwtqxVyGt*Ro#K4?fn7S~|_ z+7nNP*8FkGg|CdE{6-OB(Yc-Tx|N*S`$;EdM}|TO+z;f}F2j0}bvhWp>}k>A{xtK!WTmbcP{n#syxFDOHZKZh` z(`ku#K3Ikv?8FN}hDenPHThGU`KwR*S5uZg5JBwWG<j)@2?_ zuPb!G0_W7kv!=b#Se^`U!cTkL`FtgYGOV&6Q{!c`OfJ=4!uH)A*n3YO?s=dW?VVxD zj0~<^nrHu}Tx8vQ`&+Pc_ZrHUReZMC-chXI*o|$wR%0_|R5FpGcwxWg*b)mz(VA*O zZ&w%k*ynmE_qy6!(9zn0bZZ*1B*(uvX%Y>}u?cKuBsaFBbN>KZ_jJM6<45Ep?XVsU zpwMTbELYAT>&HuesfZwz|2DQy{aO`;FRZ|Rv5shly)qd`JxAUylOi0)LTf0O9%UIj z%80&{KWavWs8GUK4?pdW?V|2Qv@ z7osK4-ti*&d4c?#BR{v1pL-kp9L3Z`0hcdkSx?D8%_XYlhYrd3{#9{o-Oq5)jk_GQQKS#*V0rK-K`FVPtbG==?7}!RBPCNW;YtOuuJURweBajUv`^q9# zo}7no*pEbSiV@iVij)`n(OF{-$JXT5G*WD@@OCn!B`PtfFUO9aPCCZ3_z8(y5tWQ&Zrzn3`aejMO$7+0tbJd4e>43@>OmUv~((rkF z{=yM_`O2|>`3m)O=B}N zcXt3qR0J8(WPp+pP6jD~p5=unW0t%Y^els&3efa)a`{-I%91GHfZW zzR3V6q_tdEzb4Mi&Y158kK1~^TaJPYGA2Ix zFYh>=bgO(-PnEB_xL$wq98$hA;JGvqG#l>(&1LJG2(2JPot73KJ9Z2&zW5@ZefC*n zP;`xf9(drv_XlLSlX4~FoeXytuWqlEo4;ecqx{qIOipYjD+Ro^FpjTZIe{;oJ&NZ> zuVAHG#*WqN@Pm(j82@PhqxkVPd$2XxW%7{GY+c6ydZKNxGeyi?AHf9s{^G(sa+MO6 zIYFJJ1~$vMla#&}t-SH(x~Fv&Hr8WUL*@Iv&LM2+8gTX_A+{D3xC)h_iIpWRR7==) z*FLPL61*eSio06-v7=)ZdgD}6W7L);uZ0JLmgQFSWu&7CZ0YRB-EDo?7>J|4=EufJ z0;?E577S3PmT_@{^0l~vLNSB!;xZP9yY|1+Ps52&7_njn zE5jqGj*KC>n8iRifz@rDXyIf~6E+#xN~I!7q+z|zN$1==3R5$%7MGE(`?0p88(NBG zCYi>d!K>C%@anvyd}TgTSi>PI_#urG@-pEhv!Z>wisPN6w9k%nTq)2)FkHTXZgK^av_X`*&0@s zHKx_5T&Pa0Pn`;%>OdvDTFW99*I1};GDOt1eW9s(IO0bv;X^82M^_Kb2H@#d>V$D0 z>qI)#IIf23=;&#olC&BRJ+uuU{_x%S!4L1o#~#~`$3C_w}s)~hVTevq{JAoJQuYHtW4^++1M8#@qA$?J}-K!r6_i<|w4>QUl0$2NVjiiIyy z5xGX3;w?x8W3Xh^l|4ar;7A>b^(myOC@f_rkooo^!mkIZ9Co4g@eb5`+2=?)8Sk>i zEEZ~ufX=u&PNne~s$M79KQn$r{Sj13QBM8E7wVHCq$_Pmq+3v%^C9=@3Ubda!awHc zINgRim5pje#yieaLIqgb%a642PqS_>_)+^7Gnpd(ddH3t)wrwSyTpEC#Ssg!4XrRL zXDTTD#RB|?EJO>Or?gV>vX+e3tzk*hA=KqZ`(`S8`_c&ZQ>ouUCFn8o#4hq2^LH|C z4ob*+7{eViGFQHB@+?(bs_k`HKvz$aycb1hcK`#c!szZzKzrFuPc9oHlXCObBkU7# z@=C-kxwUH*=Bt*Cv)!mrR;VDdoJfcYDCaUYD$K>Q_T!qu+`=phhRTu&7Ju0NAJaBQ4eh)tMq4(p7C*Fq-ePlQ8e`p;xZs{Y|b;&NXFp@Ax*0o$B>?JLMA7Br;%)vTZMo1?ElgG=H z6H>pUoD0fRE__!A?T@!2ajF~TZvqRKmyzwwAo@TMLG}d+8FT0Z$*IE|tsy?|AT4+C zemB4G4PaGgFYIylwV4`pigK9zsYR6uuy5#AYse}H{3$+<3hPOjP7)LzCret|ZASC9%V8K!nK214G;OD0ke z8&6Q3=2|p+2F@{XjNpB1TLhu7Ge4FQLuo>OwyQ7Pkg4*}qD%@qbq~k`j5zpx#D)E8 zZn|pb(}U}x=;{L!lo_knhRvZDL4OjXBl6gk!O^0uvk_#Ut&w+1X>lxN0S4J|? z4&QVD#g|u5d}ayZF(2Y>3DO}5OGXsdQG1+~tPE;#;zC&$I8a6XsVs1nJj`;#8mlM> zd|@AA`r~*V93;N)XPO6B$Qw2id*ev%NFmfsVJQ2#`p9`O9h%^kd zEE$H4PDN758`@$!%Sfez49r-;`)akwG>XHt?93YT9OE6+knvnqMSLD1YAx+Hy89_h z$UmV-4T)rwav+6riS5F&<^8X5Yk7|G13lQWw-enQ|Kcq+(wyq7TFZIP`aa_&tkcbP zxI`#onQ_&^0*SZ6F_T|3mtwgA_A_JXqkI%T0*#?+f0bp%LGswmaZt-<+3tp)OMI?j zaD5P6#5uvSdG$K+a|ijkm18@{Bg#+h_g7`32PogWx>M*~6+=s#9|_`=O!1nCVR?r5 z6@q9@Cb4PVARc<)KK#JP?!x0w?8L+G-;9Uew*iknwvFTE{kZ$?9qdDWF?U z520`{i~Q3|2wYI|03GQv6a^iaYT0~5!xC-jq;%W2=g6SU&XF%3c;La?g2n~c z4)+}y?_^X`1UHckVH!a6tj;cjl$N&2@U>;j7Gnt0(pCv%EtQjTPw8aTbJJ)b(=Ulm zzE0j+5?~Nv(7r$Na?^VusU@v)>2{50+Bj2+la|g>1`uxT`fffdOYvoh(_VI3&bxj4 zc64`loA)}+R(QG|f-(p;d&9|)r)9uyeyY3b;~L{!#qD-bxe78k>SSG&<u@@mR$4{rROiT)21! zm(H9+Di+7%4}B0n^7w~w@6N4=C1Yk^Ja-=tc6YN>qEIg4;_zj>^!g!u`^6XW;`7hr zkDvS+&R@ES>`Dej${N`^B);s5g&xYM!F6kqCpKC#5{Wv?jwJs=ycReCGE3|XYMcax z!U!aZUovczyf9^qmT$@<&B$2+@vw_Jq$k5kYn*bl4}HB|lB&iqkBs0=;{4UW{3`zRTi?KEzy4L6IC&BaQ&Y%gvnW%+NJoMgpp0JC-;Y8*i$DA7 zSMV3ly@W$=p2YRBaja0W)Gw-{NwPJL?#?#!wzeTbP2|a!Uc_@RAHd5;j^ogU>$o(R zL5Y)F&3Cm0LP#g88HHECmm({<(Kut}2wf|v-?Gfa795)J!oIf<))my1lGG2SK1LLoZAxvKFUfW~5 z8GIb$sTY_wH$XZYCAX8bZs(jJ={E*2(J8{dVD{-_-gePOfePyE?4q%K#k&)T^|0JH z<7PQGDNvE3q8IdQU$~Irj~bOHxqbON$Xi|Hi4^Nj-cXyHW|`s?(QCjHu4x1Zx$s~SFP%&k{n}Sc9s<@h)0ubueR29;~`V5u)b_7DgRpbC*CqN zWc#+aB{+xbF>|Q{2hQM|-#CGnoqO>Up~yLtjd-{Nu}Ip`QX8e0 z*w6C$g5mX=?Z+p@+)Z+2qF}Xv0U=FzG z-rGi=B8!(qd&-kof;^Klb58xBB4lJhR#`G0X)f*=ClC4}%$Hq0z>X0i6Lqdx%XVU4AJ3xA-->lx*Riv(SmIkK zaQ?3fCY?O9vnvvFNOZETDLdp9SI<)Z7AUW3lpEw>$%R6`h+2t_#XKZ8!kuCG(&Q6) zn`*B#wJ+sHUE?cxj9=LQQ|zxCc-Y`c^@DK4nMb@UIFUr33T2z2AA^i%g^~{4kQW-? zHO2mxzx}bfiug zCI88*m9!E+hnxAC_-c7+SwBh^mdVeg`Vjd@2T=%Z?SmO5j#1W4D>I^@7~4VL_?5WUiL?5(x{KN~TP)%F=oIoC_jmQK z!@6`og$Uyj56yS|A-~4#g$--h!6tu>3}40a!Xn#*gBVFs z*Ep|#Y#*DvQ74~~6b3&JJn-P{!L>sP%=}t?mEgR28*q(P66WrbPu=S(&oyT<_#(da zZm4K1Z(dkDS zgP9-FJZ3}VNahvje&sy3=Ivj zEXRl_g-B_%+*jqOZZh<#PTIFcX*CdN@4W*D4p1>WLFFxhef##A0Y;$oZn*;G>w;)D zK0a=~%iwwB$PxU>pZtkguB$^ZzVel?;9vgBe`&n_-26Q7z=Q7t$g_*uzZPXi81##v zJTR#DqsqxpnF?At!igQ9%amn9@iactw-Fy5*o>B$ERKHl>-gN)zJxzJ_a-it7jVX( z$JIy;8Ky09W^-jOk6(ZGNjx=m4K1A=`1GFp@J}B582+!H`bYSO?|TeedInIUlFzvi z5}em07pVX&>tw?`QbLU=R(aLtzM3d$(i^m>C{sxaGEulpSzIZLK;>_^GYQ_;#Jz)4e#v(O|JvJfX>7+DCc$;Lh#%*-!ou{`n_= z5D)I%f|Z$T_`=hFj{p4PlQ_$X_cAB((t%cS`uh_@oALj!{{Q&VM{!SU3S$cs_`~yu z@Y{!G{!lBV?krU zb7qK)q?xoG((q%*Bk`@V!q#$~hBtteP5m5BRe&)SyZ}oSVs_*Vvx90QKqsv zeDM-q_~uc3;lI6(-~B(I!GHO`U%>DF?yLB{-{tdfAI9(emsjyWK6?PCkIkdZ`b6cW zRa_=5NmG@h0`tU+?I1eJ@T&Uzd=*5~A#`){9w}2hkZ$o)R5E@sg}`1vW~g*5lW!4s zPTfR^V?LQWpthQ0n@zJ*)Z3XMl{hLEekvepSYHb-NhV81&_V^!SbIOlzVJ^uHh-M` zuEWCoQVz*(X35Voq?FTD%(g6JB{I*lYl#130^xs@K>Z$$^|=g-4?Z-(oB8UWu%G`MX0jKk__Idos70Bv>P`hV*vY=jd{>g=@ICHF>|b;t@OL99b_G$M zO(Pv%MXppk*(k~gMa8Tr+UXQWDq^yuR9JqM+KqVkt-wl~a2vQ$#gMXmG7s%1_%M07WN^{mN}AuPzi_%ky6|O*)vAT~JtF(3 z@mle9`H_$0u$Wkc|FHmC{y$v^{a6Im9`wLwH^3}|NId={_#~yeFgvLe|if4 z?YCdSpZxw|{PFL;hX4BO&*Qg#^(*+??>~iC51hi(%z~ki?I+)oW;!V~nPRV`ED7ke zY04QAhbn>ve>DeRM;)=X#4S}rk^Jl17e?~mv>^ERLZ}T8?whseGL8gzx zF56g)WHA@db4&;zwm>GEVuhG@FwF5TS%R;X<3MMawI~By{YZYg4J|*_h5ByxjZzWe zP=$4N_$WyIpg%Q$XpS9f9c9n|9!Bby+Tpv`kLCCha={GhanXe&CeP*U3dekQP^+Cy z(T)n+UWOaV_#*WOwU^Ud748J$6=y7ng~^W`l#I^BJfhK%*;~}A6;Y=Amlt`AG?bw| z5{^4RK(M0HJ7RItC!&tPH*QhVtr$lK-1q~G*BA%hF2eXzEqt@ zp*Y8E^dpGIC(3R1U-7ucO$tRW}VXV`B8GGeih7RyuK4iYaJ$Pp}HDL(=`%{GkIFqbXimE)K2JAeE|{MxU6 z3%~vw-^OqK*0=GS|Ho7KKYsaJ_`iSk%lN_{d<%yToW#tu)=*2|BX1IiAjg?lvWxSm z9`du$c<=amp8PzW(ER3Beh!nLg9-9;3+dG!;+&51e~X3KXIhZ@2OKExWxOiKf^dy= z;8?-&Ni!+2o;*%4gMyb05w>YLyTrBIm;#r%N11PS4Nb>#x6#&uqU7 z54ClmgR;PmQEr7frz5JA#+bHi<^Z)!(vOjP2H(1H1mAl88Js(C2>yjE?i<{Iech|k z6HSnxZIn4z6W>_GD@i=NIW}o%S95ycfd_97vPHReILTjuWLra|zZEn%n1O^rijcfO zbd|A71}S-`?b*8rpZ>8Q!@c+JHwQb&@FgRc3XvCEE|)b|#n&%|3XbJaQzJ=bc3#F+ zP~LtbreAqfw(^m&Zr83|&>nRA@7a&R)q_qM@dO7A^Sgm+Ve)APm7y|Kwv2M}=F_J9 zGRmnem0^BGY6O)n1K!Nc43(bW#gG2zkK)sx{xp92r+*s1`m4Wc4x#wK2R>j7h>GWy zVFY0V4E_uP5mxz%(B{`>;-UMy?z+q5p*qM5P<8W+cOH1~*8mh7W*+F=8jvBw*moGT zlp{=8(MyGL``{2By7xYO?1?AvQ5{0FejTn)Qt|!bpW^9Pp2xGtUdD-$Ysi&qNO2;1 zh4=Mv2y2H{W9Nn;tm*GUOFV|9nOV#+R4J3R_kEi4@7|6MlnTzWdPy#^F>e3>|MW>j zK~$51O@)o}LrY&I!}QCgmbHZ?tK=Qv513zmE1W!8l%wWXUcOTt$(3WI(o2fa1fg|ZonfCKZK!mYcQW%!SKWg@vl)vQ`w{}*9PXTkq|Zyt;GlLz84>P z%Zp>a4uWub6wnF?_g;oby7$rwu6GUn#yk>GtW&c;uYkGkXN z9~wZ0^cElJWYb7{Sf&wF$|d;Qv>{QP?8{bX-|HIchff}9xdJTKInl~CUGZUla^B1d z;)>rvoD;nCw3A<}n3<%el&6B+9x*nKGG&1JqQ8w5BW=`9>Vu>dgCzSEO4Y#Y+KjeUf4mhrDD#aPQ7TS@NuDPZcB4U2TIOGt=Ntbu#1@$$_d&wsv%x~&LQtahL!}&Vj$*Z`(Z#CaV%x3!(;?1v)`2DvB^H9rH|sVrKKX{sZ6bYArJB|?hB!(J&i5f*5ZK&--kyY*^l?V zZx`;re=8oie>>j)&|Ub@V-Ms0`|dY;Mz^-6iJxSoj9W!i>P5C|iF6{3*^kUGSCoH- z?fecx6sckvI4k@8KN3#YVQcf?Oqs>K2RVzFR!2(p}pXOdPqJGrYodD|*~jb+86j8-r|Ea~2inGIes zSsOGowrKpQV_Bz-^QDY$@^;hVIbw6|d6!5=(B2+5JhZr!VS74;XX$C0GiaL$jfLWB zzr52Z4|MvijFF;kCd09vV=}v(sf&6^jpG`#Gcrt>$uUbmW+hVzLo*$sqC=g+@J3g#sb?GnedvRyVtd1r?Y z{`Vg7^R)&)ci+a(74b9YPvYmW%g>b8I!q%HCqK7q1qk`R#Lmxg*iO_**M?ka-TX=hXGbTnLcX|v&)uA#u>B>I*(RDF1esnofoszf zc=ELu@n1jxS$yu#zJP0EBUsJ3;14|Z1U~cOAHw~d=MA>CBPPBkZkj`BPNlvfqq13H zL7F)Y=Ya8i=Z??X#!-?i5rPEya>S(I|-?c;LbJ1*qF|+Qoj_BsbIpWysKcQX#|{g`X1~x%O)k zo#q5)sHX?}*R99>o44TM^;_`x;ATA1w-z=h`O{-ln4g)2#mQQnlg|_tvan6{kM}F7 zI98}!ADf%P<#G-cPIB8=c8c;N&-g8!DO4G6G?T#+%U&W{=jSIevNVg5yjk?iG$%kI z%8{Y=ZVZOg7&&$pvs#)!1@n4-8B>&zxk4UAPA+|vIZ{5UE}sN7UdKx)WN*;kz ze0OSW4Cfb?U}=+^bPI}+Bo$geA}q2!nn0AZJS~TrBu=MFIh-tJFdK_ufikb2N}Hu7 z!Kx;7IIW=x!A@3*wtlQNq zCCtxSsFbA8MvShg23|ebMH<9#{c;(jSE$%zfzhiOT)R4kinV}^n{|?`3}+GM6E?rz zQ|IAWI~X?74pcTlcTVhP!y(*NZ(2`|jz(?z;!k-5X;%j+-@kUOGRrl}l8x zs9;<2`qha>PE z4I*|Sfc|nE$sx9O3mZ0JAvh30VrwUGf|>|%@E<1)-ryMf6pQrDqp-z>e}HXnSlWl^ zmM9iuv&ddqLg-~u^$mV_mAcLN0Mcz8@NZ&6v@|0tD&pabRLoCs?sJ5F?m0hJ zCgw0RGz#otedSfiek@NG$IpqpC@-s;vvjg1WloKI19Wd1kwJ=-49t=|BCZ;r^C2p3 zwv1M62S4d;*COPZ8s=t-n4KaY&(tt_%>qRSqLCPS`je)9Y!;-`RCWU~@{S)%^DLzj zL}tm4#rZ6<`8o1uo_r^s((7QvTnC%_xzf_*3R7axP(=mZ9 zJ9=@~edMM69oV(68@u-nU@!S+-~N8=*wckogH+05RO-~}Z0%_zV>^{h)t2}=X8n0(uUkuT+)G&{V&H?yI=cS1SOs|>J?`7JHUWB_28)Mc|D9#-=CX>!;G&6=k2Q zQ9iO=3T!(TM*)i1bR4USYtgaPkI;AoficRL`49r70Fu=>Vm0|@vmdaIg@(afW*K*n zGi%(tygU8OJy6fyY`QRHxF)H1f`?Q557C*e+9?8<$duhFTDhwo=}- z!m5>wp-u;w)aym^hYXICj-0cGsA$I%0TfDE%9t`TD>B$IKgxOACO!GROWr-?$=mpY>M+ZN#mSO4`jNx$kD{fOvM8Iw%2ENf3j3)YMA#zU%)5oM zaqX%$+_iTN?!IpocJArG)}3i=-`9$L_xF)E)?k|si&&jDUSs<4NlPT1Lqg>3EI~@z z$O}qdVlpVm;1!OVWvTXh;z1q{UWvhXm{=X*`)7k#9$m)V>RE*MX-`v*8Nzp*bNUx5 z$UHR%`|dpA9}6M1KS{CZ$LN_^)V1YWo^=YdEr-Ypdy-hbR6*q=Wz1Q&#WB+SnJPvD zS5VwmfNzzA260mV-WkPIdIpOlOYj|HzdX)wuK>LxUFc5sAhO0;a%*d45$Ep4AWMFC z47rl6yi(Q-ik>#FefpmD8SEg0D@G#o>kdX$W_TH>nh6C6u?gUuLE#y-tdbh0b) z2v$&%*II_Op?uZB6Dy09I%SRX3Mz**C(c^!#OCKcWw;C}5-9vAgI7=nNxs*fbBUx6 zEgkYSUBK}-mN21%7gm6B-iPG{A4Wzek;@gCj#yEK*z7Cn3(>fYuJ4>^Lv&J*B zgaLII7GdgY#z8B}IZ6+*OL68w{5Y^`PtQ8XDV-$T)ywxxJ9@2P>Xly(VvM+qk8=LO zjx;#fZN_31rn>t4mXT%3zrsFK)jR`X#3P+(3w9x5N020MtXb89d+*tRyE$I&)ZsmQ zTd?EqR_vwh+q-`V+r`fVY0i&Gdx~z3*5qg6B*PPPG!pJT(cAHVQmH;kdOCH~dm=m!`f`_Kzx+Qoo}B;vhDUbUedZCI{$iD|aU`UfWA zz9e(5jFBup{NAK#kS!JY$87zz?%(RB!w`$S(yFwdo9_C*8^U>cC;FY0=Fav;+wd0$F(WDS0qMwB zk{s&%Oe;e}8r;0;o4+T5A3C^nn~UEivugilH!<a)d=VZV zB1+4xKQ2~(I9r+}iBwFGlTL#@gV$>G^T>=Rna0s^MpUvZU0ti-#fjUph*QNEh7Y@tD%O$dh3-k3WCs(lttp6pC(Ti4t#u zd0xUI_~R;6b!G&L6@#Uyt=2IJ<#1T= z^U6v7OJXMm=e3=)NuKbHzG$Vb<|43HY93W`Lhb?+Q7r^BR`$AIT_ebpZ}{ll{v5_U zOleqpxZoy%UIRlD^Fn+2mkX2qr)fY25IjY-D}b{y2&kme0lr*@`g9 z$6j69^wXWzHoyb!no6~*n4#1z%C;V_Po9r1#_#up=Wjb}P)ikR47@aX&8;4rI-Oqf z&XA5eW;l;}V8TWG<{MvGTd*r^&9+@a=%MS6BTK&0b4mJ_=_{>bEuNgG?nV_NucU}y z(FA3qzigsoKJQl{mWtDr&yAZT9+96 zsnew`d4~$u(?^70oI5Y8k>;mo**R{m<_peBv(yB_po13W&uP%QzSK@d(f7$!DO%=b zcM@|veKkqV3#KW*I+LHecNt1H_ou}tIWrP2AHF5(mkrIi+OdXWdIBQ+xW89qY8C-c5#k;);3QMYnvB!{#0*((z|U`p z4^Y455u0!~!ih#LHME=E*hpZH$mC#`6H5~3BwZk<{nh)io@)luvn$p!bR~a&y3J_k zV?PIpxrBStd?>y70_h8&hq`ljVEK`rKkG4!Y0Qz5h;z0MSUZS8)JMI29bi`+ zYT1Qm*Gs|Y*tKQ8+IK;Cjc3o+-<-9l1X}A1TG5gZ{u% zzWtUd2h;7|_YloKdFxY)Zmp6vWju)O8akZ4p8iOB>qcF(#5emJe~HgcK9po>(bStB zeOlwuK*W9Ul@~!n;n$*v1U?z?G>ry8jKi?{?GkTUoSPi<>SvDPQL8aMVA1^A)1$1& zV>EMD8eYz+tn2*AU4Or3yfK_Fqi9~+wEG4?QT_M!_(Ol(H)zEsWr)i6i0t8n%Ru{UtT6{ zY?%~#cKqcD>$ z4kP_4?{U>j1bH3Z(79;NCMxp%j$nF8c>NUna-n;68}rR@v7L4c8-?a#rM{|+0BC$< z4NdX<(+BPCHm0)$AocA-`LPFA_I7^4w-&wW8p+x}RYUHsib$@V#)lunC(hPsR#Qi% zCq7lS@utb2N8n{gPdv0;r~nZ6N04IcX9QJho8k55Bp%xzRi-aKoM9Ff8gdsKji(?9 zr^(yp$4c*yHz!=@8e$}kw3riZHNd#ihBDvLCD)Pr3nIMMnM1&`>FujPewK&HLhgha z{oT9@!13|v??BpZXYo@8JxiSyIXAMV(Vu0g)D(V~UtPN%wr(zgy~1ePKZ6~^o{=|m z{W1O#4i|HfE%Lk(R)#;I-`?et*rG~(!7G`6?{6jX3PUKNJod;ui^^z;&HT`UY=GN+ zXlO`2PRB794TA^L@{6;#o|;IQEqTyJjX?dWcqxSaiGv5#g49Gd;c-%3*oQhM(bnCaeqf2D#c{pl{JDll-w z$P>Y5WDr6Z%WurLEF*rS+Gatr`mfLBah~lu9`ou)HrBGkRf#ziU7{?2e34mBJ@cLA zaR`2Iad@S5v7O5^>YsF_x1{g1CV4;~p;mnvq7YY;gr3&SxJdz^*AP$vCaGnBo&(!xDjuAjk0EEwPagzn2eXNFkh z_fPbN$VeYNHP@8NT%Gb~miF^%P$Rv$u>*g%ikLAH^<6S_M?M#Rn1A`wSO_l4 zZ-Uv=g&S#4wB>KlsFj4Miy%7~u*j&wD&!=76pC3XtvvClQ? z2_P1Bc|mJeUz+nS{t(MUOm7=nE+1UyB!!voDlHzL_%CbxZv0wRy*05Ch|VJ|*^c+6 zYdAKmw^#{qsrKSe=4!gPuvU^jrBbE>q>LDm`0KSZbIR<1C2Y}tlb z$~~KEY5^TU7d@Yhfw_zD#MAc2w*?Nv2F4!G0j128+WndBGNVl#oapN$FhW?-j&+k^ z(Lq_%y|eYaZ2Mp;nJNF>2vio4aNu?1`UmUvD;dd5DXdq|mG~&kxHR)4=(^t|uZO9^O;oGjhO82W?TU8euW- z*_-7Fi^Qr_N|!4+V{2I!xw5Z~BsEh-4B%#N8!TQvN-yP#C?DHVkC?Mmhx#O*e#ji6 z(Q)vvncc%Ad$Wu2F#Ow{zvGl+NdATVSbq|qdFS-!jc_&~8kgsJAz9=&xmk*GfsEH@SXQU=8y+K} zc-T$I+u7E{0r}ukjXcX3_(g5QvO+JZk{047Q7@;;5bXAK669m4k6<$+7S*vL%`mFF zNhzSdIp(0ey2>)c)SjaM)@@PDt0^qx-Wz<0r`OiY08q3neUI=%8prss4uwi{{?&EQ zwuLY8*#`n%*Zwt*ilA|;pVzKFj!=e{@{qE5e-SAbWTmMx(X87X+G5uu*Qs5^bPWrh ze7*MpqLGt^cT9)-JHFjB7V_l-II-ps+_};rm-!XDQ|L8U=1E*a-tE3MxN(j|_~C}H z`8zV$py|gVBa;KO?xNV;SO_9oSo*%C=D0=9!1T?;KGZD~kK|>Md~zXTEuYt(-0Hp5 zjHD&Y!;SSA4sD;?x3d>x0XWJ>Dh+s zHI1JO-?uO4vaQ7GbIHZD5p^Gs}Awl%A>9vu0r zvuE9P@i3~o?_RORgOmX>S54KpwOG#pN8tTS#@qA=1vunr*1oxTou9L{%E@J5&-p1Y zEC%%z9#*$rV(Yr=8938hUX5N-t~FzJHhE9E{Y!GG0z>M*giw4r0beFw zN_Kg5n@ilA=G^}p8UemPJjrQ9rJM_nq-}hYSyfd<*Sy|Oz{-9JZ#)%1oD*3bj3*AR zp=&nZwl22n@*bTMI%QkRHJo4IV%0|)G=DG~5p(^U;rXDm8%dv&uWBUK0M)Ot$P_Af zKYl$FD8PqTEg8XkM0#+S0tT&iGK{H@9kv{0Z3`9i+<405`$7Jq zyk)!Q%kb^cXDjkG6C^`mDL!BID?1{NUq8D`l9IyL;^VDLY7z52T7wKnuV#`gYs;&( z!(d~iUbO-7LEM!!3=|m^G$i-oFe#&h?yqm8qOqlqj<`j|(`q7jQplvDJhHaItlM^O zBbA$k9yD-v4|}`%0eLKloBIF-7D>rAAD+)i?D=DRC0}2@XUW4#Pq+%?&xu-a)i$6w zt4brpBT7umt+h{!)F>kX+e3#Z)DkJPbSWmBbz?SJe-QX5V&x>#t|RoS+y5T(AQ~Oh zZhMx0uxL?8JT8icTxFsq$Dj_KHOC6q*En0aHI%0HeWXu@OP2vTz+gv$%pSAee$nY6 zApJ!%Io4+Es`@K@%=s73va+%#>G6M|bWvjnNB+=$Poxe)2|0IpF8T zP=z@0&25HnbPZJu2~Ay;LU^St9WkC7xVX2$TJ+COlSnk5YpJ{C)2-@jodj)VfoYK5 z<7Zzgr7N-g2Hxqo1Ig8RW(oa{ z0Fb3fhl<53Rst~yq*r(XSOEhlc~|jFMk5U157IAJ(dQh?m8p2@ShneS_@|az<*=6D z#wa340)t>lW1TE?XxYi5hvf+kT7+R+f9gsNNQUV1u!I7%ks+VDF$rHpOtbbB=euk! zG!mmoBr;mGa^XYUQ`qR^_-!p>cx6a0;MvR3T?dRyeZ!?blgRs4^7mH)Y0VQrYc3hR#!qfeu=k#j;9sbCh}2;ds=-Qb?sc?6|^M#I@=UOGv+@3Bo{0%jOT~ z(YfC+zsrjl)TLN4RLiH&xOIF4zl*880hs!6UXS&_i}R}9877d5Lk!InCPKILa&TIS z;R*um%k&TNqw(vst^RRjt*brzOcPkkDtbRoaI~?!Bo9PCn|m|mVX!H`JGZNTUgUGhmPV4duc8n5T6D|iRc z^L^>9t{R1*lQFINAjf9aEAMrYKNrffB7XzZYH-zG_|dtgXGl`g+C1|BKKo9h67vB- zNVwMG{b}o&C0MTUuvXc#T|guXrWsMzgZ>%k)tdVvXE)ZlJk3lN{j~HI&6u+76w7SM)~7nP-{P`e^}1#iD{Ezdr~XIBhEXeE!yp$yLsbd7p2aZ&=&cdi{Gb zHKQsJGiEl%>x_s$>)4vNv~+8~PG1rh<(HXL9>O4yU5iUgxROCB@pQ-KSisQM=v||* zwMmPLwj1a>M?|`E0kLh6G}$M2*QfTCCJe7JI3cMSZ9kUk0|}GabG|8%qcMeNvihnW zu7}rKqM-B>nQ@>+y1TpgXe}8ueD7$Rl}o*By=-YcKOgOokjD zXmnrVYgX#ke7WT5&B`h*9cgt{l@puj{e=G4jQCvC*l*lU8Qo8f3c@m3ufIQcEhiUw zI8%O{)Y|vnQ_rkc_Pv={lCH(Gm=}HOXSe{jU4vUUlk8e;^IJU5ixIO`UBauI$wr`D zYNLt!k;iLs&oqu7tG76hqOXrkQXHP!qN0owex1mR+bEpB>KG+&(N+u?H5_@R+_V0d zQQ!pig(hxC;4+Iv_)cnKC4=U;_ccp)$cd&Zuym(_L%cjiy0=>KrPT{HA!ec`{sg5p zA?DH?Vtvo~wKn_Mapw=@iy8wihJ{Y3Yav7Uvmg4UX!Y;vs-4#MgY`v4-FNH5D@$4X zL}ZZKIc-K?NYskUevqjZFG~zdkY&~f&$f-e^hQUZ92;2Do~Ru*KS}55w*D`0Hyw)x zebzsfe%h0>$>cx0#y63?|?&Vc;gl@^IC+)0V=b!;FwR|A! z->VOusKP`b`ZK!&4bO`M$47XE`%Dpbu4T5Dv!R~IDQRD%ipLmSu(99dTU`BBd(l%q zX>ph8;^*p1z?GP|Sm%=`-d_hea=mY6%{bv(1jm)jKcM34!u@5n-G$v|pYH}fuv8X$ z+f##)^r@#j$1T9quELEw!14RM1CWY8Z1w*3mr=T2#|YK?<4N8)!Io40Hz6adYyNNT|Kz(&*`5)c z5ykN{D%?Rn%h{nwC|g@9pJgdNc1lnc`KD_?FTT|&YKmC(wWM)h&y+`0x1Wrcm{>>R z6?w%A%$HQ{)wW4HL?LHm@7n3S$@>c|%p<$pjd>g}+cBX!*QYa&H%9f!UTZ}5dF?B~ zV*%eI14>D|M|+c244OegJ}V0IN}=I9J3D{wSMVOP?w{(10To+af5Hf2&7uI;{W1qy zKATz#w4rf$cSTI+#td{Nrw!XS5b^7H@ka9(2mf?)JFp*YzkP>8Z&6Ui@FI|;t9kBX z>Kjx!Xx%v0CQm3-U1L4=kdl z`TE%K8H@pYhCa<}JBZ4t*M-S9trCdwCHUpno&7l0uOVlzs-j~PeOAH%TOYVRG~e9i zvnhKuMp{t4dbwJQR&w(FtCXz}wrlu^iTJ!}2?X)uSk{f|j^{>d_HN<-H>S~_Z78zc z>>OjycRBj=IlywQa8bx2?!l~`sbtr$)ma1jd;KbRZpC8S`qh6f(yPr+%F)pg2g2Q( zee?}(4QKoKx&DyhDf@()PLY@N%$md0S~-1oBa((Jyq{BGutR>DTy@R{(?y`6K&DVd zYOG^erNx0gJ?yUJgItGIC(#vk)qA+UonO#tO4~h;B#-LoHW$Orz*X^==wZXCM-KrZlY7Nfke4l?Oga61IJ zln5+0rN6Xj_2bVXpuG^dJK+q+{dczib#Rq@pW`@TuKg}k(u*EEb9IQSM8=RtS=VF$ z>ygRAXF0?_Me|1c;c)gARD~@2c?sPX{j}Yhj>O#zfA&uMo&LKUCZ)8>5E_qut>P*V z#M9V3l=4UrpW2ifMj_1_>H00YQ%Owlq#chQpcU!-QR5DE&9U+w>!fx{NR{9BJMIM4 zi(49>S&W;T^tjJ%tmk;9lxAOBo-twLk?-Yjqe<~ntmR02Ng;nGJ7M*WKJtv)zSQW9 zNl`J*HMYL4BmvbHj$BX)TB3JJx(Nb` zo;Jx&DWvtjN=j2@dVE*9($4?8YeV^Cr=QxY4e7!FEBVpYJMoS*|77P}8hrloaGb%P z+(xlRme0o-UB9$i8{1S(D#jvzajzj2#F+2ZNPXy@1&yc0mfEQ2^CvQ{QAy0GD9Mq1 z9*GS@KaVaWZy7+}MtB|YjrD@-dA$cCUbjl%djF6}3;p13{SW2-uLxRwS9r3Y%@=@wJ2BlX|yk!Q-{lQ<`MLn1HE zFB{eR`dAI!H%T5TFyw?;Pp@{KZMRfOk)TrmEVGW24Lt_LoltqYB|DhdS#RAK#j&l) zk%AGpT@>N8dirgY`2c-xMic$^7c#ng#5j25#42wFGCkK+Wy(cl0@1HOhZqx@QhgN5 z>!U>-9&7(Zv(#`BgMl|6_aptI&2mv7PE7*X-i((WYzk_o!S-M z^pN)&{2*BKesAp(y)^7bDaqftCE`CcLPs zDHo^R$S`f*>l%l4?hyM+9&#@s&*vmV@bt0^zp|FF0!vQaPZE6lP3jt0v})%Xbq3!d zI@bae6_w# zrm032T;E8$hhRDY@KQ=fyKuEE-aU@Yr%JSeX~dGaJLbn#oUel9-E4GF5Bq=q@xiNI z&_mDkiz@6qm5+O3S1iqdY@~j|E!m=&2Vr}TU0@P2PiIAX3 zN&Mv0#j6*zGTa6XPikkTzXIzQELKA^92>i<3}&?MiprKaMmoRG2!(Xt{-$aBuNwY4 z>${$hvF`#s(cfn5&T$BjNWm;b%U#*BZ*CrWC{c*^tq`qEL>vop6;WY|lAc7hCbv3vCU zLvg|PmZG99!y?+dA*+W^Os2mVU1kg*HZw~5M~KrC?v)m5X|#-$m>gN}KU_W=c*hDY zIVJ<5NPEn@sFE|w&-x006eA5eWW+UL%*7k3nb*@#_O$AW?&!5Rj9OM({(2Z9|Jo?B zOncT3b}?E_xT_aQY1p>KOTGK3EqltT%Zey@$TJzh7SG@S@F z?qK=SW_-(vAyhs*7lb012ZnvJvTm}FWYg1~iF92_SZTuDsDI6Lg>8X&&YRvaHJ=Ga z+(kgv+}X|ECKEiuSjD3899OuOq-y9YHQ}J9VZo0ZZThS6aay4NFRT2g)~lX)luuR} zynWS9!h}>(0}q<_TlU)sCQogQHvdaOy?jyEsX>grbP%TAJb2o#V;p=;y9H^voCa@x zit>hvnZ5BqUWjv_lJNHxKi{H6M6DMr$pW~!=zX~e@tm=@z-V?of`kBkX2p*E@vcOX zf<#u!ODzxiT2vUqkOq^t>ulw>hs`4s)d3Z1B{xn>(C==|#Q-@|{Y~kNbm{l74q4fC zm1WCNI5xK!2TL}}Whqy-VvZ^&KaYDKBm(2pe3Zo05gOxYqRb%a|Orp`h@^| zcY~mQE#&^$|Klhw3HS%S`t8H+Fg`a9qr2FB0UF}{Cx+dl*eZ;b2eD&+)wj2TMqm_J zBD>y^XjMvWVv9rSAbBhNBo=*FLDV06rI6@8a# zZ0e)1ao~Y?Q22c?W{7RY9XHD1Vv9^Da0c7ei`>-O8kTuYi-A#9n;_8|dCISJVZsh}Pq{Df<}O1-~Ua z03U{|r$$-^cO1AhE3u7Af0xbcSJ@l2I*vUo`Q8XzFva*R7F_*_7W~G#SS=kNqf%Q& zH?2;9rLUU7+637PR~dOHV!Lg@pr1coYu#ctB@N#T8w=kAm8{v3Bz zHPy*!8l6wuzb_kN5&f<;&0LjkxOg(94Mcm=3tb28s`8VKK5Hn83u7ebh?)v35#(mC zs8kl`+TuL9ylA1(Ouk;5Pz@F; zGW=^gJDOydiHjFrNVDho+gNS~K1?S8x*P@TbVdE0u$1$w1I+V|ih8~D5U6!Dcm zqFzhB<16quo_eqAK^qc;)=OBou?a7 z2;~GyC;19_CD(={E)7O~ASUkYSgy<>iU7Bm{kqda4WYx)lRfn4P?a2ox?UX{!yBCo zE4sld511E#%KO{Pin5Jdkysb{)HK#DYTJ!6@>hw68XE$V*l?<&q4CeNRNIKWrwF+8 z5`}2RlqXUTd&UvUJG~}XWch4eXvwv=tAB)V+?(k>S&rcm%4aH6u*kBMN!A`!q2=6T zERti7e65dMmqI^z3BT~X!HkuDTc5{fpekC!EUd0q$y7K3-#Dnf-v|U$dao0OXI7h+ zp?0?3XG(e-3+i)CSdb*aDxZ@D8VupMJ@*~U_97`MJ0ijPp2jWOU!p5v*5{s{h!h_H+!HX_#?b5 zd1F^oRq`gHh`GvmWLlHUK5%STgS-A5>Em zc6XjG|64Izj^y?{>IMx)Y}O@STIYWwvtO*7G2gZ|8KH48nM<-?s)WJ9&sh=PmkV~D z(yrRd1$}$Zr|PF5yfYQ^Wpm6W8WHwTs(nDEU2ZPO&9)PCfv2Nmr$WNi#v#HKG00yB zwVbFgu+6If>Unus)m*PQ`_THyh`?>@?4^SqN@^B@zHD^5%4j#__NZ-{i2~(OR+7ME z;s3GiBR@@7f}Gm(MOg!I6gqVNf1EPv4|Y<8?&r3BF@VeG4r~zjiA2|Qzc*A`OI0x3 z7t;L19*{)yGA5(Sr3}M;W?4QLhIB8U=@JVnJ;ON`Mv^71Z)fLcI!fUjj!&Zk-?32- z^S9xG4>KCKUG_`oq2M{Vi5&n{*RGO}Ct+*gEjT+o! zPgZDgu+>>T#8=E;FK8-!c~E87l{?K!^Cx_6bCbdIE}sl>T@(R=@LQ0PS|BE2zYN}S z2*BMy@yH+^A%YLvyMrw8bS`H3k;Shig+xD3x4CpX_fWAA(wr>gehjL7K0+2 znpNDLa_Ih3Ixg-P9c5ToUA=~-;%2@{9kIQ@pVSQ}4D}KY4HpE&^qc2%6$;>5Khgc9 z^5{?=B7oYCl3oydR9nn-T7}1;FOIp(gxnqji@G45du?mHO14aeyCdbA9ygrjNdj2_@^mX9%QmzRi%%o|6~$J-@~=EMTp;qD?g`KVM;=dC=0 zCYOnu+nWP=Jn#00;gJU%;ijW3MWZ&WLgjP_c%r_voX68i$5*u@1w&oH{>EBFEGJIB z)m}}V7zpfcaA!^|?H>FRhT)|dNh~IC zQ6<6PA4$By#(FuL_5AJz_PS?afB^Sb0zq8V?bUxkQ@V0!pu@sbB7f0==QCP=X)3#> z*fUpSfmmpC2A%f0>=*Hs(bDu>J9cx#y6m{Z^*L`X-pwrf0Y0 zoc#i%6rZD3oe@B(cs?I;=+*ACwRltyn0M|5l`JfDh9fMu?VuK;Ea$1|4Q?I17{Q1@ z@#D(-$v9W|2r=V<`kd-5;9|k%@e=)V&QrQK%Ei&8=YBjTnfL5eVBTVC0zasWJ`tk&jid;U@l zwqjg!Zmt+}GI>ndjzCzzU$9nhR$ME_{k{{k~;78K&*PXLpZCJFibL z5;g5{ngD7QLn@n03OUkhIDIx*FB~4$eQSO1Wqd8$F0aMw97ON)sKt>@2CXg=q;Qm~lV;^Vm6V7h1gNCxa;?<&CwspxR zFuWGH>NrA)b@i)HvEKXELOnX>{S)K##oioI(&4~d2QnR$P=sB#*qWdK%2xP!*Thdn`URoC>QM zUADRRzMlqbwumUMgr=|=ttVw(S(l8}IyG8ONld7@BR8?nwV+9ROKM6*1r=L^W@?@C zZW@F9zXoTUd+X==E)g}ChYMzxxKZOIR|6~VD&n}_=Pq8{_tZviG$ z>pk0l-qlE;hhm7X^)y`BMfb^Yc>Oexsp5*;K^gD6`j6LZht>V7kGvU_Jy|C#pt>tK|tqD}a@hB`agzyP8Z-DWOauRLqed@%7y;EwLLPT2?Mv$u#hR}PxT(BNr|~_C|HczXEUN<{!hF74#A$H)*K8VHTQ)eQ-^fw%| z25A^eUDIM}dQ}Rg0AgpB1}@MvL2=Arp=F$=0msp^2HIvD)+Jr3EwA`di`CAEU1VR* z$2@Mt=+rD7$H)g&*wnLw2WQKviGa+cc0TR{K)52S1srHIX=iS{q(z8#i6^+sGFVmY zX68PZv_*=&Y+l|M-f3fO)v>i@Vdrk9dR<>80Io^oMhguY!Nw??057FbGw zQ5`VP9#RXJ>K4tHtyK>t$pe9Ne5+bSBK73*K?>i0N=zdaMJFNsJP4N7C2P|+^ zlL6R=v(mTO{ca{%3bw+3DR5Z!wZx5j#DNLxQ@`201AWjaybI2@;fY({RWo7t+=1j_ z+rKSTL_m`10lb)rIo&sw)L7=;J*sp`YDM(I7|hI8F#BQ3L)HSi^0eK{cG$h7VL?W} zK41tb&`Ig&+l+teT{8*pT4EUjZ@PHy{N-yx(fj$V9(g!)BLo?xltk;Ugipv@?0DP_ zlT%JC;i55uC+vMNASa%zp;iwzVZ^ZITifl_$o%8?qmTpokRymH5(-L+ zLA;ouSZ6mIEDJW!65F0svS1yA==RUrHrZ{5%U{fI%7gAFH9c2`UFTNER4Em#^v%aX zC8|~6WX^eo7OQxAdQg6Bor7V+*JY8@CIk&-4WQ-m6I?*h(tFLOZ2Xr&8n~WvC=v(o z!DQ36H@b{tR)90Taxh}%p#9~tW0-+!v3RvwayS`O^rGPT5CC_yAowP z_mgJ42pSb-M`lo!nB)_Z)I9(vIe&c@S`o^Weud$8v7^&zBIwhg)Fw-rK9cs99Qet@ z0*!l?#ff^=er=6LOY0(aX8!ZEb9h6lFfpcA*Bz0*-osx-TYsgZWd$N@;g+1r@QQ?5 z{_A(HW+YY8Uh6Qc5SKQa;>>QSKSay3K!Dzpj>@x`-zNAl6Ir8_(Q*j)ozr1Fs*?AAE*%t{W< z*j3yK)BciKVX?;DrCBXS$hnB?)n9+JKX*@oA0p#TUtIOwfU@7nW3*pum!xg5_O0F@ z+y#NDB$YU)1gzIW_cucx!t~0h(9g2aX=8b@)}${^B(|&h67|P;o5AAqzm|*d1O;B3 zL_dqdCL;U^+K&wJwOl?Iboh~82|QAg6|w5tnHC3M0P%SbN{@l~>pe}YD>Ai`7TyBW z+fkKOA{hbb+nS4xvd6MqSB0s(C;gEDF|7o<~G@Z;W&i6`oYEG~EQxg+rw z>7u;SEIA+)DSXOco15EWLMctgg(s@mZ2#B)WEWjR;$TNsRnvPHo^T_|PSXJes*A5< z1|_{o#+Np-4t8C3F(lZQg?;rYbBlOS=m{fpl6%#!T=~gCU zdY>?d4;%1IJ2K%fhX<}+I7)!a(I0|J715a-3{I9RKCrnpkq%D#^4ah z(H)ktp5T>8bTLYX^3JJ?kJLgQjSccwiKMNOh@;Ba-;`lbENti6DwgEQ-B`|={rsc8 z?POWDe_nYE?}Wv=vWUv0J+Fd3BLwYnOp13>m^@%{G{geR+1(zNeUXT9gtWw<|Ch?@sOWsz2F(*6Cl?={K}Wea&4i zqJ7mZ)Pp**LrnJYi7+&1*mZ|(gqJ|2o}6aCmzQG;^GBY+X*lxx3SD$`I%`;yu0CWy zZ^m|}-1{)|+M?}x|ItL0@~~4~{h2OD#HSsCb&6Lfu)SZ1`KrOE%EHrCXRZ;e5OI=! z3PQ|lHhbKs8nk*$aTqqyx{h%eG=9^nFi3hNY4*@Ogqbz7nO)A;v4ZaR%$jqKSp_QG zp{ys(p!HOPxW}gfrh}%#3g*Sc{qTx?vrS$h7j>7vv+eDnW_j4WJR}j|T zPjK3k`Cbqg6w@2?JHyKnWn_#1x4R*)tLJLLRD+Jg43q@leeqPE$r^KoA`cbY-7~7Q zX_9*a_nHF8jJ!7W8+DyHC|O|?e2Qxnm-*5Yy%2}`0qHTr2GF$q0@O;}D4wxVKo?ots_Z<2KVMv zTB_7C^c^x3x^7zhonB3+s4pV}foIs!^B2`Z{jwb7H*?xX|5WXzLX?lQEGBkK9kdRQ zj-0)qCHUd*Moo4KnS1~M6!gpL8NpyE@(b^lC-M&WU=VVMlr+P;po3A>ju=~ zrW3>%or17}x?S$3<#Dd^HKbwB8AO}cQ_dOs@$A3`|fG5|LB;({^}|7cBMG7TxK09!Ac*ymELB#>#UsStw(nT3_ZcBy3GW zcr?Zt9_w7<)IRHe`)LU~fyNJHH%x6;i{cjTpc2HIr^RS5tUpZF1pvx&oZh7KC;=`gl6mY>YnQNF`rF=x38t$gpmW{AQLF4^Fs@iyX~BRmW7R+ zZ+ojrym=&t3){yzY#KvchYBOWoNGi@9L{D$$m z4w&POZ-Bwg?uql1m87dhyN0${F6FiGzmT(v_`P5e2@!S zg!^DliTP0yX#&4zW+VpjLL3kFC!nEz1{Vyi(tzA=tNv{zb9hRVX~a1RN+9I74l$2T z2v`J&*UfP@LuYtQlZ*wQ*b`jdWPF4*0qV z_HAAm?=YHSJZlJ9+~bcwZfB*C-*?}A_R)`i)X&Hw9?B_kqj7^YD4)?#7L9+M;o!_Y z=}1rh7)bDoaYz#l7#Q^8iaOHpOuQH;;@W9I;l49o%!jhT4Wl7xIG2hv0MEoHUKhFm zjXeGI)3$HlK0mVzI>Is5phX}W<|181+QUJYj1%JH4{gOOFf%h_ufP7fefYy4cA6tB zjt%bL4*2vPhxcYkvcO39;!7_&&b*NTn!*p9NtzYj;F`%WUeb2Q9Y^eVamF6Gc-(5s zMf=C^{|VbOw#{;R)~vOyqH$v4RcTh>iBGYLZ)#!Q&dg2Q{K8Ub3;FW}*n@ix+5Yiu zHkNK!Z=-6dT#wz#_SofH*KKBT*_Ku+wybO4=zvY^*kMOUhiy-9ueHQe^X;0oRo3O@ zik+EVwuSk5>($UK^yIDFQv3I0?ZDo>_E685WttL4RJGMw*{!B&#v8eblVQwS?IT_`1WG9 zVKK)#uqVB2v-FQd2&|ebIFZgM$z}lwghFnNV%@yX) zey6!x?UzVpZhI6^J_bM9HZKAo%O>YGu1(qj(n^a_hrG)f*7min&=ieAG$dk4ru=HU z$KloMb;4V@Q#U+4-%#gJ0-h1TBHjTZ#%5SgOS^S|mqAF97-G{4L@+4J6@f_^g*3#slpqK9}$!55s>g8Go_d z^)>}!%Om=SFvXLX8!>c4+?;5MW^}>L+f#JKc7K(@oA`{EB|e=(X_2?mlG<}i4AibP ztqH3E7koo|GeQIIP1PZ!Bwyl#L-^w#(?Cc8jRu_E;7C?$6-#c2DDZLwp^ocqMXZoV z$T8HG)?;-Bh9Mt?9)>bSEd2SyHzQu^d5gESk&{TAK#WV;P6$t4n-D32dw>hx!MKBG zr_psCLhivI$9-rkr#}YsT7HS&hCE{WP~Q4;jKekb(Z+NUh|4c)JcCl2)n6ugv5;zs zC#qlTY0U|8p4m0kOVVsy^VZ=2<_p~yl!!10o*BGVDNGw1MXlOIEv;)4*8|k#`GPy zG^B6Qj_g(l(~R0j@=Yw)%O+itHszC6sFO4*kMA$A#A_&(e88`vyp!42smZa0m&Lg; zD}FD0Qy64fAuPY3xTjHG;>-eVXp)&pFEf!_fyuVd`>Fs6eePpGHtzkyytT9Ww{ znqGf$M<1@CqCD{BqixVH_bk{Ae(z9v&II|C*{tQX5X|e2v(t8<$ya5zr?uhusLn4M<>KEE-_5?c3K7Ecp@wbr>{uD>O6Lvx_@7C z?F8x_%Z+u@vyYibSENxHYh_0u9r3t^_QQCG{fhI;8`3XdIrg&js)sC}(_DNz<6R~r zLS>@UKWEvo1?|MC*X+RuAGBRNci5@ZC+$D{r~hD;-F533=&{~Ro?WqS%&Mqd6#bkg zTL_)WmqCDXU6oRr%B5{+aL7_+MO1iALKh^{!zwY22EK^cmY|5D$7+sLGcR4bWuN@e zkGX^N`!$$+l2;`I=>ws{q0urnJ3DKC@9+IRd-c^vC8`h(?E2xY{y`nlW9{bg%V(D={gO%Z!Gm`eC zEs){hiOcg#HaIjO9*bb0_RK3l^(2HHSBgrTmr%qRd3vmhAJ;@LXOF$^?Hsx<;FVH4Z0^s7^0@#0`)&L7ZB8qk zi<-?!F!70r3KF0DIPr2H@(YtX5TAjK`QIKAA%#kLIh!D66Q>F$>@`IYUau=S>OxyF zZ*fIQpv4@Lv>4(L7ReHfxwfq;$K=lRoJ4HWM@in^!69Y@6ki}IHb8O%X5_*nb7@E! z@xxRZMo3OtSk95rQ??kcb(PtWBT<2RGA4p!?(CCt@2V{sa0SxIJeRlt>+R_A{d#-o zOuIfe1EiuhOUV^uDFP7C87~3vLjc;0f>p$Yc8qiqfjpBoUjyQ=;~$_i-sUhM`fU{t zoZ@}NImXko#@6DZ&Dt`S4izR;y0)%JoGo!sT%gmO#(J{3M~)5k zp$B49#{5ueP#eO10Qd#@;gf9#LpCf7b^uq+irTnhLgSBoHLCqeeynx~X~UTAESG2h zh4=d%d^xlg2Jm3cX3PZew(=`$B!UT@Ri?bOB@E2Df-_h{?u7=5nEX3PBVZBo;V-~? zpGWi}U2lXq7GhfDO)}mbH^dA6h!yUfqt?=PK#We9 zhf^y~%2nRTD~YjV9S7aSd=h@?rqtv`S*#f=heBWLCov|tRW;#LE~o&Mk0VdKxF&v} z-V{A?4Ll+AL`@O7V&J8_HUMb}1R|cXnf2UAh!Q>0yMZ`Kxt%Ya`jiRRkRNx_l` z<8X3~`Eq59agF>1h~ZQ`X~5)Vjl<^hVsv0FtkW~lP$-)=B`maaNPy!$;zC&oO|AiT zz|ctCf#iw{^?_q~T3-^HMD?x6>ue=Ol zB{LKo%7BM*luF>T8OaeLOCg%z-;VWAnIVju*dGx*b#m?Wk9~=MNF6EwboOu3e*T=4 zw|0Fih~;(ApSPN>1jXRcZGz|_@Nw)n33`NqGgr!o&ViVw!?^8qmGu8wc&zVMX*=UZ zNC?3pBFHt~NBlT~0-D92g$bL(w}7>xq8v+mL@93lvK~)(Ajk+%{mz{`=Vy)Q@)>*W z_$zkw$bHt=*S}Ws8-T56u5>lRiZ2X47?pZ+y;jKdx{S&}>tiEhwrA%aJAB}<-F?Sh zcF&#n*gZ$@w!4npX?GkvVu$t}wEesG+QGdC?7-grwr9s~+p}wr-EsI1yW`N24e<`` zKjd)^?mJ+I4ji-t`}W(!_6h6j>9u-Qs(6)me?dTdBfQZQg_4zLz{!&*-B5v%3U*zP=V$li6=J(exxtRaTROA1C3XUA<5 zpX<*^A<-})4Cqn1fxOfd=Mmxy4e=MnEr?g-^1YVn8(g1Y1-$sfXxQ#oe9dJHImSI- zk$d8vVxwO>drXFfSmQ~V9i=hzprze%O0mUIc zyf3&gM>N1K;gL1ACs(=WWdk@+XaphYxZ${eJ3t1VnZ)PY`6X_v?B8yY4hT`?6_Rab zY{n~52Se5#-DCUT#=3fB#nI1Nyp0f6o}hXoM1*{k$Qz*}DYt8nJ~84F-+*9n`UzOB zd8}WzsM}u*#JX%n1GlLwu!i5}>sCNmr@I}zRd|HA8xM%_wobFY3MRTpW6^B|&_WRC z*K_TRAADo)23Ws)W4=D}llurU=XKW^W8JqiJeC+@au;IYpOg_i-vApi2x-=VkdXUY z-eje5a{$Q%X;enT6n9}v4ppX*IBi<6aUWvu^qz5O8U^t*PR;8!^3Ux zOK^$(6`n$p&h`%wD#Gs?3)9nD>&NH>CXkfq*O(@RCi!+;{5BcqiFW|ft)*Ryxdx<& zE<$|n;JjwCI z$F-w=e&6YC2c3CF*c`tzPLiK{b;gTvlJGUaKf0|T`rj^mD|oZ~BE&dvC*O#3=Xi=~ zlduhAFp0aCwWvV)B)U|iAze=Q*nM2D6%l0q*>C*DzX;oba7<8Pl44schMsbz>?`N8 zxvZ@gmu$!QgcW*ocIC!J`}ViLZ3hkm8z9ySFAX< zY)AGVv3rl+)L%wwuB^smG zg$oz#PyXaj?1LZtpf69{y?eLE!H|YB%6{W$1mxOigcWZX)}oP)Ycz(DW@%~3SK49R zBRy#_ypbMd7a%_VTv3F%5=MOTr<0<>7}H^#!x%>y81;xlACVRuc*YTre8`vWy5LA2 zgn>tV?!sSffd1a;XWlWg!Gzei5ubgF?BwjUaXfJ=EnG2}Wkm0cBRN*du|+VaGOJ=5 zW4kA8d~nzf7l!R$RC!CN{D{k9JU5)xnzlT@Y_;NoW%C;6*}T;hznpFw%l)hM zuuUw>9v-D(`!2^2=-+a3SuuP}Jk*;FlB_JGeOY3iN{I(hp8iq|{WhzPGpoB;P2nGu zP#CLNSh2@Bf$$S}gBOR}_mB{jV|(%h zAJg!cC+d%uN>yCmFY+d_-NvBFD&oku>E+1B$p^q_iX)7CDTB2@ym7-nzEL9`{`e6V z)|UAjOGk}+UDwJHL_c53jz7=D=gQkh-Wc=2BvycgRU0u)tS&2!*czsqSlU!}s>Q+WHyAy4?T^r;!lE)lOVJ7$ zU&^9(&^49ft)9w-=;&0r25qi>0n1POmL7%*}#Icgdq2$j+Qy8-fy zHRWX%j<>P^e$<(=$vNq3ZoMGFxk2b94dLrJtwV^j(Rl`&AjXUNY<3a0iU%K^oB zF`jxP_A}SSFVYJ=3|9!N-dJ(#>)PDkov+eT#I+*m%T@Vsti1YJ^yDq$^dX)j->{~Z z-sJr~y~`Mb<``$n#OZ|?LR5(SbTgW@ysn+`Oongu^E1F>YA|H5yb-<%o z08AY-2^}`ll1Z6E<)8C>)!sBiL#?hNSPjSvsrA!N3HLj zP442oo`N0(JIateOdblsISueuOyI}tBs??_B~B$L_hFnw$5@ba9xk3Lz8?B5wnPHl zc06|WPvFo16^=9!BkKxt@JHw=rc1)PxFaoN9{v&Lqu{SJ)P*?|C;S<+goWcP0`;%e zNKsh5?NsdX^y7`I5w7@c$ql|kfo)-w>525Cf5x-tulpwHl+5QnZ*~4cSy@#N$Jwe? z;YPnQzUeczo25BkBCT=b`3Ah(IXN{eGLW|pL2_PAMFAM=^ee3&dXJ3P!&Ask?wjxp zIgT1&l@@r0bx!hUEPBN{0pub$q8*L`I{SueXZYKNj`+eW+`d_Q!oW4~<2S|yCm?Ak z2jbxHiw&sf4N1Wt*=CcBvTHAPA`bCc zX?O&j;gARAMtbgqSQdHU;L)pcj4Q=$vl?DkXvXW?ffFBGq8N3a&iu3A{AXKyutG#6YGdU@MhaZMAhsv%?fUgAHZV9~J-vA=SBlDL+EP6^ ztCTBNoLjP`TXQx&HDl9rb2d9OYxBBa)OC*M#d-IeU0CqL+-v2kRc4oMd0|ohb2c|S zZ;SJbwlKe7v$Le1w#B7+tJmt*SLnB~(NVki=zX?-&jDWvlhvSRD1=sxEgJ!U&gm=K zAN|oE*}($`+$e`Y%2vMjzW2Rutl*5huYBbz_WbkD+b{jnFWIqU$9xOnCqMZ~H`bB< zvBw^>|MZ{!ll|`R{;qxPYhQC?A9-Q4z&MEe^GK8;qXT*dnidShhtso`G)mg>QN~l7WsWxj?_U%c+OA`l^I}=X530CR@@J7Iq zuh3WC3I#Xn(L`8PnaY=Xu!Xpy{EAA;nR>Jh_2=Uj#(7prouY8b2P~SxfqLOsRX2U> zrZq)35dtk}Qq>ini_(DWQ~Po@Ds)Ai$q?K;V_c$TxKQT66^DN^VXjSp0Fxztl+8pA z#0f6OBRyeUi619+;sCKMAWph59dY8sk5Y~U!gjFWAMy#mfchGNisVRZ)QJWRjktvq z#OhFdfGyc`8A+l3CAD&J7GO7?eXTN6V~Z&2X7YE37g}A zM}*CFh;d`t;dwm=+7?G1@wy(bQ(z0A@H9$0G!~Q`4~=Jusr;bMMxz%!3#}ym?}Q1E zD&X2w+K@8DV_EZ6ox7Sw39M|&&IyQNQB{zG{G=V z2_fs?M_9xgPz)(#;RSO7x^*2RNwemX8#q9u;z;ASgBZ4ToHzQe`^3HkV*7PM5zoFJ(0qb@eqHn_CX>KfOD{tb`_vAs3wlpVD2!>$qU$?*odSNSV=w5OqT)asN z4p*W%ZOEl1_mkFDY&T3%KJg=jyEORwQ~VRSGAW><>j@kvHT)8ncp>$cZjHCU8H5}I zB;`6i#Ca>WX=h%MmUxb7B9JbmUFYqgt$?J4dRosf)MF#8hXTo6utMlxE;fFV7={=JotsIbVU+_B-}5_n0&+5dLZyR@At$7^IJe8iV9w|->NM91eA(5 zVQb<8FRv0vqOG+k`Qf~s9QPO_!sh2rpnzCNXS_E{vlYbnF)tvr%hnJ_8R^!?;5tNF z^41Es8-Vz?!x0&Syh*p24|U2u7$-XM=DxH2;`PmN>8!`?xG{dvMJRWjp4anRPeooh z{NNGcZY@S6sCOzo$3X6c8$~ne6R&!0wOxlj|9X32T?8-QtbGEWl2lU@P)3N_phN^QCgxS_?&6nqRO5#a&unu@$+>YSWhHl&Ad5rLy%E`fS&Z z-F9&Q0XuT=sBIe=7j_aiHQ)lKosC4LV`HS;VD;rMeaTna@xclz{EcsX!&ljjj*hyK z>pS21j)(vBPye)i^PAuFd(OqfAo10&e$`%j=_NOQ@#RF$1$*ta*SzjSLqqO3SMPuM zU;Yk;Sh;lpdjK|Yqj%0SN91258Hq$7Xg z@x|MJ`7i&af8~*WpnP!Qp8e3_q^vlG>jWx2xo;25REBa z_^S(zcewG67)+-zW*p_65W_IE4G#?3$k2$5@^J-SM}~33{?Ua&jh6vk`v&{`1w>y? ze2w$s$hCp-PHY?t8=$(8F{_X9Nm2I}DhR8`k~WY(M#E~k<_0iUpz&31oaw%yym(7T zTuNy*#jM4N-4N#arY(yRQ1|3aJPdc~Dl54toPv}3i_bFo`@y{OhUL*J!Kj8|5M?IB z^?${~aOrKVgy=2w7f9p94{EiaNveCkEFR9S98__`cv z*W$y+qMux6ycp(wYY{rGp{9Kpc8bc{yJ6xkr}1gBdN%JIGV`*HcaPH>+9H zMjk$1@gHXM32u}C*g2s}y-SDGfhJV~j)D>gjXU;=8*=2AKR{PEmPEd! zKh=pp4~51ZbK-1BrFh|`1Ze~96GkDTX0?ZF>vG|PE#OZ}hkCHmm|I$&!@olv!G&x> z89=;S_X%m@-A1<#Yd%C>3sRZhmw-IoKgN_lMt!fR07OEog)|-)A)ZDDl63Ch0sc(d z=#qYOh}Q(X!IQS5_MR4H9ZC>Qc(B?kF|biKl#&r{S&XUfsUK%t>OZQdKX1d(nc{P- zfWYJnzwiL}Yi5*}qNJ4$dgDqyD86-9R^lm7fO2J}FnEY5VQ{0B!fF~hryoG+SdBv) zdAR~N+^G!8^0uHD@i8wc@-WX$#Za2AAQlStI{1TViukN>hT6Fc)JUF0P7=g)@#;XF zP@N#+xR47L|9zS45 z2&;X;8J{Ea~YYbzAlht9N-4oC~czKwpwjqw2T=E4s^c#N|dLOs{>elwuVFa|?@ zv_l7P+5q?=`{2NbXVU1(Wepb|+(SSd<@ttoA+A?QF1GzTxQr=)vivLh9tYY8+=bo_ zW0ZX?q7-H2ADD2bv~ut?PphlT@OTTIcN#r=CJ8Vf2JRD55if*>U+NSnB1xl5C@c7{ zQ&1BAT>)vLdlOg-4&`o4*Xbfa1Dn8uWBLfRH-LX<-jRmrBkq&@JPP3o^Fe$&r}$2P z+>@v?4EIKWf2_v^I^$D*;=dLg`YL!N<6}KATs%8|V!qUuu}1u)4j#25e+QmpTEIQ+IeEi&?A+;ePVC!&NJ>g8TV}a(h7@1C=7SKW|evCq12~nOm^s znN=&!RcvLhWGnMUD=$lxtkkVH*Q1Gs)gM(qq&-xb#)*JWtbFJa1h!~gz<|IiG>l}d zN@JxQEAQCa`cpskQ?7&(AB7yF82JCeAN+yW{nMZRwEdHR@=xrSfBBc)poU=%;~EB! zMx&vIOuL~BqaYv1_#1!YZ+KqMJ@=e%q5Z@sK4Jg(AOBVCC7~iCuPZ z&we|+|DfC)t~LZSTPYHZeYK{TjEvWzbjJiKSw^Y9a=UA&Q^D zF@0sPChoMxe_q!fR=(t<7-yJjHRZ~WmB|gxqvKwFz8WPDPYfI4n&Jy=xn&b9e1wl) z@YigRUJ>NS*>ZkVFRSAKxV7^2)ol99WwO>Qg}k6=U$vklDL($d`E-=0jMzR4pJvWu z(r@^)@?lw^(uY<fGTu^mbLpZ6H zGD2g~Bh45-3RTxtN+^h_14U+{Jq|<;N%ZJDYx9!fb8>9aMj#|*Emc>%s3-DVxK~NIKlz^Xb156ZlSZR?*=GyE9lItGtW?u(B5(L zCjz{ynI0C*NeP{DU~#)gehDqh&%0Go{mZiID6Sz~^vv?4ijq|Ma85?IS>?ZO{vw{Z zl=8?a9WMmH0g8Y>DQBVTTOt+CcE-@JYGaLc47$*TAFPd@EW)YDzp8Nmifz<_NTLFH zkpRXx40|{q$_^@{r83xRsIfr})*O-KM;LKJhWeE^d0>wV+=5>)gzHeCkjjd#`BDOV zm@>q`*b?a^)FTBSS5#I|qe}VV-w`ie$uZPBmfQ(MjFAc5=t4R*C>&#*;DalYkI*NACj(4lz5N1L1^mjrHa}`X@lUs(s-bl){&XLyL-@)X8 zlo(cIf^VA-3@D9dYnrAcyQ&RxDg)|dt7{TX)rZRYP)gP52cs?24u=j3afnC>ygb2q z`(iZpEp*BY`i*VreHHvjMLS}=YiQKsg0YPLSZ^Z-(yjqz###Gvh z_5_tU#3_jl{l2N+)5u@fXyEj@Pky0w)@j$rna2o^P>WS3$;|4&>vaKg|=YCLm5sT@`gVqBi?x_xOKuB(zgLm*#Svm zG0B@H^|+K5=^#etKDJu~{1Qi<0p}Oc2M)i@F4mLqw*%l2X`6KLqOH>6&rv5)hDx08 zEQq*t)(1%P^Ke0DkVDdLTL9I?fEVbQIgLJKF4R1xG=bjW9Fnc%`1*9f1)^1Xa)>*1 z(Q!k32(Nc4J@Z1{lgJT*@h%wfD3d$N3dK53isvs|B$vbWA`ojm?RN_a0v-9ieIQLj z4{PqN^6A8IbGd{?7ioxV(!Xm!r$#?$EV@oSa#+L?2w?e$Zy+r`TlY+-p( z)s3AJq4R$IiNn<~Zef^Xt1DkW{Lvr%QQyY;na_O2SKhHL^ael4NU=wVe972 zo4zgejyvw~ReP+yVvUcG9QlansgutE-F9dRcnCVb`J-~GFP*U#xYbLNcWKs*eJAN}Y@?Pq@GXZ%b) z%HUpB%h$l0_R>Tj};++`s>J5nvQZm!X@Jf%dj#YUQ zJ5;c0*)O4~Ai)J5sd5l4Lj4p&QOMgfmV?(;g=e)P#mS9+VG{5^@b)q9DZ= zJqzI>KX_n<)eLwl55C^jpX#x^_*+wD@X?|mO@w^j=)8!KzY4%_NPI4vrH!M|9LfYm z>cS)xAsQ8%%mgKVJ+?LAc>#U4TNHT)ElA7 zS2Ia)fX=uQ%u`J3f}1?Kh8nns=izntj>XTt2vwf)OC-qH9v;VG9qBbM{9JgEOQMHz zLu_HS=E+-M=IAwW+^9*CehVOV1jnDN(!{ptY~#)xLO(0a+eHqgna9L+!p0Argd580 zjJ6&gO4*df+VeU-NxV%FJ0C;28zb=ehBTW!I`WP|K{AQ$8QfNx@zm+#_lXC&hQ3&j z#|Xighj*Z~5RcHcGTsyv?rC-B_2mvnPN9r7FS(9eg+YOFjCQ(p2tFIbWBLFgKD@#6 z6TggpJjb?0ISY4!cmraho3J86x89z9E9fX??bdUQ4_m@oazW=k04|-)>BS|_AuI;I zA!UcV9Vl!Kt%{)mNo_8mNTmyJwpUc+k&yW0-aY!HH0GsIvaK0aTqq`74 zJ|&}Ckl^I^>kxf6#tP0;d*6ke0=;&`q@Qqp@16qZFuF=6ke|1PkJSVc`oSY<)#M6< zQhAIRg%Ii&`p}KQIEB;QNeV@DAAb&4#g9ZM8o-AbVB|)g*ktYxg{%{@Cv!+{xH!&h zrF~aGv#n+6IBX3bZ#`nibEp-!?%Y54ZOm`|IU377oyT`?hd@W`9vkT@rVFs~4yXLr z%3g=iCn0u_%VNrK<#D}?kanYAfX;k2LMSuF4>*U1a34A}VisB}KzP=#@OFAl@L$U- zCgLK$cRoFnW{rS@H?UTbwOkx1xzi`&0tA2kCU~qv=l$D*&3)pK6if~Ms*49D!RyuN z#4xl?@FHdMyaq{P2g29myrJCAsDfDjW`AGHf#;XMgpl?MMIS58J&Ty~~ci|DgT!-~Dm>*#GUL_P77>-?YE}4}RW$xnT8gBM#@G3foz|MUOsE9ZXj z2Y=Ax{;&V*|LR*+i37m!IL8iy7=VAH(Xd^+cDX-WV>!Eywxr?7ld`}E!yUM>LJr&r zCm*)^-gD1AZh&J|Ud)rS06LEJ#AU@EIJ2D>Keqm|#TSDkEB4}QI~@7o=u4iv(1mw8 zxW1~P#_Eu)COKc_k^IFs%p456D&AJirK}>3#)=V+;C6NhA7DjBJ&YaRkF^?MV`w9j z_HBi$QY5Q6F~m$(bGrD7#!M-jvs$(wWr#OA%0xV|v9N6UqQ*}veW}_d*G5B*jt5nY`Ma4_HJNy9ERx13`t=bT~FD zh$l8@tZe9?QTO6+wWiYWR#G5B zs8owqtFsazOqP{uMZ$b&GwRw>YjMX%I0Adt{Ls!=ReZ9k%z4||u##eOFtYN6upHop zE@sgTekHet+PcXB|GsStgRR=XS+Z8M=CsIx!gaNERoCA3fb}=~MQ3~koLpEzqIPNW zA(NWrYYoc_FOG+=rNBY_T+fx_s^;1`r%4Q{tqiO5gk@$`bSPTMaLB}>HQOu7yJ>!C zzxv(n%92W1Qr{9TjYifQ)n3a9kJPf_tcpxZ>dQ*YdKdJE<;y>khSPg%`8V2$+YWr<%2Y#O~PPs zoYa2MDOj*xp+|}HjY@DW@dVLqTljdVkw&hnvRUCuOQHx=MX-_`l)<)N z%^{4lDJ!MlilOID$LI47%m61|~zSzer0`}5%n&f#p>s(29w<>~UWl|*Z8(Tg)@l&q5E zl~pm#DqjUxf7Qw=UU;_DQfj9(E~N=T{GVl|jPW`=4e=Hv$wuiwJ%I3SK-0k>zzdZi z3iAS2#Z<-lYT*SZ&lOciq^?W?&*5w4jTRsNz!Q2+h{m}*<#N7o@PjY4;Wg@5W`WQ< zj1%E0+5i_QxT!X6sEt}`GvqwzKFXK?ao}LAvgYNF*G>3Bi>-toK$$DzEwxNTeOb3+ zswf)MxL@qE>}tQ|8oio}a^gp-=ZtbI@zP1O$7~6isJ$B%wTtfCS@F!~lBM;WWd$8= zmR0RCyzvj~WU7)kOPb?Y6_G9%lz+i7@`JZqFd%49U7~&(Z1-A!t4A%9S9#(A;x%v_ zlV5c~^%CFoFO5{xHfsM;dd@1@c{f<6+TyFC)w=R8l{0F8I#liES5dg|W3^w8^3+V| zD{!FYid>I#L18thati8y`kk1{o&m1?6*v;>l{C+Yu5#kL`NpcoY{|zKi&pAs zLSi{eeIniDJPessRT~c`ri7ZZe)T5#~mmP87 z$6+VTeat1iYo&ZD0zYZO|MlEZA2u5-$_eS4ieDF>=6q+k%PI?TB)^7s34BdrN#HF) z6xY;PE=rE8*1()pw2x+?O4TZwR>L1o<%?n<}~G3(EM9a`?X-bO8-uz=eGE9 zt(DYQ<2fgq$Z4F{>uMY2LtD|BnhC@@apCh~ky=^NUzHEE;(ux*WD*>61+p1^8e1M( z!~x7HgasQy=NMqV<3pHbuXC-=2TI9^7wWG_C#34_`65T~ZC-NF|bWQRvW#S!c zfC#iIHVU0m?cmRdQ2vrHye{%5PxZd^pjpX~d;|=fgR9l02Pz+^IRdja^vt5P_*@g6 zHCYO&-fg!F8mr-Bq;>Hd7NIqC4RS%k3`B8^-Iy)7*EF~I%PRjU3=Y{=I0%3068no1 zy{)i3t4P`lmz7>xN6%Cv-1f{n1Z21H>5oQbuoX)ywr$&(4fYS%%+!oM@yrue89*h{ zWG>~Txl*#lrA3=vU9@GjXjP1$E~#g=v}#2ObPG!hHd|bftt>Cu%-o#K z`?+^ZR%?o37nkLh#1K%{Agr1#udLXi19#Zncit~SEu5DUv1Q_mcsqClq}*`E3OiP` z-F^4n{xJ&b!nRV3X{=29$VWcntMVZJKl`&k^Yh~#eDFa(tLsZ&`jY+1ul$Pr{lEYB z?I(ZoC*A$r&;6XQ-eVOXtLZSbvAvdjFt}l;gJ55M^;I|Cv2u?Ud9)?U&X>RZW#4*B zdY(V`xzD*lk$4aShD3~OZ0$vVZ)YqZ}*#SOIwI)G1$a#|l5{ z5Lex?dXI9c8*w9$whLW=-rnh7n{^%r5IrVIt_*X6xz?na6~q_voRuJ4*`Ch|GR_8| zELJzQ#iKBkHO1*`;%5!{v*k7?+;XY|+faj0At6fwKdaH$HX9UG)i*80C!bRuFn#k6 z@BIJ1_GdOFo;K3kYdiY+va=LP+CzW!l$d&3%ASOjnc9jy{o=EB zNRz52Nen3nE9J5T2o=dHJyvuDB~242inyK&!Xd5nsSHkGnsxX{Mmnc4t-9nXpFCOZ zEUmLG9^%HltkkwlSUFwR49w93S?a8OXoHj{ANIpFF&;@wVEW5SAqH1pxglD?*hF5+ z#Bo}O*hvwXH84i?sQuwJLHQ{cC1B^%md(+g7&|Z)Nr=^?#5sVOjFfauR%zdQ+SGm8 z+mqr@emGaqKdzzjTIz3}MTVX?h6*M$>D+P>fh5@Q?rUE zAJ8*OyT^xC^+()dW7+D;oBIq~l~I1xF5ruxh1iRNnO7d^I>rf6in^sLX~)mMBaNJ& zg{Ul7+sjT5Y7*~n&IHV{s#AHCmBx>!&@=RgSlRHePf3VVru8y>T;+ftMqhB{Du;2D zFy<;a-65bY}; z%44M_i%~J$tyY#L?8s03pH)ADg{CpbGoyNAM5##V?9p5aUL|N5rY_kQ;el#EsNL;hubYRro<;^9cMD*1-5JNw6HlJ*k$ zseV$Xls|pr=U_5^R1R%SyU{-6l@lML-gV)FK}HcZKA}_fv*dyX@{!UkDSknIJ)$AX zsi%d1k+x>7kZ4|VRa5Pp&h~4_i2jvlMd=VWGol?RIeh&Xo~~yMUCJk;_OFX~EG@2B zrZ?|rS-NqJK2>ikm98=XE=ItC~&3FtF~ zTcIUyjUmx}DjUY86Pe0X=Bg*S2uIQT+Sg@8-?#>b7->oHF0Ys3gNF#3V#K#fR*|wn z`Q%%bw$ZHUo_1CFfETB0pl98aT!m0q=7=>~R^D~d->FmQ?aZ0WlI6-aI?7kxsB6p* zWgI>i_^M~D+?sBH3+(_;Voq`TQ2a2jf*;|~qm)&pg{NbH^tL6O__Q7L>UdBmwQ(lxMwNBTz<+H-C44)Im>JsRP{!eMVwZv=TF|63jrTUbI$`)N| zvh;r9tB7pX2HHxC-J(%1DXzw__%&nG&!46a@FMYYw~{hWstjiN>p6{LFXp^}BCfhM zMGGiH8I{4fZ%T=#56MFT*3JumUYbysdft|$A@}xVHEzT+>38~E^{I>3ds|hMrU;MI zxKdk%E$7NmfN3Kqdh&Kt_Dbr2Pv|s42p~T`<`(9aV61OAe)JbFDOib4c{R1?sgoD% zh38J$6W=;+Pd@RgVfPs5@3+2w@o(r=^8vJi+<=l>XW5vFL1V%X-o^`tQbo$7#%SPC z333AZuqr-4E%KVb34*smPl0Bj1&vMShMH=rHf7}d81>eZ^|H?@QECm!-bTJjIQ9)K}J%aU{AcuW8u&xf~^ zr2zQ_83o)_f3@AztyO#JrPKEK<4@~)+-}~O_C97u6zY|tISujR2w4QN#pi1U1i4OE zeu*bk>Ka(ka$bB98HW|>7;|}P=JY2#{HPbD!vMzoM}9#MR2{{0vg%B5hA(iAan85b zVX8t04D$emKzhGtPVJFGF0Cx71&G6OR0eee2r=59ZAoctJxKlVZMM#ISB@Nt6JCGt+qi9 zq%D~(RE+nx3iUgr7N;3GrS^98@l1Q@8bT4{u-WLDrd0koco*?V4Nc(}o&oZpF3=f0 zpnlW5hEbQeb@Bh#kDs&`UpQgUi{HKU;z@rIKQz>5Jq2hEUPC^T>m3aYq~M3_w8jwy?2baHauo__iz zd+pd6d*zka?aGx&;cUWdRCN(8sK45hA&?_^p`OhOKgmy7@kMaM$ju96WE18@jGKIy z0UCvtT|ZLW$F>UX9ota4L8++mn#_SNGY}s;k7P2nTllhd%JmADu{CZOJDl-at@A=& zgz6?`7RKRE86rQ)1m)_ge^24h{`A}8g9YEQN&C&tPWhs&{(jBXO8o{P1B^)C@kluW zhy+0b0S(~_4TbGqz3my>R#>v3+`JUaY3s|+*+6#A`qDEtRFG2Jw`BdfS(_Lt*>HZr zb`Dl-&!B|4;uYI5qQO_cZn@T^4fiZrPiorw^Yb>)vtT`$1u^R-8z_{exX86C5@Mz5 z@g+xaOkDg80IqWG#Xt+&f0=;6ojS2C^%sBf7yWEHjBptA2=ltB18}6_;}IApSi#1Y zR#xBPe&k1f#6JG}SzTeJi%(@@#2UMrEHZKz zC#MzvnoTY&s9fczNu3oWY-0_FxWn9eVWLSwlP~4BrI>+ZpW+sfwA&f0)-XUQOy#jA zjF;!EK2#|ZGE_hKw$ivEG2bgbh%m3R#JfvsAOA=O6PpVdx)%-=<=G~Gmf9(ov>VnW zc;JMGvsJZ0;IS@s$VI>(2c|elX-eO2gAMugSY1l=Yp>k0vnPtSFsr_54oE2Ibs;c( zEF*NR3nQv4E4eU0;*Da$M2NxzKf?f`M5@08O@u7kPT}B-;FFh>$YjhC`AW4cS90M+ zISX|um2PBH9dY!lyeS9Kj$bfg-s3HM76YV&Ek8@Dq1KWHo8e8P1R6iPP;HS@-BRCF znlGtMUY3${zF||}nzQ8#n#32CUrjwL92Lv*4c3PAIQ^*?tSQ*EI@rY@?%BM2$=|?5N9)z)9^vDP2w zZSuD#t^B7|%Ra5qc0oLCMR~S*tsz<}7wcB72uI;iPDz2t@n)RW4|UZYPNX3C0 zeb#<5W7B`MXg7a<#wLGv(Uw22vd=KzXiVfJKPzr~rD1uMnZuyM%X$poy!>Dlt#Gai z--^ab&8sWE&%&~cO{+g4TKS`r-TIF+R`^oI^7QvL@genLT5X)EW$n@nSMAInU$9I6 z#|=C8Z!g=;|NWZH|MrwE|GP=6{-4v9`wTyd;n zwZ~Mq*VKMe%o$$c;}>ct8kT++8DO|)6^&+aHFeTQbUQ;>j@91u6^gl9Q7-Abghs+@ z5Zc>pWVGML$4C9k>WhnIrx6T(>q?sW%Q(_Qe5{z+@dDY5=J^4`Ls%!7nCQfvs)Q<8ekZZ zoPfS^RkZr8meszhKE9!`{M@3gKC8~Uq;XdijUp>49{rb5KiM&}RgJ;<|1e{f-`AM` zLe6@g6W;ZFSY%rlKS_%|Q#H+O;-E$K2P;fO+jS|q6kNy^RA=!3kE(-wqB>RUb5dw#?+ zi{f*f*_xKzrT(i=H*N9ws?EJrv)Uzbx)(HlzNmUUEk}2(0sT*ZCRb+=0ni=kL*_MT zK;6brAVj)a+pb(%vE#?C+UT}1+q-XRZo8@kyg>)X!?`=8Eo_Z`4~f ziGG^xqUdo|4y4*XpX4xPH<=^Fo7EM#X1n4JUXauL&RoUZi$ND&1zlAoFR3VZV&=l5 z6y{fr&zfYF%NG{y*o)IPc@@J9vJplDInfzBANoc=4)eJBPJLEk#WOMkw573K)x27) zXmY@~FRF6`MHpei9r+A;#)!upt~5?_@Hxo}nmW8acu@Nw127h{85Rv_UZ|^FR^K); zj){I1&A(i&IUGLUDrvk*9&>umGJAog+Lcv>S47X>%vkL!qUY=HWP0v%c@jR}x}y2_ ziLzC{v7mI4<(@3r!pqB6oGpu|(xS*kNcSQ-$cbXT&fIHGK%SLjg&R9vU~upz<_CXC zrV!d22fbl?2rsm8%vBYiV_0biW6WleV>IU=Q#Y84nCB#qLSyP%>=0vP+pUoAx7n$Z z%}k5eX%3(}aR6l_b0cpxC1Ydce(~9>cJAb=EiFJ(&>M3KNGX5KR+%&_L4zC){ULj; zuGD;+Xqf8)z6{)YbJc2%lCE{}+lE~{SF{ta-L%1xwC&t8XcIezt$(0TbyMw?gubE0 zQcN;wRsu4v)=b37*yII^+ETRU^ys>SWOHQ@UaC?twV&Fi)#gP3vLtAwthXTCmEJo? zzOJ)qZ1K_rylD$vd~Pn4R_yeN>-PN9H#7zn*6LIF(#?Df!e1H&)V*5rLwr_jOc8@ph z;O>ka+MTuiJ4{#I@5$Ki?G3wgf5DFE{{Dk~_U?lNwr8kj_a4@Dd(#f=OxfKBd+o^H ztnJ&`wu5^!c3@9l_XXQEu7M>%HLX#VRbwKYXs9^iNxOF3g+_E2Q1+E@Ks20T!Ti*x zKINptzD<5CYB;~%&*pZ*)UTYV!V!dK|8ixf< zyghId<^{0}l&zK~9LY@LGg<4A!+6%x#KjicWpR2|>#YbUfri5JNK)brm@ph? zxrUU`mJ4#M(&?4*&?6on+LXzTm42+mh!a(CKRKEIa zEs&^+fEbWvWy&|@^(D2{t8J@Xtk}(OOxw~q3DAp*9AsTfsZM>Ze(RTGC7r}!3H2qX zy{z)?6+XQZX6YytFAQ#M8_B7jG<#mcg0FlOfTgc;lRzO6R=BV|F=GS0y&|i&9k}O! zO&rwR!F+^4PXaPZA1m#$ik@Y6%#+IF>*^!nTsmF1tB=gv%#C@gsSmsi+CDUi3*-C& z-1C-_ehaJ1G)`1LE7veoAb4PoGL?lOiLr@3;SFd71G(Be2=55?8cPx)S?R!hMqfjN zeLZ7V&{$||K(gA!k?=-u8fWEV_~=o4G#F21p4CpQSdfy@kgPzNSU%$!Cn4`3+=i7? zbRULegfuBREeypi34eL}=tn~vlxTr4Bbf}Xk1p@%>B^refA^7pd=Ysrl!Yj3Ews>~IW?!1K)ss?Ar=$q|nUub- zsZQsGN3CSNir3rAmg7Y$y)1$BPYag*VxN`1AtnCnQUbnQwBnQV)^kfmmNllN5Mk@~ zFG`E&)uhPRBg`+m2zE~OS73!%mBH^^q++G7wQTisW=mfce=1eY3gDu$ z%gVnd`cs|Rn(oSt{@W;O!WqL*Rg9{VZChD=lbUSV;tO-O^39^vA5UBPn@#Ie{`Jo; zTKi4v$u>`3>duhf7xZ-6&I@|^wf&u8CEI5UsRdas^S&{FJVPhPI+jYF~`v- zcx(34crWY9DqQhk`c31z{e;HhqZ&Vx8fwo@YYfj?{eqs^l}SPH=Q;K3>rGpHV%e&X zsNKFQKKXh1YkV_?n@e({2PaD@|L|Higi2^oKdBvsON%<{nboq?(NPtC`9<-%r)O;8 zOY`C}>ib8fnEzSBYM&D?I8Cqle zJS*bxj>E59Y}w3I#TMr)_Wt)jWIy!7AGP;?-~rpWe?sh?ajiclZ1sPdZJpw?80ysG zP7BPFl*Qb}9HdauoAgO6bt;JY&5d{J(|iGXR2`ZcH*&;fwNuWGDq-#gG2}sxr|{K0 zY4vp`$UM*zFCng6T(no8zh&1hVsz)lTAyeM-ouy*@)vW2&ym8Vf1qFU;72qM?$JD) z(OBn&f@rqEO9l9WP*oaV$Rnhg0-c^!zup2=AbR)uOD16c*FClN6?e)X+#EmkZ`0*H z@x;1hcF{b$P+X)_+qI6?eEcxK7>i;dtxR0oWcQ8GtU5?aT zsCu-o2=8x~to-P_wKX1BpIWi`7Zz-Fs^lMyazzAU1#j@dMNk|2`PANi7?g;sXM*rg z@d80~C-g9;2D*iU0s2=3VBl4H<{4-Tp_8K~VCl#J=p^}EuUwC4Fl$HdI3yZ>&>ni| zKHI*1hYAu8f;5F(l&g_1alDA&#bvG5v=^SaZ0Am`*vcY#ie{lX^&5_7UhKmI(Lq>I zcKGlSyZf%YZE&z(_#tts9F@V?K}Pcztg7S6a?@sRNscJx?A;F@vLF1!1NOcT++zn0 z>~P*p3W;7Chpg_F29(F>25$vF)lq?I48c~Gzc;wkb3PY7?ipoTc?)OBa!zr)a7G^G z1**nKAIFZUMZF)y>oq=*FIdfr!4tWa;mK-N4D#nrPua6i+_cFnQ0{=7WCzI>YcKj7 zgqP?lpB=E_q3!nW2OhK?6XSs&Q6Z&bmk6=~t+*v1&_tn^iqQ47x)=f-MMq=tmcU&p zYH~+dkm`UXy1#GG1_y>Dd=I$mDX_dx%5c8d`lLt>4UE{Z9QXabgC1vaVAw`Sw%N$= zm<5?8sanut5h%f*H z8r%I?p~qKdUwY{!d-Tyq-I#`fj4ih(PMq-VfB@++5Rxt$0D+i4f>k62@StpjCh*~G zI#&O^@WKoB`OkmejvYH@FTeb$R@!5!DnVA`l2voX zYRwi_R&8c=$rc+$ThQb+UoP8HSxi_3z$z(-iLxxg>89eX)T@#Q8n&v*b+uZze7`1S zjYkb=UxuF>?z7&pUO#K6*sR%7t!kGSm+ex$Y()tzX_NsuPV{PNvdHv`feYWIYSnHm zNby!ZmdZt&Us)1vOSUAOSq;FOthx}Ysls(J@)b?mb7f76B?+jNvaQrAVLLCAj+neF z;=+4!spR=iEw0$os)T*^xxK=9-KL%k%`B!TtH`b*!fwQuVRSJ_g=X^g*5 zH}fs47wQu1=e=K;EyIMRJQc^5h52ON%C}anyjZq;smB^Cnvdo+|FAj=p;bZ!Z}FHT zIMb`Ws=k~R9n96OIweJQR`{0MHXtS11v?4f6w@keK9=xTK2x&&+jrZpi3zoz3J)@- zJI6n-xxI%KoGrD91d<-*Kh(6cXwA=WP3A&C&9Mf&MMo&2OkP0I{yfumC^$hNBCKK4 zGb?ue#;i@x&Di_`XD=~C5YiP#e+3DJ1qmp;ai3k7k-&&jRJVosRhwTds-I~J(U212 z%A%R6=>=O}Eo$zaQ@^cvU($b2Oy$+*nvZKm)kWjIShHHGX{+jE`ng_} z{LtczNIuAd@vrR0ztx0Ji>qZjdgl=v9v)ELLp${L42fQs)FY7@yS#$Ow9N*%1(%;-x4n?#o4S zFWMf9pawmwf<>O{1a+p`dg{VWyK&}*#?`82_vh@$uiR~YclX+r%UA5=x6ep9Xxoy; zJu5ke3xj?hV|%e}`DWf}9J*koL1jj~V`k2`>Y@Nvms@u8e8o;ZcFwY6X*>MW2kh`q z?YHc(*_rRWXe-ATtgw=?(ZL=Y9~v}P-JO2&r1Yhb2fcxPW-oR`#v#l@A|a|Z1jOqo4j?y&b)BOQWEg{#s^exK?=C~_SqS` ze*T(G938fUKe5a9e{@`QleV*uUA6hkQ&!XXZ7Y9xgl{QZRa32~DA8cEARfI6Pk0l0 zPW~j)dz-k?Rqd^AanRIZ+qQ4>?Xno;7G|N(6?F>r#zEZ|DY-Ooz^K&c0S8`-kG18pDtiF<>z_#OF(KO1s ziVJmM+ylC@QlwJS7*RPW5i|vyi51;-(QtcIbf@vKDqap>0oUPelF^5HEqyCxtIsam z)eBc`_g&lUt{>lLhd;7YV>9FJyg0e!t5LG~f|Q`FRc@B7zEHRRRG;``kEQZz`+m(K zEdO3sTULY7W4<^qMY(2IUc7Acl{p)Gc-W5oJ@>r*G98uHbVI;7@;Z4UHK59n^4aXJ@51t3OtjE8;QAce$*7DQgaD zC&mhtHCAkB$O@NyFLOj)^BTweFo&q^kufw*2P7Y`l7mt)%$3zoD=QUS5U#Uxk~!w3 zM`>(ee}(R(LO5TWn_IS}Mad7$X>zMdv$|3i4RPcU+niOn6is+dLo`O2)I>wrs$%k) z(Gm5HlswS`$AHid@Ljezht?5r(YRhNYn*7T_*O&Cx82;;m&n_slgbUMpHwP9zg2^}<$0R4;n zvRahT!vT8;6LVI}(2%6(or5CcQ9Uio*xpPJBvb;q8s*C0?=2q&DjI%ZmE~;$de)OJg zHax-(ioibydPgNojr$7==q}K6)AL$bsED4IbzN1vDL?fUeZh-9j2g?5_gJZ26~AL! zXKA(J@)u{{V$Ad-U~uXRXc$q6Ug6waR?23^BnuX@2|Z)3j`YmS5C~bZ(;}>QIz0uQ#3%U)mt0zWv8_I8<&?H9S}dv`q0_R{ z6K4poir!ca&RoMer5Iu{zBPEUC%S1hdo*|DoL?col|?V5BK8byCF&1#KVnGlfLTvJ zv=X+VyUZ+phYZAS7H8yg^(y?cDBi_9sd|ZyaquvB8FIMuWAzDo3P;S$%}Li0Z`(FD zW(N-*Q2CNYk{61MACBPZEsUzZk`1H}A$zgp`e)oWsu|@exkow*Wn&Lw=f|>SG*;s=pVLnC&%*qY&CV{UzgB#V)6M|0L`gU= zO0HeiI6ZQBmkkfmzsgr*pD~3_%NWo6v)}q3{{@p-&;kNLFD%g`BN{oQi5$PF>07pQ zVwW4ECT~sJ@z-7x_N&%sLlUGkXl5k9Y64%Lm*6#3wuK3j^(pt&ctt*SRx}eTjPTAs=I`^6wrG%uF>57%+)Om__>(Zh%)qr;$y5Ei-zSjjc zmc;uPXrSPPn8gLrT8fYE3Rcl+p7E7t zD&4<~>RVk=N)JV{^bjg+Q0esSL_d;`j$O=^^#2w$RKNww;hBb#iIe z{==8PVt?@XWA?REui3McSM1cxExUh@lx)U2?V(mdR`}xdjD73MCHu@%-?l&b<|FpR zxzl#$#uY1UAG2ZAu}?e{D67G~aO{}<$+1`Mv&WCwXN3RbCtkH@uUxQxDG7J-wv8p? zZWg8WNk=OK$=pF+?sTuq;{2*T_nkBL<87P$xA2}ovO$)ueI#auO9aae#b6pqvHQx{>&wN z>A4$r_42G;y>d(Wov~M6T-03BXJaD+Hawc~EzT=a#B#Z;&E3k|vFF9lA3bA_ef_w< z5nNtusQipgj1Oqi$FNbiYDq$gCSq*9S+=l>c4<=uNuX1$B@?2TqoA{5Dlb8tEqfRr zGKbPu87kSr@hKa-v)=}H4O@0d^i$9Ty{cL1e^dW_NkY@B61HEc+1Wq3X7it0uz^zp z)-x=jc%Q0<0AHSW!%z0AgtNzct@-(aUHX$bJM(*2?aH@rSow0r#>(3*w_k#3UDC>p zj4gcomc8YqC?Qn>>yfn8 zhbuO7_Oj&<_uKY8+pV|1C+Sn+>}TiDtzqOLEVFGC!4p0}U!d@^QjPvUcjksY^PSi2 z_17-j=~r_$b7R0H=@U2Y^7)Ef!;ZglOJUQ>U&4L6*WVJevUG8- z$6k2$oPG1rllI70PuMrVaolcQZ`jIWuk|R;{(cS#<}GI4YULi$(FObbXTK%dc-fx$ z=A4a>NOAEwRDB@%r?*%9ppvr-rz`f#b4zyqv;;3H-cNq}HGBR$7kq`xwX3T(abR3@ zgM!ZrS@9nh%t**n^SQvz=GSh@>~z+qzEQHd>7upX)3oh>Yt-rws!bm%TF*?@cD}UD`tw89`;nX! zn36TW*0B1&m(ciR+4|nqCtXI|VK!^Er>i#mA1Cc%^(7lUu+s*bV^(^-YBye*w#JU4 zjel~R^60VF!y@>57H#4?J8W=nn^m?+`8v?>gT&K+B>s3~-1^=xI{lfN<&X4Qdc^F~ z@@ZR`yJlG_3aTeRgHx2!z8YTMs4ZuT*?$@`_876)wTEnR=k1%1T(sk_UbQ>#-XR~+n3(X*Ys=zI z=Oo}~Y_Lz`ycd}+Wvh!Rd*abqJNDwNX!X2(=gH%urL%6t8y+0;)i3aKj$B|h42Fz` z@@q=bK>kIcYNE?G(@vv3ed>=)RbdrNUsK#GX=7bh&wC2CB-||1w1LBt)+Yv~z%0AM zoPH)}J&*O-_MV+q{3mV8J}lbYnUWGZYnLu*CKFu_r8HNGAIxYjdhSo2w$fLm$mjc| z$f+J0!?g=#d+~oeW$D-QQtk(=Z*Q*^RtBu_nVc2QDbD-U_kXKq{SQci+ShB>Zp~V= zoU*}`cy2yxRo<ckE4`@HSgxlL`XxXPO|BiW%gKf7#a&z`q| z!G1e@beH6DjZ@|fbr1#%wra6bXKAi(Pki&5J@xHV_MIor*z2#%+stHE<9|%N)7-Jb z#5`G?eAcc-j0yks|Toz=J2SMBn(^LFC(^LFX-q|!8PY;?kD?$|3A?3EWU+pKijiR}{_YdNb_8k*l` z?WON5Xuj&Pp;5+l+Ad!x+OtpG(D*-RmoFBT?xy115Y62b@0qtfqG?`2<}>}uTU0AT zZI!e3!qap1wJ)F4ob|fB^1^k|;8n>jy*AiC;w!3W=jZMJ^)LR|j=glv&YzjF>02v~ z^D8f2v*(|_qWS8kbo+|!**oE~3`U2Sp1o?H|1YoFX#3+|w8B#b++5=~-p3Ik`7$S@8krcwRh?oigG00x@CLImbDTMbYyLMyH&O^(udj z->l@c%sGwgGZ?NzXe(&%L5Cc5qzQ^+HR$yl#aDMy&Yyl6X_w`tBI8 zQSoOrujJ%0fo{G3j(rt+Z`?VpL&$wz`nEZ$0*^pXbS0)Vp_& z`#e+QLqd{MR##T-!bPR2uh`+kyKGE6h;ytn%z?s(v-e+oQ8L%pp0jU1{<6*89Mc$R z*s)_LB-=l$adBBZGi9TrnkV`S>a(Uz-dwQX`#--W8agZbyJ=I`o3>}ygpG`gCUW#Q zEBVB4759;^zHZ-q>=o7LteujK&Guc66nN>mOLqCvjCgL_cJJCQ*^)7MMYKL`Q`g!y zB3UlmTeeEQWM@xS?bT;nHYMJ2^sW(WwrX}xa^0)XR$V7poHKj!o2Nyi(yNXy*|jTW z=^R7i138y1{k&r7ZIhZ~pL_bUJ@XXv#5sFhGUk;Fya<&%Gc+g}F6+kq-~Qh}W3Rn> zN&J4%E}yI0>`cYJ^PT6!+h^UFnePF=5%1e;SJZxzMZfXLEB5HupY^k%TP?|gqoTJ$ z&1TiU!^66l+?x4kzxnI8{a^(Sp@E4AhBiAtW4k7H`C)m*@~VW485NqcZKGq>qmjqi zNG$8-EG2~a9w~iPmV63Wu37GlL?kr%QfCG83QtsXFez)gAIUoGs2mRv{ITX64q_XBq1=pEu+yEOp~*&W-)?ZCj06h$exStamp2x3HljeHk?cVjRc z9T~L!dw1Htz5A`aJTBfeVRzg)Zts52UVHe1M>NsyP#Kyh@+n(gUX=jK_j}ED?;R6e z_Ru}9SU>-rtNxbjz`;>FaClfe&w}zZy<}NwDm!YFUb1Z`I%A zaZb!d2^_CUAfCCUx~pFI-8U}Yz1Uv|Wp*uA{j7vxy zP+JK{l&REbGdA^_1mHr;hVL7+;d=(uwI6P)!W5dRIa!o1P>0Zs7(31X!`nO5x*=glViN;%Q){NI z*e4-VV=&v2;LTZrHO&jhc$NS>k+(*+VWnG(HgMl|8{H*^M*?gZqN>x{RLclVMxs3Z zOI&(nt8z4aoI7{jPM*5tLdou3du-pXgT_JbSFT*QImr*><70N<5C*iYEzM`_^vOj# zdE&g?d-olp5eb7*epuah<;ped>92||kmz%E^ZLBK{L&e_`<^?*t9qp{muzLJral+_ z7y9kitr_bdY}@{W!y?U=8(fOTitXAlVfWv6#Eu*}Ac1PPO-{~upZ7@k9~tejl7w;i z*YHrU%HL^wcJH<0uT0s|JNDY{eM9PB)n9+$#DUK9vj}=KcK-ZDDLvzw=O-L*whCXl za>MrDF(zR*V>1#cPrQEBp83wp_UcQQMK>4irI*gzE26E}Ub|{v{mQqb08UxEoDn}7 zvizWEgt53#x0L$2SNwQDg7M<>WsQ#>E8L|v-cKBL#B|E8ADgyDt!1OC@915FmKv3S zU8q{_qSBok4zkuK)g%21N zY7EYmt#H4_%&u(`kPCr?1?YnT6jxVIuo1Hy#-Oiu8AsTMj{(bu; zzwPyxrK`&g$-gzpx5F-^XR;~r;ELV4p|(<<_uMz($}0;i=4O{Pr<@lbVwc3I4Gi}9 zYRzj`7VX)mFWC6@ZJK{}+EF+B?6J|Ye#wNxnp-)ZaL2lzpLDP#5X8ap;FW z@0_yc#k}o2wB3g8(VV$cZ70^S@`sE5LZa~Dg0&xLSv{lns%Xqzu4yi)SbLypqYn=I z8J+0C($TFhrF3PrX!%;+jz4!=G}N>McOSHg`!%km(=rw@beHGLHhpDQy7PdI>>8Cq zp115u-g@_`pYK-PqzL;Nt(;q?Bx%mmQrf0)XmN14o}`NTHRKw$4PxMb@%fW(FumvQ zJMGBf-O>y8`i`OgoHYu|p&o__AQXaSnw!$9IO-+IMfd-05&x^UiR zR_CQ>4A}rHC8Y$XuBchAYn-8U-yt1BbXyqH{BlRX^@+AC?PX(URr6)h!Rw-#z2Z+l ztUeu9Wt(YRnUq5N+GQ)`M{M|j=wMt5ZJ)+@cEn1@mTe@x&HBbPmu6P%)C=dW@VfHe zr#a=`9lemVQZDO+A% zu+jU+Z2Jdy33=pAT`ow+J0r%DYFX(MtW>$m@WMo$vP8u_nikV%NermLB%_2?|tvM-T&Z(9XPVxwrwBLyxAi< z}d<|pjR!?FPM5V- zHC``Xx@@D{dL&7oG(hl-$~^R>jM% z*yGQfwdYWeF^3P4lF?sdz zp@QhS`p%^1yF|})LtAuqUFBaD?#;U8_6}O>dQ)+-)^k*SJuZeA*wq?M0!;lpv5Y|) zncp#)L7TkLnNmGYp13I8V6Vqtny*{ElCzyVum|LQwO9XOMzZp-&r4S?auit0cI_Ut zz550ZqB4aIlC_?pFTJaA!O-yXJLr0_^h<30)@C8)#NnSP$ z4Gg=Cedx%3J9E5hci*+&KKO(C?R_8FZ}&fN&<-BjVMC&`s1q{J?b*G}4j zQ)-ivzpTCY-ffmEs83Xf>zaSAUD8~na`zr?yHRd_CS_+&&1!2@g35?`>=rpooQrfz>Sx?c8%N9Pra(~yxsO47*m{v z>(fKSlC`k!+|GE%L>85DnvZP-d=z2lgp}F=4KyhMgzeg~&&IbMl;2M4lj|Q;hYqL` zx(-WN9#8}K>v?#X=RK+%72IW`Lwlth>t22X!>0SpmwHChC=onmMs zJ`F=$jC`!5!_dZxw}S^ocyes1CBJCkz_5lNd1BNcJqJ9pEtWI(Xh$4j80shkqa5iW zd_FjVe>`W8m4c*?MnCW*jQlX%0pN#Yz}&of(^u|s-W^}Zjb(xlaY#=&K&&(VUHGdF z?{woGBbo&%P%ewU_%u3Kj%;9sJUPhAr!kJvJ+pAjUYNXS-??(mUbuY0&Rx0UXWs4G zw%y*n`+(g$vES|xBV?7{s+f3H6WYk|sNJ#mpxrs4F{yF0SgYFf^-0^69g-3?VrfkT zmsN*HpL^EQxq|)h0}t7U4&Gt=3j@}euhdSHziD5)aoP4rV0-wEBlhr4O{n5$qhbO3hPT;>>XTFIN~3PiEzH`Br(bto`GI%; zfE`tx_Kl9%w!wk_KYRZH?Rj?H^@97K)6Jo7<$KK zs#{VET5Hw5b?W!~pM3V&XP>?I?*!YUc8!XsNn_Rouh|`q1#MiyWe|WP|5bws^R8u` zR@EoQ(a{kb9PYQnugutvT@p4Wgx_?>klqI*5NCZ-RhJlBy5{B5} zN7Hf`?D~g`HZmdMb3}rAL1RJl4_l_~fEs?2n!ZE7M?2OhEm>cY##i!%wFS{wLU|8cs+qRNWW{Emow2oxWgES3%yRF{S>HVc z>pPgY+z#>fF)75ORnt$>2{|Ep3na-3CK^Xksc_TZ5M7blK2pcm|~v& z+AIErpZV;b`wnPMW&0LtoV2lBgGwX(>91LB_;#g| zl;&cVueO{+X_9+#ijx+9>9>)wLF<#kmz2EDs-IL|^Y&oMvfHv&*ip3RfYRI|LG6I} zXP?HI=+ov>Huc;Y%QO*gQEs1V!R`1^>!dPwF+JBgpye6e(B5&3Crd|5ly!B~Z)i`K*rtFIE*v(S7t5O!8 zK5qF$(RSXn&yqO_#eX(w*_E6X&J`?qT3D{GSpNG(*Cplq)w0!IZdhrL5HqYMzc8n9 z)3R(MZ|$k1Es7pSZjc&A^k1NDk*|JBo?qu zrp->*ZTg}nC&8_E24#Wr*k*U^*jc43N+})_treUu&Yhc<(mw45qFZhkzZL%(9vQHb z`g~}(AX!f|Bswlv*iu#T58$q#7fWs+N|V1P!o0@+@MuZ2AURib%-Di8cZSuj8>I*C zOIvG5G__9(wRl2iKwKbQGcLPX`&vbKZdqZk`alZD!t?5=X|tmEVtcM;3rlq=ofDQ} z6;w{VZ=htE*}Scto43xA#sw>WpIfxTeR&&uXvlKA)L;y)D=nM*twk%2sji}p1|KKB zwqygjeydNFtt>vzMHpjuZnM^a>d-0sT*&R`om$7TQnF`0zvR4j_Y>-8jR!xVmQ}#& zthNM!M4#}qv!phaEYUAsHj%c>O=|dd^}lGH;|DajC95De$Y1GvA&+>UVu}n{Fd%;@ zL1m*To;%g>j|rn1)A!!X2dn`bn;4KxU37y)u@5>z=YZd-KFWJ}u_C2!%4PGt`zC}V z;Szo?=E(&n%rWQAh<_cK6@4}A!N>O7jR&{+_^;Mh?fA)8#OqqF^p#3ErO!zR&^)dA z@8rpgmd&@N{Edm9Znv>*1L9q*FxGr2S%ck0wOYaQ>7w;;jtvSX@!YT_jORpwvNaHKFL$o-Owm$LB)N%2dQ>*@=sxxag`NbJ4-CDANhX*Zr zFln{^mNlwPTl!CzEI;07xe4*OnTE}NMYJM5*j!l?KFfY9@{U`^eEY{4=L5TUI+Kg@iM0tJkWQ`hF?DMJea%axPfNurh0BpA~kM#6Q~B0Czx$zuw)n zG_4jw=oU2p6KXaUy?Ob{(tfpNqWD|j+VjQ1S9v1adj-%XM`Ml}cY3*XFZlB<>E z%;^ih#hy9n;Gyjr$C{@lROLuJO&y%&eB%wxeHq zG6?kgeS=yVIHijzFCz$q`9=4j&nbX*)%63W4 z8a6w3x@^ZJV>Fvpd+_nycH^Nz(Q{Gsyk;j(9T6S(Y3?pbPtHiTT2VXB+uBl}-E;ST zJ9zV`P3$OH|6tC(@$6|E9+OnLwxw7?N^6-P!KIegA(an~;K zI?=bwh}1@L;DM|X9g*%ZFfeGZzBuPwKX1Nc*!D>7V_qw%4}E^69h$%Ra6gQ$AzA3c z`8nTa`tTz+_}zKrjc$<8oWi-ogF^!v4+GNSBy*|1mX??7{zt}b{{iU*;*ACIJFHJ~yDVQ-Y8BXHPBKYI#XA<{mq6a8PqV z$v?~-KXF`hfaqD{5aZhFTFuU%owgUgenI1M%z60@Hw{VG7*d`kjjN1whNfk*Z7US> zuK&cd*#JR+qq+}?c6(L z>1@^J=I4At1gpQhXulGoa_)kFD2BsuId4M$s|-yJ{X=&9ZFkz8x4p*>?0r-U`XjdY zhR5X|vl|XPZU=9ChwYKyj@=L1e);d&{jlv*JmTKC=TSSf{|PZE{XX~(J8;7jQk;~> z{>N?i4ezquH@@2r-0~i~;g)ym{SmwIwuj^%kYc*aT1>!9@Txg3FyQDQX@xUq^R8z! zknx1LFd{^Q0}f)t$i)gg!f??z(QF0-W%MH-p3x9Np2Xn^#JD`kkGy+tp(Q;L%zL+J0RlVI$VeEB|sIziQJ7vUujJ) z(xEMlRZqY98YMNvj+Phg&(0mT-+TS*_6NsbvTv?UivbS%fuHZc{}KECo9?zp$M;KI z*H~6Mmi^~5MNLX+Tf8u3mrkCt#Y@s!q>Sa$c{?{ZYjgFQRh9o-O@hkQq}|lF-415@ z?Pdwe4-RazckH;qMkGb&#b_ilNEvEN38~rP^G7T>l(xO&gLX(f;CAtV_wB#M?ktVj zL@Hw?4auU?R9kh$Z%VF^!Y;)KLGc~8-f6cCj+w*;D<;!+V`X{y zKwj(oH!xm*a+_7FEun?IrfCT#xt6U|&P#bJOSzXIhEW8inXHtIZ2AWacJQVh5{j2> z`qHW<$%dab!$%3u`$3>Zf#M7nmh^L*X-aN%Y`}K!k&>?Fj%tqHwu9B)QXXeG$4m56 zGfkYFw^37_I5%k0PMkb#i%XL}uT@uzcHvCfIM_Lt?Ux|1%btAdPWz!Bd#AnY`|hyE z-*byS@!mV^$@kx9Pk!J5yZ0S;+Ab-yg`up_(6B~M<-AjE{)l?;J_!ytCT#GjVJqIK zUK~~fB^+~B8C1f*q2YeZ-;}o6on@=-u2|=gdSGwU&$OtYRV9{$jAfDH>6(?7gu`7; zE55sA?fZnY+k~l`G*8?j+}$di4@&7x)zo)&TN`RxlaHU3+bKhm;ei#V)A6^bY60`MjfFY@0!pXYL8d zYcLlv{|V1ivzlvCwtLrJDZaxtvb|$B+^XMu2d%G&VX;qf5js=q*Ch#<%Tf^b+Q4W+ zeb=#F`y}M<(HQKjie}Ycbx9oR!&Xc4Wj-fivfy8i5pYss0uvJn$sT%7)s3@#>V5`V z(&G%Mf4FVPpQSuh331!7LzstZ0wB*fi(XqP?u;NA9~cin04d(U0= zzIWfD_uK5<@4nT3^e?{K-u0flZJ!jGwEC+-IeF!?zhM3M4p{Ndk|lOY&iVm8pHhGA zQXMsit5RW%Ng*oKT2^WntaY+&)BjbP*;mz%3t6j)CYLJ9J_ehuHERvkEc;;A#y&D` zdrwSQ`}f%A;m2>b+wRDhih)rFQn&_?=Sz)Tzr~ zk@Eg2v+b+W6F{G9mRA8L$A0Zt`bqK%p*Rxm^h z{|WIe;gFjn^-D^7!P?I!Y{!{hwyST<`uHfK@kPG$Q%Afj$7Oa8Y`5V@1}vqvChwBm z{FEAYr{d*Rws75HOw-389HXDefYn7(i;U~4d@CRXp&-8{RPi&!>!ge`pV$I6t2wtR z9F2}=eT60S4qp$s`4gUp(SxnC)v|tzXYg$+Ba|_(LCcT?z`1A~`LJ{6F6lXaHZq>D z@$E?|Woau7vaRfd=H6B546FnW^>A`ZE=!qTv_gNyjY7_&z>*qPkm7vkmP7WwCmys9 zz3XxNzIQ!f?|brAd+(FC*n2ELkMmj0-0YcJGPH!0yewpnXIJXCWu zv;%GveM#GK`yT65nzr!toYnuVZo8Hztn|d7rS6c9HQcfaADNO3D|3lPQ3#wdtAD0x zOTV{b?W)>7fI$Y@5p8JrLqExy#$95}2CGBXda+^WK7QURuZyXOE`2WHk{j`D^^S4> zOKO{Ndb{}l{bQEW+|azgX^js{9(hE0BEvZF+{y)r`W1$KXb3=+azldNIq!tva_fMmyqP!ga`7d^=UJQc?VBRCD!EOY=%c zvJS@ige}Z-ju!Kp$`rq4m3gzXsQD*nHyq0QyqPU^Z0GJi+aVs%*H7Ny218=ik{RY} z=Ej=yBJK=8cji5lMW?M>0mCPPbxQL*AM4m|pBoS_%V^4h;_4y|&P!9K8Xt=bOSZJM zU^^v??!6&pBNJ^K-ywS5le590q)lC#v!#W)pC!gw_{GAYWGl|MPfPwW8{XEju?fu| z*$JueL-d^G`Zbr|IBriobkN@aUR5}=FRv0g^j=axOT zP4cy`uFa~Q;&1gQRcGeOw=q3;>XIfTKOhgc4~REBr4Ic-(OQz{QtuRw9xPf>e8%hN z1`Z}G(TMY>P%Qv{#_mQ8dwyAhu!k>G7zVg)+xx_0Sg-`YD%-Bz3DHW{8qGETSina} zR$lV~i51|S3F~w&-lFV$OmViGUtpr&k%vL}*j3<}oaxq7d$>Rk8xbq}qU{6T=Vk}h zrvTs9yPq?ziLoWe(KMzxfJ;@9eg=&+IL3`=F2huw=d5GOlincNnR!#SRU8C~w&e;q z&RMQ%{=FpKqAndO@O%Ei2V>6lngsH(9Q<1O(@uDJTQr}}bkz48CV);DbP){Bdk%=6 z$EBn7Ybq(Kp6QCsOfOiC%k? zc&)DiZun7V2qi6bFa{tFR0cg5iC8YXePY5_r?Fy#_gy=8`U*B2+dfA|M*ZtBE7L|L zB#!EdHFtd5Hvc-mea8+tJ@vkA+qedK!B|j zF2Lpd5iG$xdjZDW>@m90PFA(?BrFmkb;b||U{J#l7gyH-gt2NVuGAx)<4i!gF~3+2 ztL=iq6Z}aNR|G~wBL19Thw%@51BCM))9{Spyq`LC%2(e#@x&9C)q}9DfH>FTI(#=^ zo#MJAwwclhkM50tIHV7*C&R9TjCtjvfn!sr?a0x?wkUyn=fn>CzNbE5Kk(Fh?5=}1 z*+5^vA2Y#4p1ul2|H@Jf&n-;ZH(q6Cx`|LB%Si4-djPRiONQ|CIFKfGZ?X-g4kH7emefA4qus{CFm+k4- zU$qN!v$oWzNE{3Ng$eA?w%xXKXw2p>P1%>e_$B+3&wkOq^3wA*t+q6367+q-1KVA= zjE7v_n;7Nljb&dk#f?n4T;Q3tdR4-M+TRQxQV{CMC){#|L9bv35AAiM9Orf*XrV-% zJ~gRy%eLoW)5dpdN>*JO5{AdOaWPN7`bhY91xkXG=Jy=RUyB>5R$Y-^TQxg*^0Xa3 z@|wNy;_LSDPkh7vN`7 zcP>KVuzb$P;}QUx9E5xEWC>cZT2RuGP{6A6VzyuUA_lt<2Rd*p@f3boy(W>|8c7KR z2<*dQB{v67b0E7FP4BD586yo%@+JW+q4FVKjBlDOB;=v4ARg4Ml+D>_e#loIApErx zqQ#2vje?94B!3QjM%be->+On!9@SlOxjB?Wra8HSt#y9J9pH`f7q2@GDH1`0x~3&a zaDGXd!+M2hlmTwl;!Kc&@K#xs$ghCDQpr~rGBsknKp^C6Yr`)rQ9o+#<-91))&)42 zpX~-Ub!IZv61`}=qQnWoG@ElxP=pxwnkpUdzDnKGOW0EQ{6fQy9yw#rJpHpkH*3ASw^v`lhlc4sNsTuoP7eys9OQZE~qZ#`>fX=B4DPO9|^A6n^?U zHu?Ir<)jebc;`)Kdn6P}7)?M63g(P&=VjtitN<)*FIt|}ZPJ6<7$ZbGQW=!D`cL@s zo3CrC7z?wcAdKxBm9m|*vc?|tmMx0u(k%*aB&6(AxmZE6q;NR6Te8!3h)#Gyug|ppf-cVkwK*kW4mBN|HcC4)ScXpWVJha;q)s*Io1>1i2m~GoJ;qtH_Jdd$7 zmvQ4#iiuZ0Gg3;2_Y7JxpA=n))>My-`Z}>D>~#igASb#}TrPcRRWvugT(LqXZ~Y^E z8aE;W{{W(~30{O}w2-j7h7aI;SZY|Vq&9drDS!Wy94utRr((bNyletwa4ETavo!JH`qVD+4ah|jZ0@KQ)xR_#Su!yx0AjA(x1rfQT| ziTzS`x#XZCooKtzlkqu%j{$7gXIrk{UhOL>Mb8>zR1K z5E4>;n~gQ$B`g>y_4frFX-s4^uCsE;8%>Q0coMP(H-cu91@UL{R9^^JovJWb<`a|f zs`8s+P?f4W9F+B`3q~`R9uVEAAC~61P;0>&1=SS=9h@^(Z!_C<%T8;}w`}J0tljXA zy|(L+c#|fVHU|QG<;h!Rsl3QHM)r+ZN_?ZTToGPbO)7$fE*0M716i$XE!8C{1%250 zhWb>vPjPOka1XC>ea9zS^?hD#%E^!M)}loz%vUO%ItZM(MBqKT0PuMDzW<4a{M<<) zg(AYu(CCT6owJIdPcooj-D%gaJWAovcm1J z;D!9u>#SIo%-ZUdU0&lTjxY}@Keao|gV0Ug>t7IFi-ol4Ib8JQ^qkJA&YD9+yU_D? z>Fu2F&Se2)!*)mN3*AvK4CU}}#sGYfIAx8EDt#<@qNM!$`!q93ZWOmuI4h!&u?uQb zQT}SPLb&0YK8HUS{5&#^Gx6cKEjxJTsa@`9KNXLMKv1 z#KYvLHX>uwT+(2x>F;NIsyK>hiMa~91-cC%Hq;#D*|z1CMe)tL^EWOl?U(aQOr4E! zr$UggFxs)AjSm#bWX|U2Lz%P`{>}$NE}H;zb=BW522wajnG9&2A-9^!VayfOcH~RS zK#suBi1-BVxyXgfV|*+s40b_0HOZgenKV+s_;I;Tw99yK-k}Qdz9u;x9F6bj*f=Y` zg+uC9t7-lc|AxmiA2N0sPaN&jKaf|w@-{O|J5>(F$ORdJb_Nd&_UJ&&U9+<@wz9J9 z_;tgb_z?ZYP3Xd}YRwI1IA%WQm@qCF6geWKD>J#A2C0%sL6CsoQ6>@?ARe~S4)m3L z-}-UnpZ=hi31nJ<)mQVdH2Bi6^NlHqOEH#}wooRuf#u0T2|!7PW|F)?goqa_K*HkgRsK`fVin=gh=o%kO`tOYpWG&TLI~| z3dfHyCr0JZ@94KVek>E{4aYyO)}t=?$NKWl;Kar7^XJdoi4!NBptuBs?Y6NFy=831 z!F9L}-wj|=ZHRZk0>cm#L-NTgFeb#1x&nA2ZooYK2aZH?_yti!%u^{#a@Z}_*%T~5 z0+RTzkM=%s-Vg1$-Tv}T58BOzQ9JzdEB0Hz{cr8JKK)6*aN&f6g0o6Lsm!OA7px}c z`t@_K+wXk+Q}*1<3Hy~K z&D&?s9kGA>Cm*wqT{>;&`3flhi3+esZ-4jwAF`kO@b}vX58Y@tW|Q{p#bfrrf9yZl zCtmovtx9m3uSrPouYrMY@@353dT;U$50j9W8xu}Snjl#ngFmipxje=l^|6HRijWJ7 zN=P`=wC59lM*RKu+aI@!7pivt)SN9%rR?yFS=)D8RdWdCsBD+2@>X6gS-m=}I%zVN zm!=toVFW;-<900xGE`B0Unpkm&b#;7U;3+$+W+lm-fe&PAAH#UumAV=+E4%d3eLOw z`mgpo|DQjyuRnc4b!%IuIbhj#pHH+X_1s{Z&!9yD%uZ6g)X0*pB@+8RIMmF zeAm10wx9f|5803Z)%)zwP1`-cvc{aqM0pCQRaWgaeBNZ+8t3UT%JXUi!cr0=gCiuY zW`Mr?`oq=*Z?-CsV#N$ATO`1zYGzYs*6g#NK5YNzzy5vu^?&+t`{%#<$M!G(#b@mQ z_3M9TzxI#*MDAnupa0#n_T|rCuuEqvZV2I60HHwb!mPfkdag;yS+l&9{H%X;pbo-S zTeaCXYR$fmNMzPh4<&5pKODBikBE2Nr14eANRh}|Rk&|;M2jsA4n#RA80EE9D<>9h zjoXlqty+6lJS=_5nkfy&lKOH)(fboN-z-~wx0JEJGHzo(y~o=3v~8s_FS)Iwaj1Gw zSM^CP)n_dR^sOSfzm~S`f4|>S_et4G&s$yNx-q2k>0Y@t)vG4I^w64>-kY?2|J$f# zJ}l*Fx3bHszth7``nlqu4dh3x)DZ2%fZuC2@-xHM|060(!tz?GZi(?#p>7+km=Zk?9(XiKhQ+-Al1 zu>rEv!&fadR&8Z%MvAYPyE?6?`ZgO`TU+IZQ6Z6W?~0L>G3chLdgjF=x^-Y{-{@%0 ze*7oyws(JIuibL@b_w`d``UBguz&k+K555ZpVAmh+0sJWp8Lv_{pN3cT6FYL`=|fp z5A0We`H$_N{px4zSN_qb>=*z3AKJh8)jzXO{J}9hb6j$w^BTCZl=VbaY^35V!&zMF z$i?_3`Yh|eOs-@((K{;}oP~cxbKK7*Y~Y84i?OVg+rwf%H9wmU9HtX>>mQUlt)J)5 z&)MSKs^z$4_N+KbQ-eu!Ln|Y`TWs5EX3f?ztLjtD3wNqd|L2U2{rs?{4kc|-UP~Mc zAbx9{PpB*~P!^<7mns9ca<*(=z4)hAzUPu0U1^U=mB5R(`Yksd+#5rp44HRn0LTW2K10!r$ z;R&MUGzYL5%f85PkEJ zL*s@OgAx>M;gTe++KLUdH0Y12%_?g(wdUmwwxq+=ik4~YutZBj+$~}ae^Ybe|JHAb zyM#ODLAhd{!KWn3qE=e5)%=RBO*Ae2;k@1W3lo-qzv{cQs;ND<8E6rY3? zTYd%Ts--h4!APdrHp7y1CPya;{-a^N&XMA{uXEFkfPCTo(ddWFnUs8$NUz8(c$-69 z<_qO1THyOnNjfcOzBd$96^r0jb?2GN240iZAf*OP@s~zpO?|%RMiS&9KHm6c5B?!= zPVu1?)j(GjPUWoE4;-MFh%h{>Y6RB^44;;amSZ}H?SoMS){U85K^!!U-5Iz5q==o#z@sEAk&YaMk$vJF7SXymKr|AReStI#vO3(C1E}0iD zH8-ddSW~ik7BI-T&=$b;c7DqByn3|*=R#(ejxV0chb#ZcBY${Oj;--(&uV#2@<`nV z220{AV*87#S6c0pPU4HO+I7*!pyVv&fkKANg^ZwAR=w96W#ub8sP3yP!I0+{is+Bq zVY!8y4@}&STd9;qn{BsSv|6n8SAohWp)r`styo%uX_8$FqS;`iQohQERqCdBx+B-9 zBy4D)sQQ(yT)X5q+h({aSPgT-8@RWk;~|eN14L-XN1tfff@L$JH_biVGV7qEZS|q4 zPtuyl@a>?72v^SUThb9U2SGoIsydWKpG(pK_&CO51F}%xDIvynJ}hxrlu#?Ws9Cu( zBY)wA3aKec@zJd0#6o^Ryq_~qS?nMh)tKY9YVW7yyy)_jWlN%WmI$j3w5=oDVBF(l z4lCMybz)oNGTY}kDD|;$Agl4N@mCv={7rk{{|I0^y)ZX*n_einN+c#f+$v*b! zPuqX}^k?kT-}r`)9aaZSFU;9edBqOie9-RLv%_xMJ>ipkMH35pa$kR+^}(xD3sySi zi+MYD?yS|zQl?d2O64tU(t>xvuUMH?5QBxA_Yd^j;^Lw$FD>~bQ!I9Du2!+tT1^vU zO-ePF-y@J|ve88Dt2>mQx)swB)aB3D_XJc3XPiivs)d*tk2m`(%D_O9+Z#-b{xZ_TH=;k|Z?~OOu%<_^gRjXESp-}MU zJz>Ktvrj_f_{gx`dGk&7{wJTXN8j7Y+#6UU@#P! zeeQc+l*gob;*8JkM4UH6S+t{!TDZeaV zStazz;8Lk~Xc=Tr?2 za3-nRsrpB>@VqLH@D@R1oQ0oP*0}XhmDa?+xUeW?O#%eM2l98>Y^ZP82Kop6{5!h{ zzwhR+KC_zU-MWMeH}guIuIM+$&~7;h=B%3Ga9!|PklDW0@!J3qYB^0!DiJ2qvBmnv42FxZyhhyk6g z$-|=~cJBN+TbnQY8IS4Vv;^v!Z|}4|^?d=QA@KO^`?pzgDQ#;~^lB1nvx=LTN!arA zqL2Hbk&!^pBCUy?+l9xvojG#Kl8X{b$McpO%xD}aZ_!73yT)-_yitk;2Zoo}F0#N! z5`_(e1)eFC?x!$LI&vKGLY*g2sm~=jB#3h$2&Zc zEs}Tu=wV%f+s3?YXiw=i4@Pm-y3z12%c|yq$Rcn60S~ z>r$vHiK?$Q9lv4JSJpLGHIJ3l?okneaJV$bN&?|)xZgUO12Ay0vbG`Rj)T%KoV#cP zQnEQXo-O|%7h?fipIEg+yS$G<`g+IWB5dXNM$o03Bp(zKHaM8HLpKlFlkdLC9(dq( z@8jvICGqJs@gv1l*2Dv!?FN0CbR{;GW$PE==rvLhRd3i@sjuG`GfdA+OEF&aagDKe zR^#>TnQ-nfThCA|({3;d^T6tA+2-fxZB@LJRaL(G2N_?yAe+hiHrz)ZzSlnTk$2h; z{m_&4;SW7z?|JvV_5(lgi2dLXLMxBi{rBH2-2jC*NK=fJX0v9Emg<2{tTtd^X<^}T zLCvG7z9lQAinerQUh+uIGGkdECvEk8Lo_%_*#j$ah|3(;(0l?-kelnj zntz}tj3Uf8UK@F+c+rsbVPp(`tJ}dHtJvWulan=DSY$rb{0LuGTUlKPuIUa|&c>Bq z%(Y`X-;!cGWHTUe>H(`%G`k3hX z-S#6t{BHY!zwi$G;0GVDANs+^><53~N&CS2pAbFYZWG%uN{0TVP8_x1214I@@;0Ss z;SrhuypMbS-YjgyR4Q;PIi^xs(|l8LT4D8OwZ5vpW5uszfIJsUh!!;cwqPqli_^K+Pklw_ktcH?Ln)OAJZN=wO>`h$m(RG|1s%K1hImTD9NLha z$C-{z$tkS3SCJkjs#gBUi7DZ+X%#VlRRR#K!JJC&D-3v3vxf@42mZ zKzbT=hxTEZbQ>U1%nj(=uFFTp26K0cBX>GJ$1r|4atFLHd>Lc3)6WmocyEdx-6*Q~ zd{Fme)zY*_c+6$}!WG&*cPU|W>Stduh>=e?VC5|NlO~anPFL)AS-egqG=Z{&rB)V> z-1rP%Lf_-VVMXOZv*01<2+;P#MApVM7jkCbiuABcm!_1r@?~MF?_)Tp@8MJDO*@ND<$)s1jbnjjx{@Xn!}%$T~M6ZF{BAo{Vl;}+m5hp^UTR5 zJ9VsLE3+wEnn~K+WX5JDi?%phlmMBqfq|0kzhT(!ymy=3|Il80=&^lv&x5=CLX3MK z+-na!dZRUfnE(L)^hrcPRKV`IYmf3P`4#UH`&6GurMQhYUjf1;0?BH%Y4xMx>CZ?K zcv)>fY*si?6b%={-A$S8Sl(+Ba}$=kDF12E*=*V}%jt0IqLf12ECe(Tw%@&P(C&R;zuo`vUeVBQxt(_J!#nN%M|a!9Pwe*_|90-> zkZ}ocoHgXqeKROG5^lXbm20((l-HK6r7((^RmO$)`!bg5ld>-wu&*fo^Xh|VQZ|2T z$tDM8EPsa-C-qB4xJaE-oG(ft`C83Z9$B@*4;F3kzH#d``fc*XRcjs<{Y>X2oQsln z=4|M$aZ8<7LC-7SZwOP*sgX)QnK^6KgH>x!U__CEBBiwSV9F*3Px|e$sc#66hZXlZ zDN3h@YzTt|hs2KwKhm#9?5y&rh}eJVxk(<6%h8aR_Wn9vx5EYI)Ym z%bcZYwz^=pvXGRpp0|3n;45(0UdJUo7cWkW)m^nrs^TK=bW8I_t7Ms0 zF`UVitXV^2NMvP+Ov3VseqpN53gdl#i2tS48S6Zw2E&i!5-%z0RLXMq_gQm~lnZ6s zI;*yPO?~qE8Ox65Y~)=dmi$P{W^b9Z`lm$`PirJyR2r3?*I((rqMgZ}x5>j|KwlG@ zUX%M;)3(jzZRf764c@8p`pe!%l+A?3X!A(bDo;<@;B7;eIVeSXpdGeNX6hPm(5M`% zeWdKEvqSg2MSwCjcPXX$AZu$&8b2C)V?&~2ISw^%G%HrAu_9J|#+jLVLMm*R9~~LB zm8Fj65z&X5yQ)4&>zoK>Pa!?c6!Z0jErJ4=d+{JH@S6({}T%`$a$EJAO{( zikemT`DI{Gin02svRt%gMPsS04wPI1kH9#xW9Oh9x^>L%xEKB++IV=o-Sf~kcMm+a z)90@J2if6-fkS;H+JLVl6C6;V5#B`??KvsTtg?c?iS8&y?aJUiFfdZrX0CG5SPfM)OPaXvdDfe8CbE12*{7gk?TZu;shzR{q0POFpgf zen!a9m_P|GJy^4;p;v6-{5ea#sPT7P9Uz)ZU()y;>o*gxux;|wJlKCv(Y6)GZTLm~ z_@d%|U2&c%*zEZwJ3l^Ui3h~LvM94*TQGAP>uWSK={GCFR6mgm?#+@5V>{}jJYB!l zPvn@Oh_6YnWyKM)EPa9_zAz}FU~3@f*i|c<*Df_}dAX@MuwqM#D^@PEg}Q7_%@@kq zjb?lVrkq$Ni!@nz+iK)Ahl*$Ly;Egl3>+UHvuuW4S2L1l7Hn}2LrT`FWyzZ@@kHTac4kHLYDT{`e`1)cbB34CYSi3u#m%K_k`)H}hcypoRKAkB zp+Vz2k<;8NzAsslOI*&KKWpdDF4@vT(-s%%lEGH}gAo@ja#;r-KghpUMSoS#NM~hL zY)bqmfef0|oSjl1X)?wLHnCH@?sjg&-D7v%zsv5tf81`nXVmU~aEIubd33K|5|uA8 z*J>VA0fC;mEJV+@Gd(j8D#=?3%vYR&#%-*uj$`FG=XmF{gDxLo)Pn~}bQUd$P9*zO z%aXN44-?z-@~lcG;&_%|@V#(GylGzYhUm5_T>@$(ZTvvS$5s{ztuAQ}6t62UOKz%( zz8Z=t-qIHDXO~2TbYfVvE525zFO-e?f_mYg{iNo4Xb#(3t*rRTA?q)0cY1@~frR>1 zy{ECj{0uF3cOW*{l_9=X=L07nKA>NXdyO3+pAXw(*;2l`B>EOT%+E?c5D%W6lf2?9 zPr-|v>VomzWkFTaWj>g@z|TIWj`}T<6YUnIlkBs>ff0?lRp}gywk&-Q+2GVs>4)c~ zbF3nts!qx?e3VIBF}F=J{w}}Nit`VVA49p_vR%d$FH=3!IPn#ZQs6k4ws=0WC>Ox+ zv6&ADsh|DXpZ!Iz;U<77MI#aK2&z`koQtAeTzgxy9?7*Oe9o45MMq6Dh+q@>qvr^Q~ zEH2pVi&J)LX3|D;1-otgZri?XhZWkwlIne?T(;L{XKZnL#(wP1yDTGtD}1>}_!RS3 z`K;itr7~6)Q|Ek++18q!Eic=dxj8#|>5`qEpAp;7+rUu2Z|7O7RPD^vw0(N^yq%D+ zIJvm!XYnn}%-diyW%nMs)ply~$q84SMfJJa3wC7slAT>%vQtY-_VVNf)%TLER;zY! zXq(+WzSD*WO5SU9)8!Cog$PVMFr49zk&J+*A7 zk7FpR+OFMFYHr!)U*1^llgda5mk_)tp=@!!Wy^Cdo1Sdg%oG#5D5qVrfe{JPS>Za{ zkRYjg71Dm80jv4=NGW+-0;yYtSl%vNcB3-b)QD98=i~5+*N7X*|7VwNqtlU1(c&O@ijl z5`;uV)pK=QKDuH{XI3ovC8;ehYVvR8CAfBMbrwZI^D;_|+GHj55e6=Wbu{>>J%0de z2R8b5<89-er_v}rALRgyWnu^N6E-Att*%!5*5GQHuZQrMIlFMSDjfIu?Z4ai2xqDW z^iiuXYb;$9eQ&di=c|e*1z;L^BWZU(Fk;)rM>IwjTAnxCCGHC53wX=l$)s`@S4wQIy~kP-mj znweU(ix*}k)Xm%J6ALboM6<~gj z8$~bL@K{=k%XUA*kDG%z2%LkDS>c%xlgbr4evVAvfD}Bvr=)cHawg@6a2*VT@L6a_ zPGi+1w3d@~t7hw#OXjT14R>QLE1u2S;PV>qvsqg?GiUV|R&BYmYMtAYw&SsJYfD+k z2+^HqC3yd7-HIzY>-4WZ2`MqRXrR5_pqR9&nDob&WmHZGiX$qCY$= zqzOu|1a*=3rAvGiSd_xe$~h@DYYp+H74e!C7o=~yeXk9ol&QjL4S+_iZs#vdIp0}Z zQQM?gUp!y83+I=_lSXZ~=q|4@$Qe)MefoG?^t37(T(h}ZF4e(!Q?VNl?zEB7oEvZu z;FFoEbGiDs7v|9fGNJhK@*yP+>j~dL9uN$Rn=H$MR~#v*`OsM zy2>RAmZ?c@Ty1JzShCEyjy0wm);^*rC#59hrA+({<*+Mj87chD*XlO)%#_WoE!)n! zci6xK{nj|xu<~HV`u}ju`j(598Bw|8Da$rAhZPcbR`W`GMtyTWYip;Z^uAEHfll5A z9~NJIq-yO<(~=lEqeOJ+T?;BA671T#H%~@$TG^142>M1)fdhUql zE{VS{D@|5P7l)W*SjM>1sHHTgiQiX?ic_{r;v;jj4a;Q<_VDBTZE#4ladJ&Oc}@L< zew4N4<+iO!5kyJmytAbRrB_8!=<=LzmeY96wEWDdEC;z4gG*#IKE!i!TzsM~=Zvuq zWzEW1PFyC!+(WWz3gZOSIkEZg?SciF(hL)N@O^faEZzCTRcz~X>q21QTX)L&AX zt*~krR$j4&=Hk?a0b4y=w)An0ze>{5_w`%)$&wXDH7{$3H^gg3P7T?_Hz%w;)wY%M zE8-h1TUcGO+M$;C#)KudH#7z`CyO396Fn?tHmN0W;ww+xex;fL{FaKJ;*Nl z#r!CoaSJkZhw_emacOeimX_!2{s(WAypmTqJX`Xp_|))c#8+v@343)bH^V!L*XYEH>%-V}aB z7wldrmodD^EsAeQem!$$QvMzDW2^=w8+hIM2&K9R&&XeMb1Q1~nxCUpDU0CS7#D)^ zXGU}E#q)Dv9z4Emljqs`+MNg2m^Db>P<1^Jf<%17_@|L*u@6HId9%OJnBcZ%$dUFkm}&57_87$wVA0kSc48iEbsc zC0WTT8eqdfPP{KCo{~yqq~c?s_?(Xu=vjPUW6|Eu^eo;8jl5NW%!OQy@wZt|+og+= z-A|vh$KG+9WU{;+eSK20?uy-i|3Tr%?B!R^YCg%@?%m^>pVZ!B+D@D}=khr6b9t>| z7fu)L+?gfewIP02vb}qENWNyN1mi)K*SykdW+W#qYo5vb*}dEji$5PMG0GtiQyw3w zrq7E8I|J&kHI;M87MC@4C5u8M&}OyL60cp>xIhLI4u!Y1m4tX=%8#X(pBGORf8`u& zc1`#>i0a4LshUk*)Vy3-v0LwuE~SZ(k8dcB3|&Z}R&6@XV^rphxmshv%UK@cT2Wxsg0~!Tv<%o z%5vH3P%cX*5dEJzTG5B{>@ogDQ>ju30JC%0f=%tY~gvb$!(;Ebo`x$Erx^ zncIA)E@<94yX^gW?8Iq}=Rvo>t*mlbQ`>e*hVUJ#->LBqMVv5z_h`JM1EaFfGoINN zIyQ7LfKh<6j(EoMuEUiOjVJU4IOaEoI*fPV7$eGc!`*fGR^Yo~yu*+NtKzS+wsx8E z&eL{{cj5{jpJbaUmOP?4h_K%9_!|df#3fXhgcvnvg1HhPHWIefXkOKi(g0V>PuMMc z_E=s*Ykx6g`H69>s_rkp@J&1a(os8c{Io69E0!B5+4hl9JCN?TgS&Uw_Wq)c6s07O z4cjZ{PW$CB(`PSORl*WwvhW#0gte zeUcI`AGrBeJG6VZjjCM*;fmcqbDH`oUl@V#>PiS zH7RpW1cI+r=w5G5U~GMh=k`HcKoIUlcZ$dPMu8Fm@DOnmXnY+iO84hxd4!Phv@JB3?6v1! zv$}+(fg46`>^<9T_}(FF)mnC`e$fUpLpGY-ZvCZk&F?uof8?}H9l2qLo5*IK-U3 zQV&VspuL-G+}p>%#&0F_zVQB4DntOiT7UnbXjKZy{E`%sOTzJ-6qjY;T|%#fkiGkd zZFmABRZDV&l=%LB;WBA2y};^8(Xs@ZVxiw|zkR>megBA+`ce}1P?&1AxU^#T-oMAH z)wc3payr?6V2_WX^XC`*Hb?|c4hBAWV7ug%vc2-^F?;2eqc%IY!RR`kV+O$4#|#UE6f+HSzflYM_#ZsDJ9$P8Mpl(Ib@xK32Us1 zK3PTa(t_1blx^FsJ8kg$My;6?!lVFYi_(ae({{Rc%m!{OT5%+AscgaugIOD`4_IYc z0;ZI%iiA>bDBS+&xb;6eVvUiOl^bR8gQn%S_gStkMeah=F1>qJI48xnY9lue+3rVoTVg*(dCoMGd{{_ZN_;F= z);KvmYfBfTkS%7cU*ov{$%5sckc_GNhxrDBr_xAhW@d4Aguo0c`Rd_7#7MuJpX9)H zwGgK}b@;g85(sWa<}wY=qI%(l!@jB%rTc~(w@FDIk#foTc8a5Vqi8IzwCw1yV>UIt zAbRddNmlOds@-_gxaPtE(J4x5Qgq$2=f8f^FVmTvoED8P+FE7R_K9{69U8UKQ4Y8k zEvP;mZGb{bt)LZX0KUW6V;r(=&9~Qe_#C4I-8MjiHTqk>GfK~q3h>$0)fJc1Q>@O^ zvsfufSwE#YV#ZoaN$VROw9zNTC?1wlCfOjv*66cqHhpf&_TO})4c^^n$x#f485`X` zX7-%g&+3}dhBYTRgGzjSB5B);6PA)PcH#I1UooDOqEfnT)N=P}d~B;)Rmxnd6t*7Q z8MFH99IC!-d%y1%%k3ywyU_OSZ?w}tYQ?EW?F}#8hQ0;B2dDX&x?Q?-(GDLzV_*Nq zaq*5*ZY(%(!#=z1j=i>HSHI)gx6VT-j9c)UF1j+gxU}rY0L;uRSSr)6dCP=9%`F;J zoLy9_5AKU+eIw~(4*)wjlvP@qvw zuz0k(q^Nm$*aq$5%cra+e#%zD!p=S$eQ4P7kEmBLwk(@XAD^}7zWRdQcIZwUd9=^Z zGE8YM9Tl$~ToR$os4U?kd7Cmt!QPp-d_8GvlN~$x+Jap?J}n#-ZS2s94L&en*&7(V zsz(xhq%FIUwbZ4QH7_;o%L6T^~ca{lpwuh|?sG1y;l z!~KbqqU~c}w3-b_i+bcv zsEAVFf4|R7quVl)k^0r%VdXDeyFP+!EE-JbZF*|erl;ra*oots4<{{`AFv&J zO1?_ZFX{6BG20_qXl8cJp8w|a_O)l9QyDX&@0RDw8GKWdi}viZFWb{!Jg#~0l7G-z zSXk1SnYV9#^JVe9ONyVh?K{*zh0xc`u~Ka`);bE)@V}Wb$pZ?EH#kEI;GflRlNXom z#g|^O3zMhZXxS${U}R{gO>7_1SdnZc0>L1^w#o%VlI!+KACVr(805y}x%m~1D-7s^ zRw#*CLc)h6kWJ4XE2lzfpDrw!$k^m-QZqGi?Z?73O{ z%2yBD(Ic}OgG-jpw{2)>)E61N{`z72>X(n&m!CdqFKg~;OKE5}YU;c5!s8{MFUa37 z2DxNxZQEb9e&Q5gAnOobc8LwKoM`PyyoYoczV@1mqCbC zXKJugV2lexmhZaZ4#MC}lk0FLfS;wMB{A)iE3O#q)}#cpoF4$E0L<+=T!%Lc?EQb{ znP+^7JG2Cy;h?egRW&|g!ry!~S2<$A@qF>|bc&{jF96`@X< z=G2yO?80CPP>bzFg zn3gcHvrl!wI3+=W!`&9KzLTsi4V33U?c$LCZ>+yuZgAB0ItZeHoT}ODI*kw%SPE18RNj zuMI$>qEj|{Gx8(=+VNGvnSz8pmfx!`ejbVHBL$s0iKQtXN=eF>&mTE5V=sR5lt`vy zf8j4YWSPFIE18`4gYr4Quxh{eAC6jC0`5bPv7KzY@@S~e!oT97)Nno$bjn#yuBdYH zLyK$Z8^U+h6&}vc;H-c90BQtLpU;(++< zf`mf*ml6n$NWgqbWqw48cB0{QfI`?xHLRWKXfiKYwO)3EW@=3hQS_D+x=Sh5N&Qn& zaRn4M^~qvuLCT?MM(t`>)n3tfriMVNeX-F3O2r$Xt&auu|jr8bc!M`6+~?k#+>IQ8}eHbKUfh@>Tg<8{9O8jSt%>QF=rVm ziwSrM*NqCVY)z^a^2&=AGA1jwxVmH&361?jeNt@1KU5D4gjUU1bFJnFNn1j4T%R)5 z@JWgtSmm3;jgZ^LPcAO8@I#J zH|d;|JoO3}%#=o?z=^iGJO#eXxsIt?_(GrNQWo)w)Sxg~)EGsvmvVxGkXl*Mn8qi! z{Yrn3JPA=#Isjt2w7MmPex&|EaS!xTDYvaG{6eR!mPB6lvz3&0K8HM()NfYhsh-R? z9WFo-e_2}zyhx5)NBKHGG`Pdqe#y3S&K&cb4?9Kin!4i?dS+Y=4@nsz9xI8&hx~S3 zpKnlx8B6e_wky40x2^PO->gWMI^(TDNucG5sX0w^hyG|kiZK2e(H6>1f`cm651qQ& zRuc}Tj5JeqzZJHi_Ekko9K_vZ8}Op=E8O&tW*`aC6XQzdiJiC09c!o_Nhac4%hWn6 z)`l%vr_O9`Rv99oy85S()hsExz<^ADC4@I>ySBV$)zz}kWreE zI<1=6uDG)BPm4lp-x>g?)%v2!&^T*xmMZ*77!{3Tut$099}HuZ+Yc+%HHC{#Qmp#O zYyMGRiDo&lo&G8o2Hn`gG|!52w({q5n#)CFoDGhw0-doH4TVr_Qt5<8Z6Njj3Vw^(&b9hvt6KNfzJx2ix*<$`Pm@X-XCqHO#+D&Mz^!+XSI zF{mJ;)GEu)cT?g!+zQ^Q=6pv1A7L|+b2{p~d_jCrV-16GqoMv=TeiOZh{7~Sr{;}I z9WseN`48K|;a0jN9I7{}&+OAw1)Iz|rME7=GYO zbg%K5Vi};yV4Rd!k=Ytn8lqi;!aq81U2_9x6Ame5%4HKiM%5ctKOzEos=OlkvYfQO ze#uh9;>pa78haeMz-oS0($b$$HkV5+g15fB8y)<-cvXqIV6>rj7^QH$HQOtqBW`Lh zx~xUKaz%5J!ZPWC=qP0i3+khO@mOVqQ97+L<;Gvp0LPiIT0NiZ@1jQY;1VB9sTXJb zVSvUu)3mbWQMS**gD@03JquSDJoDLtubQKen76se4MQWe%cTVDY!LSs)+Ot)g;#O1 zl54Z-b8#~j&4LuhAF_lvNu^@L!^0c0h9JX?fmnR%bORv{{kP#78I$+t9La$Cc~?4XUQ9 zHfL3`I}vh*Hlg*jaMU(LI98`hboSSKIUR@>i${lD76DJ3pNDvY zZo?UU;X|zOrzaN|arP}`1-6UMEMnF~P^r{xc6Q#6#$c}COh7(jGmeP7ydrsw!*xY1 zc@B++?`V9Yugpx(OU|9MLpSdcE=Bv_iSf=Ac})WRvG+Vq5Mdh)M8)6=aiSkUer`Eb z82(IPOy2yy4p#zr!VNbni^i#XTeI=mTpyBqJwBc;hAaD^T=)qTZq zhf~4U6gQ@RS@9Eb6t53|@Fk2-Eq*IrMvOh7x^tW`W`d*y>7;}jj5t*>ca-0v7^CDU zs%i}}e9GSEl`pvbp!c|=pVL4UvW&WcX34@xLiGv`bdD>P{)>H@1%!93> zAY{}~u>?gUu0%Qou>Fu8i6R~XE(#NAXY|c(yj?BWFVoK98iGSZdp-aE0eO4 z(grnaGazgNgapGMv;|#&w@5F1uwf2!M{Gssd(Da_;G`5Xwwlpz^D|~&``XKP?(B^1 z+%ak&{Lo!eC^dm194Va165;Vr|8UY4=GW}LhdBRfyJ%K2lH?RkRf@-?2PL+xA!Uj~ zz^NapLp9=v99il1Xq7Y%!qm2lHp2l&E^3+}NvhINR4A5TC z@;{Wb{EtZB7cDibtCrIQ&dO0blNA%pso*SGlhB7iD0yXG!B7gaAeWh} zNeHFyvU1Q4Mis^+Ut$4nQEN!>_PMwouCK=c;N68mhrxr<8OPVsrsAPgUtVp&3#lMh=#wxExwFK1KHSB`+Bh(&@w zH|95+)CF4Q=CdHTU?BIy@ue8>v8+(wD?ja}ezcEmX9+1KQkcXqSb>Zn-Kb^#zl zP`^Q^Ff3o$ta|xMXJu5AFrSn{$EpEt8SL1ws@>_@p2FV}<&eK4Z*q0R?=88r_s2}W$@RrR>a#%K$#V-i9sWsDG){GgjqT%axQ5#@(M!7b3C zCZ01XyfdWY(^%z2XLhO;%`c*T*Y`+^fl%YLDf(yf(mz%a zP%gJUios~C=|@R*SAEOPWzkj5`qKlJs*0bptUE=UIHRa)87X-UHJ~lt!#4g*u44gKN|orXEq>lq+^pVd>stfD5rzO??Vyp`Wc5!l zz&54OQXiBpXoYdi?Zjf9N~3ToVmbJi=Mm=-`dTu%COHheC~yVQJzKbZZqfL`k3OgW zk&z<5ay|vR)#n&6Fy66(FI+67q)rQ@4}8W~#>n4!hT<|Ln|cz?Sacap87gOw;{z93 zP#X2F!k}}7s-0;m%k}Dtbu`|SE(>UEa(lDlc=H9U+7>re6SybJO>cVqfV```++^K$zbkmw?`bRw3jd$G&cOPJA;~C$#;tNjUa>6eJ2#(!? z+#Q2nWq1@fUFL*i=zu<#LZ0js-i0-xnagc_42yP&!**I$@uxKBvdWSoFvyc1{hC&L zn~X=tb~snoSJcv7UR_#*Jg&Y$PE{gsiGhn7_2H#_Fm~Aic|v*83f`$Nx2*T~_o;5w ztEvF;X62DeiY{5LDZ=5lOxj497;@+@+E%Se2Gdl;im8U$fgy*ZMb!OtnG!(vim$f2 zJRtB@pKerviWu9R<;;z>;j*rzfh#4|T#F&j>DT21cpmMitr$zP{v>}47s2>Lo{TXe z&s+KSrssxqhy1#n_*Q`0*XGCw(TjMgpP5D-gmT}J$GBG;0B{E_!FMnQ(MRwVj7Au> z8qzbc@#J&iBPzJ%;}P^vbNEOTWMI)hzv&*)NTbEg!CbVKahZ+t;n-5mHfAD(vytgv z@?h*>^yX|<&Ch;un9>N}ShlZUEvKBX1nI0*6iUy%64V zc(0=Qbh&l{bu`BuUejTpE+XdW^w*^h&)9WI{cb+gN9Gn3z!vegf9Qp#r zD;&ZvlO>KT!ZBp|1S-d$gB}RA@ONCZzNRr$u}-66X(rQ*COiU^Ut0NP#IwZZrChM2 z9)%X3#v~y@NBk|Tyje!xNHne1313>nwyM83_04Tx;LdnF#}`d@rhdk01B5H>xM>fj=18ulV47gvSc<( z2{)lXB80w)Jr!Jl$cr2g0)BHtl23#xlX7`-;R9cOvr^)hmzV9CXP&n&eDTY6;lgGYxt#TVLK=v@%6mmflQo4zQjT#!jf5YjMaAy2_Ae2b+7BhJ#R*XRQY zLjI*+3S(8{npJA*0~BS1Qk3r?#8Y=Y^q<#BDG^S&oi^sl{Liivlx)!T2nA28T z%c>v7a8Xo3yhe1+ky=aTmQ|!Uu_7k$4ACCOu|&g*<)c@N4+&~~8X8obDwbjKH^wh3 zj+uLh)1v^bQYfYAjPXE)HrZ`Zd#v8^e>-#~tx{>dS|wMmA%b z1;y7WSosLQj4}8I3VGxk#ZF#aDXVvIv?gE7$*Q%+OH*@xQ!-AU%W!I!8_1N-TSa?A zpQ=PT6n`wik!HMO%PXInB&Fj9pUU@&lc^SX#vYTk66 z8os1{3z)8etyEik^|}NY6@PRv{tVRuq>HlcWtRJ84G%;YVW_ zdiB95fH#6sWUPehO5458U0fj(@bMF#$Xo+A^0`YRIiVvy&^+qr}qvP^n4BGcY z`ZX4i>pZTU&zY3Vczv7GGrN1pVOH1sPyJ?3eFz^OYPU>4eRQbmtmye-QB~w!93lc(Z;^QLe;ghW9i$*!Zk*B)DND%ti z&kFTHp}LE9*!2?T+Q65>JQ3>Xv?`pokuMa5kJCIE4;Y`52m;$w+~wz5}2k zO(}`0kvfmcN|C6n*{iR;>U*4De&uC5cmA9hL)}iEI%zMy^rF4+(hGL_%xMv|Dxr!p zurTIf=ySx7*>$)AzzqYHCmjuev+9l~3=ZR7Adu^D9sb7wqqlbiJKu%_c7Zp^Db9Gd zqE-)=gAMnFlMi|tAOAOucf^xmttU&?b4&`J1E4}YGbRLHVZKZP5(>n;6Jq`;pJ>~D zo>NnrVoH-=K@62I*MS4`MXRMe!-^DXDlRy3AaN%x;Y@$+mL~m<@>0Bv(xjAzvZeBA zrcv<)G3LCOHA)LB8Z==d_^<*l&2}@jBS9Jz7AD?Y_<9VI_^QX(_zp8ANb|W#6G=n$ zREyF`C`>4t5XgfqBwRk>tHa1olX8lQM|GhcNdzUeKLsBlOew)#LlZ)SFISRJS(V3R z4!&py>QY8&vvN^*3_y4UM|Ah8|J61oS@;n=3n4S|uMk!B5iZz1hKh+m$u?fLSx!&S z*wWIJjg1%W*4ua5z8goq-6)$F<*0*y^ie%WM$@)yPu}_mCeK)NfKCTb)&ZvJMj*GTFKigAgbOLgKc@@Kq0?j<4(z@Z69CwPD;rfusEq2hl|c zxa82o^qu0}_vn3_aI5-pP;|Cb5M2mQ^r7NaC8$U=vAWvb#%AP7vxG09Us(tp81CBE zCxOugZD>`3Z6=LzfPPmb`KKh9W+YgqMK6jd#~2XC6}OocPLwy}2%|XTs)YiiddRn1 z=c@yK8|NZuIL=LcXhcZ>_bmyxoY8@hh>VoW*P+6CTGLXw?6H9`0U1wtC~1cp22NAjRvL2;q4_$mxt zc8xD;HmD3V8_Gn8V@n7EDn>rA?=&WWoO79?@J@Lc$K>hw(*qjqDoqrJfi5g(VtWLp zaNkQKe{i7ky$-a2v~pBWzi2O8H!#xH(jBW{@D@%o!X>Lc=}@-$@ijRpK2jt!Mp%_3 z4@*h$Nw8v7!mSJ0@@MN~O6BI2H>=dDZbG3nh)r#2s(!E|44YJn6=&+hs`vuO4`A=9 zV{jJj`gSSt9MFj%iow&RI@Q5HBB&vpGYV&>FQFS&EGwmG1@wDd}#uZb4~ z=pcv#(1O#l=pn3-%BsIPPYJ~{&^)NdctRfZ9a~~S8%24z7a0t{&R>KNaLuiJJ_hI$ z^+Tuwc!VaDR&Au;FziL+p7T+UPyd2r_!fo(cwSxc;n{F}rICZyIjB9OvV!qWd_={R zas~OT{Dyj~t+B)k7o6JFl=9jTJ#*7!mK%sMh%=?A{~QP6m8_y;B?-DS{He}|tTrp0 zvsO$EIJ@CPL0vqkio&G!WwILM>cbRA1!TqFvK?DdEejf(sPZ7vw@ZVIE)Qx$du*3g zaA=dyW=@yb`JF!BG6?(!oHM>y-2?CFR>gS(EAlu;%5Upr{0L7FKXX9nYwANkGDoo* zg}I6T5dy^<*#6}!KER{;G|2MMoPw2%xz`Wlr!H;iNb{rWQ7nk&;3rh1$2mPXjmpW( z$2s#6hWZreG?tt{`5Z>-u8|oUA-&UzIts@-Mhj@&R~D!8ijfvAh*Qa7Oov}K)K4yx zK$pS?p8!xek(->3RhOD3l(KNt()>A?72ol=q92~hwTu%lYpU&7s5(mD7OtIV%TdFe z+JWgVsS0Go6AC%xl8WzoKpawKM8l#hy>eqQe2VUZyTF5-XdHxlpW`v!^#ZtvaW02I zRUfG_?CbZV@Lv5&MR5ALbslPrS+v5?wkE+xawK%fSPJ%t3j` zcF+z+0vK|j7x05j#i}^+hJNw|`d57dv4S7?jPnulq&&q{9=TkYzrqJC49sB;3Wn#6 zc6q%;he75FMl=?TWHskN%b`BhAD*rxUQRGJV{{hR_WH;95WK{FICo*6--Xs_xiSP96au|3A?QdUt&PtXD{rc7bIfZjjMeW|_ z;48?zd{h*3bVC)`5B!CiGG7F`%eZW&c7qEySOywZbweEH6#a%?;JV&d(1(`;uOf_N zYD&t#sB9F=X#+ZB9z$k__Mj`CArE%g_{uTmgAvLr5L!brhrkyxWHGvFhvt3t1w4@7 zY{AAwodmw86#h{g;vLxPcW=C_!RGW=gEx~` zFKm@>uOCDY{$Zdv-m%K^9)b98wtf*}d?1z+-@jY%X6i_s?`9pYLx83xr?>vXityzB z@>AXi&uz0|#eis_8!rGfvBk93B~N$+{E)nf<2k5&LOdJ2O7HVY7Kn9ecLVuD za9|=ufk0_YWmIpzaNw`0b(LEh6`S*CGMrZ+M?qq#s=NF&xwTeTHBY9c;Dq)Ur40KO z?FvhvkRc2S7patl4{*eU=fN6&{^qXYAeDe3cB};Ttd7flPkHB05*Lkl-fgE zDKnhe!f9vZT zusiQKX!qTJtKD|ne$5a4!XpYbeUR*)H74O)_{B&tGBjj2-7;o7b`5H>MR*N0@iQo# zVE9@b3<_)yro61$ky1aT5fa_73lBy3CLLdI@q@;hbP+&;P{?m`Cl4a?JNU^Lzc&FG z9s34`tZ%RCyEAV|&Jj~u#}hQ7aD<(uS_KV6G$+Lwp^Umn$joI$LrMZowK0e(ET0x_ zf?Kt*BVn?(TDEjfbR&Ey!12*3^OcXX3#`Te75tFc8#+>B#d)r4re-L(?ypJ z0O*l=H#?PX|3?AdgH?ZqN8m#UY{X|J55f=-Tfi-j#PhbRUYrl)U)lLnTe4Da_(BVi z7k?rry$9zj02B|UAsB2zJZ0!O34$7NJPU^)yk4F@0{%FLHx8kU%Hl`4#6-ZQ-q0?J zMz+$lWJYyXU(!4-9bk2y*o8GE>_O`p@iuxBL$I<^`tBHFJG*!`PRMbd*%W~>Tp0g; zcAe-xT~Nv0Ghh>IW2M5n z1y=GZ4++%agRkN%EoZ@bdI|qk|0>N^zr5O)bi*OMJE^i*jg!mudp^F+8^*7q;W0@m zv+!qlBt|819+g240{kD|Hl7<0`Y;0X2z?g%iq(J^a`@uSN`lb;gu~Z_vy>D-&82~t zOKoKpq2`zH6`xvw>OhB(9{r5a&X#$U6UII?=sZ{T$SF^@`A{XsB_c7GVUQoVbh_`h z4H{>a7UKw`Wk80)nWMZ!<$)29s?Z5A69VX3*s1uVCNdF-#<2Y1c`X!h;k94=%~PyW z{wT7dy`tt#wPm%uXo+-33cdPBN-*R;uqqSR*b z;HvOfP1J2fV-VwDO?}@IZVFPeQwsAwroH+Dd#XW~PZA;@A%=7Jw&>1p3&1N1Sm-zK ztP!F79S8D3&jfXB9rQEmgf^6acpA4LdR+BmP`*Urn&a@Ri5@VN6tX4H8(QEZ0OSJZ zMEa03FliGlLq-C}wMJR-HTE<|L5p#_t=CRv;DUl19CMYtoGz=Zq|y(M@A!dU!zvMX zo(sS#5_O=`C9;wqM9ZWl9k(ZgYwRO@81(!+Pt~us#@T`jXHLTi zm*7JW(_2RKNG_Bww*kRNXb7OPd{vwLg86!*?N;8}p#@RG$PLK>gwL ztiIE@r5yNpg@gG;!+zlkGF_LyyPT!^0F*`k@prSsQ34z{5e+1)kP;n}r|OI>%^X;* za8ym8WgK-EsvMtQ->!@B_NM2z5(1so#ajXN4$uG{Kr7(UX+uxAC-envz)j%==nTvo zgk$KcNCzqwOY$QP`W>~#)Z1Z>NSi{n!7|M7Y5+MV z7~trfU|1t=`1l2((P?gGyvjO{QliXfCOC-u6{O|O_3jg}1Nrei( z^71S8oB!@N?FWDG2W?_v!sh1YY*h*`20w1EIdtfdDxS61Uw_@c@r`fT&-~2K*x1OJ z=MBMe4hjc9VjQGwIxsXmbSQPguh(dGEg-J1@KgNw-jtBVLDS^NZ}O*3F<$JX%|N`M zaEd72ARc8<|JeDIjoX|CKjLs+0bv+IB1AvR!cfP1EE`-?4)}-@1HXCaEWEg-wihDq z$b)!5Y(IJ9x45|Iw^)IXUZ6cOUCg5w@B=pclQ(|RpLm>w)r&v;dh5Cs{;tD!2fiC; z-eE9flAw2Ir)TWtS6{KmAA8&iq~Hmo9C?Bg@UkW$i|vtpnuJ!tzv6IlhSyK7%$Fq9 zk9M(g1w#;u9eHyu4EXeO`jn=o^or$qpkC73)Pz;&e8fg6W;GURcp#7vbfd0UsTVHGqYGQbL&|d?f#)VVK|nxQz;!e7uY-ZFqO4-5Xi2Fh4O6yy zP;C+(8vXK9Lg6EzQ2mw)X+@Q;G-EWbfK!EMg|4g@t^Wec_Z60`>x1t24$VKtY@i0w z(zh1o5&vYBV9-X}5I(RYn5P)Cs*=TncqPIhgr~hdOFRqM) zu?p7-_3yE!DeCJmryr3dvD;Bfo(dh(y% z*NX>t^lKP#o)6vukpjlMC-nS1URUAud<32&{{h5ZfA8iY9CRXoS*L$GCh}la#xr4F z4)T|`NPbi!a09(}X|mpdYXrktL80(sU(=vS%nb!rY!ct;LHW?7jWD*T2j?J?7DEW& zTZom8cCw|H#9Y<``=MmW!9jIqg#>xC78hLQ+F3u(%ps!XEd_5$cy-{n{tpkw&j!#} zLC$p{B!c4lxYhg`<}Lb=I3e%QzvLA%qg}L{C*bdnMBai=NcrX<aB)y#dwoz z24lOY1wqz%IYbf-vlx1z(a$f zqRbf3U$)H(QdO2nEZ1~kFm9x}5{TE${0ivxCm)=k+o$XK_GYi3*vISs-98Gxd-Lo@ zSkJEu>zx_=*ZbJx>pey%UX0ppN1#FIv|Cq?5bEx9x}G=VGs-Q|k7s-j@%bIf_1CVS zUY#k0ehQ%khBy3xhy#z~VId^M3Ssz#da&Aw_t0N~4|WHA$gT(Ep?T|VP0umSTLr3{ z<45^MI?%6ffAR+Oz60_1#*;i;zIQwem^;=dm>@|6>~;CB$SJV3E=>d7`NsQoot}sU z$d`HwV$jC#(6<42hIhiRY)9{JK)-u~Be-7(2qZ>c(5|Ab$Lq}p2mxNMzv+Md^Y*xp z;(L-18XmNZ*Iqv$jw2F;@x<;OaJYdRKy#ZE&fo=I^U_1(!r$w!zh;GE-ipP7FNoW@ zbElWt?Wu1MsBrl5$C69bjDT#6br|pNxcv@$;JycJ*REam=%bJN3Y@sg2jc(^MEOb> zITY-IDu|H)gB6A-40{;#n#~RCCt%0PoZdf8z6-_}nbZZyaIN83PIRr>=z42I>;?#l+n2tCxoOVzq zb&QUD$dh=aiTJw?-`ntpBUF9tUn}@ZB*xl+UNs(?qAd}d2$oeB62L?X2>zlP7t89c zzzZ;>C7LQlbS1*G97cbQ73j2FU6U(Y4TGKd3I+qt)hdX2GC{FYg0Y0bOynrn9siJ> zCLzwjAWvx3BXoHklertR>Mg4*iLWSLos~sOLwe2%VZv5+wy?Zx({l?ppfO1rR$8#_ zjPE*ZgX0Tfy$)Xp6ANXL2VWd%KC3)9Q-~+^%*j71hY_y;K9q*-0i3zvxYX~^I`GO@ zGfh@BN0N)L&d8+QP*bB0stY$ewYhDo(iBdqlbWo!u5c(z&e^Ky1$EDji4vZ_9k_ff z)7Ib@y_rWYi{iERCSB&gk)|8oJLmU6hnhwdT=YyFzm*U<%YJ=?hrG&F0ByQg^ir9Uen9s^vp*a5d1E$RfH|#Zxv18 zii!)k-TV~q@^%IPH-z;x@!55rH+>Y|p$8w)p%6g`cXwGHfpUTi?-4?uM9{l_Lv6Tj zCf$cptqbtc1qk&}Fz+$Di02rso20k2(7ytYqqp_&Zsab+x ziS-wJ>pH_G35#XmUUavTGjf5&*?{fc&ZejEA6d;@gT`u`DPdO3o-F8Xe~^ajeU z_wNRHxjg+>3=?0Fhi~_DtkRcZzJz1=%59|xGfzmljsuF&yDP!#Ifqy&g{7T`1qeO7 z@jC$cA^+eu{{L>m<~js$+-woF;n%HK_Yra6E0@+IML)H)C$re@UL1Hje~)2r9zy=z zAL|hF>J13A1|7sPLvX}2xcAr*<-yyXdtns`8F+mi^wZ}sxsYoQcxDP$NSAs-xn2Lu zpj&3ZL*xlTrit!K*a#u#Py(O=dT8xxsW;%whIjMypS<^!u_=q#uN$yc+N=4=t>+iw zcpv*4&=c=62#Eqcbo<8v|BbK>i2fn$N(lLEZcE4?ztB&izvQklzBk^Z|3+AN5A`71 z;I37#2ybJ0{w_hFgIFN--gsZH;QD*-@9t9#Cp|jDIyf?7oGpMf3f?Fe;JvplmxIUe z3EH3;FYF4~x*b6L-Rmc~BAXD$|GItITmD8K(FF+g-~2ng_4)@0`CkcCCRQlCQ}#xi zHsEr)?tKTO*XFprHms@)M);6VPj7vD5GG|Z4T>&?CJaFsVipz_?DfO1+eI3I2eFM9WCll$s^Vg16K@>WdJdMT&x4{v3;@Z zXe8q|bpT?SF&;)a;_y3`8_OaO;zS3Kw%0%U(YDxLAclhv@{fkVn165DnC3cs?*cuD zi<200_{KaQjM6OC=H@xRmA2sn&>M6p$0Qo2!@y(G9nrEfNQ!qciTbuUEM)q}3XK&_ znbmCC*3v0kQ8{h#lWbCP6d~D4_)U3!YiG6Y+fb9DF$^7UsOqjV<1!5A!}`q@v#>=8 zdK4{lo2@FtMF7w+Mle1&usu~Vy`A-B%j&8Ji>Aixz|HK)ds%oMY_iz zSy36~cGFfh{#s(tRn=>`TQANF;)X`-EIAdFsWw#EcGy~^eB9usCkq#>rTSH>)+}?b zpIJq}xseTg`c?qPs^aQ5D+o!*)->vq5q=5ds44KucT3`^NBrw|$Bm>4b0FqsAW&s( zMA`(5c^u)Sr##|sP7}g{AJjpb7!Sahii;4+4SB)W*bj!uadW-`CQ5e-_tkCD#rSeQ znF_p#lusDXZe4uESgcbwKQCvqi|vRGx{&$-za-c}V?j7>ipE*Tg$t|Ta{51~czG35 zPF)Y(1onswab{TcxDpX4*M-o^LC!V3pO@ zT-?bjEuA0g`qqH@$ya2_t)n604}Vs9vnqlLcJb+TB_d&}r^^^>o8vuZf+OP9pc@Ao zk3ZL;8|Dz7J>_4PZ~PryOxv3-0)FA^YM{?vT*ma25naRsE|wi3lo8IMQ@^lsq$$T` z7YYeEQwBK7ih@~br!>lfvoJZQGLIAQ$gq^SEm{zNfluLjOM)kfl*FIpRE91j{U=(6R&l)FxIj8TDCKJ?-S#Bw6U{K+o@;az%JuM73kt6bZ8L(6%1 z(LAAz>XA1+S{HAwt1m--nu+9CQ_jj@ z=6~O3+dZS`y9cq>*8~;}PgP&PUyTcfg;P_U0XFgZQj5Q&r`Hh&+Tz)z zgEtMa=Qi?a3!m&yNVif}RCnS0d_+0s3TYd1DQjfPR?n=eD^$O<@<@wL)IKCMnj2c8 z|2Ye-Qx^}BOriND)9%h&PGF%8-iib?122<|BoEeBQ;wt>Js|}0l4DG#81GHh6JF64 z1vVAGp_FO|bAlYa0T)&jijs6Jv5sy5&Y1i}XKHU)?e%BLcy?W#k(?w6=)cH-XlYh6kbr<7C zx1J{EL%!6VyanE_Ze7;v+vB``gb~LJ=%yjOTjyT5>~|~^7b}RPY&|1R{WoRtY2ocm z&z|X93topVK5*;5yZFDV+bWG5Je59Mr!~U54BF|Sn+72P`t`WN5duWa+wEyH5GA6%^wgyqyaxxUhjdl8VEsyt))Yw*OE^XpgfASmB1D#V1{8 zm9sthtumReIddvoBPm>IGdK!rb_BvfELQ;Qf?s-QUH@rOAP%HWUq#Ea!JdE=O8Hq`}!@&r7EmFOHp zIO39~8_(a#C%S+|kH_R4^Sdg9+VwgPI(EY`9Vp=cmT(*$EyQ8CQ?DoAaiow4|sFuVBcJ zUC;gTkq0W;-^556{NO?UJoQf^D7?b)!6~`T?yU0bB^9;U@9hP;-&-m;KDy<*X^tI< z=F|~IG&7a~VAu65-W{yH;nzKH>kSr}Ynzrig{C=+$E;Vt751*R&ywH`^Li2mZox0Y zbQ&H}bMc)jQ1qNC65MS6vw=CmZQD|klr#ZqCoMqIpO{2o*cu!dE z`N8+>B;!rjTggVdR=(vW+Y|Zs@;|dpyw61cX%F{YEAQUzlJN-&5jH8KKgwo29&cJT zQqvT&g@sB`dK$^xV-K>wzqciUYjc{`F^#nG%pwP7hT=Xi<$ZXTy1aeCN5z`@B{h03 zf=DoodoAmXhUjq}LvY%f!Ly^_-KiLl_QSihT}31}8XT|NoHlRQ(e=~nu~)Ej%eGa# zvMpP%r6^nD(Qx9R0Cg*$Q=P9R@Qn%@w698}kV)!FyO~F3JYSLA9uHfJFx+n(a$AzP zSc*`U6NXCMUFpK*hjI#q7p(uc>k-2m8m8<+p+w@R*3?D%khio**u9LzlGK!4Oxe!a z*6lg^#=}xY>EwYIL${A8;*IFpov_$2v}@{jYIYc+LuE8RArO5kEBarLFX<Le1ABO}X_i+@A#89-_RLmMX4)hC$QcIVx6!BpAx)o=BG@W_X$)`HnSWH9Pks>_T+OuQ=sh|u z+^B2$QgXW-xc)FAfCX@OB=&MCf+l9g6JmA9*6F;)5ilZHxUeQe3D~70kRAMh^R7^( z$h-W3lxO-n?N9Sb^m-Wt$_zyoSO_iJskyCYY$cyy#p9OX)Rau<)RK?0Y*&`ROG|(M zNB2dd-!cX5H=n2hHyyrz8?SCCb1bp)**my~qw7pB>P)LlFD6Z^AlL0Ggi%~RU#+>% zbHG<8YD3ZJs=p8sm}h`UptG5^HL|dR4Dw!m zNBSn9pOfvZQ&YK%!kP1zKw&|}^s)~i-NuBNEL@z)MPE3l>lGW!$%vd~O-wb#H5Dw}Voj?R zR5i3dV^I1)G7)QtN!qn4eLqVv0m|-dMAkxI?T&N#n9 z0BRq&6?aK7e?XPgg210-K;X%wzhV-DQdv3~qO||WV!RUU5;HWfKKvcYMs{gr;nI!? zI}%wnH>W)Twj=3M*7i?|S|5(a+;Y8FgrjY1-8ZV-KSw5}uBBD)CMG{v{)gPb~Y)o&Sz1 zn%sy_&RlXwxuM0%u|g%f>2bv)!43HA+w70?hLlwii@En&#K;tK_ziQUQ(vwc1};V@ zP(hsUE^X2%bSb!^KW!zA8Md5x8*lK|8yLePDU3?5TdsAZh>iFt>vbNtR7wr&oMA#2 z4{wWp9f+GB9nU>4xSFDhkoDKzDm;^HdG8$JmT~otmg?VlO#NdEUkK`~=|z{X_%X$e zDMc>>V7zN?2{4otn0#Pfb+#Zxm8yX_JSh(zN(sF*X7-+NT{YD#+B&yB-VGgC?E(2T zDBkONy10<-loNVwv`yO!#GNX>TlL>1r>IENq{;03n@Ia`FBC9CEt?|YY- zSE+Fb`~YE*b#s3ALu5hsxrv8ZM0vD>2ng$whijYcfdAEn?uN$5G=9nUkQ(`Ivo*cg z$8)N};xh)0Zq^&V&lqw@Wk7p6su4X#A?(V^V4fF3>%%2;N%ETP#x%*Gl5Vp+^bb0% z&!bc_ZI>ayF_sxs(_q}TfopMw_1e2kwI%r*dYUYUsz7b<1Z9Da4YcFE>5k6zL-(T1 z133+gvLWV%fp62HeE)0EsH|Z0NsuaS7EimfKp#LIcg8C?dj`4P3%p8 zK7x^e)-k8>K4X{W&}54s!_Llbfq5ZT5Z^X@ql0|Pns)QwwiY%8coj|RcNom>YB@Ba z4#c3W*}c~2OcQ(D$DsaarTaH|z{@+lmipCRDj<`N?w!w_d(&$FMP28% zRt@9gNk{Z-cIXB=Xh^$`J@0nq_^mPSQq!WTXdRyj<2!-gNF zTb*E=b<#WM}qM0hm&He#e%6n z6Pzj8SmP`~+y=-(`rCD1-??4P&{=rgIjSJ|l|FbHT&I1))cH3KZPv|Xk|hJR(%V#J zpf-Ejx=WA2YOIR+i-VSB9(rHZRM>O)@=mWl(zcrqF= z>UkciufAWIDV)_+y!6c-GLmtwjuUWa5?Z_2VPOms3(%K_fh8J+n5{|}m8ZS?SQC1g7?`|QyQx^}L zy6rRM?sz)TGJ}hsXrbEgaokg#^;Z!X{}bA+P)3p+u`|HzhUC(pvMu14;PQ~G7EsJ) zS7JElYV;*$HNEy)(|9BaDH;pHTU!&^KN?>U++<$$vuuu2_uPTzc`~58OX{O7NE+>N zhP{Gi?-v8*{cPVK>Zvb91Bosvl6R{Xxh7Q}PV*bH(W)=%1Gw=Bt} zvQr4P;1pX&vT!*99AuwZg>=qd%rf3tn@&t5uDayov%E9%O0LgfR6|Rjh^BIXLRCX6 z-uQZtLhK&@I>1)Af|TGOU+n0qEE9bGPP)ppnBm}GE*xrc>98vEGOS1F>6F~1rp`zD zqj}~45!y$)T~{0R`4Fov@B7*C7niFkc2+M@pV9IX{+iK;e^sJ~p2&WW<)(RR^0Ow0 z@$!cp4>b$l)Uyr6vQcrE#NF0ot$?nwVpA_`nS|r~T`z&Ir@--(#b+S`_nts;H3iPy z{d|iP3zsDO4BG-z4`$ll22wpilSL6w6kSo0Jm{f|(D#^4By=40 z7aI0^{pcQsQNv9pni-?;EgFzEqTH-h^yocPEIMJth0kc2{gOXKWowAWLkfJ~jQXG= zcV~R)K9I!1e~ktsKOz>;%DOGJ2G}|*57sNvv)`GD+O$`&wR%TiMSBuOiy{Y6H$6Gc z|M`AOca2B^PHBbwUg9FL;GMcZ;mmRcT!FMPafA_#32snyQChNk55$-|GM4EY9AY+F zq-cyf1WeLTjsMcf9R6~qHgTjtSUG)OtMuX>W-=C1W0m#(Cs@S9%IX$8X$g_F4mc9L ze(SUHlLyq4@J(mos7wA0i$K-V`^KBI&1>DO>6=T_t53CMABp`5(*X-HPGe>vnjB9e z7s(>RHMc67_Nx&D_zjFO2PLEuRs^qeTUJ>Qgd7lre?@ZZ%GDZByVo$E+ZZ1Yp2iN-tbDMJKQZzk!$eLM7L&Ki{D$eGb*GKI5-~h9L%c6j9TwGiZ z^Z1giOWxadAhaX2`8bl07I5OY2f4vP7P3`*qutjSC*mdeh)QVqcARj;|JKojrU09O zQt1O6<)4!mDSuCyHd{yiT8hgnl1O!)OBf3vS*)xYz`+R7ZKlwtPoHf7s=X zSiCIm+(q2+G7dkmqAizfxtBUiiR)<9M9>auybyY>P?GsUiE^`aa5tA^J+bfGtKm&N z9!_7gxrbZRPct+Qds_bV35(9ZyUOMbFn^a->re0FdDo$6{-J?0gPv;Q;H3VICXmrZ zVWP5^#uu$q)cDLYrR4BOyQ#lA<@O)*)y5Y=c{hw^qQ3 z8>O?lCp8xrN1n!z#n3mg@vX)orV*)LWBoH9sn}?i@?}aF9hX|A>J$HjqKvXxRAQHt zW%How%j>v--SAhPqxqE5R5}-7U|g(A%uT}LEH*7}m7!HfW%ctJF)ja#7d|D;7H*Tm zpG^Au9SK$+cLlTh4SBn#{GwA8gvCQCf^wouf1I0m##Cyb^4HWGbZy;%_9a%T*=sV< zb^X^;$tHN0^`9Sh19Q|{%7d_1bj)ly&{d?w#4xI&zeEWb@gvGDNdq(UQXj4yvdtrJ z2gY-W9}^1%vCPBMOhfoAmjM%w^@ko(r_~Hw z?zer;&Yko>U6iCEy+{lo+l}VJQEm}Zems>@6B?KIn>8N``$tb-&F^`-X0ZlOYXM@hrc~k;`Yk_G=ozzoG+<%!OAQlS(D`Dc(EwOY#j(pI zIbL^(=y)^3c9-_ng~LjFS)Jy-01k1)y`90eD3CHQ3%hoPpU3=3R(i=3Vrl}p!(mM3 zzVODUQEAPjgMIEFO#FZ!^ zUXI!b6W}5|(nYFyVcY8jdcfA@Z8`uZapb11lrn*$?oOq#nsQo;FI!HG+(}zCE{HCj z%9(a_(GxV{!_;-ae(*DMZVl}Y6;Qn{I#t~c9E%e_s{vlsAeeHEsK?0Y(VBZR{e7RR z(;<&?W`C0pdp+(y>^Cp zwAE+qiR6m*`NZ4I9PxlQMoaRJ%qgdE$K;B6NY~c=c@MOoiDRJ9(~4Jld$Y6p@V^%6 zK3h5Adl?75MNsPeRwijX>HBK1^6C!iJR2wx|AtbQrbI`viVK$Jo;lt)Mh`_m=hB#^{c!7QM(AG1$42S$oL@TeD*5+ zj<$BO`0g6_LVZ}j{1|dV-+oG}N5#!L>hHB2(4)Qe`*ngJVeS3Z#!oHmcteeNjZT$X z7EVXbzE20WAu4yXbq3hu>!CqS4`;yX2kc>A9Oq8|rMGxHj~8xQzs!Lpi%wOSMDtJY zU{jITA~tHMX*Cx3v_Lw%&!F4HRIf6{U;qZe)LwY$m@ zV8Ic!xu26HX`nA)8@skweYHao>7rvt()a4G};3-TaH?{TH-?o*4~#= zRtoiDxZaIHtsd}o^ip)nWnM09$MEUtGL$eo_od;|JR^Ee08%S=n+ML6&-N@EzL)va z$^AfEtE}Pn+;oF!-St1}Bx%&0x$^>@@+=8Ro1foHRM((=CCV6f&g21WCI$vT+%@31LmiC)AODgyE zsE(Ta_t$>vM(;LLPlpa6h)mmQciB6etPZI&iDdOi*nLAKLP(Yu(zunP7F{>LFKETyCl4tpil`3vUy z$7AXJ0-i7waNdQN`Q8fjN-vsAT9 zXl8>kf~Z$L1_MO8>RaqQA$Hyq%G1pEyLkAi2wh`O7O(Cp0E`(%lUvHoKGG{AyjdWn z&4dRnA=>aD^qtdyuD7m-zbj$mCp-PAzH9$cS$TOwDQ`=R48mxBmh^~wjQ%49fd|FM8PM{&G&mf^+Q2+7@;X^NKRG>%KU z@MFvQ(t>tiBNGB{5NMTk*6^n))2jPKWqyDO4`qxRdqMBcVex zjhsag7+D-CjIEnCIl(0`pFogL$1RH2%M@5d6%e%H&AD;CiI7kGuYEm&h^=E>WD?j-@ER9NSLi2Dous_z1JDkE7nofGl>S3mg0 z>|s75Y^io44gDc>M1SBZFG8Prv=>SH;V!k{sPZ{$Aa3&X>g*CJazRgQ^oo zfv$H^8}{NuNzQeaO~{k28S$R&zCH?Uw@A@mjm^l+a)~Nz)b7PlYR2d5Gqou!X3mj@ zI`*CDySZ-bZWEK#Y8wbUY^20krN1Ac4-;8`CM_ocmY|B(&zZ5&>LBR;gJ$_&+M%Q* zzbXZHP9^gj=tNF@D{#2E7#zTMoRaW;jTARvs?}PV&1!U-QPW8(Z`2VsEHU_m`&hjA zizqCcL6P!q+~vwnpim&OH~hDo`GRLp^LR#x@zWSD_n)M|-;M3jzO*JpUYrSCzCI>O zY=#*_g+|A5F5N6D{s~oyyPx(R8_|oB3qT&%8WYJa;ES~+ zl#9i79JFuXqDWeeP8oXGC*mhOPvY0oA1!7lo9$@`M7EW8cUk%v{`ILBLXF(&kB_>i za$~0M&3HCFR4hp*E+J`A*u`wQwMa$a&dH+}|uF%Rd-Myg7@_3G8g#R5y7r5Rshy>(~9wrHh=G z6N0089eCxt>#NGl8MH-Wjg1tL3GG%l{D=nO{loh8#>sF2|*^gU~~RI8=O=ZS<`Q4 zw2DAX`)vsCv`%!p)P-xQjS*|9kmm=jSO(vhoS%gL3%|U8fhU~_?izD1{JX}eM-}yg zXol|p{Z4+C@BM1A8>$Gt%Nk;Y{;*XL^G90U2(Ipvu`2h^HIH-9<++jT{Sz{0?;Pfv zR}ly=0<~D{d9@O`KPYL#$iQQ^17qM z>Srt_caM>hgN8khZlqB2`FK%~V`$Q8fbP_qCpED34c(UC%__X(u1zOVP=(O^Gbu%R zDVXmyx!a+aV~hW2{v1a*0JGKIwk{uKdE>$SuxHa+qeb!ZPdH#E)Z~V2?RFWx2zu8R z@4jl9l;9aR<*D1|&2;+K8{Z@fGa(tvVpni$<`+t z&^7C&VB1Ee-Xq!*Oehokx7}E;@S(FP+O6jyJ)7%Ce zA>vPg*_!#o+)-bigk#6Gk?xJQD>+1WY)yc|l&+rN7|35_;n#TlCOQ#LC z(jJG#ZEj!`#8=Ry+1B)OJh}NAaUzYd48Vm4Y$2s!#PL1D8Pe!4GCLO|Hy1!S5El%r z1ekcX$M~f4pYN1H8#QBHu8Vr+Fv)1;7rT1&%X9#4B}#BE z9EdmX!Tir&;VG5S{z7TzUMA5kJM4I~V@lvN9Sf(U3g$w67q0k@MIR6r4`u@{)o=P* z53X8A2J+>YSOc!$0jN&7qo3-YUQ6V8JcN-eLBhD8m&`H#fB&l~&nUY5U%bn|r{XL4 z>w}xtWCfOCJz{mHFT(#U@I*zUH1b)ue1%Ck{C;)`e9jZ_2Xd*IN{npU-sY8{v}{(2 z2rrmQrAl0=)T)`lCR##K08?DpEvL?>qi<5zeocfDsakM{M5BHk4?F3HT$`DS9+6ZMR|;Rh=&JEiVuR-n&RrmR;N_w8$}6xXdJfxfM^gwKw5gw{7RM6c>&mKw1W{k=BC4M%7Ti?Z-Z5r-3ms5om^S zm|n#UU(xVfz|{@_vw?D`I|t%JWWCM_?n`0BI-i@A!^6Q}+*oP-t3z|aK-V-`_bb_{ z51Sx;Td&g~(mkjQws^s#2ked*pMS}g3`i|}qJ4!N5S*)Z?Ru%Z}!NY$iI>mCxk`b$|ViXDD>QVtd12>Alz^RvkKO(-xBQUWy{hJ$yFvlmoc~K$78dz(9IYDAM2h1$_$hC$H zd=6+>^vr2t9@W}^Eo+)$xfCZou#s@xK z+E8>_&|;&R(@KW7`|guQ*WEJsi*E}w-^gyW3zh}H<<8}8qJlC7i!$I3EpCgq5?bKT zOQ#+Ayzr@&+6(A1G;@2}d!_WDBOI$;Jv`Z);Dn|=-I%6G+isOm^M!;kt}a(3nF@A! zGP{`=8*9PedpL+LC?h$={R4WCO)#$a9cCU^pc0L0Qg-iAHXppO&uN^>?Qx8pwd9r&SrZfkXdh+B_$dyiIiD!%aRzSKi741W}`7jbcU!qTs9r8YD)_j95M6Yh@- z2fO+X?yiz9s=l|D2ZasEl^u3G4?CjhWK|~DNi=`c`6u)!;$-cxN9YKBjS6rfW>T&) z#f6&QmS^X%e$!MGkNWI0cS9B!0SnU5CIQAu{Gb{8l{nFvoB&Y|NQ?u*OZx(ijibx1WvNId0zCM&M@wd)LM& z3|F{A4JIXDY}a0=W0Zf~>(+Z~_{XqXiC!8L&hB3%Yr<5?+y!}53YIq3@%b&~<7tsr zAXegkHtyx)ec7CNYrHRV^d8+GKA&~(9^w`emzKiVOjYFy=nm?aH-EJm&Z^#wWvS^o z@Sb5?teVF1LXJL8?Q31;xFR9s);@yCpFWB{gnc@b@?Q?gNSC#gh^3h&RpACDJBdFf zr73+MDOu{8<-1K{sQ(Av)2$nD8tVhrkZkk=r5XRptXb_CB6uuX(NIzADQwGm_|M77 zn!>7-;HV}?Nn6qy5>$!!S@!&lK`{%`tNc_tW-c?#zrl(mQ{kSmNhSL zszL&D_%E8UJbgXC-?}T{vxH0oa~jqPpAjsBuP{oa2bF19_S(Ofr<3$Ndyv>>Gt*Y5 zIf1go^788AJf*3kOYld&w4h)0pDS!$p8R!Hkkn!#^dDqCUnboaJlPA%bTSc)dL+B} z?#_0Miplf~D1~sJ;k{&=X#!*E!_peMoGTH$^BrGB+}pkwTql0nd{+|FPW1CAsS~hQ zGsk;MbMa==K3gWFS+S#3ffB+P>!6i-23k>tUQwILQuNRNn5qc;b0^)+lywC%AR|F^ zJ#A~~dVQ;L6mL3or;BfOq;dUSFwst+-R*klFpbolYAwTULCNb^0PG-uUy!0-} z8PvFnkhfZU|50Ke)DNkz(qW>2ztnYJf~-AyK1^{KAndyus@Hs^GGF_59HS&lc4lyv zfBSiaA|P3ba^X3bqye^EQGL$D7fiBi-m>n_5$5IT`_pabC!AV%&?k2{p-^JGQ5Qfe zJb3kp^zOQX>7jk&N!yNfscW~q_(lmfSu)nP=_RF{ ztQK9RA8tL4Chu5QT~B-lIFdR~KHb$6j8gN)J@os>?DY0cyN&$oSSdfFQS^OUOZGycuN?s7*x|@BD4bP`t76cVfpAK0^ zT>q=#@s*`Hjy~=2fk*Q_Mee%kMm1%60Xz`~^S2=rqB52Z|LS%}xEjl|CFWPVk`JY} zZbiF}vcGIz7Vyz{>3h3HERS{TUK7(#@0^rB-JoBXG;@(*SvML2Ga{zDB@GoQS08kF z9MH+MzOC2s(xq7SfU-M#@6s+PywdW8GY-Bea(3<@zqus5| zPx#OP{r5s#QfP~B>wftC^0Bh3Db<7kN=2qYz4p-M>}R@;A%-AR?~7-^sBfsQkhxrr zxPiuwfTIqQn~bSmdoe>DZ9LzhJ5}~rneElO>9y6BVZg5AtoqHVItqvnzZs=6yW@uJ zPLmLJGOx4J*>WPID{l)>0=z+_ajyQHcz=O?Ws^hMuPA}iGhh0oP@9PV2fp2?QqaLQR1JEfD>{8IERc~6stuH`eQi)Zh*~D6tq?xF?oPfeo-&e zJ-Iy89q9~zryKTWh7&nO z*CbrAw_KhZE}6DaXT~+m+>K|asLFIaFfe%_dp5ZXM@90jo^LQ1`MUgKVF)O@0zXJD z*y+&mPgaU{#^}*`TOt}59I0C~@D+?V1{d9x_64Ngt7nZjnetwmk1o{AN>63|z1MYJ zZxwQn5fVj9D`}l+|BD*1plVHgh{zCi)Sa0RgDW*zur3x0=t9B?|IGw+u-;d=QRIrY z|A_eXR3!SHjC#7vvf&rt^-lZFm1tM?ul-#ob|xj| zF&|?j%)Up!v@X>7g$|RRkwG;Ox219`u?O;ZpENUjic{RzC8pG33K#my`r5Tu_h;2{ zbkr*$PV=vqC%_vQiC$-aW*b<}JImsZ%8)3#5`dv;Ljg8s#e6O5zC?m!CR3@cg#1Y!wF0@K-@72+-)R!>go%%eGVmT2F6F3Mn|`5%$&DDzdAwho#uymsw<%G%q-W_orm zqKoS4-~8=N+;$ZAfgM6}$IQX(VS3{|uQkPwdc(C-WEzU|;U-LU`rExPfVMC;b}#$~;L&U_0$ z$~+9Vo``0w;x0B58-uNX9f%z?s_mQrfPK+wO_t|G%!7cJ8T|(01CtBOFtn23p$YTo7h0fVv~w8*vz+gSa4`ArNB$t)U-x- za|BgRjM_^wLV&dlmtUWb{ysJxdv)U`W>{ zL&mM&+S+DbBaGwc$(-I%SPoea`Af4l-JO_QCg!cwlS;jAw@xaF`msF-@QlZOZHrEq z4UzaEo?W>Kflt1vCM+PXJTM1@7K9D(+y%IaP`HDX-ck3vh~bFze)~pKJz9rpHzIg> z{}zhB?j2fcKMs586iUSZq{yjOWF=^Tx0f`S=h|7C)BldTf6W|qPm(UB*&3C@`(Q2; z(c&YujH4o>mCtNQq;2o2>=Pp4{~kinIO47xfm0G?2^}(JOLpp~0fAVef8nl6#7ewp z=5NveoAm30lFL)`pHaL0$9j;!I?zIm_G#;{u~u=NF&3Mq8?n2O%2*e%LkW6&THNNJ zFD=l_32f?!xkF=_VyoL34t?a%tm+ZXOzk(; zg#4^h3xAn8;SkC3!a>LRYK^=+gi-mjJf0$1Tj(yS%d@L%>4O@`{P*OAfJ5uF8AOkP zpA-;!Ff*7?F20jwTgM;88@&z^Dsde*D9P8B`W%R-=S?hkvYLrgRk%3|tJ5h>GAK6; z;3*L2&DO8^SZ#GthSaICn0&MF*K65H$VOWE{@zm8A3{NRR%Mp&O*rPDm+-G?(K6s< zjAyUsb`RZWVsNPM)Aemo_nh(B3LGjEnE833>R#Pwt8%Mw!`C5qC zdAoRrbgajc^s7blJ5To5PbuiR1wUOrBL^M={M+E4E!x@_d`4s)URpN2*c>(=T=^(U zDIQWY>EiBy%p@RPTB#U=8A{DBuP%L=U)aE$%}MxH1u_w4ek8*v%TL41#4$bZ-gwrXb(fX>OS`@z307rk9rYCp zKRpG~STZiQb-J)+fsLTs<8+psVpM$mGwB?uN!z*UrD{Om)y@C(tbPTnj(DUaZk@4= zEZ=vI>uE!tNrd{m;zXhyKoY`Fad4hMtjmX>SW{kb2 zhs}4iY_|HAE6hS6Zdz_EvOF(O+}G9~l=AZ%=C*?^-lt_|oRzPKew}lWbsBxmJ1i-z zS|?h#_AqxW{aqNNn?rj1z^(^pytw6h7N-k6kKqBTJY?r94D6eRS!fvO=i`H=GC*Ir zmZdo87Z|`U)}JvCCYSwsr_Y-soIZF-LX+Xz$?Z-vy8vgY^_V{kTyR0RxH});5`rn! z>vfDbzv4bG*z$o#^7V;koPz_($7{WB6C;MiEsi*~oj4PZCY@v42Ly)I4PR$f!^U+{ zD~1_u#c7yA?bq6+{RK64b#QP`EL%;K>Iv-D7O89aq7Ih0%RgYd=k`VUg=wX)>GsXY zqEDc{`*Fg2$F70L&#MMKN(GNyQ=s|~d5AG9l2h0EW zfau}V1ed9`^dg?{VV>)=sr^ZpLNTtGe;sV^(b58#7QQkA?ag#-4Nylr^jgHdql%(z z24%wscc@%%5}&Q`^8qjauq-!!V~;bUor^jejWuMiAMp2{aC4tqOUbU6t~M&?=H~gwU>}{7_!;rpZow!*92A@i-``Vo_jVoj&f{ z7f(9ldb6A%ptZkD``JG&wW@eE8D4Vdh<9YtWTOAwtC;!)bRT5$If^m`9eAQpAnahu z7)O#X_6$3>;IU{&1H_pzplKr`zLka~&neofe;v^ZiU{cqab)`xCbZV^F#b;Se^>sl zWM3k_p7!C2M@$?eNF%f&4Aix03k_Yi(rx#j0{( z{crtDlD@%PT_+u3Lt)0&o^_L5K1P1+Vm&|gcJ8nP77g7j9B0*4CnP2Lt+0_(0)M70F~m9X6R?iyF<}b^M(JKWgci zM2;wD;5j*U!uvk`)M~G8aI(SVlvdQtG!(OjoIC=?k?4x8JRAx1iE6XtrCBL_c2wUq+f&PPy3SW z!Ihuo@CsA`q@C=~OWnf|>gMS~%Wxe4veJCO+VFSjOfG10%J5IrhhtX92iqaoqbaSd z>MTa_S-}_5n8IVVCI|F=A6t*`v-=3^ z8b#jy3UqHWPS3Ej zXVa$a=TRq}kM%zP6!DLK^<$*f?<#Mr^Rsx;@}fHmjP2!ubj2If!8Qidn7O{kIh^8*?s(q?MQlFBCz0k3eORuE z4R5>cMf)voz4KDN%SarrK<@UrJ7Lo1^OQj{92=(0EK`rZzo;DXM^^{?LQ>^~=(U>q3!t0;nb3V08`MJG(p4N`iq1#+b7+ zac+FfH_CuJ=YSu&?eLUUp1H>q5cqp~?z!^wc`%}WTLK&IvtM2Dxm}>;_hp+m9_mcnB@`(R8bn`|NHqX?=o(71I$nfpw6$c-mYC!v5Z3l^UNYdk3 z0LV54friF(KMW5VCP-Ma$Cjo!R=W{*NYVxN3}xBWYz)@Q`pLRau)^o~&ljyA0;b=5 zIuBwIB`HQz*zqXrz0|Rvq^Vn+VxDsQ5n^R*Wd3IT*Poek8CpV#^cZkx~k6z04E# zAgdB;X1S8JpyH$)(~po`%BA8#=2AY43p@N_s|zrJ8GlTbu*oE>6#;*aNs1JzL+D&v zztGc-y#4Re@-e_?mE#r1@l*LvOh7qu3z5Q+2*Hh`ZrM}So8h1)VzVq;F>D+cM+L_W z2f2Vj?ei(A+p``M$yX7<`DFD?nWB@0{w6Vv4DDM!?pSXY_x2e`ex{&y0kstwhy}aE z)ABj_b@XE&UwN|cGd8C3NC<-ExJ~2-*MM(T%u^M=jm+Sue=Ij3{dNdXF=i9QJqF@k4=L zq)+$c`m3pQXbr5;+TQM4wGqy*lJacIEefpEV{Itp|2IQ+mC61^+GJ?!=k@>Q#FWpE zK+8T1{mSB^b8A@lwAbHiyAhw;A)H0S;&pS;*fH^EGmez4|E#`?bE3^wI}iLn77klO zn5=3HRAxJ>;xQc!Ctzil()G^z$jR>?)AbD5k>U*{-G#8&$`mdXXmRVveQNMRyaikv zL$=Ps!fyTiTKI#_M3T~q$q4A z;^kQTvJahHhimUhC`}oQT(Dxn*Ti4y3k+8mp*QsZ?;)*7cC^2y;>=5{& zF4x?AFjvP%F89yM>-oTYh^c4#r&P(l)F)}Vqn%L!uOsh02=__ul6WHmY2tO%E75C$%NB91K3AUOkXOsQ;Po<}{t+XU5Dx7y8Gh48z8qtwG z25P#^K6KqESqkoR*;g2pXBRm#kix&3(t10+k&N7*BmO!CH*{aSo=Fp>_YQdwPV$Sm^ z?eWA>eBo@DZ&XaLRfnf!S+KAU|A3rNdkfvRk&M2_Hv*Zo&4!G$+Z;>UYyTf&;^7EO z^XOTNXPMa7g%y1pwXszs7&7;G%*+@)2KPYg2aq20-PlsblaqeqB|H9cN*`*FZb&?! z<6av3HMc;z%t9=U9Dlm#75T+;r|b(9GS*O^vi@NEf0=j zCo^o{l*Goyu9bU}F>vnO5nPMlBh#h;2t= z3dVGdMjU>QlngaXqLJfRwDS9Du;c`d$Az19OB%U4nMWK6DRbY?q2=fd{qMMB7vpJ7 zJ&%n0M+^|F#Q1cKK`FkN(Bwqh++zSasWbQgKUBSSK+|vgJuW2;8#Ry|AxMYP%_vcj zknRv9q@;vV5+g^4zz`5Y>28o5DWG&WNOz6+&d=|8zn|y#$M*ML_wLtypX;1+ovVfK z8a#};xJhlhSs&%a4Q67v7A;rk5JCJ|L~cX*MUpgen>dTb7V6o|xBBJW72oRzwRUuo`{#xU+2L z8ifj$5mVdT=H3s{dt#u#>6HF_hO^tAg3P^Ly+*`O8HjoEmMf)5ckfTtby%~99^nMz zO!Z`)i+XV~s>50OGV^CQwS-#e&yMn9k-1Y>edS(*4gl)tWBz^v7pFb4TsedFKf z^bPnLAy(~JPV1j;Vc20DgA6bs|UOg?N#xui{Z zuGHbz!jnD=^ISLgwYCeCe z^4p{Nq4vSSy@RLG75SdLc$Of`V3KttWlS*Bhdshr~w-f=kW|V8u3ELq&UAY`YV|ZQ~ z53$ZY@kncODLSxhcqQxi>}2M&)up|SG?Wd8Pa2fXSJUt82Lc8Cz;MB(NGaJ7`4`L6 zClBOsd?8gpgK#D#iHo7UT=Omc9Nj{e?BTv_2aD*Mv-(XT@FI8^xu$o0eqG}cWb61U z%xqssRZPH9>Xm0Zr;QU`E3pvc3iyIF;Unm zpo41Qu2!$=DKnKx7def| zAE(`{4uy>SG(sf4ywoZrzZrZM)+VJxa$C`2A~PCa2-U&&tQvPq8;*PyeiQVB>Yodh z`y=nl2Lf?`C4=63GSg{<0QJmd1E10gc&r@Td9Sg?_23!vj`y2sPPNAd;(ZCPn7bEF zzlwC;gz{f%NIZt zQEEG`@$&5Pqv^N<$H9|F5hX#C6#`Wh?<~0u>mLW;Yv~es39L9VjVeSEB^8a}tC@hK zSyPZRI!pw7YjI@V@X$gF$MLDO$6%SGFLZay5_eyV)^OxobkYdT%UY+Lixn7EpVAGl zn`329e}`{LtGK~7t=`&`nIWNqwIjIs%QD?d632eKBCI{y_=ySnBrW_G0v4&d5pAxm z4f6F6177LJwy8YZ-v9sVR0JnkFrWo)h%piam90 zE*T3cv9q|%)$4$vLs$okGm-XbA{5h0Z zt5fDZ%e_=Ku!GBnoA>6BX08~L=xu{BAA3b~%h#)sg?ohP4$?;2P}grJTbgdMOsO^# zErdyKq_Fk6p0vb$hj~8hpudl43C)1;o(4?O#0jM@#cd_nAaO&Q;{L;G$lr|0eOdiX za9+!q@=GG5IN+b_u`1w25^r#(m67^f|ba+kkdZ_jf!JyT>@(lcxqnig~ zZ~Vq~Y3$-jZ`V4BFYto!RW$++d$k2o9~p?^2FV*D#?r@F*nCu>O4*Prwp(Ju2#aV* z`rU$mw?|D@xItE)lY+GYsh)85vQW>vyO^(hf{S z-v0WMrvFKggQ}#ijjy(3c%591DUG?3Hx-?U*|oMuaa3;1X;341m`h{I+J52d3vQnR zDgs{tUIR?h7$@Ya)_}_-0-?6q?BF|gCW08dY7V>n@4CHA>iNag{&kSpBO_&wex1(6 z*f{%uAwTy-XTJNFRPoHv_;Tdp!-PpSGAeD&UFMw8c*`#g;YvhwbaEzl2^1X)GRKB& zH}g+$H2O0qA~jUukopSj)b*BS_{&~==~Y{m)+CBqNK-eg?nVFeX@22-1p{Vcva^(G zdKbRykX^+USrRfuSyL;QaxiObQ!kEMuaE8?3G~8Hp}a>42L_FJKTT_Y8qQ93zm?&` z+{P|>BooO>=n_vaWup*7Z9x4vm_quKHw+K=l6m~GTZsX^e@F-*gB#}@qY3#BbiR{V zCO)C*RN6-~KbqcAK+ilSBy?K^=Dq8H#IJ&>@w^<;%`S`5^jdBixsJl&N?A3CK6;kY zo8|b1U-C`di~==aAAmBoZ;l4i`V33_`xY|B?5 z#Eg)Pt#U3d-3&16{gP+PyCiz8gEfirf9dL_!UwE? z$MtR_5S9yWcY1G@L2Ds_4qozYmOIz=n0SsRw{L!NmpWV%%k|p6_wqrc@P;F?<>w6X z-mLxJR4V}lUD@*A?Qj3EZpWGlw(Y-QvuV(}ED6u%k$>Hosp9y>g_fl?0rrgSe*Lk} zN~$`gUrs5nq6gRQ*G9p7AdkL!!&w&gN;l^k_Ch2u@B8Bbh!X3R=zGSqf=rAz*s z7uF$H*59QP05TCBYAF!H{~**$s~-evDFfF*2{11^89Vd7*?E4B@Ve$T8}}te{=g#+ zUYp7kLi`|JPdd7wG-Z0zM?PR8Vpp>8a_x-3TGRcDBbz%MY{o}cZ%9<|yzjWXM&&6E zIBBi?J+-$eI!U@yuZA6JAh-VY4}3Joc);Z@Zw!D&WeO`dI55$8=~w!H1vnf&}KGv_Nm?@*!xPsn(_0xqQXuvFPLZeYXGnqJcq=H0FJgh5Ge zjKSmAGTWzKDbTLHT(SER$=pW#%ExomOp!Kf@b;CaqutfWxq_JSReUu)!P)e1teE*7 zx=^xWkUq|-G(PX!W#C*qK@`vbwkgb-*|aSpb$!2TN!7N(ev^=W`J18G>(#BdJJaUp zCiHi}(gfe~o5Oc;monIww?CdWs3A!9oR1-W?KY~hGK}>Mg#s$4>r0Pt?b*B(+)3ls zeEWh5W-<@W0*Z%Zy6DE_-zJb~J2Ro4?BM@XyiCXzuD(*yt-?I0T+1mSQfrRl^TdXQ z&{7536x%_1bWj)R$-c#ge`=nGc0Tzo5X@$zXZy!5D_MRs*_hXQtvHjUEG*qcjrwKi zdiLo1=(!WNs#mn3o2aFRLS+Vdsz0xoH8nIeD4IDo$BauMm7`R4tKqT_FsBz<{ZQ^#8JkWR z0ojA$1;OX6g?k@eubW|;e151$on}Ej_C0PlZ^4+{a|IpF&6zI^*pwPWcZ=_bRB~HH zcb+j|1Kr5LJZmJ7w5w$j8uh}TVM%>q1z(&T2k-HKWcOAWY)5)4ln=>#`YI=;>;thH zZVVg=o+t}UkE&>?XLQNjF!#j7+dOQN9?=|{Um6OMPDqhtdzBe=)n}>ayfV)SThG%y zPNFM`Kb4%qA(CFw)cSq$a5G6}%z{K4ERfO|SGafUv*Ij(W3gSrTV1BXkQdXX_Md+N z5I|)AjOLVn(z|yPx<~VI^v*K-Oi!Nk%%Eb5(eyxHX#lhO@H@zqI-*c*fi>}GXTXZo zQ@i)*NX%JG82T5ur>-xUEUy(WBTn=pSD#*o5vjX%4G#MYB6!A&qOV`?y27G3W zUGd0R^DSg{D%Wgs^tivft>CLugI=#OSVR|!6-%8mWGkspF1sc9@0a%G%`VI&1C~ML0nhHnmp|P^ zlSE7gSSGV>E>*HmJknY&r)zwku=AM2c0+c$-AlP=Bn4OH7%})ONEg>$$^h#9{TNG^ z)X4qw;s0-A=6%B<4wbAG6>ABI9S@Zn&KcA0hQuc!>6~?u; zighYpR9H@1PP>8g)!NhJzON@ya?x<<;-geDyTvrn17vycnD%`bf)k_OBR1uM!@QiD zq`;w_1icJ9h1S{g{K9IiV@bEcP(HrA78$EW9Jv|o$xgZ_o9U2-DC252G1JtdRVVws z4{iSWW*7MN>~J@~i&|O4g`81;pL^4nf{!SSDYmP$o9z?!f50Igm}E&jX6Fg(q$B!7 z0X`#3^?xmt@^aijypy@8ni6=RWxy4F;}%Nc#x?FOZY^x=zW&`t;kj@X>C5kLxo8VG z833`yhwYl6^c%#QhrQwxLa;QA|a{r8ny7*@TuW?8Q2u9+>`Je#ZYc-=engFf&wXbq#Rc44usj^+cil9W@+ z5Q;9%8>?8a{W%(xC7m>~#{HiY1pT*>Y{_9OHiRiGe111RI_lbc$Qc3(q5dprIsN^0 zk>helsPKo=tuzg96M-8RQ{J>H#9i**Le=E+Tgn~Mv)|*9`P!6R$p)r|KeP_8vIgI_j+?YtsyE3wfjrSzE3>ihs*WhePKj0Fwjr`?L~jE_IMICqy|Vp-0I+LsTqd zG6hpI*=v$oL^amr(u|N=(w|i<%To_0S!5JNMeK9w`Q^#@vB3Sy<$Oz}Q>>*ET8`|I zFO3|%B}!TQso9+X4>8*8)R@~m2j4AbPkWIzu_#5uoFpV9R_mcJ_B#{hq_RN=r1$B# zi1xjuDpBzz(vB>&nL+#O3z}$5wxf(?A?$mG>jS4PFTBwS7M@#F2c_rkltmO3Po?`$ zF?Rm~j*|oKi+{Jc%6rO%P7$Y>r_|+V=S{5WzYhif6li)SZH=(WmJ9AaT9Xk#A01UymMXRAQ)kC4;>2BUWeEHn3Ut~~f z>+*)Y>Hd!HgOBUi<)$bpsv0=on+1(4?Xs*iz`_Cp`S2dVaCx!EuKSZQoagFC-=W)2 zjOyN*R4D`N#tn2p;;9%9tWcn4ia4Q}G!`CJ{upk_WC!By(7Z4FV= z$4xV>P_!~4?oqy%v;7a~0PtDg(9_Hhc|69noSB_X++2?IdV2@_qFti@IpA!9?t~wWmHB@ zz{CS0`vbJ?rU+Ofm}~kG7*pdgIpgeqLff>7@OAu|DJJZHb2&wlG1htKF==9pCf@l< zN=k+{erC~z3iPO$fKb$Rvju#QiQBPqA~~KxWHy4f>WrLQ#^1GGZRiIqH2ZvXDvx;O zjimR22*ltiLArU2Y9U6K9n`?Vat#UGS=O7o^SU-CkvSIyirY)T+ks=9B!hjR6#I)b z01l-iKd#K}AB|OSSh4&ZRzKj~4Z;VbD}^uMYM|C*M>j*t*Yl>>0qP@Ij+}CCyFE`! zLy>Dpjy~2v&>@u?0zB{XDekLOn*R~&z_BFW&I>gTdSmZK!o0H?p4sc?qyjSLgDNt&|%B>oFXoo*$I5U+#lSd?swM` zHDWE7M9@RZra=DY;iR8bM0Z!bZW<|2a4*N#pWN0K6?LBP z&fD+hJ>oIqG59{^43)~7m~E2L;hmP=sgRm*lF&A(WtnvF-6ojGDw?@_Ki$ardWS2& zMCQxS(zOS8W_J-Bp|VN=3}B_Mj3{Hg7rbjg9;B2FB&KBxJ=PhMQcW$w?c{t6dOE6{ z=af>1aP6Vdw7Xuym7xx;c>DXSq5#P422e zVHDR2(-kY?iU_vvyD72w_P`q{vHK-aJ+ihiTxug&`oA!zD+gRikkxZW{r>Ukr_M+( z1=rDI^bMroSWVKB*U!?JA)Wdedbmnmb}Ix0K5e@3OE6YCXjH zKf{oz>lpj@nEY~?n`P3x!VC0MP2L?b-U~=qai8v(-8Z)EIXPF6LbnuL;UZo95QUrf zW&o0iE#K(@Qz}#k`+34@=OJEuQE4ORgsp}Is3DE8Cr|~68|o4U3o8%f4~Hq6Yy(Pq z$;S0skCR{b>B~pgrVmL1#M4k+~ z!U!dMsXVsSb~Dh{wIzGq>HL{EJ3N55x}DDqv9pLq+kXG4KhbLmV-H3I@1s9dYajZ7 zQC{JgPG4U=-QoLEGG!x$`QaR0Q&$lf@ELaxe(DE z5gKvn7Q@zIrh%7`rwkpSc#qS8W#U~LE@W_lSxJT^{DD?C-}G@EOuOXztBUput;MI|w1y(rCmokn}!ZuJu& z6|u$g3GPc$Uw?PKV}-&Yr0Qzv)t8AeyX#&;_ybIi){Pdgbu-D20eb+9h63lCFGJeU z|3PmneR99$4qRJsM$FJyIz2zQB6Dwc5{*YkKmB6k+Wk|`qjpeD48Ly|=4(QpI~ud8>wq7 z7tm$b7x_7&QQ?PGnOKMzwjgtV{V$mn&>3Ld^f)1Ps$mmB%d&SEdUr_J9w`i|c4cF9 zjp=;``opmrS=fVckB;!HIopbo29iY}I{6s$2;EuzRtbz-NLtOqLq;O#x0$`kTvf*C zOWPoB0Hi6De?8B>#uL7O@>7+9NZZ~XbkIW+?VqKv zJKj?Kc7B$FL9Ox15`&gGPftp23Hw;e479^R+tgY6r(e}enZ23`=^e>g69$`n%RbKT z;C!{L6u#YRV~ltL&Hm!18GB#t#9|?-_T#gcqng+PA$(o=`D&>>D*Omwnjez}jt4S^ z7K9@nQjKKrko2AAFrv|Hc%s=`w`s}bJ zc|`VsP@GVl>@z*uvZK&ET@|iUvS2Q1HLf%90i~AW>{#=nweOgiN8q@$KxBHcM2BEf zzWsUkT^is^|N8Ut#L7q;1yp7}ZYEY2E1XC{@_0)^RI43Uo9W-5Y?B8w$-L_q)H^Th z#B~JYG53N)jt1A5*2XyzNr?v?G*-;nlAlSpQ4n%fq^T#~8?QN6Wl1|M0g}Rx0A(qN zU@m6GN9rw)IbM%+!fVBm6G_<{1&%J~^D`^NVLOhBV?%bP_yCx$?8A7`iUrmEA?dw& z_T6d0UMp&!mX7iKFv^q9|0rtjYsR}s&x-K-LD2eTmO!~_dJlY9h3J9KSGq{6t*$r=>IK;^ zBkVTLpD44U3%q5Y%T>%rcNl8cG_<21$sfOfF=wL8nhqAOHnW|}yt}zp+{~Fz(#X3c z{LWipGZLGb?kf@1r%@xD1t$rCMR2+Q-W^1wT@t}nBSs_PBLQe{EoBs&uiqc^JXQl? zTiRO3>Q=3`v-AC`JNW|2NRymI#K_wx`>sfTFM(^`_bU3*{* zv5d~~h1+g(8`twI-2u;(Pv_A!=L(srG$5sqIddKjw*@)?7DL^l*Eo)icANeUZFnJg zpLX$K4|?77&M&RLIjE_r74M9MNgpL&`OWW`+VXuEjj|kL*n4sTlj%QNKmDx%4>#gA zYn|IiA3YpTSE7a^3T9mYp}5g6K><9<%T5+MB&Y84o}|?g2^LHlZA5(~39fa&_n$5I zZ*g@|Y;9doil%;hMogT8FyMTz&2_#65vSMUjL;phhH(-;$*8$*Y~=WiF(4`ir3}Jf zj)5KkN{pw($qvcEQR{`5Q9l%|AH{pWz_z6EDBr==7?H`%Bz!xjP;b?7h<3czIn7dq z@?eXFh@wpscwqooiO^Z) zf}Z?FStE_R==;_SI^Q+d!ya_-+5dT%Ee$KJBTQVoDP0#OZ@TsTl&HV(XnHazBBgTa z-J{!Bd-{@lB{pCc%_mt0(YLEYXRE1imW0+8HnHUrq;3KFN2CGit{IFI&CsxfkZd}wSc9zkb?JDEBEW{1~A@rf=zj$yPVgiDNX z@JV)m9TuPXI+*ED1vjz#{pM#iO_lBR?zE88`QTe&r+!Gr)3=tm>~U-n3Q5nbv_!2oVLdZwjY+`mndVI= z1A8Yk-7lFv8W7Ok-y8`-o5t%M&npLbqk-P_qBJl+hlyG0ciU0<_q6U8HoA8AnGam2)`kRn|diF=?kWXi{z-O84GP$nLJ*(3dN zEh-O7noHXG+<)z4v+DGK5>xQd=g6m#dM^N^;%9c{_)SB4QM(Ou5jVsuZHhVoDIgXQ2j~N!vn{6zV(gk(g6;tHKNPWc1%Ycq z(%AS3QGT*gJxdRp!4=@1U6v!Yqt?11BnMeAMTd67Crda@OIc|`d22E;KQEMKnKja*?X0mF^t9ZO+#)C+ffvF(0O64Bb(h;x`Jp@memW9{!Gh;@ zFwFWmpQ=X{5fwMKj1cD4sSlJ(N3BUS`@PzzeDIZ7ki15=uqSC@R0S}(UKnhRPfP%rB2tcRSn6uyN&T=Ccfjmt2i(C5b3<)>Q zy3R~k*fmCmzPl@X1N@$t^a}#fx@GtNoh;D`4LG7@@~8MtvG)|VPm&a+w0S9>YIVEb z%*kLqfB-n(EOQJ18a&#zVrqDxNDta7?;Jn*7Jh3u+MiQQg-E=59s=3^9eZP#F&Mh( z$_D>XoyS+Ux_aV&v#@vaB7nQehQvTxjkZTtijhnB&p;}HkjxLuYDHaLH(wU|DRv^o^!59#iLk3hyb>v=`Vxl ze=JE3Ee}@UQF1+)(<8AhdkHAk6~bP2Fh_M=o${v3OlnoG^sj?L&Ai+p7N095gFlOS zy;)#(jXd`4Wpb+>5=^?grz`>Cok91y|0yQ9h$eLnZY^)WbDR3nis}=xF;<<*(|~ju zE`HV(1#2!MmxlwL+RDzd#qtoSE@Nh;1f-PH*6-P0u>#CpBNLmc!fRL5g-HD??!#=q zI-&POBxgUIBJ*$~y#8TFyGC(0>E{COZ;yzu-WL>59Ne&Fr22G8z?)gJK_5(4fIjsT z1W(Z5nGi z^3cpEuV!~fabx=j0qy5peDimfB+Y{-nDDet!51XOONd?}jWsaQ*oJ0!sz2E}qEOkI z-UF}+I4W0@!Y(kxSWd9g&)-aW!%+O|pO6xt{LohEt{{w&ws{rvcDOHIDAMcmy;vGPUVG^;on=6_7aD zBOeSt!Huss`R_~P`ki*LI@lR3l@wtk9B!8singnBi>6vNf4>EizRBzlV{KiN~SB~iDpzyl~Z ziJFMx8Pag^9;@6pt z&QoqVWwkusCfiFA_`PM6SPn=z8HnygtoDDg4(sQ(dhg9^e8WMDWD0C*POKHeK1GgW zliqLn6QYMNa(T{4#(Q$O#dTJVdDec{71$DZ>W4+~uS2gyD~xv96k6f)IZ_?ePQdHt z=ySZ+(&ssJ{S&V^Iv-9NF4mV(4tV##XKX*e+Y_xvDm%KtsnJX_$vm(3tf3KJ%4w(6 ztI1o}xv*L6Z6f;ce-|>uo7wkbKgKAwp(i*l{Od6!-#)Mt->HU~$m51pcX(6wh*(mf_AIVVO@v@>?UYWoFE$$;J8{j)`)`y5u}L{+^BQes>wjjXDQeEcwKDFl z&+NaIh@4*QjMYK~NnRhQ%>K7dYFusmo)>2@ZB_`h2cj3E8&)zG?e4HBoBw21JcK3gxKUQ6eYNO>OYrOP!%Ou0qJ z$D#>>Rk|C!>)iYAvf~{YBWucM%taHTd^%&B-P?&jE(9r09c6NRyd?4CpdvhS!A%*j z3`^x2u)O6Dc`t6jtMmxZQ`ZR&<`iC}`^b-6Q{ls#=pV8wFUsfm@zeAdL+nB6LSbRD|C$1vZxb|1kzTjk|t{JhD1GWIgF?@Qii5$N^P z3?W-x7j7pB&t2H)hbFIsSWJdkl$!w^d&mfm)c>yPz-ON;PuExnv%kF>3Z*iNPfuPR z>sK0bP&kf%e_iXc+)wHk&twv=8HOYFJAw6XHDZ^>86ZW@c4~@Z#}=?9@6VZBbG9RbZ8WQ3V6{sMj3YV3FasRKQaoIo zo%J1Qr#>AHF=v#qXMb3T40=RYI!f2`Bcf3{sM_|$)Je--FIKir0E3vDb@hQqxvE5x zPLF95z6HMOe_jaCp7Iuh&(jiZ(_hRKIn;ksWCC@f=x+(AL>MWd0EZwLO&hy*>O z{nn9GWt9)!?9!tQcB@dA#fb^3fq90$&EyZPeq|+_uaMb)$QdskkFr{?ij8x=@{ZMl za6512(coFwra$S@#AZR&FYM-F;BdT{b?oiJ&e2tMb#>SE8ftUZ9gf##H_H5%2YA?~ zaA)BDFB1sEUa0cXdyK}1L+IjC^Dh)xZ9#&Cdn+fGVUI<%_n^#q|jxzXf z<#@xoyFq4jp*ZKixLEjcnx_3sb^|yRaD!DwCTAV~uh~|l?J9=Hv{^J#1y4%o&0zUo z#@uuKwVr{mFG?;kWa;qW{_en)wDm80>N%;4~ep`25ooI%xB%i#6;Rf+5L5y=W2||%%6hncYiJ0d-bcaw7VcP)oR<)bl+ij zx$Q98cjc)4rr{JmGlt)Hw!U)JX4c zN$6v!Hr2-wX<@xGn87FgW(X8TShRE#CtXZeQv{ND{h?sUqG+L!2nDU%M#37$l6Ca1( zP3ys>fpynbUlz3SA3z$RXGLZ;v+tUCB8l*V=u%wK+=~(r14y=v3C2I_MMm+Tw;VwE zlKF99Qq1)o@Z)ZA{ zjN!@f)Z5InvvVWPEPN8mk}|%zf?t}0MIp8GXyy0UjQuK7qkP`?dA} z=0L@~yl&1-jiMg#8&D0CRM260z8gZW4zrnp1j@!SeTuyU48rUcsAtM8IzOw(WhJVM zKbcwXGVYS?Dvuy%@9~2?1N{V`=W>gNf387`mhBwEBYq8}K1;2|$E?pTtB(U~m3zh+OmDOhMD0=i+4BR&BwJ7YiX@VZS~0@ukm}L72eE{=El94K3K1 zgZN9|GHtb^@4?8|B~8BB$^EwzV>-?(N>9V-&TCI@*MI3g(c?j9N%890hRXMyAM8Zx zVK+ncilUF}riXr9wvqlac%yK7g=i~SMbZekpPs8KDq*KVM=)nn*J~cSm?q9I?2?ki^h_QnbOm*k7XV92C@S5 zfEB8=IFvR#R%!=-7x2HfHNN~Q<=$>QN_vZ`G~ak=RvyJSWo-()2aNQx{32xp;Jw`p zW>vES&Tf&q!r7*x@y*gf5XI~6ha>L_1C^m<{(|p9)UdUgksRzk(3^FGe-Yk_9qvn5 zuS7M)WWS# z)k=f?Z{ca!GMB%yO4aR;k&%h?1qrScc1yH|`rBD1Xw>E2bEcpyfc-%aO3 zMrlvr1b~%>@b2^03fD=Dit2rJyKw7oVSiQmm4%CRmTgiC(jc;?Dda;nJp;F>awVx; z&=cE*Be>sn0Gf8Q{6cTF>8@PQG811Yv5|FD59@1Tcfo)kjmlh2wR`m(xU172*CPY| zMMtna%F`&w1*alC);t!^_v{GPOtkl;Kq}LwixUm5|`WBsCkZq{s<94*d$_4)=i#_Y@Km&;WPa zI^;Kt%v&npc8s&16-80&X+GbySL;a@wN%*D+Bz(XHx2yIwf-T{b@y%fl(6sl?iOu0 z^MgcTik^Xx#_}~xGIHW61#YV+-w(2pjW*Z0&NJe?X`>8AFnjL#{4C{@c)0mGbd*#& zPC0~j)N`#nzo(tbM_Oq=g072qeauEf;^v4CN6qlBN2y+!(UNZT!&CurHl>=|QK&6M z!m%+fTq^hQK^t)M5W|AlA}+rhKrfW2bMI* zm`%LOlr2oM%1&zcPhJkXhuwJ108jP15->c`W>iHmoi&&!CzL28(YxPK&x@iXR_a_o zL((T`;uBvF|s2CtAfe06*H zG6uD3ZBxpF&9~>XdG%MNUM*bvF&DLUEj$EXr&sI&$LyMOd9AZHM0k#Pp1V+G8P;sZ zyGrFc5O<+Q`iecK_Me#CnrQn(*-@BbR5It>X%AF4I+#khJrFt-Q(c zB@qYczbD}%xw^v#Jk>;}6q6DN%A->K=MZk8+&ulpw7M7Kw2@k_WlfEKANS^~hI^uE zY*%|?;OzYqBEmCYRLM(TZB-!<>T0%}DSefCrcV&818#G7XnAqHa&Ko&Lqs!9E*Srf zRht6MBQr&42>W^ZsyP<(A^-|l5q+7WZ3o;o23R(5hd%~Lh4o@*jyLV*Kd^aPngXamc<>=C+xrBAKVv~lEVWOMobVDRABJXyLf zGP~{fM@z0s@o`e;qVKVe5*qQMrjA@k_~MjTXclR(E*99K4}4CeNTp@|T{D0(%hsbU zKK}*G>xUmy#&jQUOIm$b>e8Dy77L*ZNveJ({4_Kc@CpzP1E}HR8m`Du`l4mHg;xLB z#c3y_T1O(M!ODgt%1Egk)6wVa31^_|PN9o+j+LM&ExNFHJTZm*#*mzbF*X9C_ANAw zEc$Cbh!_z1J9+o-2%Obl>W^_oBl9bETf#mgpVWxVzfb%>`2CtScN6d(fS6;o`E2}nfGt=z>{cmPmQ(&!S7QHde zV;Zbc4_NRedIE!A#~pg;!kqMZ1-zTBid$iQ>0#GXP?}woQx$z)Y$c*9v5YGD`G}PfWb|OD~vsPTOc!D>P>aSSdLUXEISYzS=s8 zhDRq0?wTjsYVdkCPHkPurvLAU3H^v1HNrtD$mzn_P|` zXqBKMm|L&^t8W>kzRj=cMUrdMU|3=KjbfGs1}19Y4@>JL(>!-PfzFBEJkX3UtUB<& zXwuZNGT1{m>T&!8qfi|7d?%YO6OUAKF3|xCUMtT~L(Xh(@*g~ z_50%|If(CZ6pke)eVRi$jW(EtZs9Vug3m1yAyP958pr29B()!o*{B2X=Gn=hTBM8i zNREFgD%Yn$h~*ia4-fHs8B@HGWYb}d54k7@YORnzsmI4>XKmA8Oen!0Iaw*eBXWV| z@JAOX8$&hZ>Z0v$hCbPx5qs3SPpZ#SYKdo6x~?L-?VizfJ?xm-c$txVncQc~6KYO= zoSlG?^|hc0i55A_riambGWD5q63u{q+trYB4_noX2LmGMndmx|U&$N%Ynicw{FW@H zM~)nl2UHC2-RvO4cf}jU!ys7hQEEynd2lYHu*pp>y?;X? z%v%|)a=Y!4UhFvsNDu^GqU(S#4m>h}-Z?C3rdr_ZEqB$ig@C)wfEkT|>)+JrJ;(Yz zie^79=bdL(58f7JQKi=1O$OZRMSoG?P&ia@v3}=&yJLow+5sF1J1@}fEYWtbo|V?G zL~_ImQuSZutf1iYP$6t5ZHJ(GV>Kc%awd8JDJc!|89A`};l`!0P`aef6 zt%3*PQ?f}}LA08Cri^<`%1lK9^wM95;Qy$1GH) zrIao~xv-iHPwbo~SfZMYXNfn)+ z^?TC9CndYBXLFsQ(V=nSW4_z;mLcyEF_R=;XyG7m0vxoV_4Lwr@6j=3m$=6~NyAjb z^h35Y&i@8_!YCgI;1%hE1(v-$yag~7PUXmbsC=l+&i0q=*4UxTgXl`26kxWkW;USv zL15-QrsFL2fl&0P^Uy0MsSmi&Ui<@zMX(RO0ktEr{UfKRrKgr)9gH|_MzYA*X5cR$ z1G~o{2d`({d>mDaQDjD4LeGrjs28{oiu9X&RMrFJETfJp@Esbw{!D6p$_R` zWfW)uWDvA1DYV#aZybll9o_-<6gqC3BZUVdkNYL5a2)#S*z0fTW(04hYD~M64q!(! z#em~B8M|=NSI@gYxZi&#u0r_TpC#bDKi*E)HOE^J@y>5F-PCv>YR1rzRmds*4e+Y~ zoo}AK7B&{xnJgO{I}uwTbh*s5qod+l;z%3MT7^u4EoCF`?tqySf?-jL411wHXlmgT_D&pnecpzv#_nhl+j^&-+4V2D~+bZsRg3eoXnM={Kvoi74=g!U@>DEv7r%Ha-P z$dgU_oZ=Zu7=37@0(G}+FOx`bBJa+3_;V3R+W8l#d$Y(A(!rCURQ6OF+A0Tm>7GYT zv}GEBFJerJyXI>OX%$Q+XodGK&=fyXBBG0Mb&QOHOLTWF9AwcPgYJ^E8h)gD3>ep| zL=n)z5MAPn91^JPum&Kb6EhO5uco-c9pR~`A;QaknBwj84$OGc|EgSG1x*(`C=6Eh z!C9BLfbaP+RyPQw0&?^sb;behwc0B{2zE(#r>}k9=a2vagZwLCnNh)8yX`nSJ!(BA zT&FZC*(rXyvA&MOc{ruYeC)3}wB{7R5xQasiZ69?`-7p&OCy`|kbm(a?+;ArQ)M^Ag#K}R$P&UlB$qw)Um?i%bru1UrIe=s@=%!>Y9YDxA>JPd^ z?ajRl8EK`E6LipkKwP=_15?3 z2=dif;e0{*38!0EQAKTh%+7X0SB*5T*RN9!+ep3Z3G_gQ*)pCnT)LOeH6l!h{rCXez6d72k{x&2v)atAf}D;1mhj;kBrjmCLqgA3tSyoh;O4+@EZ;rl0r-&fiAQ1ZSjwvw9By zE=n4$-Pvq2o7~#^dBq19FvTBV9;j$Hbu^AVc(U7-$*I8cf=*j090{q%R-5GJ)V`Z| zxC7&xt_V>@I^O*(luXbqdpLL0_(1Mtvh^mD zgy;ywV!A01dhudfX|px+JeHXC_GQqX4I(1ku6iy$-6zF{!tt*b6BVgKeizZl0R5;0 zC!H!Xg)>dod|r06u$uP`u{l(rZ#YiyAjlKi+r0j4-T@JD-0k10(R;jJO}+g}x8v%V zGfI?UnF}2v=x+43>!A1p_GzqG1Z_K%9I3Le+7|A7uj5 z?!_ERCVyIdUc{W~{t=M5(`R98rfAmAeuqg-V);=k_g4ZpYmAZA!Vlch!QwX6LJccQ z9&3cvL6E1P^MHlSM2jxrW-qs1qSM53=|xeI%boWtIk! z#ZiOI=>rEo3K`?3MtF}hHlnBDJp_d_Gn|KX*++&8wRgtyG;;60HhA7-b-@iR$<|sX z+&sfC#3*+<=BA@h7RoE5c3j~WjrLl4^y^2BZQ?dNS9S{f9(9}*JFe3hK_&&dfmMdC z&m!%0{^ba*p!O1&4I1$qr5N;f=rUM5?gw zo|PU=brwuYqCr(HwC~?&a58oL;7dhJS6@`;r=||7qxX22{?1Yep`Q#m+SuCbpedWM zX_x}6WQm(&4fTWD)fd)Ur2G8Y<#n(aAW=1iLQUjXW%4hQSnf;e(#g)u_dytr< zY_S&lBu(cOpK_a8qRHVdK3(Sn2K{@N8R}w|_s+l5^g|-(t}FTczR2*;e+f~9Ll!R} z2Jz?G)G4MZ?x?8A_*DAX3G~bF>>dN0j=NP(<4?~XH=3-^k{;KEFVy4syPsf#cLVLt zY5}umwv345YN?a9HjAWfX2C^MtlKoN8eutXayrh0a8}TV%F&`J;G*kFv6JBgQIU)u zQs5ozpS{LX8p~Mpd@Ba(OLiMhqN( zx%Mm(%qYj@_G`cl>8s!WHp=}WmpWGR-e6wjY-PCnCCBFQybt-Ab^=)|6Xd5RXaM9z zwT}NwkP7KYFmN$DEeh&37X@`lJ?(hFs!%o?A~RX#wR&W3Wf)zJce2Po{3K!#2Gv_U zbAe4y@;EY|m=uUCYl==jvU;Vv?D4sF26Zl(+Pg6#;gJtUF8q5$1WGClW(-VV21|_5 zK4SsmntQE9Wq15(s-zAHF9Py4%bwDx^sHrgRr+xG#n@+BEjHUo_ovqNh5L^ND z#}jWA-)yjF$Q-3KBVnB%lUKEB_>#d*L zBT9k{HLW%~r7bM_v@A@2YtMYd9e3+_R~pA*#AZ>YpG*qVy=-w{B;8~fGFm0UM^|RV za!khY_WCE>1(M-M>jGU$G=|!+ALjvn1vWJ-)$F)5{YU4;=jd@EC4rhC8}82rrG9P#CG(iO?kl%@8#bs$7LI{@zPGemwPXtrH(!h+8j&FEK*J!!f35o zzx%Ih4^O9_!3ra>=+AM%$fRfb6_=-+-)fY{7W4)UF`$uYc?G*t&Clq~VM{R7nDQ!6 zHPW#DmRUl#w88!@H)KmC`}O1shgy7yyy@T-7R-p1lz%R!h2E4h>dyYaSIOG>&+a4V*t}t|0818IibYX*BvGEaRmrq{-4SW2>QnXC zB(0T%=GyIfN%-WBaZic(N%y;B_Ph@0GV$1ajYzgs%$?u4bj$r6sw6J^CH1KYoHvOS z&}e99g<0pnvHEhM3N*{l&qJ5&Sy%(`YM>#bYI*7$hcPRy$UKbzVt3i%yCS@nQKa3n zd;EM|0cTWE-gneE)OL9eW0Dv{&i{_cbgc*e$;mVuJ3A2+l!20Io^-3Y8wd?Zd1bVY z{~i9(`M7)v6hTm+9J1IU-fF781uJ!-4+c%APForBrL)Vt%gl;$sxYE?C0 zGyWO{oPq2*Y+91Ax;I~*anZPW+l=RJgtPU%>5Dt+tTkmOv`6Rm2bHZNH2!z$%&(@V zr!&+WrbB^U8Pd4-OC(w^Zj7kmc}s{3*ZVSi*-W|d^X@k0j@a%e6IB+0`MNdTs`)xr ztYi75BOAQP>M%;nmLFl|b_a|Tl}0MHdKGr*`Qt@{s{R5Ys)glk7GuaETnnr5^O5pT zzwDu=gTBYeAsNewKK20ZFFo{QbZCU$&iedTX~T3#S2_E?&MH5?uXOw2VwCrEL3@}U zK|Q8{M6Mmv4&O4PMogZ^;Rg_Gx?u^ADvudV!Rn58fw1y=*@naw7Z7QE+%qUFM*(+}+~*%UodXa^p6IL- zZ%C_d2_9cigZ%eJSwsy}$i1|@^2wdRQP1a;fh`lmx&V`esZ?u%^W<840)XDCSkxR( z^S?`0v50^U4VdPjE*ZS5ZGT?3g7KxM|KxjR@PRFP+4;Gfg|#(cxEc5@&9pPnLa^g< zQxH?l9{VcrBCptAw@kV0>B?)r_`?mGOJ8=avT2U$Q)TGR$c?Em45qqTsP4&c<+zoe zuRdd(IbE+WN+x$;uU8hJ9S`WB>fcD%^|nGL=B^fQTx;0%%3I&^TYJ}?mS9VQ+gz}$ zR-Zift68i!7`H7G*We^0 zLqL|aiY}su-pIL=VTQfUD`jw`AL~43*Oq?t@pODec9}0daQ^a|cBD+{oo7-@yvoOH z`2t{0u0981F6f6ixqr|`+})SmCc4^ttp&ya2Asu(tPmbNo++koC=|LNyj91!*CF%o z@Mg=mx7{g`p6vI@C6)8|IAGXNl}$ZI8}Wj3IX36zD6ijJN~^JgsnOqK;FB^7OG?8n z6=NiPaDi=R*m!(&Fblh>sg{?Pn4t*vt-rGCI%2} zK6p`U_qysSmm9v83&n|6G;!a%v+Ukqd$4b}~Zx?kkmCM8PbFDhc@iw;DwLDpjgzM_!egS!< zV8F35?zDrl$Y+p7fw$dgmzOl9)pP2*f-_4Vx3YZgs^{fX4W6FF*+BAyV52WZRCKyV>^-Xlog7ydk~ZbK_^k0hKE%;+tU$+Z z8laqQHDhxk)_!Nn2yzgMT%Rr|xk`OZ-K*l*u4I0-RD5DdfKami=wQ#4da{aZ!Q_W{`5c82Lpj+536D zsKsbr@omfsFkawonYEl?>D?yH4|b`L-TmZaq#+wABF;NGP7W5gH;Us3F$dADv74I!yJ5|J{fGS!0DQM{WYUV~bj^VtjsEK=?w3)_-(ucU2 z)_^9Bg23`n+%GwVEZKBt9^;h6ZsGrVia1e7fP{buWN&?m#(k*+1afW@O-hZDFIx57 z#nm(L)Mg#bO-+@ zJ?j;1+n{eA#}Ay2T}da#S9DFWBSgpF^6h;TJVriHRa3L6oBF2u!OXJEa_!y7`i4E= z9MsqOc9CCE*gsTdNxZCGB-rS^@U#}q`>r>ZICk6q@q|r;1ewG=pZh-XJ7dc|QY;Gz zM%jp^`IVX_WPTOrw8UU-IqTrg3>D2X8 z>~|JYfL5EWK^brvcOPF1(0+Z@aC{rcR%)#VG_tK}=uViUSzxEn=hJZK+7^+sE`{4R z?m6a4fK}fRw6zy$BJ#0O#A+`Z^XF~fgHlq>VwC)*A^748z$-{89Ro~1#hfUDaiB^T z)^82?1^_;qd&|tNHo+wj$w?4YjdZ??&J`hqCbm4>#+y9lnMPAswfvPCk3Q}i_-~ik zz&5mwPdslp@4|2*$nu^;#>X_JaO+g+Kf&r?H@Tdd`pT%zM1H>bK6 zJ^eRBx@YlvCEPQYc7iv!;9_jun-79JVc^n`>2ovaprDAM&1qucIOWZ;#c_3R!$+I* znwz4En_oAjC$Ljir>ii0%_>gAuyNzujTy6S7-gPAkNw0<@z6T!x6#pnV+e}2qMt@_pe zCfS;Mna09_G}$yds}EuRl=7t7!qgN|KP>{>H@=RR;~xbpoYkFIJbH|k%WYqL{ zqB~#C7;emV*4)p?`x=9TDGPrOjAFXPIUayLu)Um(-Ej^taq$DGcMz1Z``~BKeuEz1 zg*%P{FwuP?7n@B>-^wJYY5hLn@8M2p?|B>5Kt>7d0%oO$spVy==A@BlAi=1aN~F6&E!yii=|X|MWf z1rI(KZe)1Na$CN<`11MT-^O=ZTcOGN=SG zU1l*Xiu{i8Jx?WxNyq~|UlH6d{>PjlaknnOk8%DXAoFJCkn|jGl`WeW8imxUvJmH` z7EDt}hZeUF60wUKBxP>MXg(P;U|@P(aen+zmDj%ZW$4$Bt_2LziAxmhv0dtnI_2h> zW4fQFsh}-A3Z=Xa+wRcc7Mw*roSLSY&3L&zCmK9jMfS13UvVo{NTPDyM1IA2G(;~( zNAb;NK8UM%s|f=f>JtTi;Vs)Xq~XWFh`IO6pBIe@#h?k?w%pqaGVdr15)|l{8tXz7 zIA%u}-Ng`+MG|+?^&o8BMGE%7vY-NZ{D816qkdw;KWv6{(6~Ul?Pl-P&x=A&>9FW4 ztMuE&86cF(HwP&`w6hfzj>R$j#^gICMm?K8*c?T`&@AKmisSpg=6*3f zaY1n+ahbb_+b z{8BsxW0|O~=s!Lt&M!=CPyUiMrOkQd}ko%U|5kxrPOAT0zWol)9;!e~&ECT8Za0ew>ZVQlvV>{@mNe;xm}WkS;r98&K~`bW&(&BS?R!ULGb86h_70j6zAPOTxbW2R#|HSdsu7@zwbiHG*W>>QAXDOrERa#cN&fV$>XW zh?aoecc4yFoCT$TV%w=&plzg*^}1~Bw~0DOVo0AL<|l>6U74EM??QXT39^1D{GaD} zfexPv-P_A=w>3KpF8dm1(DpHeGveA{U+60y0VuUgiYsk)HxRo1Il@au%O;Rk>eI~4 zndnVNLG^kL^PQo9YQQ`GHuU3y*uw(Ivg=3M-P8}1T5wlP=bO`r8`GK7cHT6x4x~^C zuJj;q%`={Mh=`K2-sL>$!d*(Z>ta2Pbw%8rDB&)}aUz3?}Q)pT0TGWIA9V!6< zIb{_i8nF70`M=C8I7sr1JPY4+FS3tSbgTzo`>Rmj&*to?G z9rAkG+?=$eAn-3!@KI7LIv&MKq>=hW$pq5La%s?TN%La}J^KJDD&CA07|!fvip`bugujPdAb*JT z=oe!TOQAsq81_S#vOSMYJdFXM*2V%$9@V0l=cyRfkNr4z;d|%|0E|+q6$DCsFchq% zygsca@&H-{h+k%rE{(6`FECLOq~bE- zTH>DK6%^AYt1bBBh7lrELk!0Q!p;-j zOFfoTsIqdmHaV^H)3j6Vr-yBwHP^F>0`a8)t!ld5dfYOR*Q%s`T34$yU@7BUT+dRz zGK-e8LWL2xBgR7`1@yE?nJl`;1ZzjYi;N-oJsVicB@M?82lT}~%CzJ0Af6r8UWDmO z#@`zQ;n42vz&r0PFhIqiCV;S)tfMa}yI>m_;c=MG^>q7p$~B@6Fb0SBm=TmQXW@%H zyKM4&i;#j}=-q61wh8ZmCgI3*yoF|wAd~Yiyf(V-$DDWcDT{F%%zQDOfuhpZmgjtv zzCZcnX6Ny(G^~4Q91$jOh4y;YK}+uSLUMyCd)^*i{01q)tV**(YguaUGxMK0<923XM)54lQgc=-HLj?LA>aP3StL*3D&_OuXXm&``8wB< zg=_^NAPAQmO0cX1+e%SGp6(a5kWA9hKbj|2EA;&yC<+X=K?L4v+ICXJy?ndg?&pvs zNh*hnZTucJQZ{waNx#D$P7U$|GILsSo6o_CU&=p}hwc+lOsS<9(yzc7(fnE*`h|Hp zEHab&+uFoYcduyC<_oa{kQkTyKV{Y(jPPgxM^0rt*zne|gqYj<$8qRF=N+)%hIR57 zL0r&6xbu81=8d&gNp~_ePkFuNO476OK4G&Q9lLS&n#H_VPJ=9loAbZhQEge_#jJga z=A`}`u1pC3mEz+lY-8Y0`%8YWM3hFZ>9X#itPR&oxaRS=FfsVeNVqVX;7<~t91rH& z6xlCz;=gRfYX;F0?uG6irIFS#i`?8=`{iZu9~cn(5FNptB()j3x3+6`!Y_vC0_fh- zHPfvUv~>tqx^%Q&T%|{dP~e-1yt$#H{#e0V(l-> zS{_RFK`n|@&KL!Y?!9{MqI1JG{Mnw!rXbvRYp6w%4&l#HwGJsX6^BqAG%QTTGs-B zvrw4jOnpmmn=)~P`g7Feu;sXA$-AQeP?9BfB7GFSUp&{CU7bb~R!~n!bv|Y#e@Y9Zm@+6ec~fS%(vL7)`Gt6J zu#7zHBFf`RDPB|gfowKh@#i7hqPB6Z(Q!RnavD4%omk8IX5+BJ7Ah}m2Z*Xr+j5HZ zNe+C+9}0P3<~?miB!+J7Mkf)H8S`?7U$F5o%`gpa?acF22`0(pOz4pt zaaz_v%~VW3pBww(Eqj^xVbYKNZ0X8RA`ZcP4xARugy%yF>yckR`BmKxB1(#qUmU#g zQ@2L7Y|08@TNcMtJ085!EyY6dk_=z4&s%BE8m^BDw*WHQk1N0r7JrM8v2$0`bhRgydwHIt3v_po?H2-{u3bUE2Wb=_X=3F>XxaJM&j?!KZ+_P&_ z+uiwgx#4HH}iDh81G>%?u?KYL&WDu<&P&V%?sbfE2@lF+#HwBQnj$<`pEbj3FA zBXe;+!*Mo=hmN}s6Oj_2mPgl75k-OQB1L?nA2`%~f(v^e_9It##G|>Q5^d#;t#N+X9K`&HbXWI0QDU&5{`^D+4Rb*$XqF=zt zvQ(!=V;W^swHxJekk8w(l+2!=ANQ~y^FCYvQ(QG06}~lLwUiUfW4_2N=rmnS*D&a` zGftWQX?=f<`EUSy?_S!VKdH9RX-R~Bpbb(${SIF(+0c1>?wva~hp!;w@4RdSOdDNx z$EE1e9_+K6KrIcxTv%KoJg4H+`+%l>^o4X*PJ}0WA**8nLLHD?t9T3 zC_@hZ&a_UOF&IDL>=d_4*s*Jk)Q`=Lm1|#1x zM46^@cF#ULzXkb}xnb7pqThY|ExKnpY7_6(>RNR#=W~Jl6|_pns>qE^~`Em28P9UY;62wI20C@Q(1@mii zdl=s&nK~Hvm$wJFC33;4-z4(^HlZY=Qj=fhk@*apz1)2`{iRk-mxfyi6@{_{ZewI# z0=HFHN!u}UAwpA6uka=BOAVd+rXiNJov)S<3+U=Iq6ew={6U9uSp+b3P1jDZCP|+n zKUzO^2!#4xB(^4|%%V!4u#*1b?|@ehc39pKW$h5=vq{B1VXI;p$~1H{-#3kq6a(!P zQ?3(aj`Pw*ElW4!WEuDO2#Q{T*_IreqU=v5c8bGJrIS}w;>!;oaQsw?uUD_%P*TBL zI@H(z_n-Q++ujt5Q@6J!DQ?ew6XM2?xw!m*0bawWEHhkI#7>5}KmMP5DFG+Nk~{)! z55Af$>M^E%8U-=~l6{Zjf6*I6Rshi}AjAqAzco&E1YYhLcBoek*-U-wS|dAqUHU%Q z70^%ARUCX{gGuC#F4$u?N+moSwoo-w*5_Mqcwp_?HM2hQp`F#|mg(=igJ%5|JD!iX zo_*wh!a@*b-qt3G9VWSVjQQ%VfgpL-OiOAq(8drrZ@aVzr8&74LeS8oQui&g%a*7L z^ju0xBFhmd7C_Rj}UQJA0cQ|?jzN)_y6Xwe8%IsEs@A&75*!^tT8;n)8|)$L*B2i{1d=4 zMt=^a+YM|aP)|VJ)Y@lpse6iFnr)uav&BIU;+ly z@BHt{2v5I19jjKL&F>;|C+k;V0yk4K%QLnoq0WcOO?eFwDiBF@XbRn=zh|tReUxd; zq=eJUnD~%ICeGuL-R)$oBT65Yv+8qGRt@&V&Gb0!7S9&XD2T@X-_ae=_8<6YV`k8~ z1T>QR=npg+(DsUbGeK+OK65K=|A82J`H?;B1FKds$|lpYE3e)>;WL{>Tmg6PXnbB2 zyC(m8tZd6G)grl5HN5vvn^ZZFatMCM)D zF0VnmNUqG1yhoEUHfUa3uc{ww#Dg{F!Vsb>@UAbTL^VFU;?qwRj#9sJLaXo0BI8lj zSx^72cR=%1Xi83O@lWQ%zrUp49cbtTm%FHOJjy4;QDrrffB)-w(?_1zka_K=m$?CW zJyORl@@<#zUK%YakIT_9jH@iF(2aR_&GWrDTzZC4R@+7c^=-+uzng{{?4t@n)qIBv9+gXgQK&vmWf*Y z)>I8ImJ8fDrX%OR+HC#tIKQ?C4!dBKn~X9B-jO?BIO4B2*3ZSm=k4gwkO3|(5NO8j z!_Vi-b0KZnA6ipSxImfMre+rw@&;RB$ZDX|*?28BJ{0KH0AeD9&}m#umyvfPSg}zg z-A`EG_ssDF;%u7ZCeoT!wXoz#PyG0LnZ;(@gmtO##j(Et$p-6TM}4*$o(LW25thHp zlbNe#Jv6cINze7lLbx|u#{<;i;USLni7Rz-aZ~K@$jD1t2}UJ<4+XGpMD6tSG_%lo zsa~LjmLB12>g_t3{>R{mKl%&sMJVUqO---5mcIV)vXP!9tL$IdNF$TU&~bJ)AwDqO zNZfqS?^&)H-t{0`87R$t*9L^OmP+`0R`v%~Q{NSRt8oG#=ND;;NI(t-c=bhB-!9UN zVtYQaQ!1yv3w5H4?rNJOW z^^Bv^&M0X(9k7gvK%wV$6NolEt1gV~u5_vf(~iG!DLrD4g^`%40Xcg!MMjHgT;+Zt)k%$fTF=iw6^Ajyyj#Y zx^;{23snTT!U4ZTjXktQwD4|xUSUhZZ$MQ*l$63(itN)IPNj@?h63cpjwgV+>-va; zX&gEx#jp3aM_DTkM<$$#ztTO9KNX*c=ul|WW}=*(q>pTC1p$0I@cMGek z_($zhEUpOW$NhE=|6gN5(2PZW{Nwr_cTw*hkr~nYM30@L(~Cyur1NXx6V^>+aSOm~ z{CK-EzPwgxKbiW|83dyDKC%47G+p-44SI_)N5^Q<=1HVFROC=&Bhl?1Z?Oja%;lMe zA^EzcD1gRcEg#2O3^Wpq2LRqFAoo`8dSEpB!%W7!VACnnA<2Nlnx(*IaNTuXGdf`)x?j zMGr?!?`?G^KseF)2#9FOSHEU1-)ts|5U^d+VsF|Canj#6yaeV`lUvN}=B)(n!BESU z3a2^kc)TEJfO)(F>{Iz@TCpJbn+#re`VnU4=D{Kdq3rvOD_-ZPzE*#pV3>QCEiY*g znZ~?37bxz3KwlS@%xPlI%6pXGpjb40cP_*HO!)7g^DrbaqV7Ywl6C$Z-?+V zllhRm{aksnU<-KYK1@{DL^w)^lbIbX_l7tTY@npeHHx*VLN^`eLR!VhsnU>ulhWqD z-Eqpr_HYk+;9#X<58`@&>p*>P01G-xK7%wA(DnZ=*h1*aO}Yscp^U79*_04 z!xi`x!`~2E$-09U9U^8cjDJyRRev(mmfobNJNj#^YdydR_yk_siS%~hvTSd+e#}qV z%1_8>T%-r~SD*KX)=c>a^ED~I^W%iD#KnwqL`yYptZju6m>?pWr+zB&00kn<>s(%6 znqiFJ*d)WYBy>0rydz~iH^D3Sr4=DB7Q6S~?Ypl%6(QQ{h*FE|C|Nu%pRf*zbc*@# z+y8}MTOim2X?qVVJgE}-NvhyN5FpjcxS@q%Kw^qsA`Z|+uu_x zMw@uEyDPg0uFpm)^&JJoszQG{>{;zh*D2MSn_HIv%o>@wf_(OIl|HN1dY>zaY|uqc z50eiqJsA%O^Ht1X%8u5Gk~iY$;au>SKvB_)@%~M{>m|KSU-0Gkt2Vt$e*mQqF1hs$ z?d?8vmu{8xy0~4-#7}xGGJuN#*|+5wh{H_-8`OGch(UCh)QgNAzU$pk8@Ml^WvCe> zHe5|Kltlo*)8ktu1IsUlfeHZU=pif$>v*Pb4n%KCVf)c32uhn;F4R9|M8G-xS=M6* zNR3IRNWPYTuhHr@OWe&Io|*F~+2CdO`NesdOl!R(5DU(EnQL_r_seMvP%G+Arb+Q# z#45i!_i88dJ5+~JjhmlU=Pm8u-rlb+2J|Fhk_esnTuL%bdTkU11tGA_!ec%#ISE=S zUbNde0$qYukPP_ZTR?M4J?yMUul!cB^)zfOfNFApysk6~LeQ4`j&|(ZZsRl6yyew3`4s zL^4IRaB@`6-h%?EPH_4<8Fr6}ckax>LAB$?CeRR5R8hOczBjDZ&TBJWTe&lxCKX~p zOfT~K%QHG@DnkBM`2d2ud-42#MnS;itbj=+StNOXf41nS3}DA)cqR?_{*oX`XG6oL zLYKMzvQ@FUs_oJ01gDkslOKG4s|M*yzL2anXb?opGZkHofBJ+q4l6Z6nRpwI{Uh}x zSkY!6<+IB%TqtcvNI4uXust5K4TSv7wWr-~b)V|fc6$w!GO2i&Brj;BQl>S1`uu1k z{Z5co+!;M;5PgmvE9OHjFp37kfV(DK$z=xM5ChpyBPFT~ysP`L%1X@BSARJ&z$?MM zEoQG@gKV-gZqiO6KxKf)=uGJVL29`T* zQ@(4ZKCF^P1C9>uBnjFc;}YL_vNI~?xxjZI;R@3(gN?0?up{87Ms)x$lFa)m;FxpE zpczG>D3V^wU=;H+>pz%)?7zo|>k*-*!%n%=U>ex@=nDJFAkmjuwNTe8<+PP4FF&81 zhEaTRDht@Rbm@dPaqU+DW5Sr&6=ex`z|K7Tfu^jJH5W9W05C;UK#`q3{;rHUCGoW^O3Bbqu zHg(f@rNk#$Daj`JU9;ZZmAI-s0t{C0rxtcYjEp={a!xMtZ-+BKN?&ajeOn|+^y8k)8amEJ8-ei(nWn`~z5TE&d^@*ieRe0qE8tN}uN|x;Gdh zFg#TcgA$+fy?iu?&$WFk6%+UZ6$X|5>mezUxy+4T%j0ny{x=*Z!42L;$LCz!IK%;6`ckJVJj@Y+xKj??-rfO_b ztR@jG9K27~aWOzgpz7uR6dM2n2cM^we|vGxmtQWJkvyx-g)+=$)1>Yr1;QwM`#{YJ zrV~8)a7A8S_-h!H@y;Ajegh07TjehLZT+6t2h6=(JSeDOd#Tfw=Ea(Vr}?ewaQjx# zo$T0tS?Kx(Auham3oN*02}ncXE}@YsW@{03N9rOM3H|Qvx7{2alB~cfM$`kk%fALpf2;60{rfeN9?~M15Zs5ECQ^Wvl=YBw-kSvO#AVJATRv6eJOR9s7Uqbx z1j|uh0UI{(h{f=~e3K#~9BdAWrHd-AEl)Sp3N;6qdM2O9n3T4MSnzCR3{Ft5X-iyYGQ;8F9S#0H3IxwrWTr z+trij5pK7l%EG=nV+?bRTEqFsUU!LlySJii*OPeUdo>-Bj(Pwf_XV_N9^Xm`gYP=e z`IAb6zi_AhS^NN%>IamhI|D%jAp@TWn*VQx*+>I%;eo2um%E}A(4HOt@P%ERjElm~ zqmQe{8v2ZXB^li&=bLrZoSlVSN$eszx!>A7$-#j_W{R1pcsdOeOjR6I*f#L-Mwd$C zUS&O;3u#83+AUb^`(+;=*?9b7e4V7PrtRUs#O*l9Lq+B5yi4SJ*eO`RG3xp_5jzla zbe2JTSk;M%X+KQ)6(hMrojfqaCx)9wP5zbp9X3*@)Ns}-B~roRY^#kUzf;4eljz^E zS+5faJ)_-kWG^3`*4>$TQy+3IyaeQvgqH-j?!A~XX}H%jz^`+byndDp{S!Nv zpwb6u%u0Inm7yf5rY1_c6wIE|pa)|kH4{%qbT?R?w1AC~beKg2|G~;cZ2Q22+*;;u5udmUm52&NwH@7Rx+^6`Nc}*#~XxQbUSJ|LB(n)!wleW zzN6%Jt9{9X&`ebI*t=A(X2u4ckM@08raFN7do;L>`!%wupdnFBv6Q+Dk0sGwr^T@z+9! z^jKPSN1smAon>R?UXF?%?#t3x9+L(KeqO^<$X9gYQ_3svo8a-TdYToM@ zZ+?Ic1UFrIVCy)E&#a@u%)y3R4AOxwx~0RYtkPw|i;FY346U?f4P^rN%ga|^1nLz{ z#{GN|w<9bsxrss@qR)Msf&J=v38{WT3}9s_Z_n@Lzy{6_U?uReEUEceRt^g4gZKu{j+5u57k~l!+=f2jAID9R(c;+cA>g0T_t;DZgz~~r?W8CK;(%t+`B@+pEz{LC+ zPKC*3){I6uC_ukVBp}P2igt%CnV;~_K><&X0H#Ic)ya%aR>2DiR-%-_NgRtIpBAcO4G)4vyj$d4T3 z3VQ-u%Q#|2m{`Eul8Np-@q8q@y9Ap*XND#XaKms_@$NInM(>LM6FLVjSv2n#XnCnH z09Iu6#^cxa_HjtGm zB>@2ei^LIMoo3Tg{zgQBb@lkYx@?}$p}E|>oi$?JdG9#1s*;k#G*)+t4N_--2asi= zBrvh29Fd&o5egKPe2VL=9q@G4Ym(!OlGC6d;YxgH)L;KhI)jFu>OPa*PCtmGS26J6 zyqpL6%re&AUa{@E?+S>6I00Ky zAYNA09ha|dmRRnDED*lP%gy4|;Lv(0Z}dE_XT?Cy#3vfVXFZw3y@&(Fzj)V&I@?Lv zoyEpVt5D72uVyI$pPO}yDZJ95Lh-s8US>vdhv>*A?&Oo+b`=5qy&>SCtj zZR;EVMCNB_-`sFmR58ApnH=m|LeP=D_$~9V@4e|#9_jCFt#rAiZfwpyqsVh(+wi)<{n5wU9}x3sh>13=M{15#S=JKb75nz?be>y4l>H@G62pq0kQ z=1i0yA4JQyL4)$Es>WPhU5#1UfAkA_Ck;VY!Ym}4B^b82#{Wwlf|(c7U2g9#RPDC8 zeKxoX5we%ahiv_JD}FJGVx%22G)C_4?+?f3Ng!H+?g4zaGEgB??umpJ__xyQR@+zn z-mY)jJDABp+x(%cl(gB&Tk9^}LmU+cx<4TWPp zLBnE!p!+nLV<&&vFc<_FyM13(&J6bWqj<|n>VB5u80Q!k&xeDTNwAFtY}_xpxijO# z@tY2CAHG~2v?|(VLJyBRBqEN(q|i1`19o@g@;-Kq;^=aaP5!FDP(rU;{8m4!xW{pA z{AG+X&HNss@8_81(8&^fZ(h)Ah<0$;D1B35p&Gi2Z!a2N)47wU%>!oaf<;xEm1BDwV@y;Zvb1!b zB$Ai~8tv`0+#wqIF^hAqcsgZ;v0BT_Q{2}{ zZM$P7iRmx1h=s0)gCc1Y!lx3*)^ei6KI6fUD@7I4U&WsmVxfu1hH)3im7C_HJIi)! z3yEIlIa-3n`5PWuywI6K27R?FeURV7DFS;QZOui$32xpE+wGOy?&4{~oAn3#`>Ef& zaK8cKhe85*QXqJTX!IpjpU+KVpFF1Yf6TQNQV)l*d2kp|wv4$*v5^gcK0W@Zn*BGc zxHXabz>P0$8(FrScAWvzb3Fd`~iAfj6 z;(iRD)Vko#7#MXRfg!Z8Ctk=T7ipS!Z(vBbP`h9+63QRf`$8a*)Wrdav!=Ta4(j$z zA|6KHwU>g8>D1qLcXw-LYZ2`ylO;EM{{Ein%~Q1sB931lB^al}!^2KNRuB^=u~9xu z>^3E}RZrlE#{%FFTe|qu^cLh;Y}&^Y-(xib7=Tjv_Dy3~P~MtXT6#~QO2G*FV)Jy< zt1gsLsPV?*PmC?nxI#cC^SrZ>DGFes!=jO1hCt_hN2of$XPLN40Xu|MvcPireFU42GUVbm|t}lGng$DM& zp+5;na^hzhYLeM&;onD@(SYfuClFWK16Drs-nUG5UfEw+Tv@^4>shPhS&xSt_8sAc z0xEO*@*aF}OWwEqk!Alu8PCmdXl zX}I>?eg0ND6>O%T`wUo*v&#>bP4=bVNiyvHxw6f6#5+Pe0wqv;XbD#dD=vV=C4d@s zJ*3!>S>1w9Xg-hYv&~Z?5mgCy$`FU-iC}13x-JgN^|aR(W-~lLMG$w*MzvcE};Qncykz^xGZO2 zn@J3QyT;w7p$qH6qrdl7p4Z)1oUFlyT9x1fWHyht`eh!~XPOHO{#l@ADjEs~lAj&K zJv2uwh9wDcl1EZP@}H2buCC;z)^aEYs8d0bU5{MBzU|i6iQ0+bL0UQbNN2w5*ZFlS zxk_FdO2Tq95aH`D`d6&kL;C&CUi)~FIP*m*D9QVRLbso3!~O?~=i>y%OdkEQSz_+C zl4+FwM@=zT6gqBG2^92iVgTxshn3TI0$-pWTUOojwtU*3@g8a#S_0gi@-`>SsAuH#??XjM^ii+z1<^CCj&XXn4?}3UQkKN>2Qny)nW7Y6 z%#HfUXeMemlo@RDa84r*z`3PYYNv}k6E3o?yDTZF7J7t&6Q#-)gFDU#u$S|{fW}{Y8G-9|$d>!5nu@g-JtKUO zOr6Ukp~1-G={j==njg#J0Ik{)1d7NNF%{0lbJQ#W+N~3g_$alf^~Loc87i~3Bjw?f zxjO2f(^-g(BDUyksAMVo!yOM)a8hgd$}pZb>O)MeE3^B1Yq%FG5b*DV^K{Duf;_np zN5mNBKJMP(1?A-MzgmMN_a9iwvB^zxq@LH`25@*FoHSe9N$Z8wKt4jEAUSROS)#sz z$?3-Nk1>)oy?oCzLY@g+CzkG`oDRw;=6k)qv2j@>>U!RQ3{C3gW1d*ntHgXsEh5s+ zVw`AzUe5&Nq&SN*II9bnfQ?y-BxMxHU)56YXPAtv-^~0(qad^L6b)(%b&FrAN(RO@ z-G*QxL^sX(!U4cKw<)`b%eNu>X{MrCOvNlrk8=y$`$fXp#(#MSum708QoRfXVthyF4b|&aXxU_Wvzz zT}<(+{q4G%w}%H-3zX{DxH{-Lx{a8{DI^Xbje+5x^Xd4awE3p{qQp8_yF#fQVg1v$ zjM>)3ruXF8j-E({%m|*%2qFMy0^bua$hz{LO7Eu0!*%?lHr#7zuzl@#AXAzSdKmwF zfsieME&uO#1x4jYOY*riV|bI@Slhv)uniTEv~hn6f`qh(Nn&4 zy%OmG>*()UwR6CF6PA20Rl`wra)ehj3m#ounGFtaLjKS~LI19Dv#+lb-W;tKr0w%_ z#^_L3Xf|oMI-mD^Ef#UyvFHq7I0RJ17thTzGvsAOs+Y_ggI%+=OF(o=hqt;niWks8 zh9~pOdXI*Qzklyl_Dj?3n9r)2w>#vy0(tMx2s~_v(9lole#oVN7>L3Mi9WS+$(vq; zRFzZGVIbF+e(ooyQ_<|!gxc|32)nhn6>hsSG&;*FG4Y*y<=5y`g~+eH_v}kZr77df z)-l$9UiH}V=I3AUFUhEHUUl|VWaqO9;_H+SY?Xh+IqHvMh24tvLzkB7g1o1?Jr}Iv z%=2-Q>f?|MRgq|@t4(;Nk`#jWSCfW1Ww;Wo*fek{(mT|+@CtFS5q14=XHQGn3u^M? z2Uj_nT?QO3gzwF{>3(1A73et>izm48$s8t0Ba%fn{`9-nFw~ort#DP^`QnQDq>SWW zU@7OfJ_gM+Iu}RuifyId#78>4wtRyqq|Qr6;DDZO3#;PSsqO;RKh$n>(-_38e>bSX z0L3ykBFIs45a){R{v-rIE-5dgZagmRPjA)s&`*yEG8-!^`*|Pzw%DvQ7Bl}9h|+~% z&vau1{+%Q6zFTDrWrsN|ea2H8bF{0~nWfSY@)`^gpg&1-e~hldBps_gM)aY*lT0S(ezppVRnoPnL z)x)Iy%IA@k3xV>fweXyZ+ln)+Ot4a$7MaQIu;$`8g5Cq!yR66>spr-y)vVO^DAze? z-vv*jM3J|XDW?S4EKDxI$}Ty{F?pPcumG4^|Ghs&k!Oq3QVIK8s)-^BQlzY)HHLAZ zwPWz5!h|g6+v|e!U|b5m(h}!ux4i%9xBFDtQS_X%-~C}$UHMOXA@b>e3d7IHk&Zw+Zy%u`nD2$e zdR4~9ki0l_*LN4P0n1|T;y z&~?bgju-$K-s5W23G(4uX$gU$Wfu_a8g!x~>3Hskq?vo$|qOl>jiRKQa$dS)={qEcmND93zmDYBX0 zkuw+ntK9@YE{)#MVMRArR!{cY<{8t}yUUcNN`#V}&60ma91?6}Yp>{5o5X?DmCbKj zq%1!V-w#*GlRD9C|FugY3+EZ|BLq9u)Gt8PcNbcV%8bqsPR&`We~*&rpk&syVgJi3 zgJPgz)|htteVF={#Q9t`(y7gs`0Rp^^SrY7ea&pGnZNwA5J4DF6Lah8WE3ZDoGT2)|(6N4d#W_1!2noEc z4sd2*-ioL%j|C-!NrE-ITz~xrpaj5)FY!9AhbrHButj3>QSbLs_8!f~I@^2H8`efo z%E@+5C#O*klrQ(Kjhyy3extq4J#SsPOi&TXa>KU>2obDwePf7(d*H}>Ul6NW} zAQ+!~6TH7eT?X=Ysmk|V-*&FDYkXK77$nFa0QQ^U(|N?nIvWSh49mLNXGrD`~NlwnqVmLt&Ms5WrMNbwciHQ z%e)020`E^91x=0eXV-Kq-i&-cxZ+ofpb>HA>LbFEH;_a6Av;Ukz_$fv8$?wr-*UXW z5#$fQusWQ;!jF!Qc(OPg`$b6&Pm)ErRwCNcDNZ>DIA(w)n>#GQRaI40T|9$|W;L!t znDyvOi0k-MqMh}N%xBmxspl)|{uH12MIgVGD9`>`S1N&m&nVS%$xhB=^l4mkO>gjj zH{Y4p-UPu!{~zj{eidR^RyCgiG1ro6|A zr(bV9I!I(N8$)m!e9(pj_l#T%ekOO(%r-#-rn7^QFq?eq;86PO*^oAP1jN)bqEZER z8QIf*8khk95-Su0~#UDW}w($ZE3%E;OuCea6il5TTVzjBX zqcd1GD)^vOZDeFucP~f_~(E494oJMRSZTMs|Sr?YvkR_0ANACBk;1(`-U7a2a9(Uk z$<3!IGK{c?jfS@Ihebck{Bb=pM%mx)k+HnK59r!>;egb%PokuW za0GLF+h@x6z>ENuxi~kno!)qI;2z;Mq-Ph3SB~(_L>-eJ`@dgG+@f{@)~gStB_+P@ z^nN0Nv2H*E!lO{jK<8ChP)UKdy6fJEIT9$IsUFSH{dj1sJoY3mPC)V);msHj|xw6L|bff_ZY`d}rL)H_aS57TjyL z3z%|zT0f2SD%&hqkqvIH{pH&oHGP`(t=uZ0<~A+#Bw6CKO%?%X+J+48N(2pRwl+U( zzLJ-3TJmC6a$h`|6r5B$4?A{o5wGoKZYP$xcz%Z37zZB;z8d!Yzi~L^dVv3I!|ar! zwYAt6(sl`${<)Et826hnwmC$2D8pkHjXquF*cI1bh;Qs&(6_ z;6}74__JU30!~t27ny@8pVS&WOgY(kh7@$HQklr|T=d{%^a3)?2!O4*fp&#eIo2`b z<#aIHa<}-`^@`8zt^+lm@^q_9y15;=lIZi)O&Irq6ZGu%S5OK$B2agNV<*JhKMBrh{LZwClAUVdR14dc}>_Q$B^e-+SC?u#3%hUb;|I6jr!98f!v5td4Ht-bO1zP?f5 z66QVzZs520^}u@#-@d5`-5p+W13o2xi$=V*xaTM6&M}}MCh($!2SI%!Tw2HEueA&J z&!(?=qq)lyidu8INGA^08%KstR%LlFYC7<-aP~sbcM2dlKM>Q6ezMGwWlQY}(;xY> z#}_{`JmUlMA7Wdt7Ew_O{ER)3CM+!LgFkVl*yz;Eiut53B1^DtQ%psJz>3x`e&7GiE7?CnaS+oEiHkviQ2bJJzL(y z1_ux}4}#Ub2N3paR|$&9vRmvnhK)O7HAZEvIdXY?>Sq(|p6fekqJBWJ%oSjl?z@O7 zH1g%+{5kB{QM{v{zm;o^m@ZHRcS*vgOp5_VE>Lu*Uuc!1knX zW^4@hlQrs>mg|BNfhw*@b`0b-3t8;r^t+JsL^dFV>hc;`&pI=qWbc!9x8iRyGRaIpzI4W0GR*|Pi6#Y}%v8auf8Els4 z!?l>>k3}%SNS9aegAv&p zK=rZFhhkam`(!zUL)YtI3`L=X^U!r6s}9aRZRb(5e1Va74jkb{;HhjDH;ZROLL!y! zPT^m#0Yz*_eF2Sjt7o$^XHXbc=q-a+tw&_&_lSH|~+ z2NwT|ff~H9z9+D0eFvhaD(3cn>#OzzVbxY4Mou2i>=;jH=duc4prFtdHu&Hu5Gl4$ z)X&cMx*g{CYv|s{){{YzQpe#H%~j-lki%f#xN$|A4*E&SLiJz)4x!?6Av zAxFdVrZg9v1{voJs#2b?h=NWW1xc4ga?E6HM!;D*3ak=DdpHklPJbiq><+0=$Ok(vM#GIm$fj`IdbD}}0U_7cof(3-cNk5t#}72S;D))kH^kt;O9YPU;B%LuF;K0iGr z*u&rS937!?as}Pd7Q()CcJfA`)m^epG;IXpUaJ~Y8~H|(DLk*RjX&9AaVtd9Am0YC zZHTGc2K7e&F&z{1I_zYphTe$^X|ZUt;M=ngruwBngkvG?@Vo3k!WMV1 z?ozGFhFr-`&s}M);HVx6Y%{IToP3ASP;sxcMD<-CEUv<3j?Jn|K~C6krJoH+Evv-E zVKN;&eK9=U>uly1>ACupM9aQ`)t|in*}2c;Z$i&Qfi@syUFvPN+5mgNWEau+d-_HqDZ)S-DH11D}{3~ za^%NV4c^Eiu+PN2xKS2u`s{8pJ3(t(CCyw6jfvtLjt+cdxw=b0$&|PaiC9mheivCF zTpAZDTEbcB{E8)5p@en4PmrA-Vu7D~I8y>7jzate_zI0a{P+{CkW>8y;$g5^zS_*Z zUJzE&et-J{hem|QIGlvldFnmK$6iB=_tvv_K;5zMfDEn;>hp<9Il>L^esrcxCZn6Y zAFMPjE$p{c3s}qnZIZ33y3w=2a#AVK)x{qQ)khJ334G_68m0?o&bRU-ot*h3wR6F1 zcRVTcS=z;|CKhqn4*kW;;#W|fUgl&zvh}-ryDvD{PN%|gG}td0PeGM)pz{t?DjB6~ zXl00s56+iUXiky8qR(e=^s@K!(k*Jihg@jKuTgX&mPfhl1xDsx2S16BLTn3-#_Y8g z0hYJ~!66a~UzP|p+}TOn-ZONr0XSIi8J_PNa?BvlP7bG8*KhaWgpQ6l79chP9ZLa+ z8UdEAZTDtpm-?;SV6C!J9p#d#RAC_rb{#*pDjr=B3`We2IJQYYSm@8M^NY1D*_<`| z4e^@`e9q?1FzgxnK#^v#U2akF`sabkQ<72v@c5 zt*iU=-|Qi6&g9^CXhGBZR-$`p+=$=*R*Rny^FY}TDGA9V5~icJTc zrN6~>zmhv$xokS3O$xJD+kr}-gbAL;3u;dg1rYIFRePaOX>1ydxH+2tyh&%i^WNbT|V1ju6bURfh&JiVT%DkTi*O8Wxb`OiT)4_=E zKa8H2)}ZZgy!UWs;3UG6%-X+XC#X_N;W#1!}mh#Xg`R zQ}l+oVI(G~IcF?r4_od>ScLCRFX6(Ww%wf^{dy#oTc254RqD7#w;UO1_Zt*0?~1`c z-=-R!Ya31=-jAo<72g-U;9;$*gH!tS^M-YF`G1&PU+3k9rl~kdqfkppiV5>ZoIuRC zf;6n+a0Kmqp=RvTHBNiuR|;HDKS=bR`!HWNOF!sCdf0{*oiyjry3ClcW zCsL6k?bl2``w+oY`u?-rGnXsOr<|o7m#zl3SYKEmn>*C|8Ri3TzH=-vq&A20b&ddSxyndILIeA7VOTeoxeE4-=rsmCY${1d zBltzBS)<=b`Ga6vI%jWhFcBlns3DEnl`NEwtm4O^AxdJ-@R`>fqavN^6^kg}FLL=8 zegJeoN2y*d_3Oppx%O0Y!HOyVJV}`_+^{->U5Sb<^`Zv3I=sr@EutzKJeS8(Jr7GT zv09_JYFi$aM6QiP$r={55_u+1RUI}m2lt+dum9;kzRs_01bxbEtnc1_=@RfjrPU<% z5sZTVa_rTf7t$-Y?K|j4pN(rgjAkadnNer`cK!;j79V>IL>ZBu7ozc+DL=TzT|nkX zpW|o5Sp|cbbr0vj&`gIdWB7_u7?&jsRX z)G;HE1}n?#;ViMp?dCZb&JVc}R5!J{bcMe)y^~Lr9mIBtaE?KqWW*V3uN~?to_|;* zG^i-SZRZIkD~%Z>74Da5dbuIRO;$tA;A11y=%Kj3uk_33T$|dA%*^CN8Aj+I$J93% z-)fd``pYbZdVW<(N^06Koz=px+K1ugchgqk1nJHN|E{2FB@WB# zt0GZMAwLRQXoYg}^Gf6(b`?vT2r(+s28Q63W-_f1_)tH!PTVa;0+oW_c5{AY>VhXM zJwEkMzr_FbYdF*k44$2xZAlX~vq-!cxzFU785Zr*(9HJ|n?-;wX`QLAPUV$M5r}Gk zaYMp%q_JS0-+aVf)L zd8o^)!;&=QYa*ToE#GvTYC*Exa3y8Z$fD%0SKD{}eW-@HV9 zhUSKENq1;{wFEWB3dk!V4=VKRL%3(BrD*&LO3^!mpnjDhf4gOAw5<6RDaDsbjjIN( z>Y9-yA6Ix2@7ejU*N*Y!e9xoWFH>B8{BY&rI;oKKNuqv%(R;vMe>*rlEQYfxf=oz< ztP?jndZg5(xQU+~KLJ%GN3CY(UWCfJW8I)CXAOvICHQIVi44*7IsepQ40Qg#-$vbg zvIAe*0=g@<+>QgelM8_JM+Z=2nmIQk>?{Y%bW-Ed&S|YdTFBkqpSfQ>KWGk>X*)ET z=6nm+_-&e#^%1fBjb6oXfLSB0Xk>;?HC9PfF}q{lZnH}0}_A}SxBcxaq z*f461Eui<7;OD;ne?x>iFMBURoMK{PWakd|biB=Pm^P9WT)h^inEPn*;nt?xCnfor;KpgY; z7w)g$)>2o0Sz#gf$1oyS{UQsFCaZulhk<8p`pbg6cz#Ew#?tVakjHUd!C28 zgDW34_sa%1v^zRuhXQaSt(`u&6YPo`Ylswaoj(?bZXG7$uBGcr(|dw9pE7u{rVF6y|ZlTzZLcIf3e zrE_is{4XyPD<+w8WkkAZvcH%W`^0uqXbUG7uLL>o%(BToK`Fkw+?7h8;&ZtuA99*R z(Vj+wBBr5TSYj6hlM-*#((Ev zEBWm1bOh$e`FNtP?2l@KeUmxv2icmD6`WH;jPb4TrIZ?9cx@89s9%`PTl}n4r?S;} z`Asl<2)k7M4c)IncTellGi~Wu5fv?+Z$&-4=_ef@iBuHLf(xVRX6fjIH(EZ#&}*Xg zIIbb?9vByLeYUnM6r&xE9iKMHrZjGAiG*L`NRN1~M|-dN-pfE3edMOxcYx*?xSn!_ zo{yCZ&hVn7LCfGX<@|8U8nWL|Q&Y1XUUjBHwIP3)<8`6G<=$x@@jS~>x`0`Q8gM#! ztyqVG;@I^*HnVp5T=m#ysRdQ>R$0QBvUc~ItEWE&=G$jKL1(wnVwr3m7tK5-m}?9- zea@EB%gRJUf+YuufziyY?GNv2n2vE(c1w8f68!Fm{YIXT^EmlfL3RwvzKLjopZb=z zuvQ7|lWDASYwoxE$b)Zs0wyC}PoANTeyJO%A~P~>(Zq(Y>{IQH8aeWONzKyx8d0nl zSzeiC5bgY4;qD0!N9~OIe)E~l&NhThoA)U!*=*f^)v6#^uy(i>==Y-xbq5N^9nWix z;|yH0Px0&ve6$Xf$0=T-zxW0ZhjHCqVUN|b=SUSRvfxYR&VM)nH>M6%WYV)svBXs< zTAywO$dmxroawU{95eM}92K7lHi<7RY^BD;&DZY3S>dg!;*ES(<*9ei|3#AGdpq#$ z(_{Q)<22~I8Y%ZnHcfCMk}1L^a5mr(a|H97P(^y-=XGi>N2Y7c?;XoGhjO9w7M)Q~ zC%*shA>(MyCwxqpIU}pPq2n!h73}NTaRPB{Xt_v|Jj8(Z#0RW*#*Bi>^{yZ7@fjL6 zDw%ISk7Rw)aZvU9#kt9=f_uwz$EZo&O}>IhVcvgRywhK`P+bPaA-@(q zrw^0@!zs{py7-WZxobEjzp?{VWM8hrwld*sOeh+|7Tte4jv*b_ONU1B)cfXZ9|CuP zJ92EjY?5_O3-AQ8-Wg*fRGGmYp|G=_E|9NR64S|pv)KHgB0zdd>!7H!azK3H?7{`4 zhD`Nm6WF^*<_-?#oASq8`GH5X?m#2l>yxJ%iFs~tb-@`34*`dhAnovSLrIxUbbgsY zlCu}nB87_=)Sz8{IMyZ|-ep&@IBUY1OE$|9j`j%Q>Yzm~Qu`!ZG_C#-r`D9GUXS;_ zb4yzc86r#L@^n5VT*Pku2!uu4S$@W=#73y9P{}zL_a+3Dxn$s=sTrXfO+4B@e>t{$ zKK%mp=Vh&sj{@|Cq5YA4RT@u5d1hN*hnmu-?csZwh*|6-JU|=)^f%Rw;~65uctVbU zwRy73jX7$HfO>fz(D5dQs_jk`4XB93j7f`e#%L?ihc}D#ZftC@7scNy3b*YFFRCVd zM|=dW81`1FwL&`R%zO7nzc3}Gl>gsN)THPjRxZUW3aeQj13v-hBF`pe-+w*?hTw`K6X)lmGX?E?&{Bk;V${r!Px@iWd>blK&a88li+ zj}@~Uzuo$l$E&RMNYP9q_K}vhf@X~7uV=a~sPbTrO*v(qT5T2z0cF28w_uA3Ie|nS zE@t-O$K>LyM{bYDU+!F%B@g4xU%JuTDxZaSfmf?XmCcFVz3W?Mm<()%|GGAnL|xgF0V~^?{hK9?~;9~QaOU$2_j?CSB%Ab;3YOj!y5rvKLu73E1pZ@fw z;!t2vNyg8cDj6f5*UGd9JaeDiy)yMg=yipA!%RiUJkb%NpssgftMYKL(x#^S)BTpc zF*iYIV{dA#C96}Uz+U`aU6V*hFrGArD5E$`8BOHwbL2P+Fl&&BJ9gh2@?G`V8{vip zQNL~#`I!SA|9`VKZ084<0!pO^E5$K8x|2t-fZFhM+QoYO9M41Ol|Dv5v`XF=%!u2_ zfNrC;f+ou13cE#rp@RI7O)y#}3#_D)+s6+)h1`d}#f+aA<} zE6JKNvfWOhrC9@gC?(fbVtvo{emhO7S76LL7BQo@1&)kW_RnYZSruG4oh zB|Ln_V%uFtSG>!Zpjg)x!(SglA$zzeb1vSMq|^YE^Rch0iS(^A#>5v~Kz7T&a(khh zDGAy;DIaoFc`z>aak3lw*AfV;9bzECg1z64HqZR}~}bFk2V0W0`saj*NrotMGO_z~O+^il&u zvMXz$eH%xBZ(iQ{j@r2?o=7Dx1N%iy?YAVyNK@gzH6#B;;1v5w8eRK1>HfVE`Kpj4 z$Ez}tRtPFZ9?mW2GnU>4ljN=>hX?f>#}J1B4&{jZUDF`J=EXR+>A7D8xrrG)V2<$0 zrTP&T=XSf*2_#t@8!j;n!)1=tB|zq{QCnM-3yKTRtI8#_`*H$82u-603{kicKbr${ z_Wxe&g}0{OVtTQH<0cpmcGK(xpuut00h zF)YwFs6Hv(i8l0oRxu6xfYs7mX!79fh)`yxeuWYUxDNv{V#rhcR;r^5peKE9tA zY0YNiza`(t(YO>WRMZ`_R0eD%0YV!A9=lo;znjoPl>7brucCg3^9OFva(X{t zJ;#0&b`Twl=;Zd%-WxC)Y9|jB&1Pf~9R3kcm7sCDZBSF1B9g$lUTbJ$h-4ae*tJ7TjdL1xZMMQ5adp@A z!C2c(Q~LEjD!B-T=j;m+4zT3N06wmiO1+;9V{Zh3(E`#Rz$dZYb;n1Wpqa6^SpEyg zCMpg5N$8?0P}J4QdBhVQZyJ^i$3yS?@Ht-M3Z*L(RTI1U5bA(8;U+363lMRQ=67GCcH4;zK#brr9M1{% zN1FN%z1=p@5}Vb(lihKoZufb&4?M`$mN9I_02Vm><9aai5O*E@W?pz~pt7MTfx{A! zb%%3Xsl}qc#l69}-D0@+(h!o)_lc^igBWK?{B2v5@365D${G5Nyno?haMt}VjPMER z&M_UAhXjIl2W@a^8iUHIG404_d^gg_-ur!$@4|ADa*mgWGZKDbJN>{&z)2bDSHc;z z5tVXc>TE-3@?b}ZHSMe%Bl48+8nsKvRcggL1IqFnLWa_&tpVC|P4Is_rwG_cW;0dM zGc@gRoeqvW;+g#6jhb@7Sxiym=lN z7}!j>Ec!VvGo`jy>qzvx*8OjPYyA9zptoB?;MAM(??BOPF}Y+JvioAbp2%&z`jy1_ zr`ffdOtx26GGMXly?H!R^>{+`K3Hpq7C#ka)|~o^?}ZI*(k1nlI>ClEotq{cdjhx9 zgx$d3ldA8LI^}U&eGMN?;AVNnonnn*IUWU&0}dpG`nN2UPAVgXbTv{$St0Dvg>2Xt zeS4BXNgBaS9);gRI!4sx3vDXQ&Nm7A%@9r*O5l5ho!pcGpNmRR1Qm%xs&X?&y`=po`|!VIa&T0owc40Q8Ea-w!rbcs(BMBb&wa$p7j) zaip{6biPtenSaT9jMQ8(0@j7lLQW8gN}4^5`t8`NZnwsd*|fA0%NltsWm;6ToR#2s zWchUrd;Du;8UCm^iYmd>oK^gH&;2+4Nn&+9k5M?aFr1oXc}v&`cV+S<0?u~#XPC6J zsKm)T9TBey(?s!702CC_)imq>mJ_8_QcXt&6vYi#Hp*EJtpLY)i_Wr+iK(Q81abyM zs4xfa$csbpGA|0QO1D@l0R>}=EjR7eeP3J*zS zFyO;`q(xZqrdCz- z)Wq=`dTg^m_ifFu{JIUIs=dn9wZXBsrGV^+d0Pv3mSC2){w?P_I_G;uYkoN{C<4s- zWrqLt)7|L!_t<)Vw&UeE&vtH9HsK!i+pYKaUK?nWda5RHc_tG=QTgrLHF>i`=H5gm zYIEmGWPR*obWzd0*t|^qw$^!d1w_7{R_%>i`bfG03Lh&vg(lKpNlq@ZEuE4x%oTNy z;YIW%E@rY$>W8JqEkTf&eHp-yl(ohxYNst~JEu)Xc!Go=mBVr(gBs*LyarfVCiAOt z9qhFb{Sc<5RnWCZW%A%#MVCo#2^s54xPDmmrC8)Q{I=M6m7miJ#9KMuKmsvZT4|RL zjr)lDZO?2ejw)uuB-V!N&PuzJr6eSOh9MiGiEG0y4z_4eRIRr0(ic96x7jGMis*a#uGOIh!p$*yg! zmBhxzMub0LV%#W9J~QZC94^u^3(Sf#<+BpMr7d7U;%HrNH7roOri6jO1rc+sZml8(Ogw`?jk+2TO$IXm(Tr+D`lc)K3=etJAOnAy+Ozmo_hpkqkpD;E8*kT7U+Rf~%aY zfu^Pi5{bC*kca(Ff>Q2IkjgeAv}5reSFH-_*_7Q>vFxkkinv*~u6=LZ+;+`2bZc`H z={~j)9N>8_Kw}g8Bi|v#WYXv{>#)+z>{e2_PKAQfNa_p?+(aWB2 zoOZwP;U_w3%YJccQ&BxH709a?&4;%ZiUxm(wc8egn#fdQWNXQY;}QovVDC+ia_am3 zb3SS{-TO!VDZLahnWN#_LVrT&RodQgT40E^OMNv&5%Jngms=e%cSkMx zR)7-Jm{OVFRfYXezH6+7yz;(Dm{VA6!V1_`ydG6BH$^=TOV2)xVx^lej@i>d53| z9Qr2m@Q&hOs7ukFdz>~E8yV>dkHLMeKbWdGDlvLb8CqtAf3CdcJxNZ^s_vvO(rQyM z1L=n5nb>?K9S*G$_d+9&Q>nrJijfzfiSB(ehvnO3)6!D9*yh6d^!tS&{Dy^H-*~x4 zx)Qc?zkU1hRn?N@M>uAbgYf6;)^`!SFK%?kJ}I{6eTnrAzP}L}aRDlrhmwkm>y1e( zkM{}YzY$i>o+54=E}7ioUoyq~-mk!VU2#gfKSFh`Dx;}T#zC`Uj!Ro<& z)18IdzKw4~-T&UDjtESoWRHGqpNzT}!ow^_P~Efo0>j}F%@;l2+tO)c@7xz=d&|<* zQpgTN{Dcu~isRS?-L%Wp;wADsEPcsjS6ksM;jz%kA)ouwH&O^3D#E8y$!FP9>^fZZ zr&wKPzWe-bHNJppwnCg-4riLRF+T}Q?z^*T+@Ins^xDt8ABQS^)WBy>zlb@tZXFeQ zTM(ndplb!MDg`W&Ahz9v5HsQ<Ypk>Q_aKe<(ljyiY&+NoVjvg-e+I?J%8+qmzm zD4o$XsH ztT+Zf+WXvtk1|apYhGDE&RMoxsCntifE5XLA11N6xD(HK2kjq=O^AXjrOe!l*J86? z`7Vfq(H?TlkvpgZY-%5dUuzgr<1pFWZt~q$_c!D2wogC6Y*M{fl_^?g+?rV7YUWT2ffB@h(KdqX6Q4Tk0 z>L)%r;y3^G8KK-maRV?qF228p>i8!&n%$W}w_j;xQWEQ@~zeHUt$aLjYWvQIa2vh?h-z6bj4o96?Y!lClf@>r9lzR3;=W*j{YiaQH(ihsO_c)@Q+r1yXl2#Dp*LAcL13Pof& z!c`uNyirn48q%KA;#XgIqH_7NKFt%XLgXda-`@uFlFaMWavmb2ih8)O;W`DlSP+7j zS}?BNN7zhM2gch_nW0!U-9UeOPJL;C&=hZM*5y5`=HKnXU)(czwCS-Jd1t#caedEW zU`2$o$RSZ0sWeF}q$sapN9|!!GBD2ze8az-RJz)WH zxx_W3#(O<7iRr3Px_Q2y_nnBd@^I;beMzRF74XjC6&U zOH5)<@`x>zV1<#F=u$$SyGFK1c`SgYHur}xk&sPL*<%?ar8BVd0+w}2brK|K{EaCSmNa`1tuW{aXnohlTM+E}rC` zdisxDl5dQpsIxcIfFO!Mnws*eN?~|s9m~m}Rt=p#6+3erl^}bZnHq&sIGMQX0XnC8 zc;khjT&L=5_WRvOTs6n8)Me%VB6`a{rh#=u8jOcZ$xo0k^mdl=CpZ~m{$t?Q7FX_tDOA9(JI=`#)cxh6KkcP@{?8efbLiF_-wS|})Vh^gVm*7)sGdcJ9`Ez~);Zc_wlPudSA%XE_VU0Y> zdpSu1d#WK!70da&Uy6{#=>-)^P>yxuhj3;pu)D) zWavy;3&X$9PyhP$i!8+qE7E^$3`!D58Fw1wH#50}jYrB{z{Y#^z6W$lZcB>hc)ABZ zf&ZS3sOIvB=t9iD{`iGB7qK;$#&GcwM18HmO8bX)lyYS3wbH`8rGTV3rCh7KxqidM zx9dM~Fxqfr@^s~!jJmgF+_0YRI+MHeFqbag0V$om`VzWf_i~k#^zNdb;L%qS(K6O) z3LG@&)#-SDR5f35E8|i5p4UYo)w7C;u`#;I9F|FtIQc86t3ern3@zWhc&$PdnCnNc zi1+gG@?YyQzFEtA9Wkf`mDVtT=)I_Oj<-&dG)l74jxs*1(S2aduO6l_Y$a${6W=Kw z9@)%!@b34Hl;j>jyDo~7cJWv$`(thokufX|I>}t{;MaP15TP5u1sSP#DUuC-rh1ge z)~@~&k9Y|pZGU@}S@+b}oX$~-K@bs+-8Ulm?G#gDu-Up5eo4qo(P-rk>PWgglcZGD zMdiqfQOmhBmyN5uVhBBIyFKP7-Ga6Ifq#a20<83pXR`im0^*0cT(FYwnm6?5tk2KSFY)w*wmZ#Ww$vmB_=JoD9x~8uS+4M_zJpz^ zKlWMO7lZF_Z9%ipYQ1euJX3qHclG~`6G=Dcf>v&Oi@b~mUis)ZW%)iK+<3942fCg2 zujllKN9lE@F1)>QZM4+3Q*i%yU{zzLakQvPKo!?R=Den=LgN1Ahc zT#@ppU4y^zA23>}091ps34jWXy|drg81UuGmyeGw#y^Ex$AS)q(uLm#R9!fscvl8x zRxtHOp-a=T*^c~EIajgN-jlibEK2#AaL!74A0PN(;Rhy5#U}R-)1ABacA?4h*8g4UsjlUG`NFr)J4FRN z$*-xl#oW&>1^r@RT*O|MV(MSuzs5577+O{IS~Ut15tV`j{<=GMrm`Vu*Fn+UBRMl` zxyEfuub#Ji6N|Be8tpnM4?j4MXQQK}oYtLD#B`KI4O{*H6u6KjASH^jFXoIYo(+72 z7C-Ob2YS?F!Cc9*lO#Xd&@gtlwVfr@Hz^+}E;_gmD~|sr7FbF@MvKsyOjA6H_(rcH zhxGZ)O+UO<8i&J8Z9ylMMzvG?78m26GdIW{{#jqARK?%Vc#IMvn$PB6xrea1Sr&`_ z9qkp#wO#qneECJ4;PGFpG#SZQ@cgC)U+hD|jT(B5)`lEU{OP5$(ciYLi15SJ4@^Q; zeuCB~H4vswO5-OY4y`8)8r;^j8hQqr{y8JVnq@-f$4RSoXO`l&>gD8=dzgDJD9vK= zbQru$?CujoPyg#5j? zb{mx&iF5bP?WCp>;QzlFjdYK1gPN!(3thYreNf9xv_^!$erJs{4;bhn!sX(3INdZ^ zMiZQDjB1iV2a}pC8{fRb>&*w!?e}4b=J*AUgiNx4UqwcEAd zKkG}@k?bLD@q@TZGFpD{9AWwxnV4GbwB|&2)}?01&gxHqU)vOXImxhHxTAShdnYHd zfkNowYwNGxVZNubxc|lz+#3RXdIj`5a?!K`Wxx&=_AdW&m?k;-E@nsH+{$~#tLCT2 zo)92)1v*RoM11}kFvjhLd8=K!^3iL(RewBchtkOV!%Y4grb0x!v(xL^)F%=H=ol8| zx|2*GjXJHB&6s%Cnfe}uc&(r88SrP8*B(0dV3EsAXwd=5R!~J~kS=AdGVIigms+@? zcnfZPq596b-1B6=@V&NAqy%q5OMSTLDAY((-4}kKqXoNNvlgsk#g~4xMf3vQEPqFH!}u8c^1B_+NzV<#fTUdzYTdH zUz~a*nXf^SO!B zswtE69gLId$%^yphg;68?H%x)N#BC1z58DZ>?>2ch(^b!>=awm1@xCnm3Xa_p9p{X z^hz14mzw#Lc85XhX2FO2ZY$w%nD}xm%|ImA#p{?uAk587fXJB(FnN6Vr#bE)lnmS1 zCL2LQzbK3?VduZ9?SL!+M#i{f@-j66T53yWLp-%1m)Z6&TR$eqz*5dtgqqwphH84n zZczO4Y5i*qOx$JKna=ci{7FyKade?!`OV%|#alsT&YlZtFDd1YUG3TE^%1(v2Nhtv z+JsFp{v+Umh?WJx;?W;Ju1(Ie?wBkdq=p<7{3NJ@z94VQ6iaBkRA?kq+z|fDf&P|A zBGlMT?wb^X>$o>`!4zC1^zHNv89>~U)wt0rPqx&xR2ILXV@%JI%5UiCG{&($S@4M8 zKoHf;7}fpp&##cZDn&}!?-Wt7xr!80%@i>Ez;t1c+MT^UBURNpfbj#At(DdOy0x;N zfZfN0$Sr?XpP|e8%T1kEg^z1hZ6@`Z47nx% zfz)|<_%oy!N<&5e1K9Pc7gePu7c#3*{0mD~s8f1XJ3Bk%yBq-vKSA2xbjXMG3kezu zM-ePuul)&gp&`^#9YOmdz~aA)uMWl*C;qS84+$NYa#g(eGP#a}nY__f0bfKf+M^_} zP3Jy3!d9Mn%EWY{frZcN$k5YW@A*%giIxoRf`JbY)Qu;_ja+co zMk^02nU-Gkh96~?ku&KTxy;b^*pL)H>Qo_$!2n9E;`RjK*YMNYZP0hd%ZDz>Xbq4+ zJi)9nB9aXx5WGNoA44c+V|^D0lD^}|2U~?`w3jZrl%1bKr}tVM$&_m7K?B~8oiYsz z-`?E~7mmYWZyTGj+9yJVI;5;`2FFUj>RUmD)iC>QwB)b}V(p7f=o2FBb4({Uqjy+h)zz3RW?V}<%V8+_F)l=&C% zu2R<$y|uf)^}Z-)SYdc*1cCJ#edw=pcpO{y;SiKHU}yDGVZ+NhB*q+a6LU3BMT48& zf714N8>+74pxt&F(1@QGz<{UOXiN}5g)>XnGAa13i=-giji0=w2rlIM7^nZbGhz5e ztj?)HoUU3ESRE#SAIF6dGB(yqfAi)~GZE!gmsJayd?DiOnALosUR3-fG$Tac#%NF* zDs5^LTka6tWskLFZ*cURm{4<9xy~vO0CK8r^zp$-rnS#M|6^|he4bEFZ8+-<)tL%K zuBmHUP)ou{e*8+0u{2_ug$hK-VE0gm*8F$u`ZG4rkNy<&)BvnbSO5Xw#x4XhxztW@ zIC*ftGRvShO766pP)BE(`6?L*gyZ86dnez5=(zkZ>RKsKAHa@}OGv^Z0>E>fd4~A! zokg--`ZzDI7h9JCQf}xzJ$;Xq@?O_bcf-svJq9pFcYZcb9C4U)*COpt93kdr(zxAQ zZ4)K&PNfo$aI$wUV<`73?|S^fSR@UdzudiXA$oc`^HW1GC=r|heIuq zre~?<0f@MW-n^Wg*3)C0Tts1EZxHUtA15<&StRb%4T} z3(H;J?%<5zlIS+#9MDx$5!GH#eti;NZBXJ<))wwQB2J|-d06z!xVC6O&*hf@~aiiX`!iUnov&Ad89wB`0Rme&Vo+LV%BU*U+v zjV0gIsQ9O#l)VLZb})d^3cx3_XT%w(-CB4X!CA)Re63+FitpDV7;m(E&fI0+f*mo8&WF%blw$_)r#j>J%_dB-w9KMju8f2O>j8e@`zVx z_}&L?+D#P%y7iD&f4Rh~;(n%vQZ7@R*2S5fF>@CXfzBQdezepK9CI0yU;i@a|NF?(T>i_9l!j>P zD#9oAvg$NB`RkF5rUVDhs9I z(w!u`MMx9U-J1G#s{&1OyI3??MYMt926b0^yc7D*5N$B!Yp+iTF}%Vom;J#aXBCX1WRHSz`WV zVcdWrv^|<(NM8L$lydsi5KXQzZ`zgil1II<+j1$mZ!Eu(;dtn1agR!$RD?DeTJ$@SQyY;a5R-R( z7NFhu}bmPTGd_wFK|aHeVO} zFn#&?%h|+_M@K|0P(ps)v{QjuvgP{@m}-X{xT90~$gGs3W=h{;;m4{0d`-p^{V@ZM zOb`ruEp|7zpr}ZMkt*J{v*@+0=mdN6(}C&M06-}uu2K1BByH$$-3a2N5WXNJr*FBsjWMexLDv8rjh+Qn(gv3*cmF4WbK!6oY|+oIy3cS#mS5?e~;1Dn9|%%A5>?jL&lJQ@%n|C~;`` z*Vg-%xGFfcW#SjYRa24?e~W8ntFcM*_vbt%z=~vlCro6H~W92kfVfd7X zQKxi>rmLGh+=mzg43t^B02ek%A+`Oe$3|23fQaL_XFVH_CJUB>d{+Et`-+1dCHl)AyFBQt00=TDZy4_c|uxejw>yG1A# z=Moa!x!wutP%b4Zr4vq4@bMDJ0_T2QirL?%zv>4*7-wbW5KnKrf;ziS_k-8uc6}-Q z=5R_uqi2**bZ<(bbES_6%TLAcMh zO4;p4$Y4$y$^$Vl``Ac`8XeU;0%|_^~XVYXHNgcG`nDa--UQp z=dJ{9Nc~`2hUz@z+{uBf(hs+OeB>&;kbs@qiH*@DPNQ+##R>N=fT9_QZHfdZ1Y?qnCUitiDs0;TrD zf^~n&*z>3?)OgS@FPvmNAG(hQw75ya*_Uu08y9>5Y_Xwb_1uK+ONcsor0Vd*h`u=s z`j@+sS7}LOHEWDs<1bQ4C3eIU<6HP*A%{) zy%H!DM!$T@pX|4``gvehh4OY=DLvN>YJ2Ah?ppG+j&dv*D@g(yM(bCP4WyB^@bT&w zrSAZJ#jiBI3cfO4;0sDT*fC{T4@kKYl@O8G)k5dQpTKskz{NJ`qO?`|R0+4KH%2(R zOj!>`KC@=2d3v>CesA1MgQdOG%Lt9uIYV;BlIp}9yuc5H3Y4iS+c2)Ovwa@OsOhNx zZQXRTOEwrnDv#HK9W1b4O_Q;nv!(NsrT6~w??=v^{P5;^QOO2_1Gyh|i50p%x47k|nRNs;osA%S0|F7-jThYU~=bTOD z6PA%4Z-lK?epfvQSyl6ICFk+&c~{A)BE!B*|9VLjyb!$G-?uxB%Cxl0^n9Z^F>hcz zqGt(^@s;S<*pgK-29)7r)3=&wlU7fj`0E)P+a^bTgqPR_6`I#gGR~U<(1?AHKIOQm z1_(d8lvRisKT)n^3H&o5UTAAxw=;_Q1L(msV0rjF$mMh7JcRcsXZzTH=VC-Y8qA6V}m%aZB&ih^k zb@nT#H3+R*Mk<+PaSwI(z-`6!g_uGK*7=_~v13APs2&S@zRaTm zTV-FaXNfiOJxbyX!XeI*^5lyCeR+^?-yrtwA(k6LT1$r**U@#qT2;F1*%P|1 zVC>c+#XXk6Np+`X^!4Pxo1v{jLGcdWa!~=XyL{>y>kcEO{`|@5P6t7T;@?I44(mP| z3CeX`L8oe_1(d7`<6eGhWBj5vW-M)JM|9FU+eYTRXLF(65*>$w9OmIBVki#;{|cOk z`q(ya;U;dbyo#!itS<>({W9@>hZaQ#^yBegcQAJ4DY47s2WrshFB7kB*MupmndMr6rUDY@xYre0D~HQaw?d zHG1)xE;~CU$_{c)j(~gPhqb!T9J?X)x z>s}eFuoF_JqKVcQwO3MJjf~5au-f{rRc3*%p*}M(NR6$J3t~I|v!|)@v7Y~H;4$YF zWGCl;b5+vK?jUx$;HSjI#0u}oIZ)782-VZvDYBbE&C$RNpl+nBoWFf@NJ&~+&sftF zbbQ`bIv5!|xdJY_0`cnU%C=-(x#U>BLW|^-hq?x)ws~Tsqbfwv6CihKT`YxHlMzt| zIaM3`t(vO*lx*++cD+!a$~#afvsdF`thiTOBPp0qo^%<#m!yA7X#^E_7A_KP<7q22 zU}qI{sqwohO<~NDS)~iL{Jf+!M3cg^$p56%P(;G{GtUzh+{6Px=QdER9^Ut+G97fb zIfq?vN#!0YuHY6*?Lj?M`s;+O&$}`LqYZXOf$SdTCZlJdcw4OemXkQ9Nd%C>k~ zp_}|&9?MrsH?kuiKwMhurBYPxM0PEiNyFQ^)uTrUjjGCV9u@c5>TOdZcJg#?48GK8 z#pd0(dV26@oM^+rPiI9$ITmc{)l zdgG!M7&sC4Z{Sx9O(QK8n=s{_fLR&tLp5CcHIeTvwzvm7iR)O%z{nH^m3~w2MpV z%en5<<=635BcET*QJz_!-6aw#rl<+kzHq<_eBzsjPC)aA1tT)Ad;)%)BSv!C%!otYszYFHm zBs3he^?mqSu)R{o_>#()F5Mb2klozi_1rx^f2KuOa(dRt%oFs01s!x zVHWkFXR$2DA07>mQWQg9Ox%T^-3x~F8C?A~7CYx2!x<^&#-L`*MeFP9;Yv^G*@EjI zxBuvuIeqK<#~#=Qr{{l9-ZwtVW137-+OCRu-$$bngAc;047wkjQNV4R3`&Cg;kY#&fzPR*Kxh`N!Bg z9n8y1(Zy!eQEmA(Rf(4Mfw~gKbhFegcf)GgPpVuNh?(gRRoXDUT-Fm2Gra%RfUJ6! z)&DXf3iW>XrVpa|yws<)&&w;#y4wSnX*c6CV?k+SX%Zv2Uu|)L%Yr(x8DqUCOF&=! zbJz#?HN{ZYRrbyC!^m%hE;X>We*d6xH}z3w_bkjo^_>Y2pzzVV^fKB(Tx+OR0Gt*u z7W^(bB#x800EZkQ&uy0u63;8dx<6#yGZ4xg>!>VFOm#zam{k8IX%_Pbe`4BnVa>f{ zoa}8@&0KtO%G*%))#-JWQ`dg+@hF?HC;EEErkZcXh9#gbOaiOyg~)p2I2_ZD`3 zulY?Kqknszh*e8v_0zj-wr}kIkXod>N2emk3*IaY`WoXyvAtt(T=eUwB&_&Z23h&<)V2`?IVOluMvhyyV@|)*VNL9%U@7=|xIALgA5imFGkw=e zf$H|-Sq+xf7c?ir!L=Bu5`FE#e?INw8y(U^@6M~80!2z8TXV6x!Hr`fVCRc@n40ux z5-D`!ML|d)aV}_E9S4SaCtyo;dVLLx2rE&&E;3f?bo1NjuOo#8K91L)#_{WSXJj_k zuZ;zmnZ>S~)eH-rt!=Wgd^-m2c$m>&H-^v>s{QtLcyr68_ZB1W8RBDgYeXhjacM?yP9IWUnrfT9JL0NO^lO zh{`aoc70DMdG|Wm8N+}`F$>LwtMPlFu)x^G#bqhbPF{sw;5b8^R=~(cV#dz{$-!@4 zUf9A+C~Y1G$`Kvx`s9XE`meFxBT#5o3BY)LMU97!|G+pwQ0)N~SHU%ppYebDw|Xd< zDTTb+&L8xsyTN2C2iyxJJxafouZ>Hm^DQ{@h+aoGc|H4mRUzt=jI_$IW+qgMi^HG( zqEP+4Y3e)`hFsH?_ED2#_hgI$og^G-;;^tOAnEuTb3QX9ls(}*3K?r@tsvKf%QjZEcbz+b(_L{ zbx2&CCjb#u0KMxHLxhDiNJGF0s_hWPIIKrcn`1ccc_;6}4gBTV@OsWGC@*kGl8zPW z6pWR*azbUhS3B0wKYhAeg#j@Md%}DM%peA)H)FjoURELHfY;t3ian(vJLGa(yQ31Q zSk%h+4|UH4h(%r-EhG`j#FO~Lir@b626Bt5D?QZjyAV=-6reVu{Gak-%E0|ruch}9 z;ePG%hF~QB*8@J%=Eb?*t*lei9L%0Ua~}-ZAiS?|o$*K-#Yw`-taeL@b}s6oNu8^L zK@ESaU;^sSpWnYr#hgaRLFM_^^65Cg&X%`9o79F8$D9XZCvI!n@L&FA0z1*>$JzcM z+gAdTCNqs^C+iWb`GQ-Lr*m{oqyzZ`M&+L$S*OqAya>d)@i)GM%5>mv1;JIH?twB+ z*E2%q#DDp5))j=>{WT)E2rCWlRkUs_DtSqHH2IVA#(;Ib^k1(dndrP3S94IzTIW@% zYwXlop{s=~Z&~CgA;03>>&LBt2ySW2Pi@s%-txYMDoh1O%0s*$RCLcw_)car4tNwr zP~WZjc(wgyI5(E&#ei#skmmOqd7^*mGA{AfAjv$I`5>&CFdAz0+572j4&WtbK7ym0TjDy_gpME?3BA9({(h%jc%#oK? zdi70~u%z(FqT9;~Q;^cR8LAzSb3JVRstUifBLb?ca;Vz8K}Iqx)rNicA34xo$|lpr zSGnwbv^%JO4cmO{`t|ZIz*-KSuC1R6c1nC>nL{1W+*S*n4ku^&IzD|gy%%1FgwYPa zH$`75H%+zsa{}04=?y!DE|qQ3bP?#+sx<}k85e21|9kW6Xy?0Rpu@n)A8*a4UtDp6 zPanKSV;cbS+M|jI|7CTw1SF(ZaqWxrLxifUt3q+VjzJ}SB_Y-9t~5pf-^27@v@=t? zD^0sSv|%elDai@eph>O~BD3~cXUDkK_y8dS-Ep_72~Kc*Q28jG4yMZ4p~9@{ip{Ud zcGBw=nK0PAjTXzy@&?5p7aa&_Q+7V^8&|2jRMVm6s9gt=`-t!IBw@S$f)%)dIzBqD z05ieuzs@aR=rXA5bp5?~1024_d*2Pu<>a8aOn4E5;uQxVwAO%;ohPLGDs|i0yB-2;7**IFQ2IlKwORehif;|@(fR!M3C)d= zn_a#YPKJM*z5hP4tIn#W7X3U-tqyEaGC=1;R&d5jevOr0=1G?eYy>Q|OWI-2=i9D* z(J!Xppm11>M<7Ml2waY*{cjuyz0;opK_7d##0WUhz=fAKr#q!tljO541*v^VtH3fm z4LjXLVP+%PSPvV3PO2!C*{3>DRgV>z&AVoP(5?Tc$NaR+)V+(_3-rwsA1wLPDB}i$ zl}6*WlGt?OYTRS02~^8gJ(J8V)$JMJf~hIFxvh+tkzQfzHqgH1_8-Zs)afM?$80jR zv)oEi{W^o>1#EPY`Rs_U=2sRqiL? zjpbjsaSe(;nkH%Dv=ErM^~yl;HDT|m|8}7Z*|#mxB=W4=U5T-*Z#U`j!+$itX}}a1 zw~Sz+o#dl)WQFLT9z!RyO{Z(4OPn1XN(mAi)E6aQ^R(eLr|sv~aY>vc?Pch{0vu9b z&N;b_eD;m2#!HDeuj~0zSl{7X!juaRQuJ)DgCcq{GriflSb-cYK2}X^xl|w-DW1Jv zy}OG@3se2%#y`_oD@#x35^h@u_xWI^n|eBt(9sTxqY z(Yw}kcFMl!v%-%tjJI>{@ucpMO4uaWB79^7j1loDr9sF!)1ewyrJENp;r`mxuKeQh z!JP_|?_JD>Hv!*3np8cPL-o*BLr4Kt^@l^EMUFYPKnVAW3<44Wwl%6u^+q>JB~5%;6)xXT?xQp`?zqX+d(}wUGnrZj{mdhxaANq` z7b6+C;#diGC`PX!)A6K~nCZb?<3PUz_>!g(+ED_d^V-J0%OT}UI{c*Tw)W3Hv{fIk zaiv#frMF9|lI^4;m_a?pJT-1|3)!Krq1_4kz?k0tbyqjDHKODQU{)OoSVkai0uRX=;!i;Eciy*mF&xM_;^%k>H`2`%_;$0XqeKuvX@G;kVBtmEO^Sz zb_EXn5@o#V>3I@n3ox5DqC)+{djwI?Gv;o5xZ&~(X@Z9g%!Fp8d`Xopph)N_-EHK> z1qQ!Yd)4!?|1@~ap3*gLgncIwI^17%36x=k{9Wj7xL@>SiU~$a8Dt+PTqxZ>@U@*&Ms&JW($Z3>g1yGBPx!=df+2-@zd5spnir&U7U$m4|mS{eD} zDzNO3obL#xaqAFWTXF)z`6yLPnn1O4a-Qbodvg41rh$t}yTZ4YOQ###Qrx~%&R>8@ zW>ZC$SX211PUUE|BkwHKysY@`89L7Dmr58*H+mnZ-|I}=f3~%Lejm2!&vZhm@_Y?z zqW3t{udy6V`iqq^SUq}$XJ<(Fj+^-Y>Q}zlluQWzKtTYAPQd8;^zD|SK7x0tXZq~d z1ppDPwDq_YeJeq}`-$$Dp(ec0bC1HnPG4&~N6sNq_sKR9%=SQGFsiirNiH9dTzHN* zRF|=MspuCK4{(m@(s|rSkwY6}d(hOpgsM1tR)_ih@vZo?+@C(u=USc!UO8qyIA@Z` z(~;3~S)ySegv%H9;d zQdsO%ZY$C{djMDJrgm$WGS*{|q;`7z;Q{M!|Bj7PPO|Q>^9R^H6Tvu~d+@?uPqAqy z!avp|Rkhb`JG6uyb+|phCyUC_kZAg#HM#JH@=7NkTTWGw(tFB*zn}lBQ(M)Pa~a>H zob{(;ZiT4ub3A|*+blJZXB6Dw$Dfc`TNxAh#kXM6O<*iCgn009!s$WA@ zE9{fAZuj!aGJo=eFc@;l^2h;h3a9cWnC-L!WkNKhQk|G(n@q;f;{=DOthkDTUMO*v z`Y@YjxY@9b2doS}PlC8OXH~2x^!4nwlHOkswtHu~Mmo^ebK*s2gK)C{3V3?RJex3QuY?f0yFSviW|H%>1*t$6uh_#b7J*51j1 zx0?!%ePx~FL5VCRlS!xgn3RviwdI;181VM#P_qZKtBAm6F`uJ~S!{DhSE&jA3$}TA z1gVq6P%M>9FuL#Gcno0--Piv(z^R-pc~9UESl?&&w#2b4H-gxv=DTIS>5JR7hA5`= z*@k=~yX`1^{1VskqU?H?_?^4-%Um>S9}=3?KWe!2-&<^(8=MKt9?=_94Phod!RzVk zd^WqE9Y0=_jy%ijolK#yfC}Dd#-2x1Xz|iLH6a?qT3t147gJ3jBex;JaSf2H4boCS z7hJCQj|B6Rt%fbW9d8!+W&xgLOW z@x&CIott5t?eI>L)YH{7@YIYM85uKv?2!oW*hY#(@$R0Z!;f5I21~iS(}5YQFNR$F zbE&ZOsA|Vkr)NB-Rd>b1tW8aIVc>t@VfJ;0;jVocaVi8-#>`dN>$&mc^ACn>&(Ae0 z(;-Oj&{+yzlYC(~z-La@PtC+^`W+&|v!c_g99E^3nVKhfVB%YYS8D2{0+Srf-qA33 zGAiB+p<;1h@b_ed?U(}A8B+BU3LT(mD6w+-uEy(FU6$X?o1IitP?Epw_~kR?iJ_)v z-jgRE3OSN1YTu}X{@SS6Ct$R6TN=tdW z_C;m^=1sMAu;8#7mZNvwmo-|^oMI1pP+2vv5Qopyx=y1IvsH?{9pRH}zCN(&RqrCy z6YoAHMCZwTa0hWj>L0v^{kzJ!n7R7|HGk^(kz>7+Z{TCR0f!?8wb(8+q#a}Y7}93N z-J&c;_vq2&?wcx~yLYdWC67>?rwnoTi|b7B z4yW#KEIIG?C(HifUT=PA{_`wo}AEGg0q z7Rb+^R+Wa?5RvgBX3wCni?kRnhfi^2Q(EE@*?LCtLB0i-Tyr&t`q9c~(-gMVUb8B` z*m;j2lIcK%f7Tef{oey6Da360*UPEZUszWcX19Cb0lSjeO-7W^sZ2d+4pkF+)|r=$ zjeDLZ&&0e|H9J*uBPy(L}q+EBnS7`yn6f!|$^$K?M7cQu=);vd6coI*lEW+#bOTw&l3Z!8INu|%q?NjzZ3 zIm_n_?)H0O#wQ(}Mhiiyu>R!GjUH!2l3sxsE4DCl4PLRk>$S47UgZq4ldNVxpR*eh zM~095R98r0`_V0#JCFOX_y6+t_V##lbF=-sl=Y~e`NDj3o=P)8F6+;aB*0v+5MGFW z?-^2P+|)l>U~2tEuec1$mx>v}YQVe7xY1nFg3(1|1*Udz7M9T;W1y$o*}<4^O|W0| z4{}k%048VFJ4L2A_wOc0rBF}P+zL$l5vD~*nc(FOx9VzmcwN}{`gqwYQkjJLy(^qw zW9T03i%uXv^;|_MEml@x4tJQG0Rs_j(?w(4@Dwz$pAdy`Qu~uajd;d4;-*u0|9drF z0vpb!MVI}UV#!r$k1=X#O%@gB!iCg-4K%e94TzI7ywpA5^N zhTZjEGjVcuCM_%a@}&o~RKvkMX>_o%Ej&CTXV*&4ux3Z1W$kLh-_xYz3#_p}3mQ#n z1H(Bv1Jm*Sm&~Lx=@j;}RUT{S{9ZDMS2wm*21}2wp{8x+$k?#qW5paz$7f|-0*TJ; zuYT*9>obuD}8VJWt z$0SYgVrhz@SA-UnZ$A?Tl@ob((`|_<54Z$wPY}}Qf%5Dcx{hbtv3-B>^Pb<{L0)ujM0<9 zEJygcxYWA`^x<0S53_fkJ~VFaaVcbpSYi21*FMdJT1~#aZvE#lM!%~5*r44waNLLm z|LMPnb#shZt3Bs8$Yy>OYwznSC@fmBy$RLNT(wMS;YuTUKTENKc%!%VEK0hy#*0qwi@4O8Wqse`P! z-$mbVTCC@~r!gD1jdfjxC!l<|P;U5TiwLMt^lKY4IbakgkQ^a1J^Ka@1H}!3u9|fL z4WcJH-mWFvyA}ym+7t-TDVDlfOX1JX`Wtg}sk<~KNkD6XQfa1i^U(UlA=-TkM>^eM zz@wP8{ogjQIhH$RnzZi69ry*`&nId2Ipfo39yv!IQTvJ60KNqg?Pr7T69tu3RTJSm zD}`_mV`SqOwbbc{J!AqmHQ(5P)8**Y6f>SGlv{h)IwXWB?^E@s)8=3A+OjR#bq`61 zoh|I|sMSa5@(x?p6=@-LVNZ*0w{GTtDJlbYZhiFg-#Qpzty?`iX*#nbTp8c`;kzua z)=P%C14b&jHiG|Z8L1(5_1vmQ%pHXb$CN$}K5-r6w4!qSAT%vBWu_{+JCHAKY7fyXm}Q8{SJPZclA-~0vvI)usLe&^3gBG@a(xX%c5f7ulZ)H z$HY{btE;QCnFCM$70Bm`yA0TZn%IT*QStS3D#OqC<0&FLaO8{l(Fq9)Ch;mvt!8?m zuf=y>$7ScmKQ*Hw{^4SGwaTKF8C5LOJ~`h^tb8DDSL1ivijNITPHE@d8cvRO+*vLu z6szy#;LTFw%5m(>FhHWW?k8!H-&eGcl&E_2h1#VPwv66_H35QbFfH1w)Cnh&Xa^sil$A( zvBMsIvJU0ozEvEt*RsEqW74S|EB<-K|MB!uF6&BLM(0OzykAU%|K#i&FP@NvoW?+k z|E70=nj19xWP6-kCZrwzAD`+z&}ti*#VIZ=BLnYFx(!jdO(hn`*Us-!%D#r)-kn(7 zrNB%LT86YTeEF1L+V%M3#4zx(XP5D+)qyQHL{Sj}F^n^FPRAbjYzO78fbBWEfZmnf zukq1~O=Vsd;+Q{*3t?SJDm3kCh>iP0?S$|IRSdy{HwnN^4g?cFa&3;BH6|uT9O%pG zP=x#I6#`Q14ly3t42+t2@TqEHQZ6pa$jvf}o>ZYKLe!nWt}LRKe>H*~29>pp7q4lD z5t-VpZ9G$6gC^3Cjfe!CIBJgaGurr7RqVzT+wI6 zz=9c06qE+7uvK&CR{K}G2M8O!Egvueb#9Bgk+Wr)F@;-N*>;TZ3a8#*mYbia6-RJ&KDVe#uWt!+!WPQ=MCYg*%|Nyw3{> zM=6Hrr4_Lz3#*Bu#fH3i-=g!oyB`(PkIZ(+twKMptKhpY5@jz2p5g+Xfm7A%c(Vfo zJTxrbHM$DWNB@ti^Nwcw{oj8}6{W=9BB)ZUXw8VdYL&KHwO8#uW7ZB*q1B?4*66fi z#ELzlD5YwI+N-wMv%go~=lALR{XNGYJw3+=ckb7HKdv5Af7f6^J4do79s_bg^#7d!-Rv?K}rSN@J4e6=GWL7P{FpWPCgdt4x2k=BQLL z_2>Lxe>$2rE}Txs4g9`15GWh95x&uyu>TJx+P(T0^Kd3C>_pH3=u5MT^ZOq!U40aG zvc`acR9GUUk5%vYAl$AKfD$Qj@(S}dcEvDj#F8sA?g zO-22HVA05PVH1!{|H0k?Dhz8_01tsc3}T(lUss}B?Upsw9lqinM9d!mK}Ndv8@ZIZ zC%`D5sk5_A65N@*#t%R>eOlUOO?^92*n&wSkG|j+5awj;jIMD=lJ18QD`f46He6zG zF5s8{2pL^c5p?-&bb!#O_5;;_XC#Mzz~PQ3QpesBmtM}G4$q+P+$**}M&j?wt9SbE z?P`7kiUVr^U9D#yZ-@DoE!lD7xz1_=p&sM+IPicOK;L0{@8&MVk>>}u8FfFB$i1DE z9J^NL?^j9A_H@)H1@XEsxsQhpub|LMC2Z{W?pa5%6PgE13U!En)wyH&ZH_ z)934S9!c%Ak+pIG_ra(^2#_`-`|1lM`r;|-IXto6e&+S-7_*=;GF7Uv{p3I_--MP2 zB*zIAQ5(AM)X}b^S-4B_TmbAhNoewX%ci}0fRc%$7C9}ubu#l$~~UbNBa?N-c>p` z#0W8eHNKwiZ#M~5*Z|t(u9BoysU?Oj2wPDSHrl}FWft&-m;2kO2&kKb=Hh@@E*t%K zB~lo8v9}IrTO~i+DWG1kSmRv>#Uo$zXZA<=?DijT6KL?Z=0%XzX9Yy&s?rUBXUns_ z{G&ITSZub_hG2hoks7}+IA3lkn#%hQC@1@DXe}eLK0LYaRG=j z-Y2+w_n$W9lD|-E=Lt%=t6oND5i@EtNO7Ha8k(v#t8p3ePpvef4H%)3!ExA8OUf=D z-ZhZ7dan2UdX@Gyj8l16Wo-7`xseu^135U<3jvb{u*RwpU1EzJqadSQHNP=HCuTif zf`@WPc0xxdb@<2L#dEuS$o9Fjyz`WlhGGeEYa`?qdsuWt*YpnQ@y@ae?>DFDZS>E5 zK;@@3iwn~~)I*R=q=uU+bQ+EUT=L72J#uuRK%~)1yZ42lLLSYL+wbT)Zpz<-$!IA9 z-i1fxDC9i1{~@P^Sov9|YiCp!Uu&}eb@Y1*oo>3=F2|Ul1t}Xbbs6Kh{ui4}zIj%E z@2H3-=^7eNEiW57*(=a4AXyD@DQF1Guky6?v{)}T!rNyJm!chWLtD8cc5Tec51m(k zuT<8p+{wZZx>dZ_uac|Z^w`;-=-K}j0v`9w4Ra4q+n)+-TyL5>htQ41eN<@&@pkPS zTt;Edsy!?~_BU^AX%-9tE0BQGU}pHB$fBa1?*mfQ=8EA`bs?%~^pu%hvq!w;*IpEt zDIE`>TF2e!l(-``>-l5oW>aMFytSi^R1`2}0f~p*bYXFbJWk_hA!3`Y!!v>q#osRg zq9W-Oa2A8qJ*&!7>bxxtS88VeF5?Tq)3(GS)!q-tn3-k@yAQ~?Hsqg48Vp}O!kJjn zx6(*-o|+n}a~h*4|*ZUoB^Fu$R5=PSf}BsLZ_gVRS>v5islTm%)7xD#I4n z-A~qTi*xGodN!qi%-wu74tbt}oG7kYZ;H1fiYs<4sI#8@F`kxCDd5~ol||g(=UK>* zO`6$%9y0*}*LI}VQSkJoYA;YBF3ifgU{j(T-zx&mH#R4N19n3nIM?A|7_)NkQ5cU> zCF!jvQJp?9s zF~_0T4?VtD_o#Tr^Eqhm;tt^$2wiL_@JXBGVOvLkkEW`pRW13`H%})lXq>+glC$P! zu3l`-f8#i$ah2Xiqu?hUpk`Xqc+Ubds%VnF{x^&R;BGXp?FJOECx)=SN+3Eq2S8L& zqd@nkJxZzZdro)&x)`xh?J?=*50w2G;pXZ;NC$}c`6$k;iN4v{rp+(BCvs2~U z=q=3IMp1BLy2}-xf0WHBjmuX3&Vx(wNwcGA%*_bltbMQ8PG^kp#RU4$8nsi=1=;TT>V?J8h541VD zT|0-PuTP0}m-|5SS=_B^Z*RZqU*GnjYgCpOYDR1HGd3R-6+b_MU?o?fpX5=vWxx0H zr*uMbBGb!P$k%$;5*qK|zu%pd_E;%%OfRYw4Ko$y|14_y)6M@|h1>ANS+VyD=NH$p zUp@)JuIpdNUVm!<2156Q5^uaL$G^sZr=5*m|Dese0eKYUtifx-4jWl`K|(-aiFsJY zb}qVym1MnN3z_ynx}3F_BLaYmI);S=6F#3LCvp4&n>SS9nS0~>J!X(+Hs1?v3chq( z^*l(W_QG?-(Lj99ZPcv-8r6iQ=>cXle|Y|vQcA5q<|z*upy%M$~>UN=gvC7dd5 zKyGKU>|EsM*csHx;vB^6bJ?~cRMw^6B zEnEu=$y(TA+xlglB@V!|qx+wEZ_EAQK0}|Si&lhsgo+1AgWXBj>kt1PCSwj=+r;jN z#wmcZsXgq!5InTrqHQCcDm`4sC!S-{VAG|U-I#pe%50xG6vOz;!@Hx(pLr>k75{A$#tS* zX0MiFAM6pKvHjj^vs;)4lN)t@zXySOL+Fr-8(LcgM5V!f0D2%+9mEZG=jnE``}z5S z3iI}wZ0DkFiuA|$@2ZRg(no13>MtW$0n+F@>V(h|0Dc7y4;sY~qgBYnTNqUN%QXLY zBJRe_lpUc@8(I10-syJ&q0K=PCm>i$8E0R_xmhWMN8!tUjit2 zS3e>&xw0cLvA7%=z1cy$wBT(fk^jBLUY(=}jKVd;rvJ7*T$E)HeZd-qmt~4|2PbuD9uEI>(toC;nhum~r z9rmQ4cRagnZ|R4mpZz+en)UTbtMo6fi(_q=JL*j0Pk8F620Befh>0{}^-u_4`ilN; zBO!=l%u}-B>H9jzoz%r~Plw`dk667(Wt;D4iZMd4oi7j3Az;+caK=;7e4;FP6t4!z zIi#8G@v$H^pEp3O%*+B+*ZU-~x-OzYYrH?=+_xS8kVGR3pzamlSiNK0>oyxtR5>bF z?Dp!2j$XT#P|jKFiXK>8GM=2l@5mCK)AFta#r^tozWFCkj(FIEtUM1lwQ8hF zQ}ZH}Q!T3A;8)4!m>$y9AcUfz5@^9>Kohd!8o#{t9K5t z^lKNu4iG7GI}Q^=a=`3E?L_v8Py%Bf(D+&TZr?K7? zCQMP~tHsU%7!WRCn1t!xVk1!L>@QYKa^@t++#qS40)ULI9T!KC0#n( z;Pe-o8&8bH9EzADBx$TdVt9O?^e&tJKf0W?~_)Y3X!0 zCL9f#Gn`(8f84YiXR_V^z+e=-%%8pP^KI!oqo8n^g^P*y_idK_SbGI)hFxU00e5`{Z{{3E$``qhmnBa~dGtl&JaTFq$ zn&+%H2U7%vEcI?W?yoH+)UQX1D4V2$K<0d$AX$nl$w;QuSnFw+yDw_KH8NyjFwu4K z8{NePI6^vNn}o7OCGQ!}4XQ$IiFEMLHs(-S_QMpxM_U6^V%)jFutYPG)9#yUXtYQmatkml72#eaL8Q~^xvb$EH@c> zRMQ1$2XG$|u?#l7V0;x4cWDOhS8J|Qv?Y!In3b|z^hiV<-X?uZ);hB{1vtBmZf&JF zRIv4EB%EX}!z6)noEV@c%5gcK0)mn4)VhSt2|jPs z{D^XhdfrQ2#pIz6B9hcX?T>3N%>t>qf$N9y&8f77mC}XkS(OXx_mYo8pHh4xpp80# zs>N-&qncAvpze7gbS?jQ?maEFwOwOg~5?{&QNiZD*;Uc8ZS$Ipn}4`+;+bzx(h%qVf4_(ES+N zmZWKsEv_IGrQ%M>-7Yg;%gWD*X+KAXAE{Xs-yR-$>}%KX;9+60e7C2^$12kw0%p4R z8uZ4x$mNU#pf29M}CeAq~b;szR#0!0=SH5!ti6f+-gxp>&~9J zCNHgN2<$o!N@7cH<4)P-;KeupGskTcp1#RGoxB`SNZ-}D7n1|mYBz`AhI>>~< z+avBI#k=xTD&Z|i;31s4l5hPslvd#T7BEe9bLT7}N(tquN)@(Cx^`-Ivd8^@+?_9M zCP{pgzeZC$pooZSN4rMJ~V!QC)MR@X%g6fQ5z?8{u@$^~HbSD#?uRk5z}A1#_3}tE3iWfa);on3&ox_Es1RmMw zwkI62)2JqacYpq@n&I}pLRO#~sVM@ssNYrL92)*PI^C@7*TnxEbh$%4_JKO7Mx)od znG)Nf*AO|OkZp%mlliuS&jdR%5`%q)o_z}U%00KZ`4k*>Rdz5CaHC)qf!SJ%{LKwQ zz^7G0G<={`D;&*inJk|+mo5EiNix-SE1(-F@D4UzkzmFH9*N8lWrd-4V!28E&nVOh zAgMCGGvf&C$*u;s9grYPw}bZP_IH-DkssEjMR-hde`j85Y%CITkb4faV;gs>8L|l< zH5R@OF~5gGuiw!A&+hARoD%G>)Cr-lq*?JRf-AgyQ$LlxUO0VMj zfjZpLQ^IQTn@tM49*<&x0s_QX0@i?SGP8URU3EOpfg3#;zd@y7X8HGyJzGkqa4Rfb zK;?D|YNk`Q&xBQ2rYtPSJf`d##!ny4pZ&o&hQ^FJEW}Pov5D*N1c?F?;2(5}BQ@|) zGN5^O03)<1cwr&0m=UrxSh6dEUcUKU>prJAi*{VQ+!3Jq)_&Z%H2;!!uV%^p?{`+E zK!1_Q!BW**Y60;Lv1Mp*vTu5zvgPlN%h;J)j%r5v(Q45~lgirIX5Or4E3t!Er>E%; z8S`9G%^Y-3jV!xp$#TKJ|MeXMe*FuCWcP}oaxT2|YX2*>L8GtkV676ku*<`T>52`9 z&LQ(?t+mM$=27?RDVLEczvy2Mc)i7$$~K7U-5I1 zYCndut3&E~nexN)l>VLUsIIw8|4Q-a)*}z9ao@OC=mo!l!sgSZ2PfFK8vm|*aNp#z zns%+8v|U3(!_2c69MW!OMAl8>OJVo3xIIi^NYYiu_F5b{f0%zG{Jf{2iX9tw};_|hp{+pKp8%4K*_!?cH8+`Bd z{tbt!s;c3MQ8&>SkKI<%K1maaZ(G_FSR%eIP3T?`5MLxp))z)y>ch{1&8oxS`1=LK zEX^-ySw2?z_el=2eMkzA4{57M$1)pY^x;c3!RKDkt)(fWOwE))AaXr?cM8P2<>KO^ z%8Sy*j#W6015$$>Ij7gtZ*LT2)aM2ko12wWf`D>xbWTnV&z4WDxz6nR{O}kTQ}}{U z8MbfqcwwQ_&TIiUyjO(>i+87QvIfBhP%MI=jP|N#k^!Uz|tC|PGgG?q5j2wQx{Aprp#HJ2b?I)uh zEdBIJ2vJWAj{)&AH!th&#tB0^m_Fmi-KeCcw~Z+B%q~Fo>Wn61WoN&#Cy`k>vnLld zHRq2FU-vSwr7BDRa23A?U4aO^X>4rtLIuMDJzwObvLq|#*5R4xxt-cz#jWtLUgew- zGEdb0*Ex#0pZ*`!|6gz2S(>5m=%>4h^^0;Geem#Md}z9uwtV*mRf^@Z zh_b1=`uga*vk3M4Y$`tMn`s)VSE*?pK^{^Zq!@|4Ugp3mJ#VW`C(Y$g!O0!a-J@a= zt2Fbdy;eZ%J$fZ#O5fSJbC~f12$i@3MekRa)UQa2sdJu53E*waD|N<;XyILA^NjF` zK*&NK>SX9TYWs*)aUiag)WI4n=gkkCYgQe))bjp8#2kz^BB48kh1GAj=Lr6V;(fKa zL!F^~uFanDw?#`vIsN|IvmNhBeM}QR=8N3TX}|6MgJrM5a3v^>+jV71VSBJOxD?=% zhmIVbp8DFJ9ho;b#hahN;d1Hq=Zd>Pm(ClOA8P=PCEp(>{CBypt!&kEl+K+y0b=3I zw(sfUhASpQ!wT`?*_=uZy1kfSHGy5uwK5mx2MAF^=w*o0@wur!vu$$ z-D)T3Oe3diF45!#h26H6Hl4GD#en0ey&DR4y zMQnQwVb+}dC2}No-h%yArJ4U(E>G(-eKu8RIV~CCIO#EaUgximB-Q)&_72G4F<3s{ ztEO|}XdWM0c>J^~W2(7{gYVeZ+O2suNz_M~7!WUYsH9j}DE&f;R)kn2g8JmBKZ2Wn z4&EO!X~$w2^ezk#OECwH2MritI1^vKt2TustI$zGrphc3kT4)w;k%;^qglM^JXmly z8ROJO_Ok!QzR+eA5IB-4p?esr%>7AgoZ@ER*AGQKYPq>1h14UvDpu<9wHN$a7Uvuc zniadBJH0qQCwr$j<)SBt*Esj49nj_&0uGFa*(`wfeK)okV=MeBBFd{rOOF(>%tL+z|#VspT4NDFqWn+GB{NOJ&8t&g8H&?Quo z4*7m1?~R0ZOG(r39hpCskQ8WV-+3=Gmk;io^P(k!icS{9Bb>FsVSY~00cSlO?8eFG zY{I|{#SYJlzPt+*!E2f3FfFeN|236uyO2MO)@_lSc?NCn=K2SMKB1V{#aJCC_6flB zJ^>y{$_PzT_L*45Fw!uUcF4;V6-yRVsagjg(4Ai-^Gh&I&(bd8=w;e-d9p_24z4@(c5p5nDQMB2Xd;+X-7unk+(f3vEMg8D4jVgtbELGWhZ7KRi5zv zMC_}v6C!y8*rou+ZX0V*0OwhS2QpULISck^3gA4oaR)EID|_&g_JzbhlqhoFDB2`8`FbEA>pLdCg`kau*PO$4F_R)&O$J{!kB~cXM32zD|stQuGrQ zi+?h!Xg(HG;l;o{xuux?Xv|nRU?F1}CsOim(iRA=`Vc{R>s_J4jWEaDuz(uFn7KTS zB-KCKj|+YQCT@GcHh={H;4L;DvxV{g*Ij(7e+5uY$^G%Du_pUNd-;zv5xO3xylb3u zx&=;49e#^LX&#p-+r7KDA1f#xK6(!o$yX8dglk+%s7(>YOUgnv zt@;!yfUDR^VQ_nJGN{3 z9BABT-(b-1^xe}&M&K)*iAZxYBY$E{)w@GBqo3F}zao+R`^N|JNDNRUwiOC10Q=gy zQMA=>)Cup#-c=g~?!J$}-KiiU{&fCkxq}RuEjc>T>d+$s&yU64OgkXSz{$n1s=Lc4 zzGdy!%z20nGMLt2v0%*OZI}Sc+CYms5flpie0w0(-uU7{7;FU!u(oX)KI+~V`2G7s zF3pae-oMAcdio7NFL4@p4H4F>P9IfN|3N`pSYA085{0PIy>^dCE?t!=s@_{2iC?co z2*j|GBy>q9BZPUk1$E83#Fa4@>0E#VEkzVUTP)ppp@x&BGj=P_A}<1=tj1$R>)d4< z#74iwf1J-b?lg6_5jd`N+NLz>3ef04I&qIk=9zhuyJR;a*6hGkbPfnX%(5g*nTEHG z*rN@4Nkj_@sL#@EYu=)_^Z%sTM9v-bmN*tCwh6~jz}mUiCB%ji>CSi0iz@oAoSlE0 zI@Mo}_l7ibUg8g{|9A;c0p9Pr&ASWWjj2?V(P2P7p1PkT^w+8-OALoYYDUr3Tp}x) zvs04|!}$lG_0o_GPJwh!)aiK_gObDvH0Yy}zy@Kr2j|V?FwkN^`XAs@{goQlsFI0aPeV!oE2nswXwa86!h;^SU&sDH#Ceb+q0I!} zw_#vb@8OOpP{oT{)ZNk+D4?a3(0JSs{L;U&di?wJ+XY`afZTwXb=_R1Abfm>u5@RN zjQg%i5`4*&hyA*pC|I08{#BD8sFdz($ufp$S2Hn6K2e`Q>!+{hYTvNTQ%3L_1t64< zd3aM0s*4GTJh5>aXOy?p6J>iY?{+$sQKQ_tf7}RQxTYM8H9gWzSi@T4J&=2?dIz-y ztRdet+W~pRu|J1v{`wxGd5+DyOEFhzj$eYibDB4q&!H<{za(LoPr`8+A*gE4wW@A= z-fQ#c?10wS9@!;}X`zRF4RwxRR>WIl|H%Vqy3tM&S#ZFV!4K@~=oS-BsO1r1Q*v!F zvB2}Lq||zg{oRVu!cWPC0wLM;t7D(BYLz^}-5n?`FYk?!UuVCD@t6Ss)M$GE^UFP( zx*bt2e7=baIC)b&R=%JceM$9v3o4cObMj#FV6=YHyQJa3ViR-Fbnt8c9c^gyBz$9h z-QBV2>*QIo?^?4OP5FI?*y5?0mYm?l-yfeXU_Lc`!^`8_&y{CZ+=QeOTO}v zN?@HeHHYRWZ}Ytbx%Vb)zle|FlYi!#_tS|q^%BMP5+KN!&DZr5lj;n#Se8b0&Bn5L z2LH$+M*7}`wKGU2Qb0S^o3i-;%uS{4lzNFb*hUMsz)Dq)a;49`*qDI^C+b3Ia|A}>Xg;Nsm_%e;I5khZ~?eVNJ$@GJhsu}@0Ji-$pyBD zf8^}l{%w|~LyaKQb~eXs336Qd9*M7hahg#bA!%5;(RCfpJ=!& zFNwvIK^a#Ki;_%^y;Q%Vq=?iEaIVzOmjA+Qg0>-ExeKyhgFD7f) zJpuKP+vSLcrFE{H+;}YSW9r{h&|CJO)O7^hX=CX6z3qer`i<@AF0gfw@T7wbDm8pe zJHyf)vtQukb~Ipv+^va%viyt$q{I&sxT2MPXN%lXm%?V!QII{m`@MMSr?AMA-rO!l z%eCfldjJ>GyR~ngw}~*H%M2e9_KcPe|mP%MDte*5$xhkel-@BAUw7qj?p{9(=I zaPG&%y_x6M@w}Ves0BtUGU551S3OVKq*jkl0LZB4k$H>ltFoDyY3fBXGNjFAUpM`)C-xNiuCT1 z;Fa<)CWv+=KLP1Dj@F!);-2neiTD3NW99(2gh$wS5u(Lfay6CEB z%iT-v+@*U16fQQD0X_i5W3TNKn?evJ{5~irHGH8+F5T)kV8=7fAw{@_0^!Rwz-f2I z*FF^cEnVUq*hb2L;$0bJqz{Z{I5ObhC|&^BmNNHfY<w z^Bd*DA@2nZ^AzcgX}vWGJ_s3p4y_z3EhV?i1u`Z8ARF1RAnHj8Scwwp=y1n3hlw)s zJL-ec%d5sHxkD(Z_@TMCMv&!{spA`_oCvK;VDaYZ^}5}!@)ed?x1a9TxxF@t;wnOb+v3?F0=BuBhgjUFRnAo!KfU(W4GNzY6k zSlIons#3w;zdw9g=jY^@LqizpK!c0!gP*1_tG717>WcK2Gd_zn^tR4Q=<>fX>2eTl zawxt%G!nN|1B6Pgpx|eY-yj*X5jcfO8=ahxB|9#;Wb!$IVx8(ZJmkb z)-DH{yvK>Rp~cvFwbuj8#;F0Op6T}LAA$J0f29^bm#4pjuISFR6vXFQyCNZzYAN(p zZ_5H?APyPtdI@Z19Y)>zdojxsLu`2lvZk!9!8d7!JnO2V zOY2e`jtXa0AW6MFzoTqCC{AFY?5_EA-SY9F4C4NR?~?D#iQ4Rx#bF>@OCavg8L?Tp z;)$-z!qIP6 z$+6+C0G;$e?f2x|H6z5EVHlJ_%tZ+C_Wi4}Ld=)?{^OJQqP)C8ALeRS`Wrb2W1;e5 z(6!6yN%(H9Pmj7xZQuU-{N+Z}be8tJ;$jc1{VhS&Ya)``EekHZL z4o+1jxB{(P8M5RA3;D9k2qAk$2W=$Xa&(IQH7^RK$(*Nx+Wtsr{YCaWVYr&49or2Zxbk8t5|v|KSwPLIi%tpI8p#^trX{ zK{vy}(dm2xwC++KC323TIflGer~WC}zpgAx?ydT)1JHD~tmDILG|5vF3Rri{DGu{T z){uWzI;_*rHD`$NOm@lC>uh*n%;t^R=supq(pKV@EF*7+ygc4b*RkF|pi&D*!?C=X z)HhyKT0I_IU#}V-`0!!MV{K+o>*Y{`{&Xb)V${w93W!w-UmSdATd@)3Y&GtL7|Msq{RF zbN6~XzIwq ztlgS{+_sqawnvIxY=A8RMPtCkEPoPFN!TBy?@>HnT3#A*>>Ij{PnJ7=#F~D9lUs;{ z(l0#SdkZefyq*C2)Fyw-8m5oU9%1?Wii4`6Icx$s&dDayr?oJ3?!)hHQrg@*?^Wnx zAz%s}bwR3XZEPIufGjGIF0664_UXHDv2<}EnRH1bOZ1~eb5ftV0l3Dko13%=(Kl7k zdEzsg3l?<%g7YaftXhg=tF@q^zkK`UlIK6-QET;dx4omK|KO0X@)&OUh2T%d^$jd|F+U9A;r7i`+;YFnsF6!r3XQ@^5?@35? zjWdXa_LcYMY>PP12ay8k8!xpV%V%P`{fHPwQ+FoG(z-O@8|pvM(5`y$+xRo5TwEHv zjE49AE)c-`%DW? zp<;X`QhoQKUlZd(p>`?DmDTv{ImDPo-0|&l8Z_?s#T{3R&vT!$x2`4PQ@;P*kYa*t zrp-oqSH*QSCT3iKne{^?d*b8DM?*JVM5Q%9zm(@kl>l7$(%QZJ+>jd$BrG3Ozb+Md zzMZ;a#6f{gwC{e7t|_F0tL!AOkbt3I-NjPJg=%#chE7JXDJ)GScdO+o{PVIO}XR9WY8bSN}Z$EXTvG*tbt7SE-x-_JG=FBVM9`}+Y6 z`&e9LF^GIj{o-myo$3zPL$%+%HhV#A*>{CEpfisQDjf)VQF9P#5)l#6*@0+s?*qla zIiL#+2aH&sl5>Y%I0*l3$r-vPQSim~1tBngCS#E-CA}ri&Zka zY2)uJ1pd(&utu#DVxZXRB5o)&IYx4=P+P6IlS1}XLe=1gP-?H9@r{IcVc0(BN8Bp! zo+a4QwVid(%Oe|J_?joS3ExZH?}k2c6xW7N4=+t5rZ=2A)Xc=K8hv?yE-#M~OehE* z)(7hg^sHxSsr2M1Wqx}s*3`6rOdzO3ZKxG{(p9shH zhnV!*&DK8at2oF>ta1lFanr)1e*f6hu6myv-DuVKV(i9$esiG>%3+-16lhModmF8~!E!M%o0m4FfYjbO3d?GplCAapoAxIrp=usnh4zk=V0NUrmhlux z;%s(-DgU<7{88mC30nK-WeBJupTBtX*N4;g?y^pNosjwSIj?Mo8jS~TVH05oj@w)3 zA!UP^Y7HCtkGB`XZ>(!{B?3fLZOKpFyDEJZ+9mZqCW-oMF2X0$x z+H&5N7yf9n;$@L&r>G-$ZpneT10jO9xlOKwK$Pt`$(8SKg`?C%`D@)}kD^H`lOJ|THAG2E0MAOCx*@)PpI^6to zR&C@i`fi}8C6Ej47{k#~4>=l5FL&lhX36p&Vg>%IWK?3-;!)}j^ajdU>0a^U*wtEr zI^lL2xA7o_lCgU7-1;59pgozUDRtk?Y>u7Ot5b$T0(`Di*1*NyENltveJIC41NPTBEK8Zpk?O6)eB_wl>2pktLxvm@6Qyv<&x>|tDgv=Bp^TPE8b(*;{gldzsT4)a=b9GL*hYdZN zUwt!^keBpm&#V+A<5cB(`dSxW@aTTz>fNVZND38|_^`J;I6m?@pLdaE5hkE=UeM;f zg15`K6YP_`XG~9XZj8wbg^ognqsm}efAq$^^@8*{#UnE9=Fi$J`ohCZG<7-PcRU6lb3~Es)92GCzLgv1WWoW^*^zGui+C z4gLGA%p^^bGw+0`tA<}XJEZj~iZUa)9CJ~$Ki#XD+vAGfvzM5e znZ*nw3R!_Stzf7(&F#&KKYB3-$y^F?fGB0o_>}^OyOLw?g^b{_x=QB=&>&t zm#iq-t`oCg9=&gj@W8#?k;e1sP%auR9x+~a}7JtNsr zz-bU&-E_9ziMsc}fX&9m&u>Uit^l;BH*Pdhp2y-Y^os6^chg}~69P!)umU;D4uBS` z*CY$8w24T6^$JO$)shqdv|vUgTAfr`wUz!kF3qZY|Neb1Ns>sLtAK)Zs3s#x)9Z+c ztnjVuaP-yVcC@1$MgzTX2R>m9KT3~a+YG`fATP*CNI*8XYUs|_z?W>`uN=5n;nL0L z#s!O07c~r%FEZdz{Yj(SW&Nln*Y!1Dj2$Lm?GR=(zgTbI1Iv8#pTFz^pww0$DeCJ- z*HJgs=Bi?S_58S~=8rxOy@V{cifvZ^5^7U4Z3 zs(d)0CDdgiVBB3ZboQ}1Cu992OAip2+iaUU{Wg_x(Ut@F+wAmu1I`*24R8)I6?!Z0 zQfxiIfjqaZcV*Bhj3%d@CFbjZXuAaVx4viVH<>9n-b2A)3HCKI@t{a_1A73Yp;M2< z+q;?)C{^l?JFSv0UcAt(oKb9r8ST!d(mgYWZnwr#XY2 z+HXpm+w+o}MVMp&umpTj9UREldS-|m&S!NRv9#l2Ja=kql-kZ-a1X{bH+~8F@b|DP zXa)9ayAtksFMg)T(8Xw8DDBFTY&!UgXvsYf*xmUBzfPWyUGYIYfow+Cz|%*d@|svE z*Q;#G`>j`>cit^V<{5FQRY)Y)ajx!B$|dtrYZf%#U}m`|B6{h$>I>#4ioFIOv?wmu$AKS{}&%xNeYJ$f~}*vNh9o*visqeK|4jm#fvuV=Pe8C8PDKa0~ZQQdr(g z#O)G${?|vj4O&JqwjDsi(+NM?7^cv8l-0yWbfoBwbB=^;)Z!?v&%2BoA>ZYi#F$d8 zulYR?elpN^1->SH(in2!ft_V4z3G4NLSAtdUs3ePe&$wJU8%j&TFtmRxF|v)aHCz< z>ex7RygF>W;~aF1Yj<5uKbxC2`uD=YxR4pAa&ze8NB5hLI$dN8*`B0*)xUCIkMY)Q zfw^ZN!oj%L`%Yx9sOGc{HB*G*>%C%D4-d63a4NXEvb}Ega(z?lF*Ned1ONzm z;fs<~N@Hc&HFA>q3_C%49L6D0DwR>P8@xUA_VB1%Go6PDvamlB%MUXUEB5TNiu#`{#zuJUfUnnJfy)xa%Gh zH3NF-3;S4PW&UWJJ>PeV4qjo3?d$}9NmV4BzBv4=u3o(!v}vPmRj;1$Lk;t|lsoQI z%bLQvbB|=G(6Wrx6-z0fb)diw`p*pth#N|};=J)fWMcTcUGuVyhaKY5dp@D-_#hMQ zIaw}wzply_3X#Uphd$<`v=w4=H$wNsqm!Hm$G2as>JI|oAcb|*a^|JJQ@ua=8e4&L|!D_5NBGdE;f1I@-@Hagx#Il z`vhL+O3#2oE^6C$#&Fv!O19}Aj2o3f@~qq*)-EobW|$(};x`Tx4YUuJw7``>U*Dbg zq&A?u0XOaXG=gBAc_OBqtDI!AgQi5&*0iOQTj1OMqiyZS%d{ik?9Q?Sdc4cm2R#!X5POu-Q&q0%>u3oV5a+3?KF+%hxh7bBxBv zeS0f4CBTIAm*X1cNvHGY2*UMNV!0^^<-wb6*h zdr$2Yn@;MRl8DYbB;>=G<)?5?S+D6?-*W?B4xkKuy@6U#Uuo7yq$bZ7Av_?cLyEQdbvYVQ{|+^PXOo=!ZPx`L1tuKG&?6U*0b=(9pZb02x{L^PXv??#s>X zT-*hroNQwcuGn!uE*1cw zW5_PTZ^H=o&CsRXX#iXPF|9v!o~}HPaRWLc^~1S10*d$bfNg{S%tf>xg!Ug~RIN51 zgfy{i29MPD-WnajF5cF(o~3z@ESQU9L^kGI+jnp!UtG;|YKJBonp`MP6g(@WXy;p} zh`n}254o5ps2Pbk5EK|l_Lm}{=6r6iErW5SE4HIH|ctWxsca+14(!*(1#_FLbY_^Dg*4vAXFtLo(-?3?H&SA1+LH{^ZJE za3-m?H4iDlzy?Vg&A@#T{qQsog7~IE!HB_rCWS-Dfia=INEt%@t2RcR1N?S{ojt|E zmGo+Z{fy0#J(n^I5jsOqYqz(z4n;1 z8J~W{1%D~~&p*!TXD+BW=d%^TjAi-i*)OaU8SL#TMV5_~58o+u9BPi z1MU~xogwfn9O?Lr^rnWhMqs5sbh)`Q*tS`{+Swix@T(5UFBoiPi~Qa!t^j}H=rXoP zI7_iZb|Tpt)!ld#MnFgi`n*+$x6!gGeZD>fOi?J{nO|byV)%r6bI#9}`I2q%2a^5w zF1x?|dHyev(2cWD_&<|FE&ySK<67O>zrH)fp*`_%W53gA8^rk*- zYq?k>v8Q`fJ zj82w5Sg?7L2pk~q5$DJ4)yL%^;*^?}hM>?piIRDd+TUHLrzrG>6364~TZ0Ly+1Ut4 zJiI&2ZP=#-u989x(xCXlofelCm#htRK85mXvCe3O4sU*?RHG0AcMVe${TD{UlQR#x zO}FJZ^aZ{aN0Eu=Q{Eu1A@tH&Z(hnPD4J&CqG%@DeoNfobr|t3Jfek6x`Q2Xmptm8 zKLE%j0Z_dD#3$oFjUP$$spq8mt}~6r457-gzx@He1NVmnKi>1rsh5)JpvAOd-FLd( zwVf^Bk5CRzk}duqos-Qj|0eCN)W-MUdK~AUVJxosR#e?-IK!63dp@QR!=Y??Av=#u zO6P-D%t@SJRe^hzTS?q}$^c}o14d9?c}3j&x}d^EQj>69vt9q9)8;Oixbd#RNe>J5N)br;s!M_Zs`m6;r$0y3wMUrdg^SzznTi~Ljq2BE@!(3&aoThDy1`e59=AJml$ z3TMqck(SH5`Vr@%k2sYn$K|v8qPbQAu9N|5je*#gZ~kyS4Y?i;6h9+DfAF|Hy=CrNLaWaEBL1jtp6bj6%F2uqi_I_wG?qSZ!YGkR(W{jANU3^{`)Oz!ewR<3G9S+ zuljGDp)Rhl{ux+3mDlp02p9rjv?xLUz_Hx0m*=SC&TaL3h2iZ5r&`n#@8c?Cy*&lPMyvHi9Mj_j}o`*X5BP$Ma)GBdMeb&%X>k!1)SY2{w`O+I z^|j8;4Y{C{mP@rSjP4GH3qazZ!Af3=k-CYtZ)8YM0*H_#@x`cD3%*! z;gO?A&^RAGQ2!X;!3GP=K`G+N5G^=~`m)}9I8o*{dBSCvqN7TtKnne^c>4XiMG_dP zW4C*kg-w${mF#`@X9~)dO@S;qVEAy zR1);2JWS=|=$yj?akkaO5B*MZg;lsVYVa*g-_Ppx?FAG^L$!u{48y|&**OE+g7`bS z2ljVw4T`=xU!bDO;g&Q4!M=7%9#^-~qXt-M12UijC7++;P*K``sD|Mf^Mrx46-57+k4-Tz#3IzeohcNOxP!EB;OBjPDty2lGDpAs_U1j~(i z-j#_BfKi(eI?e!lZpgGws3J^1>cU0*vHfZwk%k6VUHuWnfGfl#D~J`NMAW?4dCjq@ zL#a7SO(R-+GHlv?v~ID+Rdq%?fGosutmD3aVeRP5H`k3AvlH{zUR6>DEh%h(>v@6murmE;TTr z@Wi|rbS&i;Z79Qb7;eGiw@>H@(zwSSnX=8oz7{A+Uh-pTgtboYYVWCsYu+bw;CN~N z+S(3d2KHtuA zJi%=rA-jaxEw_%LAfMOkYp&YRfIxEocxXmCj1C~vC|BMymh!AB%9H$tdN?uMX~~EW z8TXpj=0{ZquB}#_d(>4vacj{mX34GPV!;%c+1KgwcuEafS}Lu+4WT#K(Hd@?mkmL* zsn^u9`CW7L z{0;JcY6MV7H8NO!@e~MA#ZO4UF-qxlCZTjvrN?#TN6fWH{9#rW=+z2labV%)4Uw4@ zYkduguCUVp^1eYfHuYq3!#G@Q1yuR;mYpsyWyN3sIKMY>Bh6j-gk(hp@F`2Zg|tvS zZ--JI$&$LF&?__!vd5|5x;mwJPYubB72ydnlNJ<(X0W{}UcL>AY%HTlZcIL}_m$pixVkt>TuvBz)~)<}Np}F7PMQZ(TJB~pd9tfCv>SzM z)ubm?zAUV)J)1vCD8;G4ETvgX%(3R9{j5yb@5eS1-+hfm-KHBv22k}`HZh6v()T(q zd)xa`w=$aTub!s9+f-_LDQ&+WQVxfFthG~N?q}hK-%X?H8;iPTXxc5blh+%P+AFHm z^V<2R%4WZBzoasJS%~6DyLLHZ`vQ;E8*WY*{#{F+WMA_iN?jcQb_Jbi2^RxOOT1R=>p<8FP85w*-dgk{q62Np9(aINVB| z7c_06Ert44TCZdH?S>c`^cI^!EYnX(=Lf1BZ6py18&xPY2ASTdpJT)a*GZRpoC2X1K6Smyka=Ybe|at#9*k zj=3yqa~CcTXD&zObsDG+WluR|a*!6C&GfUc=S-{Z3;nWJxJrKuZngpZ(*C9_pa0v$ z|E#81*|r`9*vjPiotaJ;I2w{>x&fZ_=55rl`w_$)DUf_`_3*kY91k0Jv_kwycE_9y z@4LA@)@TdInb#g0JrDi2SV$ZGzJikS1YbLagp>N9hht{;J=(lwo0h%Z$h_GMe7H8Y z%nk3TB1bPHxHK;R!aP_*b97xjb)2)b-+e~FBdK1&-8(?8aLKsIuITW+cSWqpYJX9j6rAe8n?@b>x=CrRR$L4OIXXAe|XG^AKDG1FK($pF@x%k`C z`)BRZJT!3XHu>+rA0W%O8x4WM+MdI?G!x*IsNnL`tCwAz zS18sx=K9S3^os-bW@?t6xV+6leAdmfcD3PC>Ls;Aa^I<#<0{IljAwI+ zQYAtsh{rcn6ha2Q-XWyzC)xQF<<+Fk(!eBmgu2h?bt_xaVX1h3d}q1P`pR<;PYitq zC{7X$>$N_$6_FxTT8mM2QMI}1WvqXuJKDa-`wN}CeDj=i!eTa-?{&Jd=O++)TTr60 zOgVEr@jWvE_P8K(R0}4k4oB`!MZ5JuP0^n>I;G6jAnXv`0$eE6em0gh^%+4T|tX0GQ%SzQ)w~<2jSEmsYSs@@2V3~4Bo@aMDjxQxDdhGC?p!1R*`|D@aqZWz7MeQ*@jxRX!Z zSajG=)b?ZLz&TF;I`Xuyz&Y#e@k3#D8HJUno%43a?=YI=JJ4rRagVhzzG`{OlmFCc zX@Y6mo!-t3dv9~PeH)=8v2qKaJCIczuo4g{BX5K_rF?}MsWF)TARaq5cbJG;nHl*S9Cpm zJnqVD{c|NAr`F_NPXw5xnf-ym;SP435*v`5Rf|uqN5Bzw&ZaUEelIf01esQje{GnzSSk@5TdK=ux)k*0)r`c6*T+6fsf|- z)NaGe<^of{GNO=vtS-;kA}8}@%M?aS8E0jFw??gf?_9$h?ZP1who|Z^lp?$%lTG#8 zsC;hN^pM-ul_Mrhe-NsPU1k>Ab4!(SK5UYPb2gdCh%fL(&3(qTAdg~PhR@S2p)?tEl&Y=eJXB3-&wu;JvTbmsEs?9Ptq=vIaYn7INg;$AXDDAa(6(izjQ0vtd zFL;F+eFdBEx}4$st4-WNr|%|#Ut$#FV$K6)H5n!1c^YocB=}F)C{u02rCR9*hbODP zejjLhBTxdOPXzI+AHdP8O-WDpz`cZb#P7PazYW`UgavRn4zB4&ai9uH0j~&@*zEs z`6QO%>aCx|V)8DfF;*|3l4pHoVwGA$f}_Pwl(R9eU9qT>r5HHvnfLttXXy|T+|DZY z(OQ4biBr}V&kLUAANECH_kXeLxog+uB|E9lsmT9HCalj^&*!o|(-n2KdIEoMs_cn&2usNeeTo7eA^vDGsHOc2A%{R%$@zJr~*^N7)GVnXe^YeU!E-0bb zN*`_`tcq9FFoFVka6|YJB?9!AMK#I|HXCAEz1@u;cHY@6aw8RSLM!WZBXs8h6357| zdSA_v+Y)H@6?6!T6j~jApb^V3i`YO6rUfPDgM7||QCmJ?Ia{h%8at-?CATHMQTtyWufy$kEtJoDM2@cP8?o7lTLX5vA+pt3vqM6C$2a>3@dRX&dVR zkb?={BQe%ET5s4Be6PuEkZcC)Cp|#Zeta?87*?-`OaDcV|C-%h{YL-3&kf;^B5JB*SIFP^loa7et6_?qbK&$gUQnA((U4nxe?1H;H zlAQpF!aF>?)8YeXtIbeaOEl_Lg{P$v+N?BP8U3cI{ZG-F^I2+*1#I7N6%Azx`IEkg zR(8A>M^d*J9dR>JDQC8J22dFKsughuM^zM2RRxKKLU3*emWdq~`@aFBfZ>+!VeufA z@O7hHxzzq*zHnelKT;#J^jJTC%}tM5Wq76pq(Rp7JO!xN7~thc{L_cXI0+>kgrLHM zmoh1+8OA=2^|>Z;W)o9_nAsT{?BBLY9!vE%*6W=TSKJsL$|I{4l!AzIn|~i6w1}1w zYnSO>I#7J5*$}}H7Me51vZ3XUMJNKn4kN}CLV>KU58j42by*0?Wa0*JL(BibVyUiY zW7Dm1r8ZI0s46dZ!6%mN!XK`_IdJ`%NU0aiXWa?r4o%HnbNMFCeSV-!Y8BU9`dzh<2*7emJ#CiWj}p`C_-yYH?Dc#QOpAcQoe5VlmaQr z9s}tW^^l}N)ErMn=i8)Q4DU>B8vRkm)B1tHY2(i?yk8d0c%*ABPom!$$$Egv!0nfZbN>9Q5lk`9EC`DBRkZ)7jDrUonSZM@&&Rj=#}A`7 zhBl{YNCHO5kqn`Hs2&Lr^J;Pmr|+-s}72%P4ue5X5L5tNM~V zW1G@ci}(E(87*G0=y{D3;_*4Kkw$tfvwll@Xqy*59b#F-@80xw{hCyQAX|h{Ka7ld zZRE_D+x3v(l)dfZKZX9TY&R4$ZXnUrqr3^Neh+=qFOa^5bn>s<2ei?q)VAJ2bb zgG6KLw{e55OwG+PN4e_BUQLHxv$3IF@5c882xW&mcv-`ADZ*T%jissBN5}m9!IGFH zeR6F7$WTVmG$=z^Q`a(aQm14RoqHt_p5q9n z=8`-Wz~7w34fexaq6u?cS|wSiHpCIUFKLbg>$cb6{oW&mK;(@$(^=nSFg)KW;lrAJ z&SpaJWoV4T{G};1nOXCos#ye{DZ3dzcmo&^I_23l3a9sLmP`c%LmA0L_R>T)nmnXO zt~QO>x7&Ekvmd`5dq>cIn$jD>lL(VKsPs0?|FntU-NCj&zWu?jaCNLxSZL z*Jc-gCEvgmGQ$R0_Fd8*b7-gEBG)(rdg49)Iz1R|IP&x2rwZAHKA z|JYS#I^!2cLDp4*(2hBWwmQbQDbtl@2_1J;%I%E7rhI0rZlU6OKL6NM^oA=81&i_VKbdcj%b-8aS0D}Du=^s=a^758|b7^%ovMEbXUUCIokteNj1CF*#;Dt!s?+^dCX$A+xru#aDZsJC0x8XWYKrkIBB|Prx^$%Ta@vWe5@}GX zv=OD;eP}o;*RNRM(L=pGD-#mZATGK;jJwZ0^vKNZm?&>xtMt)DTbK>S@AMt~& z@1A;}{FF7o-aCYX0E^rGQjcF^{{s|;_>(-;U-&$Y!634EYF_$rNUVrqc}5{U*@YA7 zEu>>(tQcjR5C~Vsbx?o#>B{MZ0Cs7DndUW`kOrnKr|sq`GFu)o*!mGyOK-Wzu#tN zJ~<&-H80XGNiP2g0Cb-^pv`LGCCOOijnXPre}!s9fwxgJW%tza!1j^aHYp4djSWMSbVtI_%*SW>kX05txeE}g zK4%LxYmj{%)`jt8#R(w57MiQdj(1|DoYHU%{6ivsO@8nvB6oGHm1Jc7&f1=$9{##z)JpSNs z{W)_+MZ>HSeaHhILQiwq-T+k$FuCariTw};B^anS5`Gd2+U5faL~r*Z^=OhEJyS4c zcoTEk!8@Gk{OTr7q75dhD8N<`FnZ*gV3D4ZJo1gY&z!kicv8jh@{ll>3JzZiQH?X! z{B_>yE6ZicH4MXIj?X25-I6I2#p)&TqvzLHjH{Aer!Ys3W>t?uL!a-H^C zD7>Dt##R?cOcj%$uJ;TVK;6&6JH!i%U<)-;0RH2Z`dc}qUgQ_Xz^Ujc`h`&qQ7Kb5|mKwMCPZN7q)(srgFOl zyJanqug}aZqZDh;bJch+@!Dh&vmV|10F)}-U&b)#>2)W(k+<0t4$ikPlV7(gv9bfo zeW%)}5%C7cSl^PRqSh_dR)W#*;b*y63C0}d*%pTVbB3N-Q`sH?{L%8h8A`*^Ajo&E zh!h?3=x5&~KmL_xUZZ)q+bD0iE3k4v_C@JxI~Qkdup%Xq<&Fa=`1|0-0h}uDN^x%< zpzwr1Q1IHh&8FI&9&iY8!pD>fpZmV%w$#o${`&3Ov{e2t2d*POA~(g&`DgbLS;+mE z_#c7$P2J73{nOnq$68dsk(4fyc_s!d-18bnvOVZ#x3j47>FsYe4Xq4Goy!T*5##&A z&4t$7C~k0d-M8K@8ys0#q(ufMUJ^;Aup|X=66eHGn1*Jc4(3GkhkQ*Eg(l@iVo4?; zX~7PISQwi>sphi88m6&{2f~~@WOHGV(<*5~jzd15ZPW`XQZ|qr*!1n#xq|CVwUVdz zCwa=|ELM7d*lFKxRlh?)hekz*I8HGVi=vJ)Y^q2Euu|Y@7yRidy2jIqz?(D1pudm2 zoXpl>xQST1U$op>Rnghh#Y=W7hScpyBWBPQW`?@I@mLOtvlb^0pJuN`RMkgrZ4S7? z>0bD0`bL`z-7tUOP>L7;D23_4P7qwwK4RUV8N09SpkB+r#)*Ej`IA_gab7&Rw=w3} z39z4IF8-s+=2sBER%e*2*wk@xoowe&w?(F?HUx0aCrs%xUg2AAH_I;h9Z=Zx3Fu^TCIT`2q3&14yr4w>KZTbEZ7=`akpk z<7m*9t=re3EAyD25@Q8B^l*xWQ61xaFMRi*gm#cSQ_be?t5R}jx#GMf=7b)hfoWsR z_BLL5dEKPl&E)<-s{++7Ny4wG<3V88loIBO9z-OO9E?_*CK~T3-|$WFvhl*4VysY# z4Hd~rj`Gs0+?fyvYgX$;x*xXVh8)H_$z_eNn2LoY5t%i2tlS8IZ~Pzi9B=`A=2hXzP2=|DRJCc@kp9P4Ur5>UqUqH0rk;uZqtiBNPnW<~-~uEuj1 zv0m2iQ=5|TQj@2~dEwSbzu4RN>+6!3lw-Ps0Q_cZpBU`eLG@|?o12b)jH0V{fxdPD zhG4ejHQCELmBP@%$HEocrM?0|!Uq0i@fu~HWN*hr3Y>NtdfY)`<8Bg1P4fY-ZIU&1 zW475;Uykp@64M=W0KIva2^8JN@siO$v+~-n-gy8m-!N6JG5)Z4LZs_-pH^ zBP^O(xZqrrSV%A^fenwieQkYWAC;7kBgi=_VS|{9ELGC~RnG!SdlOQ0PH<*Af=$Nx zw|CyBEzo^L<9;8NZ}Mdh$jj9fpq>7RsX)!w+Cs_}d+`#EUT+$}24d<8}M*}9|p^_e^ zSPE@<~*;KuU2w5wC{y zSby5j7SYfVj_~wU7`9OJN-frDFnU_y!OLZM+7j$MVi)_0+$_i*<~7&dSJ<*B648yu zb1XQOd{(tl*}nuDSel?USw>TVGrGJbRJ;AMjVbyp3PCCUkD~TmF1}DFyd9l3Z%6e& z&$P{YY-CFr80$|RFAs(I8f|6LQ4>PioaNX_xx#3H^^i9DhPyy!$>WXv@45z!FuJ18 zOzBcBS&?#4o8srldy1xxqIF??EI<9&IBgcgN-(s6th|<-(9@PTS0Dt#0w@*T3kHv% zj1daE*d2?Qu8G|HF`Yq2XaYFdsZA|^au931NGnJNyKK<9S z4`V!Sf8#}zhK-~Y8d!*m{okRQt<=QLUKA#?u)vsg^2*V< z?vdqqh-45qBg?NDkzZWGs>d>4K3{GSEAHsjZGr%l-L5JOl?yyy z4dkPGXVEB1JTt(=qviguAGe z1g04qDp{Ky z?Isi7PlC}6-nh~?SW}%MCTG|sG#q@rc>{5M*hX}+*vw$(20uJ8N}SU-VOUBU`38N{ z%Tx^INd%VIu8m~Wz#n68;>Xy#@`A5$Fwp0HL*CCDqJ2A~Z!qg7*cl%;-dV2Oa{bIK z@_B$|k2$D$>|R9op8BUV23NghSfyMal4UZy*^%fASH8uJ7<(zxxktd5ljME!pznYy zSj7I5>(0w4w9le7EN&v<5iyG4U0!0klgfzWL=vlGV9B~6WEO7Je?PaVx=m2NxSdKw z2&(-p+IofFYexxOYS0N!_t68>f{liet5{a&*2 zXgY0y;J;><-=RfPTiRJ`g*BP!IMZhxWU3(o2h}zv7GU8i?81N3RA)lr~!T>>)n2jyi*Z`9(Qe2+#IJqD-$AXB1dSR3|G0|HCg52Biu`lbL-P0bABNUG^ z&?9aBx`U|{D+~n5MJ)X({(w!6txzYdUJ`N7pfAN;WtfJsa}H*_UGGZF{R$A6%MJ3~ zw(pxOXIZUJ9YrK#@{Fe>X^qFD32up4-o(OB&II1&+mde5#<|n8rMowrpEd?FxzCQ? zy(Qnw|IelC#39vCBOrZ z!e6}b5rzd>rpng}apjZ-E{ONlk(Qr-&BI}8Wt8X)p!3Z{ZNP{9!MLaK2B>ybxK;Et zU;B3wcv)IBFYTOeo$*rEf;5TEFXk)N(p>DH3fWn`ag;@l*N2{-Ou83eqCI|1bZ*A~ z-GjY7JMG5Y%ngSc85$1SOibEv4p|nmK7(GHr9=@sGBFeCvkby810hT2=d4g-0sF-@ zTs?&njIGtA1N#CciJ6mJCQoAHp+q<5oH)=ulC6vNo7OOvGwiX=WDnKr?p!bIVPfkSjtoa;&(}yLvpCJ9z zA997L=z{K6Qx(?ZEQV)K4l12&#rw`88h>n7Gwke!I?G;-|9<(u58uj-dq@zz6Qqi- zkD+>iS{OzlazDK4;!L+2BdN4U=`Q9zrDUDt%~}~5tyJezgFx2QrIkvIvD>yT4{Uob z%%2pJfe{}k;V%y7Q5(W;}8gayAAtB`^BAAS1<7h#wn5^tOd13U*R(iEVwiX8dxDx2@+hV3?1Z*OYxG8&Q1SBljYklv*co?+_Y(adcBln zK6}Z+#x}e}eGXByfzX<~(1hgqln_zlKZJw>w0xEL=7?;bbB0(0Xh~@)rE6MFN}o;z zq-GuRvzI5jX7(5N$CvAZjWrcy3&1`F_k4VO){7_Z%^P2h?kA62kSeNr@M*AN zU*wY?B0;_&DszB-ZfuI-y$|MWbzn&xHFYUz*fQ>@O+Z*a`Q>&~xYlZtpn5OOeLN)& z0Q88bKy@U!9f7wjOfKG>VqBx$fl)?4Vo!lml6tLJJS;l_um$X6?H;c8D!llYZm^q9M5s9S#Sk2^TB1u$fuM_ev?6Hqk;Wq@8pJfJXUAskoql z1M!fSiRiI)6TjY^a z3xg=S5x%)OE8;A#+aYn18^JrSjxY>H^cc=bLX(DSmJTDGwFRFqPuDf}k|Y(vV;7wD z6;(zaQp6b=dgr`znG8V!A}~unch%o2x)cS z-l9C-V<|EB#Pk2bsf}zg5p=U5>e_P9^(i*8vXNTNrRS9ZCpg=>^x&g<=-|W#!BPg< zbX0%+4;ev6yZSq#*(uDV@wpxQM5fX1=;T8qO(^X|(i_s~#^Q`8^5sxh7ZVhCpCnL_ zH^>ZTH>Y~hlZfYBYDy4NtVU0O-E74np8OgCq!NiU3PLb8c7;GW6{|Qt$E1tRe6xM)6tN%G+#W7)_R_zMsB@9R2*}-w~#^nxFHc!sCfn+WLjMODzik+N` zgQ+7^m1N|jmcMA-;udCmE6om;>Rn|HtSc5VVk1}f*%+p*z?n-#8_t@4<|<&tD)2t2 zb#L5CGuPT8+|9trDHEa!g+dVey?ty@wCm-D*sNu=)2q#Iyk-tubyDG*U56Qv*DS2d zU8mnjW+#9p0EB2YH255bY&t|x4hxaA-c~=R^a*rmpx%X>XfU1s!9KRN(z%fKhN1ZZSb z85s)pHTJ%GstW^ghhk6yfb8sH2t`I7Nrt!;(mk_^TXq55gX_Gnb@Vf?XghFesbZRY z=em)Jrt0DMFDq0oF#2TV_sRDHTM}RjhtAt}$dD${r&9oJ-s_!@%3r<|p)kC5J0b03 z`~@+(;h?m5KO0w=n>M#Zic3Ib{n`&NQxEz(6z{AkNT!0ENd@aO%Nsm*(9vhYMeHJ& znqzk~+_8HU-aE{zKytFSHz`q$?&R{wtM&mum~g9_S`o{AdZo_7C=17?##@}-ZLmIZjq8Gd`cuf>AXcXMeZqH0De+hrWyx#%z zV{_6sRCBZNb*ouZF4VgVSXGol^aFI8`_CTJ`$sRl4n6R1gxW$H^@4KM04m7uHEjFUJJEFAkp#DZmnF3 z-a`2)J;{hu07>5#3O+qVf8E<@u(Lpd&qu%T-xhTI_@MQoRM}fHG}P&q`WHtW`Xn$9 z(V14YWLGGsxtZBZ0nJ63W{>xFph3YF#nFEJy?%>Dw^ECeKA3^?;6>#%*YlDhe;}B@U9zJHU_uV zzMu8NS?Y6J?;{!Syv}?$#m`QYXypjAaY0j~-=xlClBWCO;Sk&hnhN8n&i$%6p`uh;<2pXnJ+1o`!~3BpNK%Q;K?tesS3;u$9YWB`;e-!8N|t&e2k!(m82-G7f3tk zU<2$NS$?y@4sxI9y}V0UXuc~?BjRZcw4UYBuww42`sgGOR(+7WbdrO^xDmE1*YVDa zo39nZUAC6Idn`7AQZse5_86*i;7avJQgI36*4RP~#KOTFSmO#f?bu`}o(m7$p}p6` zbBvGZhit|5d=qJ2)m2y*hNj(SZyVvmrjK%0j$q^b<6IP?>&VN9Y6k6^ZzV$Cq-{wm zN%9Q=CF3l}yYB7?*oOZfS#KTH^#8wciwYu$2n+#yuZ0|pEvM>qHUopYa$-~0Y=|LvUjdwac}ab4Gg;<-0=BR7tG>KBoTQn`#s z7BLF4N5*#K4NtU*^DTxtBM3)^lp>!H!GC6bY{)P%=IR+gycZ~8GIc{@sTzbKM8AyY ze$rStW1bx6w822LTmSaHkq>@Pbp&hE0=t1JS>J=FM1AM<>WG)}>lE^78Bb7Jc&ycr zHY0&Dd+(BbJwt?WtiQ}E2PH6tkWD=EA|)wtyOn%Y73u3~Pcxz-tF4Qbh{isXgJ^OJ^Fc1< z@C|F&U8_}rnVo%W1O?GMQmR=&K+P3wQu-8hZ9K=n2C65}~_mnUTErcKYiHFkX%7_Uu zS}DHZi7aVAXXZOR@tr4h0Ul^v zQQooF+->R&fDheR3DOl+^V=z0$2a$CbQh1KYr$^;lwkjpM-Pr0#g5+kc)tBGH<=@5 zErP4#K!L^gOyp;2uks(3U2TN9+YO(ax>3m#=dNb4W5XPKKFrIx?HiX%F6i`igb7bZ z0&>QdpNm(NHJ`gTx7Yg8uyCM$LQ;ObG^&u%bX5T)aIb$oU!6eBVWKDh-i@EirL2*< zCnzPwzNxu6%s$_x=Taf7!5I5WT4E##)k&6NI{HilY~`H#88*flK1v&ZcuLrzSd&=k zQz1%{K#`34->s8rkg$A^mrS9%eCtIEP6S_kzj_4wMk!LOP~w`PYY-T$*gA7>R_#aYNOFQ!ytvy=?Db%+)y5 zKEjYA7$EZ|6D5I-TeJ*m$@CbO>xE3z(C<2Rg(wfO0dhb|z9-R2)~FQk>I-fWL{csF zS`-88{m*fDnqK-%oEY2quz53+XycD4r5Z@<1f|OnbOCN6^2T>Z?p==I*5dVdsB-O> zgwM)t^0V&wB8(M4kRBoD?*uFq zJhTc?PpXD9H&z?nG^zgQy^-+*%m1Eu$jwzA_Ek3Me&$(~!_916131iN=tJR#T3cAh zq5y@CEMM8%SW>}hd?$Oa=PiDt(tIVX!*oWiKm~jrLGL{(OulAxSLo>S(5z6p7!T&- z$U9*uPd>_Nt&bJs(~k^<8n%Zc2eoxmJ4{0%$hCf+R(;^pl6qiv^w!&VTGS?v+O4lMb{nZ<9df(6V4daa9 zO|Dm?PTUo7nHJ0e?}nvE`VF>RKE^A9bGTj&x(Yc0HpwMwh1uEZp0a=+D+yt1%(?29 z=Jv|5Q)pAj@%#$TSs-mvCpDqLmg6vc_@ml1Z}{-5qcG}I5X_^)P0Yx)J8{g6446Kl zB`2Oa0QntZjryfLXZ~JyFVQ8B*&_r*w=OAYj2*}nTY=R5%zauKx>6nzr3P8`p>^}2 z1;6-!{w=fWm?2cCaQP=^=T&~*b6ipT`>S*VI2$c;CKa*n!D^umQFPPPtcPEtI@+6q zz9M)*M&Y;IUZcfEOc!6zwRCnF5!f^acsVrr%ZC}-(F^hJUFOd3_|kUG!08(v>v=S{ z)S{HX`2KZMZ9lTt?%21pf7P$_l97QyvGwmeP2pRm#-G7%`la`4`MykOVEU;?6g-`c-0M-V#mDbwKfbXUb3HZTS_)scWn4Y*VtscTv0(J> zEE{DInc4K9YK;GLgc$k5=^s{xgbr(7f+u2q891rRWNdYTPiXS4T&+tZy31b1x-rwg zf2I%EC--CYW$6+5WC@NPP!uO_3+}f0?kFV)$hqI94_Db^P-hy}&j}92 zL|O;$fhYpi4`L|Ot?@@;+b|Nsh2%R+Q(byT6`#qAHkU787Y; z0Z!{JWOnE#Mu$>5*4t{>G9`tF+DhzcSY5w!Q>3Uj#@VH^o+k14<=?D$Ew8_$vNxtx zdXf0V%Nd|o@i?baQ&l!m&eJ+BSSg;$zzkuge;+QJaN}kZ%NT80_@pI1DXYEcB_-$2 zl2^#7k547DG+?IZ<99#fU7b}fN346a^4grG4)42UA>~%Ml+B-DWaq-s zNh8j@+#nVCl!g51h3jtIQlF_&;{P!il>wiVkqw{m>u77Y_w|J}xQ&ctGB-!XsDdBB zMojZY+FTdqLn{_`zdKU1vX;Co*$0T=nE<4SpWygDWcWEG zOMEiCmsO)n;2E3N$KqFgWSUi)(=fpSJaPX1?&DO-QRG|D;MtK@;|%Y6jP%+VnVuc@S3+FP59ww1a$HJCn9iuBD*3jOspGL26Z;dTqiUbb&afn7m*R_Dx1 z&0DV&{@1s&jRxN1IUDC(@H`n|8h$idD z|K@tbA=Eq_eAgo&pL5Q?DzHO}LHf``)3g~LzXFZTH;MqA6s;U0(Hz{x`zH}l^hzR? zm}5%aFBW>NgmxLW&q{t$_PjQvyhWr@Px;jFeR#J?qn=EzwMJ^v@uKKTghT#(VAh%< ztuG{+PMr508sSLAWd%nG71?P$K&ME)&uovoZtqtDsBKy}kJ!v5cq-35h-Jr<_goMV ziL!`wmRF{sY@_^uF8cF>M!4WScp@dtjcL8IwNFWDJ7kGeC|oyJ%36! zsqSKy;kp(1BnCa41o$O@7JH`W!tU0L_~!KU9MW$&d+3+qLzBz8?;@I&m|;E5<=I-@ zC4I+!L5tZCi<(*WSA8@w%zTogvO8{gjhcBL3-_*GCwCm0+0SV+nR>R?q?%wkj7lG< zp)+04ct40Em&-oas6JxOYd{mOG%AW@nKWv}mv1x8By&$$cUfzKp9T7!uIY?9kINaS~QSpeLkL+=*5Svqs~JUc*A>sGl|QeP zOg(h4{nWr+PVZQzze&}EP8dlH)r{d0TFA~cV=gX|SFIX!cRcX@wfmv0U0jibsP#%0QVNCLr-M?(9 z*=qE#_PKl#UfELIrH3`Qma$pPOv1&wgI;Db)!e~O_TndkB6EcS5%ZON2TSF(kC$#- z(Bh>ySOdIj(kM@Wq3oKrx>?%O96UZ#bz4c3n&uSa{?%>aX0j1ZXE*E{LOBpqiGLaa*_h z#*&1Yp5Mb7%t>GEgZdMuY3Q6xgncnN@ z?kVd!rlGjl}38fPa*g4Y8L*)5*ejUTw$kCBfVHx$+48HovBiF%-@ zXKKxh{?(2;C!}O%@*`$2YGTLbdv3dP=B+S{ZDWMg!uW2mZzVN@-g9l^;jY^wS!iZw zN!`MilQ@#T^()a~_0LJ8*$5-auct6hT+U^Oy2z#k~em9p=}OlJnnU4_-Aux(CmQTyg2)dU#e<3vUZ0}BA~ zFuA+o18rrqg*w2TEwJOB`+^f=t?mEX=lNM!ST?R27VVK=AJMd*OdC10wq(6;M&`Sx zk}|x9ZqLd%MJU%rBH^RULIQ@N;jYNA9J%~yIPA>#VMc;XwLe*})16uc^vH>7j=DR54%i3x`9m3P*UiUX1q>HTreCUENX$ zYU$GF{ZFZh{dZ9wb#Er*Ri13jMYyZ*r)a9hN3Jd=vc37AA8`m`>Dn+eC8Z;&_4xGy z#sIu7aMqB*6PzZlw!8|Mn^}Exp8QQv43~S8wdB(R7I}4VE7O3-S zIK+hgZUJPR68EMK3VLtQpYF_kMJ;+!{!9po=McZeXH4pz@))+RNjz+AdPC0?aVVWF z_pcA+!s~Ag>ffAq^a2VrJrv=KiPN2d?$9g5491|~%2aSZ@6wKgzJhQYg;?@4#D(#D z7AzXC&(Gb(9bY&R|HwL7@nFAS7VxXA{lw=S+{vDy$Pd+Fx4Z;Bj~)EnBq zk%F6T9un|&;e49Bhd3R+a@^Zt*7qK^U$|95(CA2 zb*A>%hi*-&Ea$BYv9&p|m^*QFw1Bb4)zR48rT)WhhqwHV__KY=y!?ou80yiUumv0Y zR)UW1lY`z;Dbp?&&em;gZU5cK^7Y=NkBAbEUe{PR!!xStUV)G1-#W^sVpJWU5#AU7 z!g3q$;u)>s{9RgWNx*CA6Zf=pCBI4>gHlrL%|9#ofyv-igT}@^YRht`jk%v5XKOjk z(=jah!%jabqZ^(YCR%>ukykgSNh5f}6xS075*}1gFMaI9((2)50Jl7z{lTo7k2C}qmTI3OJqLGIOFt9AD!VE_CPz4yG8j* z-Mh%moH91*<`W32Vo~+!EXdp0uVP{Dir+|`9XV-JS0N}6+JH}1E+}q!+1E<0$E=`t z%D8vNB`#u`8Lc6gWh)YkcWUZ{m62c$@ZnnX)5|ofYY9-C<8&)JvA*n8j7ndJ{cSZ#9!&RV!t#PdH=HCCG)V3 zST(cjCEMCu+)tmi%uwyaX0I=0N@ku*_s`8)(gIH6qn2G6LmDaAVQLu z;MqyzxN<;MzRGgG{~x2}Ge1<_n1N2C8RM=(m$RJWU;o5)-I2WMb&Ot|pf_(r55)aYZL5#&#?Dq;%Wf>L4+wARuE#lU zZLUwe5p1pmiVXz_uA7U4N4GeLlivMyS@Cl$)}hgHEcrv*H5ozQRVKgF{M*-lyIOY- zAV8s@+y;v4JO3>*v=QQ#(&OOvDoW)TemrO^p1noMQF2T4k9EpV?}j+*l?OaiWlf;O9i>t zv=7c-rup= zFDAb9K8N>~0-8kp7U?l;V;JaF5~AY&N5f-&{+1d76nWz+V%0xmztl+DZF z6G2xr!ymt}w*y}3INO3&ABxC-PO1{#KUX(xSjvnfr+p>Qbw5Y>%Miiq3%PZ}H_uaM za$^raU`3${(LH=kbBSmAQA&!-8}pf=TMgLhsjXjn+{nTKOgZd=G|K5hfmKyi1q9IL zOTElMFB{ytmgV4j^c2d(%I7|0Cgq0TU~j9VmLh-c<$&3}Txz4s^@pUaXE+vOpxQ-h z>2c*p?9=Odq^sRj8BAbH&Tpc?lyjkPT#+t#;j3j6!%W?R!()4d&$tjLIfR*8?hGg~ zPmfSYMmuW1QTETfgMfLXfa$iCXoi-J`Bv&OuifqPo79&2{g|8cnS4;-Meu7hTX#sw z{Mzld)A2FIf-mBMpv#`<+#(H)s3Rsk>EPMvk|T+I!)}AsQvwBX3R-3)3uxVW1XKl# zsxWKg^F_XnXx?zp|L?wuFT(2)#_^avs(F=Z^J?q-u?p6Hwucs;J9dq*hApYZuhx(4 zH2HIlOsfxd{J)5Yhf<3{8rCrx==QydyCP~1PmOQ#hRM^?14fxdrUNN8oemdx zL1#T2nB=u9a*#hN!(+J}W_9L;$qWUcS9y8VoMn~MyDi%sAAwlivTm=bJsj__=*HD> zw=t=`tWv5ra=s`h_~O|zEbAHoLZ^(bL>e^$s}u$|^O-N@O{Q>;10#;qus;$}2*nznBrDe8qnGulJN(aehWO980X0~w4pnT6OoO)YqVJjj$ zC0TBvcl^6T>xm{LzPtlDIh+SvLR=wf4vj?l%9-70BwVpIh9##J{*)$On0u#X;{_e> zc=+7`OPfhvoU`tmN2Sx-tD_j8Jles3E9u|}lZAdx*mhZTzJ1n0h{BYs1LMRvEi?=v zp<_Dw-N05%?6plB+US6??IP`*qT*(>4Nd8>HYPh9LY2TNT{PC-9DngE6cmI(%+w*t z{Lp0YnvAYS&->t@kcJCS)87T}73b4UC<$>a3_1!B@T}{8peYJM?a{x>A*s=Ox-*1L zcdB+?++V*tS4&XlcT(hpSae^RK_lfbW?f1FC8ZVk?Us1Y&dw9iAqI392)w|>L_rfG zqa8dRWV#@uE%02Roki6o@5HDcDd)2Z5F6Vj{&Sm09Qsp!;>D^-`7p$7D}yEA&J?D0 z$<1X~6h6&Q+DCm``+H{qSD?yVX5}S>+DwPZ?So*c(pQ~)0_89a%P*HEU6ShHQ^AKn z7H{nMijz09B+|3zN5{6k)wRC({F0q=pi6rHWX>_0eIe(OIOR(w^&n;A>=W`1*)pF| ziA{~bp&^SGq*{lY?wQ%TkI{2lEyPE^QH~okYW<_pj&N7ZgAfk;p^H80Kf*Q+rIII3 zYwQitBL`AQdYfb4U1<4XXJNRj2Z!@UVRxseh<5S27+_~xo>YcZYdW|E=E^n>J~!~ zx6yY2cQ>;BVT*co%Le<`Qa*=Wz<5xCOIlJAnOYnsxq|}6E3;rn*T-X{1(t(MMOqa{dd(kyBdFc5iGn%*6~Gv z@+;h|yJ>6`#9W=gM)45cqRta8RzZTt2@mH~W=bkm1k)e!6?G zzM>JoP~(z6!ShL4e9k$J@RTvmMWZZA;K#QDA3D>!#D5k)4XLUHu;o9(X)}`gjuZo(xqA>wC=Nlw zRmfv7j|byK=r}KT-v{l?yR(Ia%KozAt*{*Bo2f9(KNjjO)X3L#N0yL_7mGLNm8Z}H z<_EZp+q_%+E}k*lm*VB$_?bQYz(X)gIHZju5f;a&lG5BSj~5)uB_LqAL+;uqELiI! zF~d=7Z4~Rwn&6;#{0}m8VEVy?QNf^UWCiDbtcNAGD4v_WCPQW!g9{>u1MZ~Z;b$Kt zjOUE6fZ`NzA?D%9cQoGM#o*||dGdkq!njb!%L!xoP{+eHn4stCy^(YwdVV;Uh(8Wb zl_e)ztqTtsr?^1q`nvhCmlz4NtZai84(uMYDthS1z|6U0n3+7Z7gy9Jw01#4a=8~E z)kRmok=hc5dgM- z9u@&lu>F%fPdy&P3`;r-VO+1kj=5YBIiVxtPa1hFx%~GY=BZG3%pyY_H#Q~GG3;$j z|M7)HmWB_-2&L){j+5ndKNZ}_Q!!b&evFwi%U={XeG&cCB^9L`{KnLxc{zkEcji5K z*4#QIYRu!X3!*ylCVyohTt9xy5`>Aww-i=HLjopOxNa698Q`!APLa{4k)q7k^PWikDsD*HYIVz-ufFlh2P3abkc7 zei!`8!2XXp1-J{Hdcs3L4yG8_qG4_)y6g_dOX-EfmEsW%E+|D$km@D3xHweBXO-mt z2nZNK6=I^o^bsAl9@)GXTx991TT`?ah8cFgw0~pxH}1^iFBJg^hf<0cI|P|_xLh8; zC;u8B_|KdmBHL$JzqIDyI@UwWBerX;t6!|10*Xb|@3Q_PSdU@AJdZe!n;1%J9tdb% z@c&(u8Hl(`0JZ~f+wO4py{`yYTG}psJrMrap&oY*x28UuZ8x|ew5<47_q(M1t}Fkm ztW}b>yXVdO0)9{S{o#bjmbTLe=T-i94A--FR~hGNcc)Sw({26_mTm?S;qI-wO6Lx3 z#|+mSZGI$qx*y)>8UV2>n*;o^)U2_8k`jJEBwSbW?2t^Cx=@(ZzmNbE{RyG8=d|fi zYi`0cZ5VTs%`|%z$bdpgODnm%yZc0AU3J_A%;-V1-;#b zns|A=*!RD?`sQ*K1f0 z6IoZ65J2^V$aI&Aqy@|s`YU<6%t?1+ZH@lVNn-`$G+tA=-FHOC@?xULkTlgOx6NTQB6VcnF~Z zMQOmH%jaWVf_{pJDp}oyaf~qx;W{k#)1_j)WtAGaAGI!9aN=|#EUQh|=1rkuAsoJ2 z3W6>hQAD*%2O$FF649=+m$w@Zx4p-^7zRNm2A)(d^EK)qb<7sf1v>sgXpF%EgH=rz z8I{vH$DFBDA%xg`;2Nh?7GrO!&-U*+xL`(6sisXB%9IGYj}H1*3fMVlbn8sA2=`)Y zo(=7m9^1v_7WFdDKf3H5W=$ULMo9rFjAg}{j&Cp(QaI6Z>xo|5<9Uh_064q*(V5gh#0TkiJSdzS}{;X?FZwrib5cs%$Y2c)lm4U zcsqCw>P>~YCzS(Hr|GXo)>o;;+i$)!eXJ*^dC`IKUz1Z)bgeZqM@1`r^zf(_eHy*G z+#$7kZ#s;`QdrE4IloTWs_Zitzp$}%Lx1?mOu1j2PFl2e*JZSJoACpNUdAX#_uucD)@;C|ETUb^cddAE! zp4qt&X>@JE(~`7Ha@obL=;IjG3;Pa#BtI4%utL^$E<+;UXLG4A7H!ghd9oetDA~5( za%Sj*IP=YFknQ*-`gx1zrV*9tJyh?gsI!3qcqVUU?b1#6T>a%I2B|>C8eO-ew7br} zPDizxR)Sdk+x~Y@3|e*39;S$Mv)I6JI?#bWW}g?r954XHPSNK;=vSQ9Q;iccH3^YOV!xkYfHTZX)E5e_Vu?6&8zEWnDvM#*|d@W z1GoJ-J2&&(<u()qw+dy@!zpiMw7_aEhi_rZ++cwa%bbqC4#b?COEzm0xSb~sz@m_V<4S_DoOM~T zL3k0G+NLR5ZI_c|K*0)|O|Aea$D?48(*##_Dl=Z-^th?Pah$IyaF*;e2{bOgag=_` z0Rk2KaU|ab+kXM1z2PfPi{_N*M+`3BZsxS1_+j zd^-K`(D4+wBqG35{So%}*$Yo9(enbLGC{U#&64l4RwlW>`D2Xf=(_6n=C&0LmmgWx zX+_Ct6~0$1n#_H=*Qv|p8P!-c`C@!-uCP@lnv`}EMj%n0rl`;Ul2aX`@m`4YI1S~N z$_x)QA(eR=!oiJ!YnU7?blTL)Q?l3|q;~@85Xp7526$Bb(39V?5e`!!yUshqUAn`_ znDDdYoxqy2+2=R;`&14lH#kPoA2<(eN=X?I^3i8UsdcU_-=^Sx%n`j*O?)dS|6RtodaKh9l6I7_KEgq-9!Et zexZ=9CX&tHwQr51o1zn6batnF@W9V0H(DCB({RKRT;o;p2{fe90P^;Gfr&Nw0+WTN z^pd}JJ6wCbBYwcN*Rl^v7jk(n@hrj)Lc*xBLgd6PPf}s`WAX)@7muL>x&6i4-ei)(foTd;!I%G^VKm)qUeB!GI-$i3V*;fli^rmC-oIrWXe34 zXUcBdlp-XO;sn(;CP7Zm?d~e#~4yheL2Xl!)Qz~ z)qh&gkQvNhoMcx)JvC^i<2`yg>cXq$Ye7tHxy&U3G!n7dA3fHuGV@Dfur@=20NJ!rt0kAXwU zjU!cGR%XYNrZEw#sp_$aijh|Ts5g^gpMt|kVcKNa%q;J8Rv_i9qs;YOY5-GuqF`Y> zbHyQ;e=XHEV>5ia#`TD1mtxFp+gxW({hMhkY0pdu>3njsY!>quU4#&LXI^(K3ZyZE zj4`*p(-GLFJ^l)+FqqT53nL|)ZDoUB**fgblq)Tv#PECokL_;$Yech|1`tm_2~HRm z5a_f-=thotXxiN>tCs!vIK5jt2K-F>flm2St_pW|gjhLUG~f~Nnz;2d!r&aUTeBOd zcK3VgzJM0;x#Y?EcI@PXE*(YQKHBT(f-XS2LV-Q2IBCO1=={Gye947GLqP}#(W zg{3J6sNyNsVc+e6)pj|5?US2VQxJa5#VnJ;6DX4 zK<1)Jf;#lKIfb@NMzCQP5p(uIU;NMQH4NF`(Pqddn4A zOZ5-~>|$eh+NnzmebDJLS4Ntr9a5`L7Ii-w3exwXv$6o)JYdt5_ej? z9{7X0EM?ToZhnf;8yfD(YC3P6QTxPKPp@|gC3Iv3?>_aQueHa=P~J~|g&=#o4{Hr; zr9?nkHRk_J;nd>W#L{Qw42J%2Ojz`3*lSCCwJ52VqTSsbgBjHtWA`-mB*;8eH!jhY zihmkaU^<;(x(ZN<5VL+YtwV7tyV8GVP(_&wJjdM1q-l6Q`H**gTWL}DM?)_hF|6_N zM|d*tU)|NtSZC^g^k*9~<)wKP8t?l?MzSd-h;-PhtKvHW$*8p@Je=Rn(Nj5V_sj3z zCCjv+`&7m>+pE5v6)W+Yi4|jhtXoOSYezA3N&^Ow*GWxr2kD+WP_{wg*+f#T z6o;!G@8eL=T!E-wV(S>kcHN2E#T9lI%wGG)e<=kK5J7#w;?`o00I^ zXdh&#uy{QbgFUBoXnu@=jiqj9f5Tf)Z(o6*_Yk^YuyZ&!jFKLlJMckKWHB}t<`jbi z4L%b`er1$jMTT@G`u)~46w7z74Z2?;0}sryvG#p)ugl(_?ssZT*74tMReLrNlugea zrS{`7n3a^Q_s?nD+?knY?dV@z1^+8gI6Uc?+uy?{V<$)NVR*i5ffbr|nfJs?cb^tr zb6R#&h);MLM0lEK*N1inqi`IlF9bs^A?!YJs3uw;4(VmBXCAR0)$*mj@(JZ3oCOKf zv7-gTe#=$%5%>V29XLqDSf-_ojrHz*K!?`g-8=I;cF_;MHb+?l7HmoL`u3wXkX*60 zh10|4n&KAshDD8x|5{|Z0M88U#-D&FXk#_*cmML~rg za7ukdRA&b1?l>E@9}hCHoFG4ev=xz9PU}8WZ-;f6zyFx%G%U!nQ26B^|4T&n z^-b@d%Et=HC5;I8g@P}#&rOW++|nf##vrfsn13b&%q6@7tgd=|TobGv9DcrmLY0?u zLUVIp7Z(@5RK}1+l_~_V|2?n)>-&5YGxf$yNij&FV38`F;3L4Q0mWjX!TW1LyTt9K zgr?|?SYSz>vsfbwHucZR7n`3G9T7L~$kE$t|CQagi|zC1x;tUN-RV1?B{ZZCiDABq zlS?lS&aGc097+Axpkux@^KpTbn*a8Rm&M&9Ja^4@?{{d3YBDS)gJIC6>56r_2|4#U zT{Df0xl*(>fSEhSt*VM=Rpu+E7FQlW!OUhve%AavPk8lYpy2I4jseWU1^yy3IM*j~ zNJ_VlOF-cSmtKL8s{UdT_<7JBpz(=B=-m>j0syDsw;ko+phDiQW>L6dnF3TX0*R|L z64vHS-OT;}d0*m8ztn741R`3h5f4W_M=@d=_Akz|(>3My1aiI;jW#W^LukYzFjxX? z7z}1X(73hba&RqITo(Ujquuq5;PE*vmAKzRb(1xcVZr|{aHiR_3T^4^OpG{foQA1a zjJfj@MfYmN=i^H`?bjopVul0Q7-tkX$8vQp5%A~n9kQd1N5nT1H3jj9mS_&n@E~qw zdv1XNv$HywW*%6F@M{VmUvH_oL;v%D*NH<3fnbb+A2IuvJmI##6s=og(Iet#TSmGp zIL*&E1`&ae~nO&}<$-mMI$Rf+N72K7vpB_Ze{TxA7A_QU3PSfbA$@624h>IUht^$SxU z9VK*&urL*qQuM7$2Gr20$bS6ZQi9x?^XF^IKioQWR>?2TH?h;n)QDoHd(-CZtqAzI z!*}rxRBpIrG(V*=E3V_4gqpQEe1GzdvB;)`LuCUVx9&>^`&WzQbK|(yM?|-Trc6cr z##Hybs~HMvN}#dV7qM%BbN$1#nEaAzg;gqDdf%@gX_jbqs=>wl93W0>*N&ImNy#=@%5OCRYsy5)_aonbP7Fghl zX|7>rSxS4;AMgNm)VFxV#7$_;N^|)ks!R{M*i_bYy4&=UCj0@#xXBkm%9s7)lg;@d zE+NdCs)FU)7hILl?1#`GS%#%Le$B9Q*`>M*(Mm%wt!_ovTuhBC$=q+q#q%W;-+oNZ zEs>su9nFu$IcFQ4ZJ1paYD$1+tB!4uc?_Ui%(lILQ&axLcll|1E!D2}ROg{$Ep;b# z0P{8R#?iuQ)^sgltsjUEe>)pkOE9NPnO0YuRcqLJrC6&HG+Ow3E1A1l)_WebTVFJ& zi}?3;%w+EE?fqrT&++Fq09~N<7i!EOuo0U`JAKVuAC6t-(k%~!nG!sKPW`gNE)WnE zy0m&4O#Y(uqx2zsny%Deh4OePH@VByhm6lE7|s)ZcW@_jV|se zEY6(%n;K0Q-rD#YI`lR^GeP|MrGB0|Yvn2tT`qGJq=k!W_H3$_h< zwy!2vS$mr9>ip4jq_ITI*kB08Hh4rcj}!t13|IAURN2kb^!%*3J@_%^z~amW<;?OF ztsJg#t$OOM*z(aU1O;y_T9U#-m`apHQ_I-i|(h}%$+=3wEJ zZPuB#z~CI{x60DLb+BjNk0oP1+D{Ay%On#No{SkOQ`HQkX z%p~Fs3^%aEnMyX5@7p?owX2-I+}N(`8fSo}FAEqP>U6rGQ;aA#^yynO1e(&(t(s7n z%ZQ6HelulWTD$nU+*c}?y9uVX^z)-36gz$fLWip)f1WmQPig4JERVnwh11)4&7=-Y zn~vXizLbzACD%4ss%BQnZ1Cvp$9-Qx%$iz5^|n8ODPc(W;sz8LkVArw8hN8`Q28XAP2 z5D*vuHjBSmt_ZYHMidNv-{& zSH=GfaU#_JNM83^+#AP?Suv?iI4(EwG43jnlAtvW!W*}!o)q#b6dtQ$Q?dwf^_rQ? z64E5-x&Uv`aQ0>BZs5Th$vk!3hzi}>dh?tN-u;6hEL(~E34h;3JrPPcgR~p{7IYxQ z$Hyl!3Ob#U_)84-*jfHD*~|Q}-jO;&U1gjAMi3ukj~R)zTZ&)l_?&kE`DG(y|Kf04 z&oPVizmu8!I6c@;iLvSNNCEuCobYaM&6{c)pVZ?F9C_BX8PmOO4n+KxoP&-5sUd1^w6WHvzP6-^%$0`?}iR5D=Yb()u6OU#E-KD7sA^E!n=c2fL^O36^hVj`2koC z_6&}`!*oKSQ2%QrSw8h|aS(n);J)8kQ?wPl5tHU+2@1SBX}e7#!bwU>;yAS_k&{1f zX68H;(sUrO_V8#(Hui`hT(i!(? z7-mjkA^hiM1V(F!O`_A=bBVd+Lj;Yb(_{AEMcWeqjP<7@)EkJr$YXk?jkm5^_pc7N z+ithd6O1lMeeTZu#Y-DDUYHI_|A?^T;mYpYJLqs>kmMSO&=KX%@;kITqOj|=%KJMB zJ-@+-bx7nb^4+MLVz8@$Tf>OGs(|>CwDm1u z+3eGs@*C3@&1;j4C`CP%gOXrZ-57|PEtS?B(VVV##HVIi%0%N9!1B^j<{5d#eu;xG z%hK&BW)1I;CPN!wHV78i)^Il+0blduw1-9+;Zo*u!gE{6Bk!Q4{vvjWKDQd~W!8y$*@6bSNUMztD!W=JrKY)Ly&^ zNwVId<6(NBhNVHeAaZpREvX(*QSsP`vnLq;cXC>mOo?X4o>)_b@0D8=6guciI{7OU zMs#6HDL>1(Ig0V6;R&K=HTbn&iglD#Kg&%OcM!RvRL z3j9PUzZ4Smds9u@)!|qVxS(J2UV3# zCGH-Ay7-wZb;hDW+US__GR$rmW*uoxaUo}PhZetu*RO<7{uPiJG`X$tm3b2QZHD^w zO~4y<$VldwFzWKc(A|Ss@w0}}9@l9D%ZG23vvS4YF1wM#l~iQPHS0}>*C=sBSfJ=# zc-Hx2`4nKJ&BP@{lE#>?LIDEtX@nFcklQE4$?Dj}~&M+e>!>)D1IJz?tZt2NMTO0O|d z;az#kpMC?qt?m7rrTvw%+dpNFcQa5GctWN(+mAigL3@3^qF&BDwoptBJ(}M{+hTa+PUUX zlx#X9Up)piw$u`tZ<@t+s}sUHd@xII0{gZbj54m5Qgc2}4wYOWQrZGHGjGc7%h|(a z*?o-xZU$35!?Av`?%aTS;CLpODvJt%QD31epuR)Z5*_wO8^gWsbLtZ&-Zt~geA`H&m9Y(%sK<0`xPy*Tc&AWg z7pmiS9v94#s-p&RJ-Dd46GbTu`jzjyRe(5ie3G>M5Dn2@#T*#>-Y zyN(~nN;_hPS~{G8-|!!oFTd?f~0wbDk9 zec~&(cfOBAn)Y6FUgH*FX#l*|l7vQGlxcTii-x^Bwhi;IgQ@UjqE0LSyv7DHWRl=n zrfbS`?PXoZ`J3d;I)7s9VZ*bFimEbs25Vr#^E-3*7x3G#^y@7i1CNER0$`V|H}h@O z9DeA?;m?cn4sDClb6l5@fS17gbQ4UiEw?9S0>9;r4g$!MH-*8=V{HPh=xB!f5Wn)L zrPIBO8qCTSz~exq&5iM$9e|=os#c;SGJr~^{qG-?PvbqsSGkp_u`jGDBBxodRAOEe zim&q)-F@Kh90G(7xv=&yDZh6~f_CGA3@D z{IG|9edqHR6;W3lCny~FdH z4pE%30C{``%L=B#-zlVwuc9DRTzq^%{qlLnwoA=VMcnPV)jmw9Ufy8&h;wGocV6FL zy3yL_*H9Bj_p7{8=HdA;RsM6fK{Qdwxop9lFC4@4Wt!2Eh+HLJ`RMqPY4adGU<#uF zh!&KgjgHX++$>#h2vDNKMS4M(n=p%C9HZFq9~r zA`L@HgMbXEbPSAygmeuJ0%!Z4?>pyt-oN;Fxc9x+Uh7)Fb*-dM_W>i7Sr2oWH!rEn z#iou3LzW2N-o7;|l2b)-&+HZ5(|D9{f*7Hu0eT+!(pSeo+w5TZ^b0lZCiu!`X3Vw_ zq*4Hn;2em$@%Sl7*YHY{GEsrvk{jGad1-w_uZax7WUdVGA!RR%WSt5(7A}VUmYYMb znl<%7Gn{q}pzTHl_I|xLLD`5uZnwK#B2A&!Goi3Q>kT#w%{ENm+=IkrT#}scdb+p- zORXnguiUTyId}S=XdvmID)X9?oOE1VW;E&JLM=?%vU#rMJ(zm(*OxvS@Vhf$c)*J> zCO6yd1?=kJ1+1s@xl2=S&kip4y$&Z*Vc`4! zvTu=k4=Qp$a{uC`NY;C!r{Tgs-Y)eH*E0e0dvRYfnThU5m1^ZY7Ns8Nbu~^-SSG`7 zV3zcK-#E^DqjOxyFm(AUFM^(Gd#&O5rD}NWnd8XmrVzR=Z2!7nt998O=v(P>)zszuOM1>Ls8V{H`>ochL^$=zXEh=Y=1_ncnIn2f z%{WVdUcG-Ey4yjVW*zb1xrmCd41Y?AI)rMu{> zlvLkH$ZOckm#CAoG;qiFU@O(Y^I2T6Y{ zcPbV*%l>gxXgO9a*$6J#7f+-BwTv;ig2%|QKlJrM~#5;Yq|0egoS=*3F)K|D+#3vS;A#?rzes)tyd&3 z23^Lz@F~KH;$|galPxY;WfLxY0#mt?Yp@}7e3M}T)wbb*k&i*79ZgqN$^YMR{ykN( z=V2LaSRS$m{5&DB(^tIm3?fxUx! zQ!j9QHpQ+vYl2x!%m~G+(~s_`uY}vsa)V0a8WU;e?Nqp7`!F(wS6yo^J9NRn>1L0PhV*?)3r>M0`lh9u-M#KkQOs0FR zQA`~`luZT6R2A{&sCjKzRFsMs>=j3_qe1|m6ZIo4%tFWJ`6mXd`w7e#3Rd+(4O^pE^!4^q&m zYU$loHZlM7cZQMTp_iMbImfsg5LVwZGn^A{axhg{>FSs!h&+5I_~MQbYsnIgxi{9_ zoj^xABAM9=&kZ^Z;s4&z|6J8<;%?*G&qRB87DNj;oy&n7`}Z|0GE8XG6TuW zN+=k;sx+Yvt&5JQo&A^)bi8!ZOrEq5lxcNA=W8xFKAH z!i6FV?u=x^<-RG@+8X%Qi@EH zw!%G$$fsr__5chJ`o&L{ROiQvyyZUJ%>>u9kH^&lLd8s+?(&$1q%`nJ>trCaAw!R9 zWKoSIU+Tzkvd&aS6%F6j+EV^-On5QMH{X7v!F*gtJj?g^c`<7Mi2q_KTIN9B|7L|q9)k!Xnf4GHAsPU*W zsq)40$h^ZU8zAI7|Ace(t8SeMFYigNfG0_c8n{c0CsCMz4)d1A3a+%X`IV5}$eJKo ziNPO~x^Z73!r4M3xR-HZNp!(~J?BOQn#g*E8fYhdHSay&>a!4sk`F-&A>}cO&*U%V z-vW;PW28io)3`POifPZ?C|)p$i>HHFqO2;N)* z;;#<)>iX3#jov*|dq$Wlqb~Q=mHZb46NM$|KL@?8jEEpBynAUk&I1hxFpDvthu(}L#3ja{Alv!3bIF($9&kkaWn1S#BVpc~}(8c5J?d_&r zbn|RgbLfxh9D^o%jd!yt{pQp2PpnVo-1#dV<7xd&U(h^$iOMYTFb(w0FlbY8agQ~W zmW1{CYdjZk|A?xMOU9N&gvK%dG(^l_sk{}~y=)t4qR|kYTUOTCon6mu(xihmaA{(ddQWtN|4MXMeMy;V6sXlc`QwMk{hrqQ{u7{owthAmPVR zv&?=!t($*(#@FOVjz%ICb8CJmD6v6GH*9Gr41)kO{M*Z#oQ>u9hAx1~+&Go@RuJy# zwjsPMW9(z)q-FID^XA+uuc*cdtrzY)TV7>ubCBCH*wIIET3SbgIG8c_JV_$RNWg*-pz68yu)Qe2f6>rne@JY()!dL)t!}C>9mTF!i26rYeIn? zn>?3?CpDqh`dJ)`9ZER&p61B~%IiIW_U%x;q?^MV<#yu8S@-;@Kba^8=^oPg&1C!B z|3wPPKi^uN^Ci6V`?Li;2I4&R&NI0?<~;$Ja<)m(zC?=GNQk=uIOEln7qUs_?7g}7 z7`JI%P{F*%de!N6*G2p(uxMTo^{Pq0(xB1A%Tlj%zEuKz6Yv67=+I^{=Nv;uI+tD} zIkI(^zz#c9X{A-@qeVKPA(5VL>tDG)EeA&B|2 zl$q|1>e&(tHh3o-fh2_@>gL{J)M8K($Gs}YHn_eD-hkm0K^er<23J(Rx=w=B*aB8^ zL*WP)+_1vZaa*(HVhJ3oGHD!&W7-M-nXHL7s}^#ZC;`tniNu?dq22LW@wjRer8ZigeT@4!$J@!q zUBLVZpQ5f(t5}-NWvc6#G#F3m`xsOrgXt)`uB`LpwKv{d^51uA9!*p;kpKIsJ|qo- zb;n2;gT59%I&0}N#O#qi72bIR7DD*E@Ix!Da0Om&zNoApU+IGuE!5Ik{-RB0X_sp2<7PQGKthw{6nl+te1CVg(;t!De{68rDoWp zh~TnVda+&@EFvNd7ACGiCnVVt=|#r&1zdpi6bQuf?Bl>4E56=_XS4wf9eYGUMb9vlRcqEwCvT+WLty~>r?zFP1M1J_~(4NcOT;qKzA zvdrSbvDuSG1teekA-ONHuJMlDIn705KPk#e#17D^8hDpIL#phsX`RT?%^cy&FW~

)_!0Uc-FRHJJWGUVA}TD zaJL+q@_NjJ$qq7@P?n zGOm=?QY8?attgc&NRK2RR(5ia3M#DUP4f2kq?T8}LTRyty~R1?r-B;>&H&qFgm_dEaU7)Z*ALco z4Ur17eDfO|vWV5^7PEZF#Q*-XoZl6T_th%7i~W@OWAb>;4lgocv%&F8k;@O2!L8I1 z?@FUVP)4hsRG&eqkN2_Bi^dTN$-oKmin9Rew7Vai^oTvC*8JHvX@x9o=q%f3tK0Sn z+KyAee|(12MK;~2YZMev9>qK5RQ~Md9|tosW{OlNUWdYEDn%8NkPjIM#$UJMP}(Ee zAu{92=dX&mjFa7vlxa!A)oj0z;CqQ2AX55etQ>6P7^w^Y^t5ysKRF{2)(5HFc4bTI z;QmHvj`fpEIOLsy1FG6Hd`lpZ?Sd%Gc~gm1)P~NgktSd?3m|z*%@UowTtGR7aN?Ab z-8vHK?B}$>$_%^8QWx0kVa~@kk#fdzr-f9(VZx2rCndp14Jn`Wsxp-D zumhLn>q}Yp5gb<|h{(Vh)Dvy*xhLAEsGAHZ8kKwm`xXh_{T29b?pH@em3ZBkCMk!e zlQONq!AiQZu1-RVj_bC;94bhQC?MtYWd&vn6V^4w4pyh6N>Yg84I3*P=ySy#jg^J*AQ#|8r6>Qp9h#{3P=z{mM=PsjbMIdc=&t=(n7>WMeCTb_ z=@M=d`klZCdL>C3cJ}iqf8Ku}__X^v2~DNlZsj+B0-p~Yu{}lEj+AV#k6%Rcjc_!1 zty?g%zDUoS-9UayNYFBA(QiytkI|g|x-r?`Uqqf&>Ej^{akErCn{rx3!=5` z9`ZnM-EJ*0I&}=(upRSk+=nIa+uGrPqF`vH)UM-`LC*wpafcMgfoH8zQZVk856fBv z{#W6KMI8V2KVlD)BFKglj{cDLj91U2OFvX|b3)|3)71O#)JIf;0m)x`?jy<5S^ZK8 z-yrcVeEvb#6DYZ-9gRU{@g2Oxm~$)Te~AOQ2W}%WSCw7#aktk1@Q=c$Gt+4=6Q}Zz zv7Mit-QAx{p^srq*`0-H&jDC9Y<7H2R8+>5kZh-3nZ(do9TDIqX3t!o=*eY%_OU&+ z)6%?5KS(UBW(sj;-my?SXV*K#OyQb3oz8Td=-K6Ca8*@P^G)R|!EQgDWrKjWj7ytu z*T!nj6yZ+mc6IZ~_KZo1dh{KPYbMr`stAfTfdkwZ(M|&VZqWWeLOg}oQMdNj{nz9Z zq7HCBI*V5YZcluV1L45V4W6vgm+mmApwBsf^In?&ZYJsU%nVFgx_x*K*3l6>z1Y>w zd>M}YB`g(3BSWyUe#9hibwqu@Yv-reNd3u+JlhJxDg+IfNaK%lqg77?wrPh$BQRPx zJY8-6VtXdfHp9#_E_5Tm=*x{;zlHO!KmRUa@j5NLm*4+s!+ua0o&`P%e^a((#saoI zzXRT%rHx{)a6z591_gcDBGp*;$$HPEfvEImY)%|{#h>3>o-VWEj97aNJ}cMS@PSGi zf6G|SSNe z#dyZs*ZP$_(kp7X`!`x$w~YS^W)wv2(^wD!-2p<)3#(5HK^)Yp^N+CP2ReP`{^W6Z z!2LHLU)Q)ce*L65Iq)FIxPG$Xqe$g+<*c1_j>vOt3f~hoAh9?Kq6m9{>moWERHJ); zj8}9SoFQ3RQGb44xuj@rGpMCdB)IiQ?(11g8CU|?#f z&UE5Xa`B=``ec$yZTA^?@0g+LNqoQKKVJqQ-aj0`HYW7Q2+g0O>tIf52X~5oi{U?) zf=vuc@;-vj)TxsdCS32AukCRfY7V_BZM0$5+KH3eRz*<1eKyOVol&8MHO;fVsZ*yn z^gM_ccPN1-gbN+&o*`YgrVCV?S33nUEQgx=tN|PHZCAN%F5lh_q!xGbqZ2`U|GeM8 zQPl{@jUTo?aO=DS_gi|u%53>zjmEH3VsI?3z4+7ErxIJ-%pbRPm&2CnFfF61M~H?}J~NNsufWAs-u7`Ws6dzjf$+ zPYeC%*&SEy)v!L(xQr6%oA2lqhKYNSNxvrfTwU!k?0u8b-GSAUD{=3 zr7SyhHu!AOc0}qZjWj8ru9uBSyy?5Pc>KdB`ux>i^FsN23q@_F)Tg+=ew&_}deIp0WUY$P@>AoGHM%RbhFN^nk>qN)o<#6- z_qUd%h$C44P;`^S?wz*ZK5pulc3xzZ&J%qsr?opeQK<{8ucHdI!fO28t6uAIgrABX z=SPZRC`z4^CC#3H1bJ3Kz^N3k;yRzP8 ztWep)66abGjk~2KJ+E8TyO?%g-rd#HgP^0Afzpn3{CBJ$j zqT36=C+`$6$4Ai^n(4Z8JxTTxo8IvlfN*g;Q|P6NK6>Oj(sj^a9&}jJf1xArjbubSy4~Nar=Y>k;j{C<`0*CU za>^A95E2p+6H#6xvapgZGO$l4JWQbp(<{q>Qt*v; zbX)q84Sft{@;z{n10t84BUr`EVmxH#sJ`S&aAG?m@73^FG^xzXCmq=pG6?OaJH^;* zlyI(3f*bN(L$qQLvekfpM^^{jgL@C&L_CcM3`0}wKjQEwv@Y%VAy;DJkz)TdP6(O( z#s?aot{G1!RJX9unKlg0H|Ogd;IHDS>GSnD`PkrU`q{mV^zMVYaD2YI)ax4WDr@mK zTTA`ATLa;{k4~qyx2jD&Vn$4O!)mjavON8VZ{=-^3jyI;LB^z zrh>DX!_>qZR)A@6Y4=3ZE%26tOtnvOm1V#t*FNF6qpi1f#W>*3xOTF{&|%u_+6d^p z!vG7h>DbWWQfyvp3wXc!9xTbvME?Ndl4lv?saIk`nx;gJhF8Mw286sd*^p)HTQXKL zWagju6%S_-yu{aM9UB@R2{4%mt~RikD7;(5Yi?bG&Oq@4_h z?@V3mEHBb6>i{2`#fI(bw&4}Lp;TCyC*CZ1L4R(LdaAjZBK5c^+#Ja&=HA_T+E_CS zSZ2x20kg`#?a*_L&aey2+UaR?*Zjh1Q}c!Mdtg|_+kYr@tE_bAuadj#m3NhW4d4mw z|ACrZ0QLU)Whg*g%=mbcZ4#BKWa*Gs8%w+io;)<)56Gx8YZ)i*T)e$rYzC%sMg`js z6t0q4FDTo9Wq$2B*rFBMb&-fkx{XfYzu!&G`%}C=Fzp9eS5&WLdkuT&iMR=AvJA5^ zA05x|gFIFmXa6aa{3cj3T9R3^JShFe^LYzshMC%EZmae`tu(fYHvHjad-XljXS=+zF8Z1=TYt85McMDtZ+C9fGP`3n z;vsKkOZuCQ)=U7_r2CPn2=QBBnq2Mb7we66AkZ^%nz0Dnr5;m~9vyU2 z_m(5D2)S5$-o&pX#EcCKIh*$jwAcwgYg^o174Rv9iI&L8+_VF)TTD;sgI=uqLf!M+cpaS0`1i#H#4=bpHxp6D_%vNj}-$phtJ zy&io^l7WjRkNK-Lgnp)lxabpAp;$g>Q1BIc@w6T8`*$pVH}7U5bZg+OeOSK*NQ=A%ug(Muq8^oq8L9n&SRhxnfux9!c z@o)y^Yu)h>_`ErCvuT5rnAcKf9I4wt8{8n5wasYP7Z@Oy@EEuwI|rX~iehyA+%qhD z`=@NmXal4p8~Z%A3xMwhIz5vNt(8B@QzM8uf-wtcoim2rhw0XrYlZQG7#a4Pzx>oP zC!4)33HSyS1H#@UCwkVHu!XD7Gxh6YJ+HYYXbr3-Rp*(E0f80)dIt@3dVVQ!P!U|c z^z$w!#oDF$SH<|yG_1yoEsb`jTdiC9+$Qtg_itF%gvO9CWY+stjdHj^C6ccsls)jW z%8a3Qc*tpcUwo0^Qq;t9gqxh?Qgrsqji$2M8U8fdr_;wL*KJyA;59+U?8bo}Bdclh zT@#1Y6KqE2ZNx`S6bi3zUKJ5OGV#MEXq3 z*n7LGc@CH-H;_d*kubxZ{`P035BsRJmZUFMQwI};X8pRZ%y|%?q*wM-3h^`k{%h%v z+3IR>rQxcTKd*j2?b>Vw{n!wbfG1MbMM>AzhKW(Sv1jonVf_1Pf>8;YNGwb}gmtq$ z^AgZ=J~R0``hXBAgtc82&?-z*@vafqKt)Ca*Fg49_Pou|JB3Kmui zq;PQ4#io*y%;Xw8iY+GT5UFd$t1)TO52H9sI>*6TW7tXyx}s5gaT_Jfmm%r$cE%c_0t>SF zx%ER0oc<>Lne9ry^ZV4sh@W7j=(0+VD%%JO^_fZ({>UdItJbs3VM?n$nk3L=YN+4$ z)VM-Q$Du?|FgDxCaOk-o24-g`X6|PaY8i@n_ypZTjZAm+B-T-ras6usGHmF6*vb{f ze27vIMpCxSkJs&CGdZc!nd86kr=*Il<~r|7P0bf-3BA3V15FHQd+z~p9G=;lkv ziDBpzV7L}KcC!M&s;8UUp>uIE(0A9Hc{gR(0C3)sY6TCw-CMktyh1``mN(lmmw)N? z8+C!e4qbxvO3uhpMRJdgo4+J3n)P~&Ra}=ST=gPM;w8BR5l^@HDm4Xt+&sY1hW=i= z#t7XafHlDDp*QPVT4Mxw$R5k{Xv^BaY#~rj4BQ?(uFIo&4b-DV55M9lF$aF=IQT;1 z3WnX~&7g6Izhoal#=zkAaKFmJ$d`)moMj$kBXO-EQnqnHc*{f#m6PQ8S)CE?QA&39 zEHw_}Asco-FN`!U;asc|AiTIpvyvM_GmmKWv&^qpQGzE$u9F``hfk~w{}~;_yIpWB za6+P@HRQ}*AAx-*!Jq8w9s2bUp#r{ut?^&s71Hvg-^a+$uf_nJ&C^mG6?_=bYAgWg z_+MQrJ7->h^)oIK-m!kUTFWf+*o7LmWgx`I)>g98Y|hC%M+lXh(djTE8<;9xzvc!* zPgpiYUhOsfDv+-L#GQJJ#^hgi7DWRD1OdC3Pc5BgkDkcd17O|lgW16luflCBjthy6 z`=)Cp-%Yd(AQWYV(7u&!(t_q^}@aa(9ojTIH>OYeb1Dk$XPrjx2`3e*Z$L|eIA2-v#Nd5_mokI zuwqf>=)(xA>t^P5F0MNic*ZTM+FfUXUF(Sd(0;eqUAKNb2OE{4S8lgeHE(d)3qa$l zxTt3x8+PNoxL@{xkq+{l*a)jXJ1=CNZ!wlP8n^C*z8D_^JGM3u3iXXFA|Xjm4g=Kr zecN|tl|Nt+yqDfE`VljqcizWi7P^qjlsx0yLGf-fW5($Ms~?bPJ35+OW|!@l{gFb< zVPZQ9kGmGkGNsGiQM_gB7RPSwZM^24O0tabB& z7KjYrU7#HBLoWme21ie72<;-~+v2510PV>xHMM9OZQ2-)Ki~9Hw?@G1{TDgyI~`50 z#%y@hvpsfLRW~(UT!OwE_FV%6dP(rbH_IlIob*s5P9`7gA1a4Vd{Lxed^SJOB*^b2 z5t4qqEGd?88T13x1sZ}7DRvO9C={>aINFg6INQ-78 z8PjB{XaiM&&9ZdTW&bgfhCtFpXHK1($MaU$Z^2^88C zM(}6sO`wm*0YbrFIz4x^AtBKAF@nkH>v>c>If8<5jPRo0A}okHB1xuJoib-KU4lXh zDKC7pGjdbe_4r%40(4g%svxK*<4IvrX}7{4Y1CgjW9qM*p6*5gW5-uUB*)y0mMaYL zSn^#v&3}CxDc%dj z22w1s#(J?7(HM3c{wi_*LaxxD<78xbCU5%r73y6L6VPcOFLg&wXLpPY&kMiI5~Q!! z%t)NQ$cowh6HkWrr(;2!(A}LAr+Zf1fG~IWm>rgJd%X6cK{?)O8vSj%GKj0u?#6!Z z@DY%(GS;-E~4qX!QwvVlaZejX@BA9;_7my#5yO7!Y$js5|0s$ zeI1|zEfG;DR{o597I6M6rZko>R_xS$1u0T7sr0=d6%hxCQDu!dPN5qo8-F|Hg2Ly+ zrzhk?b_T(9;og<{RKJwvWHH`W)vW%H8sQF=5|cNsmF2cHEZ{XzXKYL?2W!(fOXZ!Y zv_6(&oXuQ@hnq}aC_R&t4CDm$=wdZH7lsx>8z_NdRUs+e6`rv%pwfx?#xI6c+|duz z9^*Sjc0a|M$sv z2Y3IV=#%b-xJzZW}$=`rpp~l)6d>{J|{aGbEl?NO*5}wtGIYZqx3GNf zM6voCeIWRy#4;44#LkTKhGUhwFC99!=9^EFlby=ye=M)h*4yd#*V~TDCFlVzz-k6y z4}~G0Um-1yha9$IbJ#nK_|CsR?Hn^C#_;cLTiymNN+ua58B-FD1}+AlU2M0&6N2!f z$o;I6w&EW+^2bP}RY-Vhndr%XmeH}8LG-af3X;& z&ka+_qtIu8;K*72*LMlmf*@CcqDEKneod-;{HQaf9hO3*p^0;4w)$p{!D;_?gL+Bt zR-+mt3kxEWkQ~*6OPKP2Q*}NrGx?Wp*8RNNf*y|Crcl|(z>rW6hp*Lb52vB$IO#nS z42jL|#k%DIjbTDS-93Khpd?9)WP#2wlDg#>@2G>#hJ^(gKxXu5^4W%3)_tu=G2tUL z?%fhPRy32}7**}#P-IY|vAbVP=2*!RNi^=9y^$&iU66N`zoE#ed5a5oWk2O57ncZg zw%cH$^g=~EI7G$Bj3@Ju+Qeay8sf#)tO0e&?P;3KKo%Z!br2)F!&+uVE-Ta7v)~uR z8)^7bbrLjWTh*ZNE|AbRW$SNdz>Ccd1=dvU6sNWoVaVTOgGk53( z*wJ<>B+d96r~h@V(k6X2moA~By`^fLrp>H|i-O3UvPI<=x&EE5V!coh+9gbeuwcA? zh=nXs_pOs!Gf&PmP5kV?RLZ<;9P1T*B=wSt z1LER+z{MzzNn!pO+)cJ~ziup2QPkTiiIi2!ZY5@GnrW|b`0^g(hAB{Du`^0N{=NFF zMZc-hc?1E}_&KK&ve!>etNpHtxB@T4W7s{FdDGdju6A?wgEHp5Mx;BJ<7upfu}HBU zaSL#D@zs$0!m(`7jk+2p($^0Hj9Uo9V`XBwV>v0cgt@_;pmHTFN>J~)96`<7LXzu^ z$Ofue2!(hQX*=Q#4E{W56+431c)Tvh@KtgTOE4!+1Q|0EA8z4YP<%+|op1ai7^^3C z%%^&E1lZw!8gMZU*t-f>AZ>cQZpuOu^VOnfIENc6-)b%_JU5nTre`|#d~Q5QPCx(k z)C;!kA`jkBylFIyH8>&85Ar@567qG|k_ebOO&gi9XXCu+Cc;~q>ALZ`0!F}aEwPDo zRYM0uT>4z|#fYTl!cRduGQNrnV`RsK*97{Z<8_4oE_eR*q|Pm{2e2%$U*PStSwwSW z_kFX>lD;#cWFB5rqbn>b>e{Ggu^UB{*)SOQC<{=p`%hTWocQF*YOtx_^f;aaRn!KPshaE6g^yUO2@4n1td-MzN7W)OEbSGrS1KOJz>@EPEiP3S@{FV3ct}^J$CV9I?MlZMZjfuMIu0wy*oY+1>X7-2y$O z-LhD(?w-daGP?Z@)6|XO}LKU3KslKZR?~Uuf4U4~MOb6c=wr-3Qtk6C- zttV5PuwAj$tVnx@#&E6AZi!Uvl$fJ;$2ElN)4_(O5W<=Q+NcjyyqydbIx)*WV;`p` z{CkP}7IJUO3>A#=2`JA1x_riReHE9w{Y@YM>WY^|$w1rWZ{$?vAFx4;;R=8qT*tO` zW5#kCIXh_w+;l&1@k)wp`HuNc4}osX-Bm^<%9+1q^^P5-1Y0_&9CV1vN}LJe%QBRk zElo6h&eqSHlo)kjlYXC>IGNCzh*(YP8n>l{#;)(V=v8msJ`>)3yLU3D-7MZa24{ao zNVSL1G3+7i2EI$SS`g@hzoxTOBk(8GK&Kg`~ejA^G!%gKqtLWfoJ{Edid zsB15@%ESo_B{h1}GaBVP-TDswmVBw`BS&Kha(j#cml8;+f{nZ z&2fXej8bMzy;qWd3})*N1$oX}`M<%Y-W<8ij+ZnYmF)T+_GeIE!9}vB=clqB>>Y@N zm6RF!%gdM7-da#vHUSezp=)U!!ypjoi;+K}JYE?tF&;6YN1wV#EEw7%ueEvd-MPBu zD>3i*SkF)mur@lP`sjeK%x8B$1~?`gq^$)1iDs`Dy3fJ71t{>#LnUgpYlbK#z6++A95sMYFU@x z_(s6PD+@>JG*;PtN@XaX7LUnH(lfI4f^I?kAkHyDY;}4=3;z;yI8y-D$$IuwXr*YM z#U<7}QvR79IdsCBN-kGI(H)UOK2GERXbh#vD5rWDZPD>nBOAlHPk#HV44cj@q0vB|?_NoT(J|84DRCR>QRSCbTS zi2q#sNToI68u&%9bwdFlcB27ew|RFabVU{=GDAAYm67YVJuQS#IjV)BPb)>#i+qR0 zx2t+-(TTggO~As6(|64<;QT2Am+NvwWyT%dLXNU_Mn_51Zs^gNgIZYNn~oKgss%Hv zdFf!^9&tAW$w*hvKbPt1N(-bL4q8~XEMP{q_hRa?ctP>CQfu)Cy>6ukQ>5^6|u zbs{2Ny~;We78I6WFph;Gqk!P9_1!ypoqa&y)EJ2*8EnA4$cD@1Iv=N4t`dQAy~lYK z5yiH22s1Dsw`9vDYnrWxiE-p&h2w;CeK>7LkW1=6U54wL%1kF6E)H)g z23ov71&i~n`sNc}H7sk1+GcrAu%HI+^t7{%f_2_)MN9c7k3DkZ879uA;)zIc#^{j+ zo)qK}7m0uaklLZmWCUy6Ypm; zKh(A@jT%e7K{?k$)%eq=BuRF@kkr&ZwY+t0do{6LRpVEO?yktWe<$PlURZ;C(>GaF zwp-TB>ajkV(|L_c(*lbu)q>wq;Yi$A!Y##l5VMjhJdtTBGmHtFer-K~8_rF=YmlZW z1Lg4`V^at_&wtIxvvLQ?QFAw>0_gSxC8gn-=C z&}{-*W9&#m_|Bw$%(`*zy}9pUi^E#0bn|XLGfLL6!ceHLpg{gsV(Ug)6G$c9fKEfs z^{8p+PiVE7{nSuITJyKvR}R~$O}_IcK!fIVzqzM}V^C`+LjNT&1W`NLBO%r7^>EYv z_OV5rl=ngGfc$+qmRL?4ICXKI&D$i}g8=Y_k-xIkW&^$*p1+d%Xg(59BWrpQX~`WJ zID}wX6(cNG(X2Y-{P95~=Rs?^o!FcEY_FvCa?1jW+j&gMfe|jDEtVP<W;A*ba z40+@>n=w*7R%0t`9UG+vvRIPZ?`q%c)5yKFp<}(3QxWC;S4zlB+p}c2lCdJ&OWsW? z_q*=Xr-RR>SaF7Uly^5}ohK*idiW7qJQJ^96T)n4eqLVDnl`RHU}QFNAv_bP?InCS z*N2o~WR*aZ?N~UqHIt(Xg1@=goIgN!*cR;0)}g+>hH{gKCXlk2IIVD0z|Ifu03DKs z;UR|Y^6I*1TyRyFpYh{rIJH61Rdwo&zKWXsr zSsOk<%lUHrqT-WiVy!6Xag{)ytc~hQu#$V69=~|{yfv11)sd=NP4Qj2E#5EyNYif` z(xc=eizJ2OU(Jdx8<*!6ym!#U5w&JVTFh)&2MbP?#kBATwYyO)>qg7o}7(B z!51V=K?c3hY7Ps|9WznRYBII0QGsHBQwx;u)%Sbg_*8S~+ znphhmONIHy4u`36r(3L^XgUkKz|)<^NJ0XbXCX)!bob&V9_zg@LQta6pl4#$9RgsR zFh{)D@W(+s5W9kh|J+#X8_&6(A@)@g#zd_VHHz_K!_jiiHLg{Aeyq*m6;IP57W_h% znogGQ;Ld>W1Cr#+hVmCsXUmhWdS`q{6%*<8OfVH_-~Xv zIxkTuq2>pp*G_0N(B%s?bp>R8PJh4Y_>Mijm!dBF3+AFyQg+NVGB^(%fuD~^td5+X zf8ID;{W^VU1?oImIr%92j)qo9#{l;Lzo^V32j-mKCU@(ugrszsOW)7U7^c;!jh%U2o3dUB-uDI1@Q*|mN}dgFP4asOX4>ti3D|$l;aRNd~m{mz=K@@>w*ZycdaZf z5o}+$DaJY!f(jhXV>h^fRaCk)(YWBsQpmVa=I9U^-3 z_zJnA9ci25cM|aH{Wm@#rd^wrKZ$qjSQwY<9fRk&jW3|v4?E>FSim&kWYE6)k51|t z>xAE`Zy?$~0O@<3MU)a_6%@q2DM&<^9f(qK5loNQytoP=Op%_bhzH?kLgDw7aMsC6SNr1lgR!p{B`?!A7a7j$dcaf)z$7+OY zp4ET;`GoO4Uf^7uo=^At`VQ`gi2ECgExVJjbDdlD_pK^A7d#6!*5M40fGib1wDF z*w_o(BHO{jIDM^o*|f2#`MgySS^Q9~Q)z`MI9|`LzSf^%u$$XnfkMugx^4i66uYcj zH-WQIku!2R`j1r1s`k5V9piz>@g#$fasiVoFI8`pnuz(gU+G-d4Nan)7w@O1o1Lz1Xuta8-30w5+?uNIHuM6z_r6xIWmH$tdB4VHvC70& zY>nmUeC}1=UtH0rSEZ_gikSpv$7%j_pt2~U$g<9%h{GU^6k}{Xb8P77kDuZ7#vz

%?(L{I z&*5aV9Kk~wXpr&WFpr!4Od2bS8PVsJ3UVj4i$S*gO1DDcIp4Ln`a2?Cc`K}XBx)Ud zu>AWjUmF`m-oDLC9C>TG>(7;4aTbQ;v_G+>3d{~V9nU;S($|+{ENk%KSy3U7bQxT>FGG4T(o3d! zKdowO6{a`TU>K;<>mRF`Lq4qVJ4Y%fQT;V8fmYOK+{crnkLUX?kDv8J-?a!Omq!zf z{Gk*^4Oo}aouw~fHLZ=jK4ZCkm9$sw*l@r{eJi#iD zS;b!_+L)w7B>qV&aL#v5hGpZXlgzDr$bY5YTTetH+>&ENbHxt!vGTcY*Z*{z zyEmhmRCDQ02pw|CkD7U3xlW4mg|~!zLV7O)u7cQxH#XkatXd~sIMFP27*WfyVg^Dk z)AL`P$Hgrdc}h#qPtO{h1SMRKg{Ulj4-c`Sq%nOzrfoJ++Fpf(sQiMx*Nc<&9xj+bTp`4@1mtp^hsk07fvTff# z-7pXll^7+6#0aI?Ktw=FML{KofMU=fFlv+n(hZUdf`~c^>CVxqbjKLY7_l+N@Amz^ z&-4EN=AYeM_kCUGb)LuPIB57j_uqGdG)n8Ul44LwkjLL{k zLztC2>;DxNX~=oP-GQWPNE%G>Tp=i#wqI?ncrW6q`A_Sh->>WTTAv@)cM-q`0gDs- zm$nS0awuH-r`+@+AAW7W$?x~}j*b_8CsVWJ%DugX!I$^5$O~Txcl=Zl^tWca8f{rT zJUvekz8d!YgIr4LxjQf1Np5~`h+RuDI;j*^lgPz@Bb^)n@E%c(Q2GHwX21e1F3-P)Z638eAg)Z_UfQf zN7F4yRfHMVTF_{3a%Ok;3#Wq6jsWeN>JP?1rNbh6hkNa&55G+p8zyw-QQKd{Lr&4% zDltM{H1W`5=gg5#+$n|oFEl(K1d1)M%CeD^QSi70LNG4+Qx$@yky!Ip2xb=4>Emdc zczH0$1K%7%U&!mXcvpZP#?H{o>f*jqkQt)h zQ=$3J%;&xc-~iFOqsyA^9epgPc6L)b{psk$V^=szX~zR@becF|Lj2Hl1D8KEm$@`I zw4)QlTcQ=lR;X-+7VizcNL@((%AOb=klzSMPB-`W0agE9G51zcz&mBkndT>wC26sM zz2^mWP&Lh}Bq?+m$&I9Q5d!_oa07+B@Tt6iN3 zmjdc*l4kZLvmX}Nfyf|v5IP+L_f}9JF+ScXWhT;3W5HlZ_8!(ad*h%kR*h1ib0e+l z5OB72!?E%8q})F1=Nj|^xSkB$(e8c<1irn8Bj_$=k{_yR`Va8ZeX9zo+rVP?_nQ1f z!>1F~g4{bORHzHt@L8)OWEF65eol&%xu+j4CMJHwNMnc@QfIB7B3bg5nfn4f#xTnu zY$QEM!+JZA&`nxdE#{GN?S1BipJo?WKir&HG6+B-8++Je>zsOP2#Jt?mI>JeNSZ^{ zn5nYq_s;&xH;teHf><>O-74K$B(UYGdaPNy9o9MywX}bSe)FD-H)?cSPKS?CxLG?m zv@L;nr=43{wJKiyzQ*$?Fvf<3reAUm6(28{6M1s9xnSdbFI zC$lX05Xb@nbivnld^x=HF%5Yg`s`JO{CLjDS1;TzOz;A##?yncewCh+6=3=JMD4SU zsEySRl2nO4eKGj8HsBdG-AxrYc&Gg+i`NEJKQ*? zj`s|Z2N5zJ{Z+wvC?5fPR)OIpkXiu%HL8SdeYF@=q^GB+;X$71eah^8#B5mAC4PPn z81jtl4`5iYFj-#NZ;}x03Z@o)T=NABWsY+VGmSTW`js)>V3q0-+|-&a;EkV(SAL0y zV1czhd|1w-=yb=m{n`^kv`TQ=R3Yd?PC1O(Nni)CHFo5-a0xn^lI>DeN#(i0*Bjrw zJ8m00@t#W}v(C0uIMK^NfIW@Jxqlg z@;)_SqF1-X+Y-p7VqW2}SNZc1Ym2*0oR>QTmZM?#X~kBxV@I}^L<-8_)Z{4OJR%H~ z+wl8_zrh?E8~@?b_DpSNU9H{zm6Xy}2D(*>H0Mo3r-k2kHRe8&brO(|AxEiLh>6M( z&N^q^DBQTMRrXCXb+0WpWcr?3U%QPc_HI+w66J*s!X?H$c*iue8{~wG|rg4xw&9Hk$F{UPb zlWN{!Jzcit65GMM-y`t>K`wcXPiB;U8?Wo~7rmfIlBX z@TvXE!(pLya9YCKMNi$7DBOym&dHnk(LiP9xq=-7ik!qEWNbAqhV8TP&5G_9|w>JRCN7*Bx`M`_zRw(jYCE%qfw~5;n#G0#l}5od*#N<*a6hk8-En~&FTtY zPeD@=wc|Fdg0Wn2DI%9V7;T9xc?yaAte;AZ4vv;hHg?O@Y5XrK-3!^3M9Yh?KOGz! z!#LY6Q2(&1_j&bppbILoK2gfoMxCrPk@y8#Y zASw}n*=1u4G*fhiDF@_Y_+j;YHr}RtpTd5xjB<$uVl|UNsts`3EUQDjc|z zzH}M|S$KL#;OTZ+PpYM7`qY>!klqq7Xfm1G|3|<&ZgZ+~Aj_{FmUXuB3`(Psx7Q8| z$bUgE%H*0q@|Mp=v)`%Oaj5}BN&yK=G&4dCM}w22@G!Ye zA1WEw>p`5GPUAG`?w#N z-_tu0cSG+NVf{xRZ1`7C*- z>z@ArZ~M$XY@_mm1L3H{iMtJq4GaxQ0{(G~N?(<};ePD~?MmF*?yBwx``}neK4F|S zBkhx4X=hU;^!W!*(A8*X%cGfP&o8-#VqtZ3!RD zC@vc!Ip*o#YS`kmT60a(idw7tsWoEiYD6JpG<4Ef-U!UQ4&`Up3~NL>1qK+e5C43~ zsTH&_LAO#25FKB>k6MkN|MwoT3SMSLn+1-~ha;(6Lq=jCL)$3!SO{f#6J-v4b<=AQ zUit1>S8Cr0-&s7=6o%1^8}Ka%l)9GVzAkXWg%s^-uVy1Y@<)-!?m6}xI8e{vIc+2J zj?W30d#I1eD1QHtXHBu!+8oPdEqC*7^Z5~i22yZ1*Mx$gT*E-Rs0Tu7p6hClS7MOu z7hJvgNT}`N-OCO_eS_bDHF6V*qj-MO$yT2f;;4a?jXy1q*!=f*Q|&V!GapOvu5d0S zvWLwD10m_R)Z{D@v%yy)8(ZI4i)JdzZP*FA-8${Q;V%KZ_<}dCAlH>zH*qsPbp59K zPo0K(2QjH#w?Qag*|AM2cSecI^%7GVD>011zB%5tGrlA__E&ZI4&wUD#ojOtG29rx zsGyD&?X$t5y}GboepaC$=SOdE-JK4mWwroWyR4)1Sga|t0zK%ssozt+<4c0Lv6LEM zfxceiR*h)R`gLp@^+hhxFSH7T>;yuZ`MqmBw{RX(KJ$L<@42Koy|;o& z0^xV&mQoi7hEbDXJ{UJFLurfIcraiW_p8hIt8R88WrJb^q-nL@_eiBE%COBg&yUbF zlR#s8Vvm&@JK0KuXCyCBB|p_ze|N_RYK@`( zA`ub^W||C%5#4U42bce~%dBvogC5v^>Nh+2B2_IK9n=*_IB7idJrfME$WtFG0?XMc zI}hAet-NuP?9mM|4(k4^aN!P z0Y73qHRCns_*eR@NB>hRQC3D#1G)QFu!HI}fA8xa>qTKas>Aia!w2gCje)m5x080l zmh{yE1|$$hZ39E{q667#Aqn6)4B&B53Iv`umO+q&9?u@B$|}g--}30sck?ZP{3(6b zlTr-|HI6^I6Qm%LnuzLKOk5}fNnU($FjNj>dZB7r%vYVl5b&zrY9G}%DT#EvP@Bs6Ma3`O z4r8fsc>twThdJdJlcq0pFBG(hXa^{Kq>MG3b^!x^^f~N-*Vlts|Lvq%lPPsB&BraF zH2CMm=203K7PRK`EW8)qRsbDkuSb-(`u}2_KoI~7g1gpqy&Gd$t*T8v;`2AOR?|S^ zRIb}6Hi()a~Q##z6`ja5^gvTl8=XL0Ac-nPysKPpM_RkRdKz{y?!)7 zv-DQ>cCMr=mxllQUW;Yv;5uL0TzSFnD!sv?OnFU0F@+-c;*I&yg)Pb#GHxS+Fdah& zC!@8M+wGuqIuKS;>LH=cHIzaGBysdJ2(0!0qF`s6DR2Hi;Z^n%ku{u34i|edp`u8a^y!H?hev2AsO)`T^ObZ zv?G2|6QX%njuur*t3t_d+deyweS2)1q3B!@DP4p93)h&}`}{(U+GWn0b*Db(&0@4} zBzX}wB&(XG*F}&#JbKK$tdqBx{pTS710zp!)&DCw%W*bX?jqdB5v*GET=H60ZbQ~< zZ|&uop`%WMG6UAaNm_D7-ETP@Rj%rB99K`CeC>Fuj2X}N7+L6O)ktF!2U7=Ta3C1# zI+*tU4rNwiN=$+)k)&PCVuy>Tul}?dU-bDG`csBm6#>^;8?w?X z-jKcC6)2V^e4ysCYyEZV3%o2T!>y=zLrB)dS!GI_weWVh)^ zdgEG!&0;!u?!8AiyB~-|XEmXARJXtRt*qazkRP0zG%O}6Is?(z-dzxW3r~sJjw&pj z{^8L!&>YretR&~1zsr~)3FU-R?j192&*dKvmHg3>DXkHQkh4!XiwrpAj#Vv|Fz25Y z+)qS>fN%4Y3ZMA<4ADZl8<-k~;+Iq7&K?MGVXi*ym7NcmxJ2O?v_rS6_t;M;C!X9k zR?rCCauAHJk&V4KlV07P95ybO-+F(hk33g+ofxV_CBhMK-3`6ey6V!X9Id&`kie-c z##XB=)N@he9Z}{vqlUswm6t3+oL6&QbutDO%XyXp1K!n*N$iy?pN`L%_<<;5G#q#Q zht|a=xs}J#$ys-$u{OQU+;{0#%)YiSUfmRAb7{N1?8p0=RLVD)H(UI88r$T0SZBM- zjp41Ht#k%d-DvKI2>P^@T!R>F#*`n!QqfX3mcFpKQ1$c$Zv{QSpUf}gi67O62Nh>c z0lt6)YBSg{!(r?32C1O0@1w+wL)8RweI%#C+_|>Tss2Er_K1AqutWB*I3Qp)-;dSz zn|^=n$oKM7tfA5P8F>^wWe+`iTzjPpy0?5`stk!zuXW7e47I;fzv3aErt5a#vz|8! zBT?ME2n8@3&p+>2D9ixBZq~#YmgS&k%>$_$q`LrF9iTZpPLVoX zK?VFKlbPW^GoWV(-}!~GnLqtWl{tpbOvtgPsIzD0IsSiQ@_E+bEjOWEOD##6l0#)Mpa>djO(t=FLvMr!f=BJIj5$+Vu6G5s@!b z7$CzFW$TR0`Tn?4;g?D4?GcgMvu65DVI--uLiX1}qA~1Q&R_2~#hlZq8Gr^a591F& zi`iW{9rQXn5Vd4I`94QR6*;}}#qV`)jpPt`PQuKce-)#uRen8d=MTVhpvLBA@m>wi z9ZTlfyBTMbZ#EoC6<%6mZ#TC+7jg^vZRk@Fa2CKa-DERR`%hD~cRmvJ?bIMd5c}=ri?L#GI#_86e&OoQ^g#ljipO}0q&c8H z`fkMLS@}D0*1rqxvl$^(fjpzEU0*Nu-)ni)liq;cfjKK8vHFJx$Yc6i2*EVYCLKYjKKh3q>G-kC;(omlrR-&&}h&L(|up7zln-iv-PRUlXz;s!g$VE_11WG>jh~myR4!tt8wxte_3pcJ+XVN@q=IScT?=Q$(pi(iswGt z*+qR;j$g(iPy1_Y3G=PN=R4*7_twP+j?uey0~~9*KEwgac+YExb=1dBr~dw3UlO9D zqLoVy+fu~pXU49;y?|bdf|^PKY21iMzu|JlqVJN~l05XNWxwmlw|CojKH#X`tMCxa z%U{|s3ZbTEc7$Kff$N{rP8<&ZusoaxPGM@0NOirhA4&7F$puVQ$W~DfzkUOd8Vw3~ zz9E0O7{h03=r12fya;yf)K>*+pAW-Ux((#)e0xbkeq^fTo@|I^RK0zL)33f%2hWz2 znm3=_9>on0;R>IF+tJs zi*0C9vNn=7XpohQ0(-|7<{&x_x2Nm@v*3-Ra~&*xbf??MKwW}8C*jdYi%e*`o> zIG?Ltd+^xzbOXNQ23?!U93>aQDYCj7yn9T4l>cp?Sa-5LdhXOPUVG&<(A9PUa!F>& zfQgKw`S@mwXV%&bQ0vY%))Dp9rK^|Ep*X=y^BX18^9qGkigPX@Sy=%u#^F2JzT2*~ zbNjPLHM3+|W)j3KD!H48fk2ezJeG3y zkDNmfJVwUMxU_=Cfnt9LyfMGpbG-DSvC3ce7k8)#xU-5(+}L7u_6w0Rc+(`ku5S*o zp*tm9J!Z%G&!`5s@t|eCLyn7cWbOpS_1q#wWHbv-`oG8#yun9BBB7q3+!DE$o**{s{|-?z8-llBwO&e=2n%L>ISSllS9yRDJ1d9jTsouY3wbigPy)N5mn{Ub>A5?p{eP@%Es?Xbsony=%q1$UpLFO z9fAb-bfZ-v!V@>s*?%4j2P))AQf-C$KVf`(W~ka_pBEb+wE5Bag_rJCk`3~^!pe%4 z7<$b%fg7+bg$g%F*|IgB^FNyPMV%C&_b$v#yo~xtU7DY;yql$cb>Zyh_)zEaN5hCt z=1$Hsk;wAZFonRn)+2cJtK?WPj{T~>v*I~WwXWLH z`AuP{vu%!!-kihDEt<)$TC?k%&#qp*T>nNE9(S#m=G)l&D>c)jRZpF#tA#pFZv2vWbkDh? z%`(5!mxXY+=fvkCqtvM@sSZYp6=lPAwh_gj8Slr1uJIg!Hw!B16W?{+GIyn3z1#vC zjx&69DVLuY?m1c@*DPmNMGa7Kokcqh&yqBj5lh@1} zd$&);+s-U3k}hY#$;dX&qncuPDyg7nc%LkK^b|M`;5XQOlU8cJX)Mqs zKOjk)&tFjMD>zbpSAUTHX}FrR)VAG^?`=ebGylrG4pn!-fH-|{tcTxSB5zK%2a)p< z8rpr68wz`-@n;Bid1So%hR(2fBj-71(}>z?(0W$LYpMpt(_3>CKhm9DMB`qSUkc^R zWXR{b_2e9`q0nCt|H79T)KCaPQ&+cdtl==RVqs2{s)NyvyAWPMFYyfMXKIH6h6_?@VECwMN|9;mG8XrJGY^$i-9@i2R*`EKDtx6l^O`o4mqQF9P+4%8b zdOEt01fH&IDWPS4YOHejmUW=cVhZK*xyU}PbkPM(;XoldGFq*{3wsVsa@-SQ#xjq# z7b4Y9hNCfeOT9!+m3FzKo(%T|elHBNsG~YvPBh<~_1)2fJ%C)vm}qiUns9GJN=hWN zAuYLWoQ6KV1^Nowc0+GY51j9W)GUYc+X+zCujw=t7D~)q4SQl|J#)!HlzKVSDR&es zl*A(fdu2c~8DW+v2qi>bz%GONE*+??<+2KF39MEQ32^utHx=v$ znrbpx!@M{aqyKMjt--7ppE_cQ;Sv4_>YBQZ0gUn&dn35F7N8KrMJpVNR6}Mfn`k)Hj3MhQs+0;V^L{&ru=F3JrxWZ z>o>X2CR3G`#I7gVUSJ{^k~YdhN5`Y+PQIKMVWcBp^7xfL#=T$Tsf%I0M}O9DV@20~ z!Mqn4n62ZDInpF`Gaj zSn-*z?4$UV@b8jhXZS*x<6rl268gr0Eu(5knvXR|@y5Eo`a?*QomPxyjJmH_JH&SlR1Ize1E!680Y-|Mm7b{F}1(*Gc;%R{7bG=bC*m z7G=J6*(fm@SMv7C-YDj;r^)Gd7<3sWz2C~DYkx^vXEHcI5FDBZ=nrA%eXai8;!qbL z$PPcm{6oO9%Op-zQuL~+I~{>-InVxVdH)^ph|nUhN~Qvi z`-=_C<)?*=(zgZ=OVNYm50v%E|nT!tK|jXgin# zEt<`ie|LuQ`=F;dgkqtT09tjN5$CnWIdnV6M8P&F;3I0y-ma#$W@P-)HX**9j>?wi z>;UL#dClPgrfttv2z;*AF3-=V>4pRg#?7l&;d-I+aLjv?xfP6yIR1+J!~9DkJnr%C z#Hd451;b-vXn?H|0{NM6hbIwEYU)Big*yqa2@J)x#i^w(uW!#7z{fXXr-f&BXZuL9aZWD_i|PBJy+9*-0T*ga z!L3ftj%v-%o|2!N?gd>xi`0-x?$HsykMB9jWrs0|2RGi48L$5M^WIXn{|SlD8IOHV zD^Pd5Pz!`Ik_tLw2kBwQnkRn~y2fi~52Wo=#p7$_URG(|^pkb@1 zvv95xUW{&qo=eg@X?j;?kqK4^cp<3zl1G#k_J#Blzz62=dVN-&jl0Z^)?ngb7>V9e z@xC4R74`bA0)M@o!|Tl#DxC&L_vxLEDNp8+1lE&Q(Ah`-MabFkf1DG@QvmgTXaYMh zsgP8s$PFO_#2i=_C?`&K2JWie`q#4=b)g*`YVY$WlTGiMa({b<@~_JS)89VX(6Ydn z?H+QqJ*fna(@8xw61^XlALMYZ!)%^+K5$8AJ_PtI$?5)hVeqg2gn|G~PwPOl)pF}74ulHK9M<#Wzde+q^9^UD{`10+@097Po zs!mF*r^c-Me(y+^Gh(Qcelc2j_R*87xLED$2?N8n&i*~-o5p9$XZnYyUr)!*h~|en zNc_{k%$@&<+1a|idVPJpm4DjQz)@G^ftmSU>u&9?D^|kFHI&4wBKKmZ2r1aH`;roS z{O z1KRPJD*S#GQw-j>dma%>i&eX)Doh=>#LLqjYfXQ|vnZZgvNvjCOsIMq#EwN*rfQO~ zZNT`SCDB#>#;t4xI2BUXM$Ww~u-8WBKh_kd-6+!g(_zSI!vA?R0X_)%oKL5#(!Zfy z z{U<%vCB};Poi5a|8K~9#tgUavX#MXTR_rBZribAh|ClK5F{16 z%eRZzm2#QqmR$47p+s0EE@w7XXvX~w6b+oBXbxBZ)+@*#?275`POuT-_*RmcBF` zKNjs*$kQpZqv3Sd>BWg_L*U_>q2lAiSX*P|LqHQd3UsgsVdG6!EWC<;1q7Si(jQ>35jh@l_hRqV^FacJ-#N z;=4tZg|@)=B%egC#1)U0==gyPW0%m0H(+8hU_*w6hU+qfuUlk%$SoDc!shFmkLm%m zn=&E4)Il)cEnVae{ z&Lhd)^UOc(znj~~LUG|II>&s+_U75+ivj((XFW6Eb?o*YO|2`-u9)(bzJmk z(4&7!)zy0Ei+WeAEhQtaoEtGgRg~h_SbuWp1ExL<#28-pbLcqtEaHJldZ6nqMBI%e z+It~Zi@(BY%&TwHE9_ntQ?O3eRq43AlqfeCR?q%!4Ofzd8xR8}DimygnA30^xS=x{ zrLc;&cn;Q^?O0++r`Nm_Fv&v>MChBpj+C3ssEb4LrY}h<*hpvV8u@O>t||%X+&$AA zb?v_?N#qww4WPMP`4zkrxiJy}=+-Y@=ajkov?1sMb{l)x_$4YjnpjZ84rRX?=jvz3 zbq+i2u9&asKCUMWB~c*gyXakyny8OXR*PfRa6c3E;%*l`R2T4CNjA%;tmxtQ*`=&K z1J3A3z0Hp@+OwL&Lx)l=(x7n+YO;WlN_GV~Bc#xkD7qK}{<28l^9@yb{FKmN*=Tz- z?H{YYcg(4fLZ`UQpU)37B83yg;>>*pyOyigS|T^6@p(RH{zp^PN3gh-Q*ubf#%D_` zz0-7$EtooBkZ$-^VEGiWcZB>(qIhAx`#reQVxF*YB( zBxS6Y9k4mPy!bp0_h%TS3fnH1)t?fS#e%wPjz2|O&}U2oSB1HflbyzaDN;LkcI84o zbuhczc?6L{-7op;i=KZc&@vfK*kn)_MOP$^Rb2vqiM9D%XtCbbU z9naZn4B(nB{S#B+=#IqB0=>Ai)sMjRnQzL=faA_Stq@c|yZmxwz_7Ws(Vs2x!?>{W z6{+ttwVo$wPYvQg!BM~O4LHoKFHzn9eo^(=*%K)d{u%-6tJE@{5BP24Jr483zT`QWg8fyX7nB@h=e86^W}P zT5LjJsuE?#^E9+<)zgG(4htZlCt!bGhr*g#!8KHN70|k26N(^Tcb}|zfNz)(nH%mq zo)X;?>eb0_{{S}NDm;Cvs|IWyLybKtQU?&#W+V3zJGZ?zk38hV(_JLYaC%!e8V2U1 zDyJ6}<+^QLvfM$g>NonSLh$s=M8~A zU|=%gY=fuvFC+`IE-TLbGw2{uA}?a9<1iZ8tOo|+U=1}gXwZErqA5j1%{MOu<4+aJ zG^W&Z{NY&USoQZcRNVzs<%08!&&GEX^0@|%Vw`}(xK9{}ao+c1Y+YUHWIXQ%pK65| z?0IoLU@44O%U|00+0CRnsL%I>`{l9PH*&LCQ)S@wHc^I~Rx|C8=8qWapkv)po~aU! z#*o;Rdt{(7y)nkF5~q|DJ(Cm(58)+)-e7(Ya7m=pp2@w|-DYWcXy((fvjdM4+U1VC zp*qf7;NON-n$-NkPQwtXv)lCz(R%6T_e-zk!7jB>{e?_IN0s%D zy~_&z?rO>&w#+NCI#9SrK5FhU53M`2tUFNhZK9(z z#}lPJQ-`BV*ZLIlReWf_NPZNsszBVP6XuJT$o?xWy&m|2d8hpP0^|A`iEcZ3>ivkw z&^=BDWh^D>ZJ=L6Qtj^s8jVibae?j2zZSJPC=ki*)~`fd@fTJ!I}8$K#v@o~X4J)D zaOE0E6he^dtv33P6};M2oKuGK{yVK2v^x0YNC5vudZkOR1Al9x%JL{MUX8w z2loQr&s^^nkCUy`Kr70Fr{%Aq7pX!-G{rK_S_EhMn)X%u&-o}2yeJLJR)}dqX zRQ9(OOG5D#X?CifM66SvUYn8Y9nADhFbEcIlUp zZV|Tgj;6;(b+Q1tw@UN8Og6I3hKhU*J@s%+Cms@d>HtWhtICtV~#Y9dnIa`JkA1fp1(>jqtfq&DU~czExx`%q3J8lWdY4 z=|A40tP|`|aQgF}r)I9En-yYZqrmTYB5?BF;0+)g^5NW9)(Co)!ut4ynv~76L$8J{ z-t%uhZ^vQsiw8a@nqgzbbj)4YZe6@7BqXf*(hQnO`0F)dB--sc0a3Ob@I1Q%$weOM z&=S`k4z1m%dk{o8xAy3=(Ko3v4Nh8@Q5tKFwReGP-((Q=dn4A%AL^z<$Mu6W)Z}PH zFOAvY5mKhlIFvf+Vt)4r8|1fEzf{;_AQ31owPp%V^|zmorX&u9(^)rc8h*2Gb^C{741aK)U_-&lXOsoLE`tkw-PQJK@xPC~--K9>ZBsFeYEK{>{W&+RWmD$4{GUkFoe6*c|ZT(uyJ z1s#swlUwdWQEma(T;K^XR-h+ zBtVGd#(JxSLKfU|g`hEzxb(tJuW|T_@YG=XRL>0pLuRd`c(gicyfGNswXbgq==~b! zw7wRNDgJl3#;(M_{Bt!8OTeXlOs02XT?X%xb_TY(kWF)Y>e$`lgxD2lUPI9wAzFY- zmJ;7n%H9u)VIhVI-44?sv5Hn+@0!=zBSFA|5Xg~9(kgk>ZtqE^T?zAq#361b zZ(a~Cmp|Sfi+i^t)Ofprm7M3%XG($Xjh)-cZPYLN>VAW!L8pAR_Lv*`z6XuJ?k+B3Hau%RR-ne+Yhz zLW>tEcu#dczp-Oq3wsA<>wnC&MIGk#(Im0j;xI47h@x0xHC@SRm%PVlXTEabnE_;^ z7s-A+v2qytkby|hNJtBK4KYQ(L=Y^X3y!=K%PaM$3{89mJTl-+;e65%ERcJ5Py znsu9rP7}GWn{M#voKwp}n<5Pk%rew8pSff@3$jW(NB&b?&U?>SD|Z?Uq;xVXdmoCT z>F1T_7=L<5LiKA)@H?Bw4<@iJ88y4=*YbUU%G2h~`JB?WW+Rt@9KNGFK4MxXd&t zwv(xl6ZO81x<+DtL50~f#z5h{(wsV?=1^>R)YVe%kh0`3u)KX?a^b_f@w-ZlGA!j*v}%Lb1CB7J32@;$V!6OmroIU}984}3RbHvgC| z?7HpNjjK2YVRSy8R^vj0>^IMQ|8YXjnq~&$H)>|igjmt`6gcMdsf7yzKPg)F$K?G< zL16#y_BL>l_?Tm0h-xay;G$aQrrddEt;$c}YU1|JzaE&gKtO9%2)$OhAeO&>N5Guo zN|E@)1cOQL8cAj0izg$E6}2@B2G!)&N0Rb8X(_}I?N$P}xtMAyWyB^hM!qwBH+c77 zm$6Gc(BN8xF#2JGy+W$vvA1)V>NF#Gk{spD`+?nX0S; zeXpO4CJ&d=s2*faEnd3sVjc;u5t7y}I6IaVncWym_z3N45ouYrzVh^5=lx%Jyi^{wX|hcZaXCT78ugPY{JBPneSqRb?BB}m;JL_ zr+)O7mejr4Brk;VE8+U7bABMIy3ti_*F%p?s!pgwP#j-w*;`)eTt-Bg+=X7?QyTtk zdSG~>DCLiZhx}I2L!V}ye-yn0jXsg=ipiJHs*;4KdE|r#-Z7T&NaUV*3k&7@-lR-0 ze`efOZSX$x#`RPEZ|XDZ9b?ajEY(K1w*vbtiVgT8J@F$Z#9tH7i=!r5B#V7nbJqr{ zkAF=}&>aLKgQHsQSE$b~e!3o0#EHt+ssi?76~F?{mF7ng40qhNjIoV*RXM`C$+0#^ zdR-=_&iPD@>k7rq`A@QHu6V|$;o4?otpz|e+yF`%zP_)NX*6XV|x4`y248^J;< zG9`j7f=o>ynK3}IitYFUBqt_d4V!N52F z{c`KA3+#JS-)>$qM}&%}pU&>lB9`0jEtl*gZ#^puw*br-tbkzRG%4{YYFS z19JsLwlI>eNEMuzVA69}xj-CwDX{y2ZYk|lJY6wU%iB2iz{cuTOuVO%vgw$s2t^+( zW|T;@Au5{iMc4-SCW-pzi>s%0Epd%GK|b;O@F26v5o7=Tm3_5VZg4z&^z82%IfgA)O z&?mYd(eSmaHv^zjdAP!mP08gUsneU?TRNZPzp1R&SeXV4t9BL6ntWLAyoX?XZ+2O5 ztf-Djex+w$LfsiM!4}6^ID!N1D=YVv%#yxo8!OqPa9BL=-l+K& zZMaT~TewaG3bhmU%g`yKzofktFheOYG1quuXV>QA>wEu7O+8f@8y~I@ryCOS%{1~# zSoHIf5_{_(8X=p$TZuP8wiOrUw}`lA-c#7v-RZQo+B|0?VhiN7vu?KXDA)OxzP0=y zmOck-sMdunaI*U6D<1)6pS_0#rY6jpDYaX>?1lrO$8% zf@XSe&+M$#PUqj`6ae{j7%4D>n%P?*!b%MA+1XYwd%6A3ktB3?T}({(=y!;WS(89U zbdc$qINg>3@jT4r$-F^Gm(_HBBq~&VFel&8AgVCqG;NMlEsoK6hnNfC{3kNF7qrWZ zPUV(J{;G7ZY<`nF=Ct^L&Pwmc#G$fPbOUD>cfd$*1Lx>!vG{&+h&r{7 zj?N4lF+u~TS;vmIK-n7oVOVh+YHZUyw<7oHNH~;&g5!$bEzJiU{eHP%EGO}T92+u* zY}0?OewP+oXN#KHXzYax>t{Gc#x34iXV4t^r7G-i9V# zN-}R9iyhCYR^uQnFyMK^xm(9y=GI;ZheKDT-Mw9020IxMdadAPT_{feIa6cF0)eWK zzg&4g5eoI*&26OgJ)E%Trr9b%*ZR3}u3e{NN>`S&s2rYtz8 zAGs5OgO6|{n>18y;%@`_-h?DZ`w1HFuOz9R&$sO|%$Py_bF#->%NfT!lz%9ZNi^StzK^6m|W8 zO1_>`9AFol5DBU2J(KH<{-E5SNl%EjQpEp%4;ZWzLxH?2QMF4gxbt3{Ps?+6FaWvA zl8M;OY$R{80x9QAVs8Yz*2-)a47>Cwvo{bO^lBIh{_y;+xjNf9UjjprJToyS*%Qw} zS^9AKnkFyXOWeZfTokn>^$)?Yc`v20G<5SgyUk83zy`}^$4CtK|i4=m}Tc*TsMoS%3mp%b{PqA$lFk z5M;tUnWcPn(5&|vF*P7PzAi7(8k@dm@Z=5u{G`0-rKC3=ZMWEfSIR>e%5mrO;=7zY zb#M*CKYJ&ER-GE>_r?BJdsj7TOfY!yvFne(N~dxDF!D!~+Z^B3n7nY_czMZDL4N7L z>sG}N8UT+2f7+MydW8gfdtt!%&sgOp6TT>1eMOk!Gj)s#eF9}vdQ%bYmf_hjz0i1j z4yyiP_O$RHz?V=5F(v%DePP_Q!X7dxoZrOF}Wr~?Wk;WnM=Dd7?gCh_B1X- zxeOT-WgDBl<d#$e|~HIzW;vzc%J9|yw`U<>-l~@ zuXh~5V$0`%@b0d2)zA1}sgBG$99mM3JU}jy_I$i?txn;rtk9eu@Q#kj>n$%}d*A-Y zDWAAYXVh4WA+h@%^R?AgaT)`t1v}5bU+75>_3$CR0~u#8wBmbRl&`+c@m|b(Ichj3 zEl{XznT~hfy17ZTPg!^n{!x5*Q>C%u#m|RLl=o*hRaVun7WGB{3`TrJ%naF$H0hj} zjf*hmhUXky6^GB#0u=|V7+aASRY!pD3dhjOMBSe}t45p3NNeU_@3gdiNOg~Re$!IT zn0MZcGN2AG@Ed~Je`6+i+{AnxJc|oOzWcQ!%24eq zc}}lrxNNn;xy@r{_uuMywyS;i->Uf1ykt)x#U zJ)$y=|M&Y`H5+gfYUJEcf8LXzw&WnYpe1YG+*n`kd8_ozh5loIO(MU(odeH~i$to$`o>*;IU|R5MG43Y;nE-32S=M#@*Ijxn56V&D@=9QyN+w+z9?qVzwnqS|Orj%NDNwHSe*E^B zuEW+=l$|iXk$t#KA!w}WIKs1R+*i824;LXp-J@)a|>dc@iOc?&Ud-`@`C#i zXy7b+aPoBToH-rZ6o;^Z)$X7p$C;`W%Lq1ijw|jPY6pP_-xE*3gK=idXC zEuTZ{2myT;wB>gsvdtKAqRe5{L5zm4!cQ5FI5-W@^fZlG$LX4%d%M#b71T8~ron((0}jOdpyVg+HA0g-8tlYgM zKg+8qIaR5Zv*!?}8J34L-q*v*x8h(;u3NDN@zU z+5OrK%~T(9+NBRQj?DLW0mh1&uOo&lv%|__K>*x9?e~$!FY8kFZl&e5QwtH7A=>RP z>1;~jhh4?2G##OpsxX3QWu)Zh^*47V(!7m4?1`BysXY@~I(NYJlwQbAkZ<=09dZH` zpXJE4)-vN44zbx$2EvUg{?2@JGrrT+5AH9wk4B3J#^NrTG+j(0liV=2zc|m2whDJ( zY-6^~_~EU1J0mWrj}f!j0T$VD@vfK|ww5Z;D;l|b6K@@pqz0_aoq(3(c!+R1%KcU* zBc3VJc1I95B59LPM~=ON6rUaUZ>in)H?cu{?yn1qwG5A|2T7OV!+fjR-{!<_J}j!; zk8My`CstMey;c7i-4%G`TrI(C)tV<8vbfNRw6x>m`dayLhV8rV z_OLP@^d(_?(;Y*J4ap>d@4XIwbL8wt&O8XXd~Ms1aGfAkh>zVcVQM0b#xiWTyhxoN z)oYdH(mLlXXh_594KydtlQlywKT~C>+2^@#WePMUZ5tvJVPW6=)qZGru}lFEoU%-@ zdm7pej?$~T;iCL*hTLN79ylsLzr)^lr|?z^effPo`#^1$^Yr{a3+-5c2atnT&lrX`K~=!rs{JoTCowGC~02kG-i^_0|A1?K2f1i!d+U*f2XF`)rt8o=!OIDIc&QOV}1jOZM zGSwvK^I}5|(rE1(}9{K#51oFSP*!Jgp+Bul_iZgMx5Xz-9DP#{!{Dk z1g4wq+M_NUkyngaao=x~+!^@$F~z~9H=MS6<%r5Ukuf!plJ!Mtx_Plna$eLe=f0vC z0+&#KWLDCtt1f8w(SBdyj0wIa*w^IJ`5K07bq6Z^wO3wr2toELu=((`X*Uni)yRlG%|Mk05~ikjE`IKRDU zPY*xgRTYlHYuHxnB|I4qnQy!eem+o z=mR)tabPN3QW4Cmt7E*(TT`WQ`ZkkgF%6IoEgB-~{}CCsBJFnSHe2v5$7bsKGQf+y zu-PcvxJ`EGv)B>OZ<^M_dDkSqVaNxVCQ9{n2&~hU7QUr%`;JKQ&s*Wg6%CM5zUmM& zoM_3wdA6KVtL?6l4<~yi{0gEe`GZonG1h2SAC>va68I3u+|VlpXl7!Vo@*9PNNDVQ zoNkPRRpLUv;s!g`D%RyQg{51kG-}a-+XDd=yhL{mVO=7{#v3VF_JtEI8W%)9Hc~vVAYirWE{`3 zlJOGxpzVsrPys1UN@i_td%ra#sM>e~9-@u8B8AWD+c){p$sdm|kS`RaM-2UAY=02O zNee$Pj~_y>3_FPg(yn2J*WD#^`6jr2g%-YUYwTEl`l|vo$mYE0)xMsuqI~=ifhSpB H=X3rA9Y=aV literal 0 HcmV?d00001 diff --git a/ofs-2023.3-2/hw/common/user_guides/oneapi_asp/images/PF1.png b/ofs-2023.3-2/hw/common/user_guides/oneapi_asp/images/PF1.png new file mode 100644 index 0000000000000000000000000000000000000000..ee2845b1bbf9bac0a5cf26e28858393b9e817092 GIT binary patch literal 387436 zcmZs?XIN9+)-FsBNEf9;0*HuoP&%PEsiJ@ggx;I<8hTT@G!Y2BNfV@Z1tHYX6cT#x zEd-GM@$9|N`?|k#ek8f(%p7B_b*(u^xyL=;YilZ#-lM;Vg@r|`s-mESg@t#Cg@tQN zNN_jPP;>D5?g!glM_CT5YJ_p?Zh&Vat09YpRg*w`Wrlw@CVHo0;EsiL-{YST_PjOF z=Wf!%?)4jwSI&0tG^{+#EX_R3?&h$t{8=p=?cQm-*g1Q+vx*z~v-+@#^K-NM-hD&4 z|1}a26nXyto)8rg`p=qwOZ+o7hJL$!!iZx8R*pA~@lmF7Y zA-)dp@N0Lb9K_GyuS$r5+@cvprzkB=7E{fd=Q(SH@&9@Le+w)c^>PZaDxoRU?A`$W zclzJ|>H|tNPiu5}qdWf3^8ZMg{$x27C;0y@-yqaHAL^J7s(12zb8eDjh-q9kW%b)cYpXa&dXN;o+WG{kHNgJ!W@jCt0uD>Zs})0K@i2YAPAq_(^iz zY^yUbE_YfgRG9%C3;Ah2ww(53cemmfHWCvxHDT4NexNo^N=mBAVlvtRtEM@c%-N`b z-OMi?G~=Lr{ruh%5D=(q&0`}Da&J;hcxm?LpYDf+&f;gCsV5^ zD88I!@ThY{eGnLKDt-NwI-z58HFsol!?Sp=_5^?whY>`TFiks6>kN;^Hbh zxw2(=v&~lI^SJaVUy-?8RhFLeRIEBM0j?;Q9bXl>^gHZ!fCXgmTVfXQ5|v=&^*r$^ElkB%40Ow}`G+OZvKwQOa); zj!km7aC^Pb=%XS`h!ZW%LGc8LY-79b>FHry5`HGht?_(bi@=hNDxbI@%b#CN%x;wg zo#VSsH*uR61x(Y(6z>0`Th7g6#TS0#tTw}&_FPdHadB~UbjniwQH-d8iR5(el%LnP7bdrA%uvirgI2G~ z3X{&js(^XQtk^%(ec;DI7bQVY$)zxfMuF#RN~@^h?x=4eMVW8tGm7T*V+q}v{+MTCUNX24)I4yr3jLeHP`u(6e6U|x~Y zPgUDz)?*^~2MMlqwz+1S&SyHW4H>lbja(H!Oo@-_;lUPeZ_WlsvLz}_KsPYIZO(Fz zlj4!d{`YdI_o6OBsIflE+ESn^gpgn5M&hSo0{h3PlVv*SmXL>%bTthq-hceEz0rji{h zV#%t#q>ALRq+cC#YMgs~06p+uqX}H1S=A3*(_h^Py4hHZa|tMjjgNne0ssTFnSEC& zk9+9o83mV359ZsI94idIS&w{;BwU=2^Tg>HLp^t}vNhFy%<-6eu3zT!&sCaUku^$C z@~D66Hh{=|b2Lv(9#)s4HhXh@8TTewUB0n8V2ul1=>i=jfvVgam$N@4!49sXz+JfB z_g-_kS$A2T_dlpdtEk=ReS5yaqcWq87@!vJw@t8}28hDgisl+ctFaWL3f&`8)%L^p@VyW>egxgK4J zyu=Z#4oj+xO-6ZuAVu*SB%Gw$Nzqh71}}lkz7ob8IYvlVU+7hiY-&kuxwTbB%LBKg zm?0u6ojjSs85NROu)5a9=LefEnB|PXNAC{yG?}Q+WdC+N)sNTVV76z3980+TmPE8yjq`rx z0f@7Jgju&i9=FTA#=&gwIZlbHW~Has4~0;C?HC3eevKoe|EU1*YQy$UZ^Z*i3ij9c4yj zD2&|t**_xI*61rI;+Z63!$C|JucTLB!UiKre2#w73{z2j?1~Gl;~@1Njk|csfo_69 zz~>+>kZ==g&R%4a$7gQRmf&D>k4f$DdgnO}C;dJXBn; zI;yMpi>i*T?o9UU14pK$nAngT?44u)&`*Wg7xS|D&vHy9S*(al5RTz@sD_-#09ML| z=TXhZ8VHCS(%s&28oNCDcf|GYz0idgXpCz_I+xWux^mzt@iR-)cBC77Wl;LjZqu%; zb=0Oz=0@3fE&k*Jgp!1kY6(iNf<{1cFYHZQUE3U{c#oSx0>T{K<$>e5desa^kB!4_ zEM@pS6o5el&TUkPYmxm{1e{$mZWN)4(*T!4E!yFgB@J>^OT&(Yf0PF8*7U_TH}+}0uNJZl?HAg-;f!X`O08YZfOi^8d;5NnGH4tDH1E=Ag- z()3|0NuvyIKi(5|Ni4S|*amU%*?G^)KV7QF^lJs`G&s!&D;v%FZe%N|zvT7I*D9qs zgNHugbhu$j#s4lRd7;w@y*wRu@{9!Kb@+%1D$xRcEGah%`L-^eGr9Sa6NQ~^>>IbP zAva+4m1W8ajHxy`LC#eg?H!xgqj7a@8!*3@`7&jnk(t90@9YK1v)OmJHlZ$}q8PT>=YSB!<*2d%A$NPQYnz4SPMl zJ|sTk*7}q9phIUq^KDCjJB?Rxs zI$(H5nFl2;p8XYnhd?Y?kly!Ys7-eRuXjvDiBsL@#(KxXfzsko!SA9W_ zk&-Y?+I8TcEt2;Ct)PnuCA{SjrQj!~nM^&&RPx!s(Qq&aaP*-R ze+LUSDAkAHr_Pv)aI7aL6c3m7%+a?fK=MAzmreQ+eXVbw(07!{j_SK=DHZzdpI-JC*ysd5>6@Mv^sr07 ztNE*S5r$@LIzX8`vr4%Z6#c6BY$d@#{1=gS5FFO698cT%gIX8Ud3$rY_C1O@g(JuB zg;HcxoO&Lq^wE1Qe%n~aBBw;$FozdqiI4OM$PJ$Y+q@g?YDhBDkpowf&XJ*eHVKjML)mO{xK#0DdqL z=5|B+%$Gp~!{MhbGNM^Hw}rb5iUXfDBaoj;v}omfhklaZ{xa#$MtN(J>9~-S1r=8$hj{#S>qn&o2G8a%u<+aYuS;CK zWJVN9zYOB<1HDj{Z>OW#)HD(Dntm}7@FV$jnV3(<*LUWNnnb|%s~kZc`ewND`VB5% zmDI&2tm4;66p;2)LCVT;_Dx8xMO!Yz+N6Ug+c)G7HDr&xut+AU?^R9T+VST|ffpHx zrxf$C-k9Yqx%H=lbR`ubDT+DYJac?CffJfNGk*0$K#sq&v(4w48M|QtU7+4${FQ}< z7HZ;g?E+sn`+>G({@uYj585sYP1-I=`PUT-U*|2C>iYn!hnr!9E0X-@_-jCY}jK-S5qcI?Me43+$o3(iz>^=SIO+`l*c~;n9A7PSYF#Y5DIUr-5PoR z(nVJSI;+sV^|a!%4?(%dNA`*e{uU|v&>}-SNNhxhikRte64|V z=L(tcH3Niirokn-=1738O{LL#@2!}}=GWQV%FDI*nFD?|f_6K@kub~5KRdb{Dm0Sd z&g1L@T9*Fluz(Q9aL4>gyP#+b6lxnf_Vct#-_~=6?tqt&18(ABW~9vwF{5bl2^&{( zq>5k+t&kJxz28H`@Xc_2qP zvEgmY=uZ0Q4aB~KUrCc-yFbd{bQkGhBdj9~C1~ETh~pB^C*KRqzp-sZHQ;~p?f+A> zw`sCYyG6T^f{i5D>Yoi)2(v688{!)wv1B}C!?jq7bMY&v_QrLXPzUa;wcHjDGo@E{*JDA8~MSpe*Es6a*AP ziP%1m#W~HbypClbyyWwcXSTED!lEQx!aXN6gHziPQU_^CXamsq;cAP_KZpbqWf+~* zvk57R^^JUsc9E_6`?tRvSM!vCN!`k14Cm1>A5>mJ(t~>{3OE->%;Q8j;ew#6Qh)-$ z&D-c(H=!abgeJvWbPQ6zmg0u{tE(OR=ZO&@5tE{hd|@*!F5Hq9dF?A3?`f*Lp^Gj5iuMOEsDi z_&xKZfZSU`zU}o#>`<*h6k#*GXC>Kth+LA5Gua0fE3g42wCyfh|B#Te7JMUBY$ZS1 ze!b_?cv=(~P=6i!~veB*4y zQnbp277#(+>}kt@Wa?}?s$TvL0=bbfy*f&Fs%wxXwna?cWQZ8~m`aX}j1Kjx=qh*k z(hcsV6XV3`ZTUIQ%s?q73%DqG2-Rt!yc}wEc@T~UX@sAi@ z!D?y}%UB4U9=zTKVCOFuh*m|Mj%3|*^Y_vAzFU_nq*MSvWdFe&hK>O%3_l2PnYsfw z(aQ?&%)&YF=f}QBfAcebdO?zef4;WW z#<=RfmJyIBm`K`11Va6=`WVlpfWL~rRVFecjlB2XhAJH2P`M1(?nxMCK7JK2UkpEO zp;D$DZ|Jn{BWvyJCYPh0JE9I3#ZJ*+LZVb47}FSD=nv`_Dx~8ws8Jh=v_y3Z*A${d zM{|FddT|MTeYikX9kG+5pA4g?tj&b+jzYU_(0G!|M6bqUs!$5Shz12fFms}aqkg2v zIv2c-24bxITEz0?UMv_zXFNvwgQy+!0Tg&8DTl%x|Im_@CS$0>fg*R&?WH$N-ZE+U zwp^*gXp!w@sxGHFM$_|(w~$H@)S9SjP#96TZa)L1o7()cCKkXG9 zH!QN?_=wq1_lVch=<8fISV`+${*7q`KSCwpX7YUvNl0(%jTPf~V=Vi&Vy2FW3V8XG zbYT>sSALR!V)mv3vEPEd`-lyz-WCH~E_&UsfHwhr`bk|wE2C<2$w_75E5TBca_84Q zn%dOUjZ*9j+)x^+93M1MGBKrEP%W8^hY)y-bLQr6U@wa(>pl{|Z!O@+hxB_t>pNwP zh5-W;u6net!Xvypj9>m~Z-g`WwC~MM;!0F26FwL@f=N{1({Gh9AeZc>W+a108 z8Vg&7Sp~m}_9UfSDC8_GOkN3+i)(qW5FZkmLTo1A3-WlQkA_>MCoARf>yeiSPj`M_7TMI>eJ;rnmN~2w&D8`5sQXjwf zm8Y8|6-@HVzwVb0N+MW}5g;y$+@ObsYU)E=e=_FXd*Uz9HkXF0;=9g|enKNi%s1-Y zw!)+YlWlRkVf7o;AKWIdWZ3&IYMKSg6p+7_AUbf56qcvn0%`Xvi`B7VH?QKavUSE4 zc@3&qSmuhXFuA;nCtSQ4U6MyDG-~uGkQJ`M0&~F>H;x98iv{zs^6SrFffT>mO2`Ur z=7MdlBy(x@7Eg%&h1vfHvsPc4Y&iPDGCZZ^6l0xdJMZs^eAaD9I`_ilrwW*4zrdnz zcd#u{wn>Re+-T6Ur66r~m zOl94~r)bJZu4(VXj2ZlBF!XG6zY~vt8((SV=cLY+I z9V9uec}?7TNQ@r5T}_YslS_gze*UpRs6fP%1qPPmn{VY{c3Kjr*z`w{AQLTS*-Bo& zfCR1<4f{qx07rxqem>0@Y|mn*L}Ih^JA78_=4a<08Z=iYn#&78dy$cIZ9q4P_9t>N z31+5vljya1FC=QdF6j&aB0tHG10`jPUZ&roO$ItI^lt|+Yr7pLH)|$oIk)FIjo{p2 z6@5Qy-BZxqolOK9z>6zw zsd~lc%YB%>S$(=Q?lym>@A>wxSX~zcVG`8!aU8~6u7A7q^Iebg8A!`p;-HvaGs6&XtL*p z?!ygqV-#YEAlkv%)}{SO;Yd*>i9-B&Iocr#PEEnYhS%K4^k?ZBub<407Biw>QU^y1 zgqFSGuf2btRHk%sII@upFSk!(6mXva3X%kg7%6Z6RK@`*gG6i`bZ0I^o(Vbv+d)oM ziH*-Faz7%FecFeTTqmhTZ>#$!hdeHRRCcVeQH#t4e53r}L-LY_kDQN~wBD(L2vLyNSJm`v<cloU0qy-P5sLl6l$htv~dCR-cS!|XOmk4U2>Ccivsg=i}M-;F41%&?;; zjK3&BW$>XJAzhuo+8>Fw?I$re=DVV1D+hE!?T-xuc0s`QY19WBn9Erq@UX4G2$M&S zb;2HBqs8bX>$kmPN%ihWtj$Hn1&HAQJP`J5gXvMOKmK^668{&7L2e8HAU-NG7-}uA zbx#X{;Hf&6g8a0f2a8yw9PG!mc(|z+JR1 zzZt0SHMfzFlc{=I>k&M?u3h<3Zgp8Svs%UhONnBBVIm=X@G@0VVyrmUZl5zsU{~CY zou`ncMQ*^|nMF|aN3bNAS0&1uD?I3L5YPsvWcAL!w8$FA%^-P^;WQ6>p(S%Q!LB5f z;d!yjR0{Y6&m#E?nPJ+reelr3ZbJfGdM(b?C4sO;CcDI!<+^)e%+BDG0jcQAGbX{g2 z1on+cTV%5J5Nu#ClM=p^X=e$V7KBgf_2Xj8F^HRm>p$2dF$e`4vB>z4&vH@zi12_f zGC_PNV??)m{L%SrcmM?w5c5as?9+Xu!^Yy6_hiuE`tbUiLXd24E|=gmS}l9JR_YnQ z?F@7*u9k4#d3~=e7glVNXIq;jD`iI&t?7{ugWE9XN_#$%F62fy`mdA7^)l=YPy1nN zls5=-AtA0(f|BMdxUAHn#QAaVTcEYWN8iY;Y?(^a0oxI<$q32W;|K^U)!R|OXn2$% zTXkcV*;5I;5|&`hpFTaWmMLJf`Kp)SDL-!J7bX?AITpD7Xx=|S@l#DtlZCpR#~4SV z$xw3gX;eIM??XhQ!z7!;jV(v-!-GUc$-N;_D+!55tQJXEh2A>xl18dpt*+$Q8%J=f zp*)f6A6LDGe;m#NhPP#aqp(=35teX_GWd`@AXi$j4Mx!t>)*?Mts$d7Aj51m?ouJ} z-?7Dm`Lm9*>&GIr{e#%O{5FP3%;UHgG~G<5hvv2xKJKS8(6a@bNWMbCNI^U-8$YuO z3rYDv8B3XavY%uQ%`Dy5X;!g%!K>s}%VjH;V_f(>OOEySe&)IEkD1oo8))zd3h}LU zXsR4D3z;dnR+rdVXJ7Gogg@d+ouJme^Z(-TPx1irfx9YJkv5z`Ook2*t{%;dBG$P} zRhoO2=oBSnp6t%2qQV+h$}8*WT67})nPqe3yhOK_EUEB_TjYPpD2P!m=Q>MHjBMg2 z{Wvv)lecBr(s&f}R#6>*B>ud^ov9bYjXT5J#-|Lga+sI2AHNyH8=&#IA&*>={N!uPR+Af_;03^VOMjhm<3@~g;Ej1!o`@y=3CmZh zw7vYv%)n5rf?K+<04cAN&C(MazA?v!*j=<&5v;D$#iB@yE}0^E(?9ryB_?b%3lmGzSZ3w-3YyvDc)=!-Y;NVh%|JC+^Zjmj z^v_R`w!z`uZ`QXhsaEgn$DY*E-?n-k2wI$fjNwbuEhtd(l3_KYaOY*VBnzkOAvl6Z zP=G)*ZbUeEq4+;4^ydXPqdA!U%s1szv;onNFw3yh!{CsFM!JJi+;H|D+HioNZSwcL zIt{8&95jI(j+nl&Zz@!mx>|bKM>$j5dmEsN&4PUt;Fb!o%~VM{(WagBiEUGa|en!2C1$ zrp@i1MES|WW!{YEV4efF7NrWAdOlE3PUPup(!;W%+LyeR-&+%HwTmFcW!+iS4sBb$g4X6Nr6 zqiH{n#)bBaE)xHqq^wQwURAb1k25D)*ReqrJGB9DxrMj!M(addNfH~#l9eKx&EM%> ziqcgSwka`v>?aYc=Cf#+m)Ogo(S+4JDO$csqW1HeV1LWRv4lDf zlVT>n(yV-f-^A}&8dQW^b4k!$Ipj!-zJo#&(kr-HpTOr)T;V9i;+hmU5^Kx>RFz*` zc|K%(5fP{eJ{ZgFDAzrYCNJ034*bgw_?#rY>i+!(hh#MwJB<1xZY_=lZRi~e290$h zF%$B;!sfxTcy$z=O9*~P$KpO_c%@8F7vnfq3BG%BKIp`*7w;{}3mc4Hding-& zs0+0`-?0+O18eaun3$qB1|(E)JY>D$MB_RD*i2=EZ*81L{7jT;w0w{XrHU+CR?q%< zrDTJomWEa)S=(Kf+I9yz+?NY)|`f=(9NttcV%iN)`?uWHFrKxUrRp7)j5-B54-g2>$b3zkdqY_s_&$L z?5=odNxj!A<%C5vAavbD=QinZ`yIDf$X6wd`y$jWu}?~0#iW<+IJQRi?FCSWUMr%W1F zPzL~>}zW=pPW#)Nt&lDKJW;*=9|fuz6kw{e{l5f z>M%;$5%$5we`bao=+<;0S-2}`MerX^&A(5$P;k29X|roQHfd5AWvp2&iv~Smrnw0c z&bcZs5Sp});3Wts=p9j1t+bUyfWk;zlOA&Blbcr;Jnr%B+b2Unvg?W7Zvgbs`n=g#(z|IDp9C_hZ(=gP<5Y7zw0vGgK6-K>pxHKSvJ*Y zH*j4P2%YV>N0Oj&ydmC}Gi~F7oC%^HARxs46;<*N{bK)+tN}(wX2EvOC^slrTLdRp zv(j4ai}+)$NgzBAyb9<^ikF<%giYZ(VSn%R9b9h~Aj$w_yznDbbYWP%_~U90^YJ*v zY?^O9cbxtDl72W6s?yh&H5K=4GXZR5Of4>?iIf`_hDeGXwCq$h`;F10$Yy+2MWJ~A zI;4dE0RjS;Xn!~_nv+;?yZ_YQUwzevH#QmfnY*TAwYC;c3UbF&ZbpD-mn?C7 zqdCa2vzz1|zhh#i|5(F&9$B^O`lI_8YjredQ(h{Xg9(ARcp%9nnADxRSlfJow1rS_ zZLy(sM5d5yR9Z|)T8pqYI6Ke@tLDXJE*J9~O(X1TV&8scy1P?UYozZYq?KTV^<8LE z+O!38FSf(F+-@z8xaAeiUGiQn;e`Kk4H~sy$O4lrwAy(B$!RO#g&)@_Gmld=0Sb>g zKs<=~kwO-i^08+h%WOu^L!J#3I(jIZ49EczO0L#00C#vF*_Nu>CH6ki>d}#cZyf;L zq)ff$Cn2$<3YGz5SolCNOdX~Ed@JOs*+|T%U~uC5SlM*h=9HzCI2(A!?IG|c%&9DJ zZU|vaHKh6{I=&8;;~#qr!B*Aym4F_LsTIJK;TFoJ@q>6BhH&8kSpT5EM^Aae9{fvN zQo0YtS{(?->ZOuN88`m$D^3RnqLVL3uUFABux2?Fg)(w;sb%zW0CEa`j|TzMeQb&A zmq;e}^Sx?lqtJm$R26Ts|5Q#n700YUWWY5)+-Cf6UgfiwO%-!Ms;|;af~e$Y!pY1Q z9pLudf_}|j@nogAki)?pu&Gyn`_1YV*>xAxt8H|BGgNbPJ;eJCC3Y-~XkzI|U)&kq zi~=J&Y`Amz&Nd^@GLm1%+AQ1hd3Yq|oK1G`;3tK1!;&`Kn*9t40aduQb{yLkbbH2p z!H>3O&Nt=HCx7lF|13KN91!wGaUmw7Wc+|rv9Bz(!7YdJ<%2~Uwua*$BmMb+KJh$+F@DSUwM4@gBTifeFhzcE4wE^UJbL?5L%P`y#4Er(=-S$lk z|6*@FLt{^?YaHc4IIy>g3a5~r;y|f`Va?IIvQ&MMpq1`ml)QaK@Bx#AJJ~XpahvAm zpw#z%{8*py!&u%P?5P~l?R#6gPap8SBQL}6*U#TS56%8=GZQewv=yd3>a)-C$A_Gf zK!D&#@tu}PX!?7yI;u4}GO;H)KB0{@XOw%Ie&g>Dsn6i-Fb`(sz=360TcSkau@@^0 z1FsRpe3*}WYrOsIGiBCfhPdpy?4Or3DpK3(t&^Y-kX{VC1R}{}RS__7H6p_B90X|kfI03Nr=3Z= z79QWMF<~Bl*Prwyz; zPIs~6SX#=U|LTR0a{LAhsbHcqb2_mk(*eug=+xy%HC&khfuWoa2%wE{>RFZNNDm|H zNowdb{tv(OUqC}wt?PizshOkuC|HT#*QYnwtDC_zmOP61 zulXeTs6Fl$$7~QeNVSPWn}d)^g=kxmaSa_SY5B^lsMEyy%a#RG9g(=}DZ{+-ELGHg zo!CZ39)}y&bqM2aEtlJSHm91zi{QP|-$2q|cJI2DLE9BZ8iu@QswPE<0aqyY8U-K( zpepk0lVn3Fv3~Ff{Bb4K$4yzuHPGOW0;4a>L)~1by%S7u%aW`jxcB~6DTA-d9ikX_ z$}zuYw?f;DQv15c&|b#jX)`{mC*Qp#&$FpKH=5+WTS{!<(7x@3*%WZlnwJmPw9j0= z7ULHc6@hGeqhbV>Jwu)pX8)<>$sMV~_f1q3A1GR5(P}66!(NEg$~Fe>+qo~(X@MiP zegk85fz?%PC-vk8rQFMelkX*myirLuy;xSGyhhkN!S!faL(MTpZk&Vn(Js=@jVUfmZ^^=DhHwNg1H zjiNDAlc(%UDZR4vGIU)z-`*J*4E5_-F1RoJ-YnW)-b){D{S(M3h33;O+u)Z%e~C(4 z6Q_Lgx#s&r^LJGUV;g932Lxbx-Xhq>JWCbZwF{Pz%w8MYfFKeo0wM9;Cuq?LmgG}~ zl^?PfKi$7uwY2?2kgZ!N&-ARcF@m}C$?bd$@ou*|&Q#hjKDkYE#f4?zb3FAw5`q_> zZ0K8D{YCl&eSL7pnv#6$hmYq)jq0wZ^|~=WyD{oJ{~h>pK|dby`^;l=%Ao>t0n^&i z{mV${QEL(-0qVBot{rqE8-|yPP!l>o;|0jhzZbjw_hPcUYmD^X`wI#Fmns{+JG%2+ ztGA)TUz-0_{T7vQMKv73To%I!+IwC|FWv@}|Cfmj=|TpJ)&4`}RamqFrH@=~KQzHl zur@-yVDB1fy!dZ|a~At2woCi^&x&{uf)Nq)&i|#$AoNZb((z8$8cV}bhhUHJ{D0G? z_2cgrK&lOUnq!U9bL|^v#_j7*oLRapwfK@f>^C5vYbN48ul!7tPxREuZ@WG#Qt?!= zYeH-s`>_wDr+3M}0F=6GQc?PafG%;m`d-;AEfPOz)9@46h@WaHdFl(%$Y(5cN+~Di-i3@mVHM+}<1IQJQQr-|=s$dAAPrlTStxEe*P*kZ4Y5 z>*&|=4_|$f$2i}Ji%-{0RW#=-7Ad^dYw^r@_O^xZEJ?ve5&eSHVIf;<}@~>j% z=sqeP>e?c>9-%E>pnBPsh;`~eo=sB7llnr~ix3=7$lF&v$gICXSbEme6J9Ui9^d?e z;ZFdY&uo7ex5H-3uisw3daa7>MuWsVll1v(rET`+?dx}29<8?;&@@D@t4dQJ{RnOxHewKaYWDJ>ed-L8X}yz+*dwl==KX)@Vp{^Z zkRgYf)8>{gjT2FB?$cK1p-9s`qwkJgnjF>o8inW&OuNS`)4s&~)yH;hmv7ak1FOl9 zfbXfXw)AviEy#))XxZyT>vG;Ga$tkdJsO6e&nmhPh|kAA9>GM6CG2CTFxmD*lqJ~~ zN**P)2NJXxCD!fw+^y%?^*^&>s{+Ko`6u-Jycm{tpx21vP2|~`s%~`nrnbso`lqFq z65DGbg-tmg4u?Obqow61Y74TmT)1zUxZNqqEn=KIrLsEm!<9r+9PS7ZtVLQtc9_o?CPuuo*$bg8*8bBd7SEUot_3f5*`%#KIyz=w8I7)9T8ksMks zju)4`y#BzwL)%*HmR<2CKx0$o)yEQ&+n1_oG|4Da)mCx&Q`cWtWgeDArc5j0$orO` zr^+e`M46){5ko`pVvWpo70GRx;GqICu9)3p`pf5s_T7DLCfLveY|8~6n$@H8m0DUg zWQ`{{#A}lsS;FM4%*WZ%B`>3gXPkef-1dBG9Mmj`h~74Ug)| z2u)bn6(xuj>P#@9?4IIQA&Hv_q@6x&*vpl9qiIK7AiwCbhtuzc( z=@JE0$cCYYe};OXKi?N>?sBIbAD9+fX?~W|Kjr|*aeb?1l|?^7%9WKC zNOV8R+R)}BHYW_jr~zfSAu0Fyo> zPJPiSB|(RiA&1K1bjZ+%RXrFGe8|&cA^w}ev-N(~Cs}g7bW=-K1uSzTYo)0N>C65x z6D&gSTE2@~s{f%iyP}}o`DO3PTX^Jy#zLxpg25Vg?pj2gl9Yv4lEu0*JPS$NByH7p z=syO2`kEU=^5HbB=LZlkdINxqx(R3b2A;n7HSDtUs;U*X)uLPWI)CAR>0XN|qf%;f zj#WJirtYq@jKVahv~92Iw_6>kYzoHQ&9EW&Yb3LJo!aQk9VsiGmXsb`Ek#@ITJNWY zR9Jpof0U!f?DdQpxVVVR6u1~rK9N#jp=>!dVqAOjQEkXH5(C2Wkv(?){;P;McE|G2 zKA_R4G3Y%VirhGuv(J51A>K_L7df!D1}Q7fsINxmqJrRQyCA{;zLtgR{7 zTgDcy!FmA+I8toh%#Yb+V`@Wu6jj11y@88+L_4*ywa@g!9R|eo*$ZQF6e4J}4+BH|rc z+&B)i7)MqQC@EH1f(KLZ;l%TlKU4|0T>Mqp7O<0#cxM$p*(4h^66?7h6avkaj<%(N9Om8no-TsC)Fg! zJ=xue23%)~4VL9+@pZ(iAKF$0Jf6ofe5Y+)H=gQg<;}iMCv5v2>b>TcLyuxFoNo1N z7qEFkwXh(at8|;lG-Nv;mL_6sIQ*!*%uPo{!I+h7BhGbdO8z(e?Ahe6ogEcVNvAkw z(HK)=HSgVvKWp0-sIY`iD*Rq1G_6UjG$D`Vc@tV)677F2|HBvH)aO@e?S%Z@p?WMv zS%$%e%4WJ7@8&hX29aKeZ!j|ltjbfH?(pv1xBF}JbonL}m8K+*a#@>{&8?q`=yxO}gvNUSKT{+lZ#Ve4=l%q#iA?vg`ec^7S>sstCaA7(nnx&! z<5ZSHqd%=lM}Tgbct9R<)={2s-a&Ol=7EI7;|9{0-=90(pW~u238=Yi*g#1TV$1mP zQQ_WhjtLmle~AKa!Z*benm=TPy>V{krm=~%u~8Srwpo4I*tr$do6M(RfU~Ipl3Izx z;BjPU1$bHZ9h_sKs=Q&)7J|C%Ibls@oW$xe^&rtiMKR&@%8XZrJKJVcKfl&%PSL34 z`L?^XHSVQ9N-*TdqmGZNxT*{M-|(;t851F_*^-lB5G{83{??L;GHK<{BRPn8bI-!+ zy@$|Luu!Jt)!T8$UqkCpOV5oagN>)IC8Lm*Vj6uDR{)3f(1)L#g+%2o7sdy&2#;q? zzdSvxJ`zVpK~bfR!w&eSgvX&y4*1Vg`bC7jh8j8`P3%cf139*-Yd>e<5mLKoa06~h zQv^ht=XqNuQv;rZ0m>}gQi0tatM0FUfIG z;)l!Ay<|1s=AMS5kMW|b?poJa8>MjxTBo0OB^E|^yu`*VrKG+_$Zue~XS<&?p%`C| z-p&dIHu1t3`oiMYoW=M10X3=D?*K=!A`E>SM7GIy$1&AN{A2EG?qk!ve!W?`?3u)K z41=H=bk6-0Gk+_}L$F0(d2aG)^V`kM2a+<_b?S$kq_KMS6@$JvuJ<=rB3ANRN$A$o zzW(J1)oMn24PccGDLj*tLAl8*D-4Rq+Ya?H_EA&6BFn$yHT!5)yOs2^(qe?ZuUpOO`MUv2Aik9<*8W`d&vzd34P&QTxZdr{ZWP-5nY4L5 zvX3A6A{z`UmS-h2vIeXNP3^-cFqY|Sl9uIYJ)hc&Y)HEC=B%te;wZ1KPrr%kkfJgT;P-+C zP+Z9i&X&}qgR5$q$l)u8-m!Gij|n*l6?z|{vv=7NBi;PtcC+5xvvFg5>0Av5|BkS} z>#eeNulto$wq=KGy$y!aTKS1XA4N255I4WI&HvbL+(v}SpfwEAC7bb79r2piWh=KF z-hTN{+fGv8sxqW`H|RCRN4(+s@*AxK_0GE_<^OoeOqHCh6!}k!CFEh% z$nSOOXtN7;c8!iX&S&>5GJ1n)4W-0c4I3QOD$|WD!;K5{(y9GUbZv=o^({@cr#_Rm z_JuhITz!(fy*jQv`B8bgZy7vY0K*H8GkmgMSEam5(lIustF-!@;XTN%KJ)DXL0nO! zTR%GQ{*2pJ56baXZ(na=wIJvkSN`p-0UaH);I%{C7qW^5->JGx_xd1CM+`fnBU{3I zpeP+;KjXtm%!-{K;VmgK=JtmlxVf2aV=_Vi$xCx9j(faU^{!9&eVI|}duB;_$BQ)k z0amPeI$HP#$MZC;lW!e6#m52F!k>-N^h(UxM&yJhi%0WS6i*XD^TkF-Y0I%nBTn2j zgU6mhsjk`^kMSPN=lVc#Jn}Tb)J=krGO9&;yvAO4#O#sR;`U0M@5Syn#K6VLP9;gY z+9vyx3DoN~!45m`MR$5BC@L} zkju5ZMjsttnoh@|D*FLA_NveZy}fePG0w$$iON)J$Sh0xZ=42dtnaOR9ByFy@Qt+h#H(Oy?hs2 z$-hueKa;Q1a~VpKeI3qI)%6smKh|1twD&^pG_MZXqVx1YeFZkMVekI%t2+u(fOb`z zd8=<8=In%She++i)IHy{$CrvGtI$4XP$bXwElPF-yiZhOmg?b!^MnsIc_n7z!shII7|K@EWkX^1( zx@!=vM|5^^)29Sm!q=r;N zx@#!u9J(8Up}V_>?iT#!e!us7ZlCq7#b0Nwvzc@D*?V8t^@|IbxnmZtQNq3BT3Tj} zz}rl!3=A30e*^v#%|9FAEBt;D+nfgj95Pqa9?2b9Rx@7Bz!^bZ<~7qn#e*Gdes-Tr z`4_9~IIVPLNTGSEp;)#Y!F#4JSb}!M zG^_B|Nx=3?n~l#KrF`m6oLZlP$hY_rY6b35Q6;-vY7f~clMpO3eJsld8@h*`1Oq0x z@8KkAiHyuPs{=UBcF`x)`@tW5Yh-qXA+1K@N3`{{3xiQnGGL_g&3wTT7XXkb^1#+M z7H+~g5{Z63jtRkoT^qt;&O*(Wp{s5+$K1t6`ZHyue5NG^Y+sr@P)Qz3YJ(INmC|I4 zjh%Sa%sq>XL3v&?R991bV=gcmLcKX($}0w&Gj=fhT^}K6~pzxG=d~} zm&n_awq$o%SxT)2ssl+oy+qm_&9+ES$J5^u?HApx9QFg*YAOlk{7MK-r-oyCagD8& z5qw3QS&H%)OC-3zB{h|yZS8lSoMAh*cW}Z9d=<(f=218TOJ_XB9C|*gpzq#{;H12< z=m$((y^NP>tjRl&Jv~R#h7vz+A_FIZn_kbiuFg>@Q|X7T7Mhm|HY;c|e@|I%)h!y) zM~U1g$AC+bOSBF=f11;!3HIT=v9n`dwO4MeGrFJ;zg~@@58m#=s)Kp0K1`l#Q{Q8G zOs1{8AvSdMudxGXjZUPNk-EN}z3cxyEhgZO=}E4}*pBya`M4VAh^?r%Zr&q*Gp{an z(eawsFsw6C(SuMlNgzw4ulh>+Fgee4k4eh4jP6{e1Qbo;^4Q;YOWO%B_qLsa?=5b1=Ir{1Am z(a7t;W_#Gdr@G|{$01tyxJBLQB6W7a4(buHUpqdB+uQPp@JwLBDbI4;@G)LXEC{h zi^2$t%E?bi2wUhm7#lndW&60TV1p@O%NG}BED@G?^Y@&=XA{fSmQjbx&9ld}34&2^ zmLf1X9!wtr%_2n~*`Ze5v<5^ycQC&6`|_(AeeemX)@cxaSV-9_IX4W}H$y z*2hi1K6d@zw78Qgo^gCL$=VT~^d9Q_Lfv}vqcJGbVFX*KD%aP-6=Mf_VtW(2 zQ(i2FaaDItHbx|VTt}Du?L-Ofi%SLX%P_k|&3sd)o_ z72iGb_2N?In-qr~eGH|Thur3xrqU?w-j#`mUF7MhY39^gEc%kC7Gy`nm8-`Qf^K1- z=Bqry!XwKQ88f}*%bWIpOd%B$Qtz(WR7byu=l)U8Jxbu=ab>hC>m^q01-g9We$M&1 zIwD|v;KJKa{mUYsfGie-3am`A!&pg1nI{^CI7t zR(1AX?W@47p-84Uc!ktVu{V^ zULc)#SRN&j*{~gfqH_#2(C)c9NL%6+9^M30zeB{_F}ya~HYp#EPKp(km0*`cHQUSy zg7pMkKR@f|UZj7>Xj1lb(Os@;YAcoQY5f8ty&5*#K`=59H zi?MOtw^=?tA*~y>Y^K@(_4=NR$!fURo|@nS;Of&?Ex88*D|5f|*85&#cslSysc-&Y zu7ofq;zI%S`WGq7YAi~+Oa*nzq4Y8oTx%ll{%Bc{O7(BRHS!@7qP&I5)wVJY6CA?F zyX%^*58(uYAoq3(p$2VS>N#lLk~0Y2VI{)j#YJ~!(?;lGXTlRMX8CNKKL#bqLoBVAVk(h)I+ z_niI$#|VVf=XiVGzRRRm)^4Nlx((s$iph~aB?z9u$0(7OnZlstqBD2=+)@6K72#T8 z=WP-SF9;pa|yShz_)TozuR5L{4o zF^9S_lnq{Z1=l~Oh%5hTRK70N-j^m<%`DBp-EdbC3gL~Wl(HlP%yewHF^Z!&nU6@= zzHGRunZgVpZ73}j3842{WfrL1Kjj+l!~`5ygZLz3&F=w$qGb4{a96ys;-|07ieOH0XSczzm~lB6$excQEr^2;FL)-$}q9&hngj z4lkRdCRfn-cjAFMyQBg0B)iNDc=LU63GkuL4mAFecs}^0U=XJLH{WQVQ`1v}!^^H~ z5N?3Y43&Z#M_sBY8AAFiMI+59ZCq?X_q&DWMQ3fL>bpI#vW0~W7at#OUoRt2KH+V+ zlH$N8UY00p5o&1ix7+HhlLE`#Zp9aq4FK!FZ1gQtOB@{R0lud%2;rxJ zz3+#*{s?{p#`+ad+>+a11n9wzA7K*GB5mKcN7Q&QGCOe7wykt7)M~%dqKlB&dp2(T z7(AlligrC7TkpP>3G#>#7F;~qGhax(&KsP(EBtFKI7YVVL-7uN+7#dtppd0hqy@-g zWx5`cMB18h0R&Cz2(L-K$BQ+)t&MR8VW)oIHR%J1g3y?&JY8Vr-BPbfroyi~bN?$M4%3c5|`aHg6Sl5upO2EHgAg z(dHmoD`ytJarq9P2H1ey(yzk^()O^07s8q*Wa#O;xkN;OsF^N(y^-^7z`CT^L>IXg zkx%iIzbsMU5_fRv_J;(%wG8HyR%%0Sha6ka1&6mZix~*nY!uQsYGogh6Ur8zHsf?5 z$896Sf9y|h*8Gi^Ks+kR8WzsI`W z#j2gT|IK@p8HMH~=gjBYi2^CV8UCQ;v$4#g76Sc(HPXvOr3{C6f5`4omVuamu&%XA zNl%IQbd@H%(k;>~nAS?GYLJ$)n^ab}`VKA%ICb5D7@eQ>|DkiM0b~+vX-AS5Ytj z)?1U4l}Bk1(Tmr1VJ5s&%zDrO58=eijH5Oex?E!+c+Ana&<2sp8|{gdIze^I>uh## zJn#)v9wvNGNcUW@;iq1jP3ELAWEWN)Cj*!ARZ&uw%i(LkdYPj9?o4ql!!aMp3t?7< z2iA%Rhvx#>!TqvvO=$apD^o=(yVAz>DsXZJ2D=eA`3YC;5N*mDTIcx2Ft(}whxDfp zUqf_5xb8z)e|9{imd)A{w5sk3*!bVzxLyG0aHt^h$RP*3Rc7CW*6465`-nR+A-ou! z=#}X6=>2`c(sPLeI=LBJdAL@z$9BOq&DxgNF}X!mJk@2{EYjY?dxB-F2uGJg;>=xR z_woH2`$x2kj$GEZ-`ni^w>#j~Xm+1Hh4We{*zovq=kfds>Z^X-{0I4NK=cOj=OcAI z?mhuNczU>8Zatqfi;|OByc}SL#s*V)j^Tz;8j4zrNpg%Rye(6yKLMu;nxP8)`@I>_ z(p*f9RXu$BB{rP>;CB~D`HM~K9l1a$y-Z&?YDzLBc5l9zGQGxU8IdLpC)u@lt0rCR>S=4RxzEMRXmJ}20+ z=FlW&;5%1?Pq2o0+#x_GR*I0m>JKdI6|=UyIUH0}KMSiYstQ-(%~Pd{_b$qHwUKWW z1GoG{@IA!ey?&_0WiQb;(%h<9#(Ivs)7Kpq<(zA``1T0B7sbfOR+NHVm?=6L71jNR zmDel6&aghlt=*kr;{F8f<7mwiEnN{Hwm&kSL|1aa7tzHRCq9%zFasg!AS{qnbi;u1Ni#^H$GeGAiBkC4?v1qT7eG(Vc3z z_1$TKwAXyLb2P+=XvXg3+pYhbT;qvr1l7 zRagNyYH^-F(ZiUmLM?5$fN!-KMYKg+R%%?7O3wSlr4(js5r!G>ZxVoyLA6!ZE~*wK zaR@Xl52}`kIF(@KqRrb?%+TnP^43qs?#pS2?AW)UWxdQHN&mz>-TUOmH?!rg%psB7 zzEQWl@y_BdZtg@C4zOZJg_e}e3WKZ&s-zoR6((+G_Lcy*U2frb_ zQC%%7)xCd%by&7!?wKBJFpr=Q8Tnnk-JZCZ-*9SY6~j?KYy^>UU!{f z!8uhYjNO>HZ^uGUj7nJJH%KA7z<<}EP?)-Eg@%%l#tT1_(SK?~ETM+W!DudEM> zB8OD}S692K(k}Kctuf$an2_w*A}eUu@iWZ1aSgmfd^RD`p=VBu z^@=H51xqzx>{d0v1OrPOA^&L;`oX3sJM&%He9wz2YPo{F__>I_U+7d@6P6!4O>ch(^_B0t6e zKMp@uYm{yM7jd@9LZgiF3g?cT0@X$fzzWiKpA}W*4>)@|^Y}>dhQtW`%qMg$MNnDX z*9NcE(T1rYhkA3p*J?` zQb_ew+jVg>>ho#sY(7~&@u~kkrSce03I9>gxu+wBOnvrl?_y_Ie+z*!Bq9u8=I;^Y zwnSPx;Uek^8E8iL4%)5yUV|#9$BdeO zRbTul)6HKm%XxO_anPSX**N`kR=Kk0>6PwE0j>%)cGl$F0E{~jdBWetTVisNy+1W+ zPxfQpxfWAP`7?yqLsRAR6#v1wQf z^17S>q*N|{Mx`(1M2)A5FNa>-dy<`>XFN>xS)##X>}2+Xl;xngBwHEij1PrauGdc& zJ}4QZ>*nrhS7h^?yU33;jI?*BlD{VL#-4tiDSr0ia)N(f33D|L{alF-Kimv&lJv_F zqgF*k%FGe(f_1K4hFIGsO5)U{wQ&X!8Bjt=d6Qyy!u8uiR(-xj?css~#)x!Y0Q{OkTplln$wnfAX{J?Tf8BZ`1PRv!`xPu{1ZX|97p4VJmP-5h%Q`=uDI9EKwc zrPQn9g-=SuKiv{~d3jE$WN>VG4y{o5Y+F|<{Fcq%T603ijmFR;tC$aPX7ya9@0raJ z;9jh}@jT$iKt27t%<6^dFAB7Q>I@Gw&s6byAQWq)jt)*_OnEJ@=G|wK6M7HhrvzP8 zU)zF6Ok5W>Qa$a|&$OhZmP6XTo9%Or>HcWk*=Kc?1O?|6&d%#5j8C~}C}tLsZ>KuR zi;nj|!;X#L7xHJZvYw^W_B{EP_yr74*EHDE{4 z-%}NEFMwbrK7)VV7$e+>^7InL1OOhN6p@!j1%6XKlBBH|!}nc58xy{+rQtQhe#dRcUwz&Z#51Ce23fpZ!vZ!^d1*t+^-xqxDO-2qT1O1P$y9brxynpI_{> zR9d(sT$YI9rX+6~BBv={1+o}?iOfW>GMskb5c^Tk;0B=PapKah1)-tmQFDDD_e0xb zP73#@fLnU4`FZT&;NkQa696PcY_$QN;-{863MhNGp3KkijRFy*3k39Sw!<3CrmXsA z!~9bL&?!K>FOamBjKgsmmd@{|{(I=MrpLrD2kWRq=EY%jZzv{~x2&1B>C!@Pf15`x z7l!!9a&KUKE)IkyiyA4{{|3|lL8E=j{<9W3pRp2nBb2d5tS|h7Y1sDLBm+~+Ze&?C zKwHP_jrgjOXOPRE8Sdf)W5J#;{k%G}b4JlGukBj4@>ruc?GSXvtKmBOCJw6BNH~o9 z8G|b@BPQ78cwWZPfaqr4@6Z-8+2u)588!6>Fags=wnYz81@Cvg^~dLtExC-;H(0wH z1E{WDIVrB)i^+h`krg8%=P>&7X{e9pKW=N*e=hf^k)P&Xm93I^nKdCy^rn(fKc@+? zc_hu`ai$sLgPTz1wXS$@U#KxLLie773vaHUGldv7tgj~TEZYq~wV7WDx-CgCd>N8*O+fE_kE}LrI?WV*Wxaf6i zTpq{XDYrdg<`TAGOMUQb_n~WaO{tpr!S&E z!Y(;tt;%D>r>SM*Lx-Zi!n4&g_Cd73U?Zd(T2}Qsf&g*M8TqS+d0PVXMB< zb`$91eS-q{g8|E@K=JgXEW`t9(?MgCOs~@qdJ`I_6lmgV{h3#aUZJ7Wvyk^zbkLkRJ`R^?jBd!SMNr?ZmVZL)fl*se^lE$sD}y~ zdM(#hJ)RE&L8g*MOLqYjW0eLaT%)Pifj!09np|ZEGb6;ANvvu71OGUS3ZoA@a<|BE9@#fK+bhGRnvz!pHlNxnFiw*lpzHx zvtl$?dWQFCLBm8gHmPlGKNb>Q-_P`I!PceJ%CTXE3%!s&#_wD1T{1*xFMgr*5OnKs z)D~U7?di`DeE@FeD_J`y0JXNAiM+}_C23j*ZYs{&=izDM7CuIV>qXdgheBmRFU?&B zh{$@G(U_0}MKU-fewNwmcfyfYf^mPZF;x5S59Rfy#Xi*#q7g z%;T^E<3M@yi8d1hxG>my8y&?+luN*+to8q@)qD!qRv!7JQ=1^g2-QP6i-Ps+riK-Kr;c-uT zH;~j$XMEd5yge>MosiDn_Aa>A*n^*26hWY;s;o^gBcTag#^QRob9?9>KN@Xp8?CYV zYi|O>7kCs>%Bc$6@as5MT;@Kwub6PhAsu5KTX)=+Y;AoT7Ubk)^Kv`kM9)T$fc9LY z3^T#kf*y!Hd;4Q;zmd0-<;}7^*4*mc7mCqu)fDG*kMei;UZ+?V!F$zSPbqe`ZfD-2 zh^nqa@2aF()C*If=XYFcK^FDRO)gPE6B7$-weD|7#FOBVT~q-eOv{=Ph!?3)dw zyH=>kDFmm0(5SAyIjNllK0NpH2-C=TL&WE8V0q^QuA$Q`!_VRiEUtNzlbR>*@IWy@ zY1FePSqhN`<3C>xV+%68C8>Pi5+3KFedd2wT0ujwRw)pC4OuYv;4B*z9cb!3^pq98 zfX8grV$l~=2ieOh6C;id{;O93M8Pr5lFEIo(v=@NUdEtrS2FRL*c!K>Jh4WfS4lmI zt8cE808#Mll-5iuTCtTY8;|wOp}*2nCL~vmtAK%Cc zOD*q-)Y>YzxWA43crPoX{KHXsN0!f_`St%=lMM2@R|jR1-;$)`qq-xf+^DqDn)OAd z(t3nfA50aBc-~e}09;gYspi)!)WRs8mlJI-(`4QlM0rCeQU2rw|>PR&R6T>?s3n7 zI-lVDFkmZ#*$m zWk;)Tte_HtG{P1Q#@G{|#PB+x9VbZx5AuE@y%^dT21g?v8Lf&gBl{rlHxuNq?`nfO z*af@;(%snLBDVX~k^8oqRGx5?e-f6`C|LJhq5Gc1N~Ls}ZuP=3KLY+baBbb#eA{B2 zcd)(_k;Y@+iimM54eG$nrQ@#K>oRT3i&s8nJ5paIn|s{-4f0M}aR-b3@**h}A)zf; zYrjY|@f_%VOO-oU+rI|hw}>KT?O9Y--#k#$AP?Ofp*dp;l(J63>9?f>WIvb7`@>xF zrZ#RmJOH19f`q5J^yjskuDMzDIt;n75JfX`!CQ~$8BJe-liTN<-VrH4h+c)I?E?#< ziw6K(7F(OY0(p^5u=`-%GBY8V+Gh^-%`y|bfNjT!iuBzKX)`YXEbFs{%s(S%{O-%9 zE@=^ko?AY^CYR_qjOUs0jQ0ik${g=boF(-eEF3(6#p0Yh8j28pDa}d9KB9OWF5Kw1 zu1vY)XX&+xy6Phyq#XrAIGzK$?{OxyzzBo=h>*oPxuCk5XLdT11VKk|sx&r{b4}t63!tN)J+jkgY#J$=tfY;|kZ3gx}hcQXKHRW~{dW$JihKvja zKyO=vr&ByJ8nLHMF}6kaMK;ZmuVYU7Mc8xq>QiF)Tf69ZKC16@?(jo}*lNR{+K`XF z_7;`nq~Z~u-TYbq6JOtU#R|H+K1xlxo~*6S=mi{bAE*|0pYQ)~pPGN_wahr)aY&9v z)OBh^^9}h607ok8tdt`g3Jv4?kUYIXi?sQfg$3KDjXdprZkdRJj2_JWiu?gNQcKnZ zUSVF7zFE741|vQ@9m{4Z@@nquY&?0O3^6E7kHPpv30JHC%)-rjq5N=&sqks;bk54J z1))4=eIBk7E{pBRgcTV26>o*Er5pbJ^JixIZ)=k(V)W>^6g)k!IPAU;ho4ERdz>#U zx-8~vta3^X>y<&)+OcRVFPUIqUtMw!f@rxE-74tHqd0noTY=;?E&3x(Ei5X=KE3EJ zq`M>v+J8Lm*sa7r5EMbqLWM^zLNPfroN82XG}t1t>eNRm{gf3p(~Blo9*0J>Jxu!* zWi#c6m640EI)Z3Ra#yQr2{1(-AH1GM81d_*89EQ+UTowHM`upFod~&!^sdM0*!aqf zT4ZRzKFg`^E2H(tR+K0|9ePrN*~_sRUD>_#SpJ9VvHz+Fy5ox%A%qb?%Qk;2C)axYv1_%ta5+84t^HelO)QH z$Sk)E+Vp2xmPm|#9MXUG%Ve4l#^#@(<#K0W|3zeM`F&~N^<<+7$;lMzM&M;`t+JM` zfdj&doPLw$;j?kXvwnO-xa_Qp^45E)o*ISmBgGi-Pq4xx%rfeJ2rIQcjLf!Nsav^BxGdp>&$ZAqZ+=QD?EZd zTf654OK%PWXQtrdIBT8 zCpGWnK#{$>4Om10BP;Mh@x0)^l$N z9Z&9PhTtE{A8x2Q)Gq$v&2)&a*g^1Q@1kuFX3BZhMqKt-%Dy(eR339)7Qugx|H5)( zsI3K8yf=*+Mb22sR^DXKmzf&N=4_z51nWNaIpgB$%5Jy` zjINj4EU=?N^b-xvowxFBf)m>K?gvw{$z9??ZW|nJ*w+Prx1GD~q55w!oMoKk4!c z(GeRpeB`rR06M4IU57`>8@KP67%CkoEsS_EUH!`-_D^_jTYTt>OSG<)6wG#bomYq7 zoIixyt2N*{Ds)1%+IwyIC-$Uix&-L%E*pgYkk>xJ*zZ%n#B!>DVeswydH zTi7rI?B2A#+*1&ENB2GxulH8Q1OvqnxxY?qV;ME%D@mrZ^cgPdC2#$bY44t7OMI!b zS?X^jP|1v)$Ge zWZtti!Rbi%Z&>53qxB0$3Evt??=KL#4~#i4RAWbSBN{X05`cTBFBns%-tXyf&!p1s zAVZRYI1nn1IynE1(4&g6*PjbT;{X;~cP9SC~YLeQFx`ArO4#`_EWWjV9F+~z%8X+>T)W>oD)9M{ytjhFoRick&Sda z!9%U}bZRU4h_ckfG?inC8r0-0{gc6r`H%;5Y4ZblZj<}O;^ zDXQkro1S}6jobW>6o(rR?cKdj)71T?)!v+-Wk&`vlb;YG9KVQuStc)t+zVsme=!HlWR9d zZn59Pc{t7}B(+N}m)5YWP89tCCL)t6m!e?hRpIeI?u&p*SFT&{E2k}xHw$tGrWGLf z3;IC`(^ICnKVeFgeCgcw5w$IsWWIJK$OWs@$_8qy#kPdKOoYaYK^6w0YDm`nvk2 z_%@o73>x&_qiIW5E<%P$S?Js>q-i&V*D8ryXCK0k3sEtT1(3EyNY7TQ3~q)cM1zUh zozT|KIAlJ%^aGB|oSUwGI`3QGZftLiGU#6c5A; z=F1|vUR;&h*`_x})t`>!9c--TteJ+BB_lM*BR5lIcipZ*og<}_UB9xhgWd$u`g?_g zUu{w5Qtvh*%Dv?MA=!eWyu3H-NU~!56r4ONwWfU^UwX)k8{=ID9Q0HZ(myYww~W7& z!3^nB-6?6wEz!6i4k^FXWDz}yvs~>$mNLY&3>;UaH1M5Wdhf2VZ+@KyYGGs(*nHL?ch5x<`nrK_@`dS5hejAYy;_G!B+xIx3pZx#Gji}ct zs6n?Fa~qQS@Y0E@@N~>IgqLt170Q$`@*ew^7*E6|@}np6e&q8wOQdz8@YXTN-8i2) zpBTOP^HZLWg_y^-ScXnw31*02;95MpJ1RoI64LibZFd0WXG0U8tuS`!+@kTuBlLF8{NFk=q5{C}g zG||UzZ4>=9I9i$c(ch$h8xMc?(!&>J28d3*C4pwulKDQ0x`<2UJ#+l}90^|Y+={pc(z za52<Ac%*5$?Z(W z;cz%g9*JvNB1P*X*c+AvXpoz~9SGIUg32-A&G!$fNKo!sjkaL6 zCiA$3i#Mo-g9z7}9dkbq(s`F$U5JCnnxZspFn_x9Arz7J{A4$FTXX4>%iNL^riM$E zUy^rbsu0QUT7j{}K@U2oXO3;1eLktd?CD<%P~$uGW*c5Rc_XFHAIM@e5 zzZ!jhIfY9qq26qb?klx>QusIGR`$Kl&B3TyI(Gi6td|ZiiwM%4w#}o=IAs|5nzC$g zxX@4bju?vt+)_LEy@+zhU=k9+=b+3+UT|d7a=|e@X-C{Y_i-=xx9R3AHZ18AX*6y` zCgU&$4~u@#WRgF*mbQ&8Yad*oJh86%iirJJc}tIH;mq>ajIPa{%&1@usD{HaVd6+I z6%A)&|Jb$Hi6hE~7gRCcXJ0d5ow6z(a;WPANdly*p+F^LX(gbUe0MeZ&0t_b+AKBc z=Q}QxT{2Vb+@|W@+m?aYpDTdK$yMdHm0ZvyZ;w;0Rf}vvQrFA8#ebRb7?5Wcj-)$f z?*$5gQLcYxdwwsgxh3g2Y#jnvsi7$IH~Xd|2XD*_%v7b0JBvX*i`$6ef9UDKhPyQX zS-~oeF^0=EQlvKEO;->Od3ej_ifCOSzaw3g=f-keNN1a1Rg^o56->&=p<-<;^yfPnT2n}m{qkcX_!}^+Lxh% zbyV(I`^y&GsZgYG%#__94d!!Z+)Sb7~Tw$9L)&9C_(MRkdeeEK`T;6NUYc9_rcoAuw|g>lq3= zH2^9?fd`QJYxn%W1CHVxloF)#HZ-0`1|BxD`6N9bJ&w7s?iLNKg6;1%JtP`H*|pn^ zAuE_tzCqDz@{o3{Dt3H0PO{3pApG=hZ}zVlc;*@E9C>)bT1+xQ)nVH0b;j4+)9I97 zS8L%4n?VJh6hXx-Ct1r!H-0}05%r}88q z1mTgM_-2r7vX8rTnZ$Sx?bdXH<^eJ-NZu|m3fiwzJD|pZM4SpXOnFB%68#lbPx>RC zWpLvIaP}xnR%{*VdKV+$5b>G)*X<(&5U%;8 z!R@B6nCZ#|UP)`ClvXxt#>A;n4|luAuE>c$6dT_W&(EKqHzsv!j_e|&O?|<`u8+p4 z&g{M`EP7o3H5>$GNJQ-UqR`h%_^0ZPgR7>G5Dbz(h@Q9BwR&_YueJtvFNjvHYgx9= z1&iGBa}C%O0NRO=%RO-Bkn$)-i$o(k`*%|rvWR?H#{PcSA9P0nZ4L)uI%1yhZC{GR zA=sBV{gsuR;LNzE0l!yaA4w@l5c1piR<1lsoR;RqFXS@h+~uTd-ouX1?N}YGdO7hi zw)7@ejbl8^Xb~8E`iaNhL5Xbjd`;sh#G!tXh{c1S4eFdJXnWwN43(7F^~}z4Zk>LM zgStz%AZjFk!Kv}@({=cj;h@jhf1R4KPo=~~4@Puw9pP*Euuv^di7!4{@uf~!&%TT)c=3t{w})USVfi|}Z4MC{hgEhDVlyM$XQbtPQB~EqZq@U$(IM-KaF*Yay5CjYA~UOwom?XX+*M8FicG^?k{?E_IvZA z+_Z#;QiI>20uY%h0A^Lg<4{8D!CC-St*jEFRFc|$kz2X%GM>QwwP$s7m2mgWU%Ug+ zPhdOk28_rSjoInDhxUFBf0b?5D+AO=R}_+*tr!|ih*+=|rS`Fdl1tFiCOKC}K34o( zC|)Qz)->VN`*)d$57XDy)1E;NQ3XaMuAN*vdhQa9N{bM5B?F`&(eKJCDuakdSv30f z_Pn4Dz(CCCJYT7I<+~#-BcZa$R|A4Yym6m&-nsB7DUjW>XI|jczwQ4zdN@etaj})Y zlayGqfYa})cl~zd$@`<3+<2Wxu5H~_J4YB+Zm!qGtux)YTJxbe%~}Wji9X&EV;Y}b zye7T-rdxgQ!IDV{zzEF3Gvqsi!ua4#&Yl4-#hTtfq?ku1_^W(?nA5wC6^md>1IUHr z+-8Y3)JPnu&9n+{ZFn;SyydlDRIr86Nk+RK%nXzM_LjI=LsU;JXOUANkc0v8j|@|{ zmR^}#X=?9k^On~ST3yv=>Q3)v1QsZ*>L~^PdRSerUE*UC5!0 z=_j~*F(Udrz5J3PPT2ufVYAvq;wI{ib)AXV*E6%4>7JTKFc zep=Z_6-O!TmTXDwr^G@8I)_W)^^aE;Zz%-E20mliMSSrJ|jW6;|Wz;Cxa&>KOnB;d3Sl4!f%W>=`>VhSwb>o!z?#j$A<*aTT$fE^o)PeWjT9XdQP{AbBAEI3-2@ihX&-6sjX39n$0Ipf z?jWDuP$RtG5JkNV@n2_b#;`%gr=1+N;?l&kl*r#R?T=r%w7$cc6+E&`fvZzf@nCl1 zyz_?pTIUJtlJk40>QqBOvDTMPnz?!7x2(-`MFWw8?1}9UUwswZ9;r{-9r}z2iV@t8I&RKU>Ftk5W zJLXVdgfTI2duttBcQj(nGES{fMyyafxY3Ze@HVY?mDNRE4D;`3#gena7cv)ks9$X{ zj8YYKa{(irt+V~C%jarfIPS5 zN$-7Ws{|TfyR+0yF6p{qY1P-3JJUi}}GDO)$Po5mQn7Xm@#`lIa4swhFc3_)# z$*#Z5XMY&^bBo3d2Uv&^xr9=?T&*=Zxw=F8a>#-CW%y(=fQ{dZYAOo9yLv8e#o3&L zF3X9z{OZ3aISDQ7DEfK2r#T~FX?AbQ3%AdYr*5SF zYDEz4qrbBJK2*MO)6eXU!_SBor3(&^qhIfcf6+ovP^^6zyW8U>%d>i&i&UU*_qOEq zK)UC7KftOvam@0R0p;1f#`-v&lDa>(f4us!e0ldXb;FvacuI>rp!_WkZ@#7F8HE}c zbf?kw6eWXD7LSF=ehkI~e-;M;vSyI0kh))@z7*vY;Cy%Y;~iQNKLv;tJiF#{9_G#{ ze5$%dN}s?ZBK+RkE(@6^kViS=6Q59YB(ujDssKU9>pG$akqLy9m<-{8Yk;Y>YEQM% zzbVwe7Jhi=*^UN zHspTkfw0vrr`M+5G{E%ZIjY-nV2~UQ92_$37h*#mpyAjRp|Pp-p82soU@$K$mU?+} zvv|Mmx`oOPp_ee!CDax6exw_d{#6<2ndZ^%h$q5I!*2CHkok!0-)vwPHL;2^R5(M) zV*t1c@=H=QOjUE>&l|T^s74qLF|zZ%m(!F|y3frZk9M>Aa391k-^{^MtpuJ`%Yll} zC9GemQ5i5dwh>spM$1h(NJ?!hu}ak{U}=<`$LRJ%y|eycvm8Fv4Vdy8Ii`QbLBslz z=+#EafydALHE@W3ljSH?<8Fr6R@9%{sO|RONq^{k?+bjHGM_FBw%3EHGq`&>4UmK0 zcU1q1(8N|{Bv7m~TK0R}PH3x~OdzYG$B2w{(!YDM^p4?VLj)m2PqNJL=!FSvU>Ff; zPgr8g%tn2MZKe3xmd8Jjj=(iQev?x`KXN6hMdQfui6zm+mgul_l*q`(b6c@fl1ehv zQuQxRepjmst09V^Up5rGfjbu$&M8jE&QZIEEe34K!b60a+{$0>yf^oq{|{Af z85UL8u7A_0#Gojh(kk6B45bVqARwSL(gGsQPy?cb(p`c|iF9{K4Im7iGStv9FyxT$ zav%S_pXdED$NC5>?)$p0>paij%?_!f+05ih+RU#%2993Hnhw%bd-q+?|XT)yMfs-fXqys8N>p%MWq&$pmA(eeUXVcML44n;o$-F;zDvNE8vWQ%U6v+fyIt14jDRZa=DkCytB z|8o@1lvg!yM#d#nUO&F0BB_!ADPkX-38YdD@nzm9Q-#B`8T@<_#dBvCi<+3k5VaFe zArp?CY5G%t*xdEDPR<SoqWDJ(uGGj^-S3+Ks6^G^BBO^M=S zV%CQ3DX#$o+iU2Bof@K|qM|{xCU#KP+Ets4!qZ#M2kLv|VM-7zir>TT?ks$FFck>W zw@Q4B2|x6K_okjKa+-Kip2-G9t_Jx78GQn!jP9iK4Ln1~QW=WOKRI^AemV{}dTu*i zlmdVa9(jxMz+skRo?_z@4kg(usU@Pvj_i&v+e$5Gjzeoexa+JocJz|zf9^Js+2=8( z6HA5^shYCyByRc?^NkFiZ+TLxt6Se_^*;j-GUy_vtiO%MeOHVdiThD(JSNCE1sX`# z=MeeUVCj^~C!Uhg2$z{->G53GrW-z?nJHSc+rC2W_qne71xc3ijlU|^I)L-*<0BA*RKSb^`+4RlenrCvHo5?D?53cI@r_mSY%IGNnr$KGQO zk6-qH-G>G^g}gJ5mYMNehCacbY7A|qi-?IlNxJ?F1XA>&t}b#%MkSobZT#JoxC;t$ z;qTjNHaeEro5F6P8&KooYGZ?wzeFU21xP`x@qi$7diL2It`WhoX3c&O`?=cczn&M( zk9~l&h>>jQ_-RTh6eL`IndJ|SVM(H3lt}GxenD6}^R7PxBxk?AJ1FtR4I|mg=O|FG zd;W+}x9b+BNVTEti9ac|l}z@4Mz(XFM%Qa)3XD-zK8aOl)&MSk-!&@jAV5SWF~QiS z;dR#K(A4T$#=AuRj5<~JG8xO$hTY7NWj*DV5GRQ(&$?YtEMTkT!rXM8W-J9> zl5BQ(*Bz{v!T#8U>_k#l|6$cyAeH7%q(1r0KT*q?zRzY}iqSRjt;$za5gbhW9dy&K zejpPN;-JMYMetxY!*PRn5qo|;Z5%gao`)7`3{H|pD z?F-}I>X8P8mqccF&S6ufw9ZcYkFI2p0jPqqT!NmEQ+Z5KZw(tPQflSTfX5!Qv7jGu zzr`iNktKza-S1_;(R-O3Jf07(KCQ(e9m@u*K zL@bvfxMow^B=N$0Ys>nH;cSS^VYGG^;V8juxX0zUj`Q0j7o<;EuNIxx9INYCVH-eO z`hOdUQYpcj=vt1`>1Wf*Cd_2c8dhz-ph$W0`S|;J@q+4Yk6!TI&r5W12=?SdZ|MsB zsxWw`#xBp1w_N;E{p}wI6F&0P}0hG=!2kcN36G~CkIO+WFI7O6{ZV^5*5 z^CzPN5+^s&0rBX!+&=_)N>rfvJio9TIWc|WQzjgS<^eVI8L_YEqBo+1Zv$s+pN(O5 zS;iRtGAjG*s~tNVfB$A|GNm+?du}jM^V#y3sA>**yB)dxcEV&5Y*b0T z_d-Bj+}l`3Ko?N0V6;3O9L=#aSBDG-lbb}ETf+}C7Q`x}rD~brxB>H8khj9qdn|rC z!W;*@VIm_jpg4u6GPf0dN59>8w_TVPQDc6N0jVn1$DXN)!`B2W{(w0`yS~xjkUELzOfs_TuHujp;pYjPQ+KTc} z-*HXs5V15r0WOA`sc;kZRZnZ{XCF^+ZqGD-Lri$G+=ksqpSa(Q+$%*w`S$_7`GVUsw^W~ z$q)Ve2dyLR<&(`MU9JZt*YR2Po*<@2zKTF}Sj9UUDLfl%0_IZ5dSgC&_uQ2nfY4AQqkge4(P1E*LY8 zm3>=6y1b!ex9%68{5zY7kk6Wz_H+FJU)A9IUZGD;o%X#iTJ{>ZBiYb|tW>T(gq07@ zSZ`L~Pb3%otMWSC1vN4)>zBJvdn8+RuCEtvF0g>8cnSf%r3@h_kjB#qX4mxDbu@qM zI8|D6N#yaALa}(Fm~KXnD_^~HI~Du;~>wjmg^zgVeW~ygUuQ2m#^II z-SODTa8R)PhjJ|)m%zxNjTL3pODcj2s1c98bVkzg<`?tJAF!8IJtMujmJ}SIkF4AO zW#%NJSw4ylczap1y~)(@*3)3ChQ3ctBcCT4F%r_Wx{sn=-Or*$e+!)#1oQiu`Sg^^ zX8Y-y`LDUc4rX|)r^vMXP+zqNTE8L@DBA3JtFRMQ&#mzLXAqY{wg#^v$8UD31;ih- z84l**KN@@t7)M*rUzHgi&x&>49i{v|fb2Ttw|DC6UR?hMiwsbesJ&R8sB>S};y7oc zW$R&U`xggBM;`=b>H!n6R0^5Q_>#Ro+wTM!XS|>Zm0w>{#I~$3V`7iDr;Nd?-$=8B z=>E_(Q@J{wwg73`Yqh29Vdr`@Hu0pt?-G2JQ5?7YcriJTxk6!T>~2wolWlb*TMk`W zkxAro93Z;w3-+jDzzKqkKlRvvhy(iG%n%=2u+o{3769NQ+bjl;3Tf{@lKpXL|Hf@} zRG0?_H6k;GU5W*TF~HLKl{9{(2R>i#``#HQ&Q$3)$`rFV&{Gw9R;%YhR4m%1^G=roEVtmxxeGe$=x1!U}$0~AF41~VWzlNq4np_9|pDFbTwlWzulTj03SHl zKVU?B&?_S&Eo}pP?Y+;m_LaKWbJuv%_&oX~jEXbSVW9il%vy%X|1PNjR6rvWkSTP+ zYE6^8(hh%7_KDA3pb2jl;C!rSCE3X_*D3HP*;lM>53?^^Za}ylIF1W94ZnQWZ~?zG zcRRhgZbMhW&yeUtRsf2OlRtleJ7T^3;eUQ}`UTr}a}HV+gs=0g65f=t#>!v8+o0DQ z3zy0`sQjf@8VdKZm~oyLXVNB?&g`&0H#$_fL|#@<^9X(CgSI91@pBrB?@vUejN&~r z4As6nz9c{h2MOO6CJ{OpHee@Xqq%&}N3IVx+#e5STe(va#;(C)?SHjxUI&6VZ)&>4 zPqkeiV?oy)^0A=6{SXxaCG~oC?FJQH-dKvIu#U)Hc2uwrm!jC4vp3I`!Ht0%Dqt2R zlbN*(4tXpGcCxN@0*rTa+;%;+CLQvfq9f=U=jTlg-1`!^5F0u&!Ho zYi}PuH&sg^ZnTOeB`6?TvyD+|8mo+fBt8m|DbN_*syhMbeU{6+!j@UZ?+)K&T}1WLvgt2 zx}wg&at<{Sw#Jje8!O$EUR(;E7>(xT#B9upyW#W=`)daT6CRl}O&t+KxSv8aNx!E~ zNo&e^=)@V*Y`kjrQ7+%T0?-Q7#ghcg{1mU=%M#x@bFDqPW zDuc(P#Ic_dwa?vfH0pd3=T&0 zuJrFB^TCqMVzvJ_j=M_47xz}Cr-fl_n>?#=P?mTrC}sZ_q&$bVJT5h~cQ^0`HIKcl z((da71sbUMY$mC%*SZK?b}ARO^`O zFtNZRDE#HSmgC6=A61d&OSazqNk60?%8Iq}y^C~s<-+yug2Zo(vx{Fw^Tj--R_tF1 zw%!^T=!z4(`y$OS1sJfdM=QR00_MJc+O>-uCk}c(<7Gk z<_~f0v{EXyt;ai@Ona&wnSetqGV>|P;m|sv`$!#C6-lK8RZmZ^6HCLXklQ6G@$CAm zsVW-e(@9MaThz*G==yHf9{oqEt8uH65^X`3V7R!#De@%%gYuyC%+Unqn^)xubNhrO z+Y^h6u{FXhBe~z@_ighv5AxSYK`8ikFzJq*^V4t4(hQcz>EgA~aoL`SC$jIR+#b7I zRXAH!Y*RSYZ9*oC|F7bEUrTV6y6qwaik~xg_j|atzhFW3s?->_*OnJF5_WLbIMp&lu7VJg>>44?Nf4tuj(zLX zsu(#Ej?VEXcWpiw`ooSF-su-z;z0M!Y37-Hc(ZrQwkZwU(7!`n5iM-3mgUh;c^kLx z{ZI8d(_BH9*M*aiadJUBJ4+d&4*eX7+%nuI6o}@=rXe^JynzsrlKVk zpJ1jJ_pzGzpqdXrwuk1Px-~!nIaWJ;h~}zX`p4nyja-4^rFxMsbabY=Y?I0%ewumdTtR-)P;_j!6Sl0m%n0ZcxsK z)lS-2`UUgrFdziSmZf9-%L?AB-d9kmdqLu#J--CH>L%Mws?Wx=GO~d;X|WGuY9iVQ z(FdO4csMnC$BcUz=bj|5+T&kT@>>6yz8_NmYblj@X&A(mCNmpigkqvuufMY<*R}2L zVr%OH%Hk}_nzlV{@s<@sD@b*Ylj~Z$Yrii?h0>C;CIuz>%^5FnV|;pLSLZRlbWNronTSA8edJr^U<_b?;Iqz25-Ny9f2wB!T1g&E?_zYK*(o}J zpdnnJ;(2Zp5Hk9lbL@yJt#wcr@@wL6C&ariSt z?1*L#=EKl-y0BZ`NRe=!(WSHuFw5gw7KJ_0o3r5GYJB}l5;lr&DoTA`qZQ@*l?(~C z31#A_clrOmFDj|B_eKZ^dGR$i2oNtgRB!2) zAk|A=ixNhb1dxK5V|lWC|B}m|E_VubA^fgCt=sHOr`~-@*Vc5pliZu-|MK&vAM`F! zzMYY+ZiP;UJeB)fD~(JR@7^@&s2 z5FrOfLk`oBba#7CP{jwIbp{&zoZ>zS9d&NapaveSEJOv8!w9~WUQ;U5&+k`T`LWw5 znJI9;Fb~H^5YAGxu&Er}Iv);$Sld(%M!B)`&)f;?B~uadWhYBxy<8-9&JgnKkO~v) z`Vw0kG~TXEs-R*Kt|t)eEA#p;z&;hi(uF`M1D@Ebv6_>m+Wi%kl#6a4oY^~!+RdTz ziF5%rAWf#jF9*m83}JWPU_ZevP@NAT?=^;U=9)^k7WDeCaSe~}vuC+&owS8NmqwS%wc z#+Cvo>>u}ITP@_CxSxFIJ`l0j`)~?O75}$On+f`NR0^Fu8yB84G8Q;dMRc6U z`ufCLzC5(N1>gk7gq#kRrvqoa8jjcI(515GcI%6lKNZ{O4{OTa)nl^6slg$+@Uv9_^R}pAHKlrOK;L_!(%`4t`>%xv+3Gzfc1*d2b_2Z?|wY zgt?4%w(I`sVRaXaSEfY%oZRh(Kmc>QZXvMg>j5dycM8-F-02N`9=MqjsCJFkzV2!* z-|mqUi3<+a$V`CKUHQ^A`N(UYVQNtR$^g^%o$qP zGatJh*`=OLc4ngc2f0>53gH(gI%}*Bruj&G^xf1Z1Utp)AAQ6zV85G0xSNi0yma$5ZBUh8{gT~&Kj9DZzKx!gk2s$SHFj`RM$kXvXgiQyj`48 zHGZpxx`WE1UZHmH;&+~TXBR%-oWGRAJUfK~{oe$b?LCC8pyyY`D~F=#zF_uG-^qna z5xvqbK0hjU|BSp?DVuOe4<^C&)S|9i17ViyWY`8|z)-lLDui4-oI6rfRTRDMPRy5- za5tuxvF8wU-3}_eJXXgu;?|jIH?$SlHKR({84g}4(MH;%5{Aa^5v&W)k(pndr#SzO+G_rwVEJSpXfzV;!|*dr z*rSDY%M6_%gF{0b&0)iQoY=fDdrvo&BVi~VGIr5P0~rtB=uiKU^4RmY200QO_}(Ie z_mJrKR~)ITwwd*kDXD5Nh#IjuE{RWpFebJ0kG4la%j@;jPFhos?;b{vPcxtU)_%A87-(y{aw`M~cS8WH z?-^q-sj$5IFll5;4E%Bhi$lUulb!Gr$UaH#p`!iAo^v-2)IVdG2fHS;&YVbbAb8LHj?UmSb2aUY5I(5;Iw z=9eZB?tcELy`itAWd~bLOz5u+7hjX_>j%GShiE%3^PjO&1^)E+;%1H2n+kpctQ6I z!3m1`3^=Vr#Db^}YuPBy<3g+HHtFi=a(#ZXQNM{ks-{1^zDu6fhfGpn+-LdvH};u? zL#%d_!eSgHY@Ap+H|$%mHQNjB#!(i5>~mcx*p7{U_yG<@i9U4WO*lPD13h#JX>TCw z8B@Pz5bPFDcWC+WH^I5!qJM&q7DKQ{W5|DV{68EpANyC^^0sh}M*$Bwo)>~XuDc`U99<<*J|Wlq zucD|*fE5Ga$1=W4)Nw}U0Q5gHL9igudf&RfRWX#6J$6|wi9%G1?@NrK>Q1aLM6JG0 z47ZUzuHOBRR%Ow;oy4n2W@gQ9%dweZx{@rvAm1j2;jgy*aW%sSP#Tn8McmARO2cX5 zlg~qi$R&1FHWnocC2~SnWLC@CNY9A`0gD_JFqhdZei9TML(6`nMKbyZoRC8D`-Mm6 zteODN0+@fZK8%QsB1#Pyif4HPT zu`C3Y);5DvU_wUv5u8F&5Zd%q-mz+l%8OYIw-uo>d(3F4K(L>=;4MgYUkoW{7e}Sq z`bQ4zaENb3ey+af>IYVXtWi%~1(Y=*xpuM+n!7V&~&$%%bVLNo*)GT=@-UA zInJcVJqKjOv)>Ks)-{e@+PC)_2e9u@)e*$dIlLc9hxD&6HXfVyit)29rKV*GyVzdS zycZ)PS}rw}U5B*(7QXj2xismQcJb@~TU{6Ze>-4z^u`3sejh#qy5Y3wtg;1;g|oGm z+6QEvPhg&4ySX_j%;%!s3g@9H$!0g9X%Lf?{agn(H}~LK-gA%o<27TO_x8hJ$1@A4 z8HG~61asp}@(0ERx{+~wvl)Cw$2Ya_z@?!H(l z>n5Hx;VBWCI-G;q#s|`mMy>G9jk(1{Zy@C=P3#DKedG#7ISJ3>0OWHDs9CEkA*H(6 z8y2+Q-f+8TMEztZXW9gE*VgA`$uw*z(md;R44eP0++c;yf#`DuH|pNzkM_Xj5>@3n55h6;8mrRd9ZmO8evlNGAP7SB2l zLmGYH5nwIckU0HJIM>Wmz`c5y-8L5(yVOhH>ZO%&kxsbT?+O|Z^IC{w*S-15sva9s`VXcc8LgTR8K6hA#A=Z~8Cz+?czN#72 z|MgWk5pmbeP4r2~H`py~a$Il<#(=NG!B>CuPX5rm4zhgCa-Y2=!GO+>_lq!MQi9y1 zHt)Q4_Irn==tAS-^QG_PMJBueH)Tbg=>tPQKQHJpPR5uQn}QMKkGgkmnp@2g+g%TDYoe=w*#T7DwD}Y|dW^dB5C^ z3jQO9;j?plqh984!KWf@3=?j}G;RwtpqDjvI++md!|ufFan)IuWllT~|gW*G?l3FJ4pI&v!=>?Kj^dLy2 zI;Cudsv-fV67!8MA{~dwKI98zOZS$LSmOJn3RzRPj1 z*b=e)L zau(jIo}HX_UHhKfqVVU6_%+La&hS-Kf?+!Y=eT1kX+HFw(W#W7z|Kyq^)F3m9Bo#e z=(*SKUz;p?r2I|u4ywlZWCz7@agr~ZS&waYTz@x{2Q%}xEXb4hS&aR<)GKGkLKgwB zVX_X(mx|hmL{Wsn?b2kEWfH9B!_NfD95sc_%SCt3rbC>ktnoiqk6A?Uhm?|+c-^7W zd=z$+r=+9am`tV3fTd+aA@VoVqe6*WH~K|4`v%-|45psd*=$B2w-k)fag)5<6Zq1{ zMnDi@s9eeyt^sJ^)W}wj+wg7a*6Ij;k5P#Ml}MWW1At5 zetl13{NE!Q5y3i{))%{MSPn0D0Z(p%qefoY!&QQH<3fBVjEt_!xV5jXkxCepOGA1Y zU;wC!rw*!y+Uhtih(@*hwMd4=*1WUywIG za*#26*8PelB^z0AD5*!UMheE4u5!sEU@kh&kMr{}p4<}ua#-@`GHS8kuZK7{>q>Lu z@tiD&s;6mo$&it3IYr6|*cTlhon1>#e&mSq@F$E)gLHMx#Lr&y@5*Zuns2e%k3tWB zb5iP3EmzBkI?DX#{b`>GeSn@x5fT&Pym|n}>>t?=c<58K7+|Y|${Cr>oS$a@s$DcU z9uH5*ua==Glyqc>uRu&n0yG&J1em-lZ7Z_7s4(jeC2(VfMqfJ&p3M%h8){At2gcv7F67H}_r2$`r zh_Ey_ke2w;&eqz?c6P`J?V|pAjz3%D>KPN-GDoKhtba8Z#KUHHG7|HD^$FYi*2U`B z0G}U`MXm;pFDLM{P=c>5QWX~)DkvT5&a%6U*s~%X#ea(531L)j9C>d3jwmzx85|eE26h4dXW=kcshFM|N5R#Mp(OON6TrYVVn01IH6bmq$7RZoO8_t z0v@gRPRlKvq89E>y1evR>f&@$O92M_40`KHcP1|i`}i&vZ~3ltkdE9brfBjE^;!As z&1wvjMg^ZwRtHR%f`{cT?3&`k0+&)A)nZ2F@fse;esQ^`$r^p2<-w&BB9hfkIW_7k;t zGW*QG%QM~&?f-iLNHhOhnzEnG;<0_mjCFq#UY=TP72x(^aEfY@2CYc5)Drd_rTEAd z18epd^o9fmZkMNY_@%7a5W=BpfyeE#iw=e&Y*NA7&zZl56IOE&u^E=hd*<%4sYM(H zqSGJ`1-AiH!k~&wLJSx+g^HxhmU6Z`sh}t^D(QRfX=H1g z03|i==?bdq96vS}E+a8M&0zpr#aaFwq<0J)*+w5F0?0CB8H7@6Eh6oOQ;8^t_RYcZ zpC`|4fd|KxNnv+_OCwH%DnMlJH9t90mMqQ1&!Y5cUgvp_IWUtSwTs|tmxY6xO=6I% z>&PPxkmG%et3#hS+U>z9xL3)0^6#^VcckVT|b2*__Zq<9|J)YQf7B5_)!X8%au9fgM7+l-t%Il{Q zB~yCdLtbYfs{hvat?uL5ct9=wuBqPss^VP!5cQTw$viEXy{wkQ< zIgCmse+h?}Wv%oXKf1QIv8zaA8Q`;%wyEFI)Aat~Z?@NG?bp~XW!Ai6jUgf0 zhffj#NpsXrnxA%7{)izUYmKLH!JuN_63f+j*AWh7;5+cF5wln@p-V z7kt|!ZtAN~Iq$|WDH-LkVV=+v@5?)qy^;EoC{`Fq6~)IIv~%b;+f8dWsewI=)XEQo z`0XDFNG4@NE=P=izT1GV-mlgeP4>Gf>-}Tp8d>|{s_wPRjD)d;b=I|&u>e#+DC^Za zuBk(|bo{H#!H^uU+CW3G3gt=C^P2qp@A9PQ7dCEhbZzqs@_wAIs%OJ4U5CsI6Nj|W zWyCAg4T(#uLB2J%G6!uhjfKoJaI=oZwPXR?&Tnu}puXy=_x`JL_Q)C2YRWA4EH^B7 zXS<-mWV$hqJ7tY2o&>A7tFZWQC>DD6i&UNvh)zE$Qu!0)n3I`LuPeM8s#=oocAT!~ zCtG0fzjMPC@``eN! zM1f>(6eyT(akXs6@go-q1!;#-xjg)CLdhferr@Y;4F=5n(FX6R0S*eoHmHE%XdG?q zz~L&m7G#hp(w(hl7ekwlktNlqu!$HsVO`^Y&FB`0cFqg_y!>eyP6C{>jctA z%JY^G?tR`{R_ff4eqyRz=A^vmJ3o;s>lC3m7r$w2TC7K--zV;}A-gx1UOP4+UPcB( zZBJ`D91k0Bb<3Z(m!_Q`MBBl8Z+IO`+w7@PjuzxYc~cf{O~}{QD^I?fZNtC_eWaGo zon0|64Sr2%X6-XAfY2KIw z#ipMU)lh*J?{otgoPXi)C_SvH3Wnq>*<|#WyOduapAE=+Jr-Fw~1a&C_ zAI-s!fk9`Q2jY#J*O?t|&&b#)vS6E@FZ!~>D<~;PvCsWrzA*#I1ESalY}VqYgK|~? zTYQWZ%5SFGTubgU%{$0rrqV>POnY%$(Ht{-{NqaZMXf-yut7@B@O~FDBnP2Wv~AKLXMPHK0>7jiqJILvsAdxp@II-yF$gPHVQM9xB?| zV#mfIy@UbR|D&De(@8RohSGNf(&yesV+>|tQawmb4AuWC(~&#B%-oz{#$Nux9=xMWHl z_q&L*&b28~!YN_ti#OtY;9a7oA?i_vxKrD?ZG-)sxGn=-5`7KiBJ{|}MKxz4ch4x{ zgBd$JjBS6xqfEcnB0bMg#Yn0{Ek{9hF>NC^)h|xEL2<=ZdmrfS>Gm9wH??DY2)FsGpnjo=VgyU|=;-)!Ykc_whw)_@d&H zeMU#knhCq`vtUpKkYJ3@)PQHczl(e6YuG>usaZfXM_Dg58>kPY3N=}r2tK90S_ZHu z-Dj3e#0Herez`Mz8pa+J2oPWgVB@EV(#;oDE77Sq z1FiL{J8buB$~1P9Nf4r6!~Aoth(rC2Z_GA)6?qRkQnfliL>qAkXXqw2+M9}xyUkMK z6ffd_=?kb z@n|J}(%U&WaW+NM+dBaxT)|?@#x7sqzma`>y7x6S@kumf3dIr-i$<48`6Y{0_^aC5ET3|nr>;9-T$Gd(i zObmf_C*cdJ-)YBt9e|Ii+LxiyZsX`3;ACMA*mu=7Wq%N{f*RlvwE51uSk&LeU&oR~ zL*Esbb%MQ2Tv;b2(PA#BjGDTLJ0;pHnxLQiU{<8R4jEQ#dkQZm&#kG*HEemjb^bXe z`(&=7kiKb+?DFmsDV}E17_M~lFrCmvYVS+^Z@x1@d>FrJ+t{xP?6;!u`?5}B6AasQ z)kYf+%2eJ?ZGQ8c@-v#EnJiseg3z-pk=`}+`$g^0yh4HS8Cn$=HxAa5{m%~JEcCkIawDc0`mXUrhObEavw)MVSiI`7YEOZI(O~0IqBs=yv;uK;x*!NYl!SAb zU(8OS|7CUl)8&^otT8lPoy@J00?FRbY0#uSEEM(L7&F`-I|#==+3r6{w+gnzyv+=? zA1a9ed=Lf=oB0{mD?L%kG&y&NH?&m$XWT8>h8uM2eqcB)tJ)fO7h)1$+wa>uQj5rM zxt+;#c757r-g?$8Xw>>oI)Y_vJ77!(NzEu-KdJaF8Z5(>;>!k_p?YIwWtE(DGW7cp zKZUq*)m3|uWxmB^D#R~bl84xo#XG(VKcj;p=!zAbsU$t!Q;_ zqy8v>tHDF>S*>%aY8xu0=py$eVul{xZEJkz8KMJCPxqf*NdfireBPVo=jFXivhP29 z2wVPKq6t3n;;~Bo%3T>lFpKSLyVh_>kwkQCe!e`xTxKN-HkyV)r|4%S^Ln^Z?imur z!oTxn#PiZ62jyIo9yvV)k^ZQ2Tyqu?)~iz<1?^sz7SsRiM_&EW?wf)fgke$H=S^-G zgLVBcq+yneaRl+N)AwlN1H|w?s%Ss}v3%dZaQdz(3dz500DJvAY*!A4PWb1;m}&GZ zx;%@ysW)-!B+&U8iRGJ6swE5vD?1_ktT1=%k$Hvk|HbG-T0 zgn3e3^D4i}nGTEdt);cF$Jplz0=Z<QEF z+a!fUnS%Y65zwZwp;+fcQgHSHVO9fF(E2?W=m3ZS-izyQp;kiU&^+>dzXP z+B5?V<$#;rwyU~I*r6@#pE}K&$C?9^tOwHWlneIapg3)h_}MDC8PxFO7{E+DSrg(NDB49585At zn^p>@6fp{ujHKL>k238uE$Uy*2>Des$x#3&I-EZ)EOIX`ar|4(5lYycMm!aU%d5I0%)U!~MnL`8!7uG^WU4~XaDz5jBWk9}g- zHmCA>bwE^|Sw_4zd^u-yZ|+H0{sr0_jybj2h;Ooo(DY%+0aNT)OlCI2|a;uGOA<%ufb&#b~WZI_;SY&uKTWS(_qu?u~CTo z-l*ZL!6y?kx`UaVN(2P=8jgLHTQBCJtJ*h5$VjmdxqgF=W>)9QSl~5rUpw#yw4eRj zmtEQjpjYgBf!7xR;0f9;n%W%AVDd-G^46ZA_8)dSA5*#Q|MPCViLJ$tW!>mJ1gx0; z%hcKB9{*DQJt^6;hOl0r6gonp?0y5VOD_06DVz-?cNBYhUUt*C`Prmj1uodxcYk(3 zO$0O}rHwKEL{GIr8rLZs`MUIVY>I*mfiF(fa5P8KFbm)F40ChUX6&c*4*0_{67jLu z*MHX`c7zEM){Ir#OXnZDkMZ{SHf&@&x0-W1{LOd1r5E$ zTH6li2b?99hk?rulV#wIt?zK-{(K9bvXuu&NcB|zH}|Znt67Gl!iNc?h5~)Hnpc(1 z13|EpV|@_ajG@!!`J|J5)&1N!%-DD#;12uWmtwJ6agdUUw!E%l0N>YtRmR9*d)#wKwPEBGLGoy`l zX*=Dl;s^Xb)&aHfzIZO3=RXv5{)9gzAqf$ZM*hvzTlYCHsX;m~5)eHb?osu@&Un8) zKa%!*v@=^%bh2ftAFk3OclgKAsw9gaLyZ)Au#wz}x!!|Uxee%P0VeA2=%`$8Vz%2B z7l1&NkgzN8dL541ItBcDOLc+Ub^HG;e)WLEm3-PU_ys|^QBYDpA7s-g7yFnudX(xF z@BRx^5RZ0N7m!o1YlIh}x=Bt+`J=6kwWz3QE#T~vpnnILP#zl>V*!{9PFkDEo)PfY zG`}*7cvusH+TGYF?=fs$xImqj15$g(G9asl(6RdEYmGSK#h=$kkDh#k!X5`CnLG4X zy;z+}e-Gn$~&bbR6%Zu0wQW znGhn=&JmRUxD0|R zp&_0Ux7yyzXt7atBpwfD$_o3TLQr-@LV`_hZeVcqSQs>eqZU{Js<% zT$=oqMo7+!obg=gEKfUtVBzfVO@ualE1`aK`0mHS{Wq;f(Ojc>y`*NXDbmsAmU4}%>DrqN$d_q^K}wZR8OfyajM2ia#`Jme19 z(<_`DjcKodZuUVG#bcreuYxjAH#>Inj(9Ubm7_UFh&kFEOO8ptH;}M*GUrk!@+i8Q z7LH4S&){joXn#?(VUx;UT^rdx)>h^>Z}mO?wd&VNAvrzzW}dn||Fk;$`&r(3(NOly z<+?NMAcC5Q{O5f{fgQmyQT+8a<_7W)d_fqVn4G+;u0@m@2pH(ZVp{(>th_MC$aFCq@19)D)8Ds>;6!o( zBU_-7Q}Ad@${1`_G`o}{Ze0@u(0)JI3cZyNDc+Y4P2*L=29y&2 z@5j_JUT%+hm_0rxk;&1>C)3GeX0)1+-n)0k7r%Wtv8Jm~pkLw+F;)hTKjq$QFPR+v znY>6u7|Ez0ot<}(1T9a7!W{0;_i>}##w;^oOTtXj^&n+J<@W@Yf6&B;$^*F-LjK`U z%iF)o>o$sY%ig%T-D$g!y`g8l-VdV~*A18qp(Th%t8uSwDo_M#{7bh9dXBOtsIT@g z^U%}>9DOAG_wvlm7Z(UGV>xpRA=fp`>o3^M6}(U-n1q`SzJN#m*u-0 zKLhsAeffE<1l=CIy@J=vf=1xg?{Y-MC{-Y)@yGa{0}qa4U&uumXG=u<@23I@xPE=cALZcZs z-3zgGMQ`$gp5D%;q-r=Ek{#^SSj@%h%@9v+&X)xq7m=LY{&>F(SarGt`R!=l|2%h; z@BSy`B00OAsxKQ}$B#SOgh!7DuB_xtLS*(;-s~Rxxir+NVx6usugtU01xA|)FcY=} zq@MhUT!+sat5+9BFU$NstBu?HTyd4xN@lW_>RWsv`u%>jg?9M$Dn4QB_)E*sTOpYp z56~PL)!HfFJBFWrj?;4z{e(7B)gjT1JY6Gy!qJ;Sv}KTTpHyL9qw0^@m4^^C0htlL zAv*nZ{OQX|4OFb5W~{kPh2eV(w*IEi`$o;jA^*g#@@F0YEH1!hn54*XV2)e~+nlji z!_JzM*Th~8(=eKehP*Y>(Uf|M=s`i(i{i<+K|`IQu#uUGPlxNsrSZaq`gUZMoz~LE znD+9FT~^OwZ&tT^Z}f6nLH6$iZSWAXz;$Osr}^ynp_Kc%n1ETmg=;={50$IZ#@0?y zXa9!Sg8j=dQAAaE677CFL&?OTlep<&VZei*Z0~=dsP1j?-`wZ@e(@$bm7`pV)An}a zz2SVDU)aC|yY!%1b6k?SFy3wqFiqcgxgKomz+4Vt=x*hou0G&;p-v8fD~+ zI5-p=uV>0hBsuSq5bR7qwOYf)!VBlm)4y)=+7&@c!AAJ_@q@hox`yE=80{4+E20m0 z7j<%1dyRCT;OR8$m?Y`pWkZN*3U~UI{qM>jnR>eLH%yG0;Qy3Ym+Ru#&H3qz;LyQ1 z6z&5x-s;lW0#i+pjLx=EcD(bk9Dm$P(0CbD^a^wP56qpdszyxraWnpLwDwdM7$b3o zw|sT$A$HFvVng_yJuKwrhX3bZ^29+ydB28tj#JjZA9t6p(r&o+Hl6(6=JP+> z(0Ru9Wiya7Zy+Z1Pk@4U7+O*%{z0=i;>cwDW!$EC+w2=L1{rp>IAAjj?pRQVy?sKcz&)VefPGasy!9}pfuK{W2?+t%%0rekEL%`$LYoD`!V z#{X$#(7RZ=Z9Y^S#gG{%!vO z@4eP~U29#>3pD>)-XNp^6re990a3ge-7D%Yih)%=fD&lR#=i9`St9T(li1gFEtDTE zp34D72b_WOXSUEyXBpfhEemzI?+^MAmrEq$HAga{hY$#_!b&>A1ekc)7XEfW3J*d%bS%=XWouwmTozL2=zqey8g^ zP^7u+v_wU@Ao+UIn%Q%YyiEoH&8b4iu3$~DJ_N!)pIyx-&kbtt&12~ z6t2#IXP$_m);B_?%l>#T)qJ8=j`Xys8@LuFykADEt!5F4Y~K$czJOP1z<7=Zks z2o$S)efmrDoZzpGe9RhC`5i5|arRhyCVO8Y-i5^7*GC2)Y~61~%zG7&KG?p`i#2Ou zEe20-xb{~D{~|nrS26}~7+ceEo{=4f973fKJWBqr}t@q16gp1+Xn+m;#2vkGWFZ zd9eDehST2&qEi(H90<{z7`=&peC$tNxk5lE*HzM}O5;tBnp6eNbDEXxmbH6IJgCCk zUUD@i0c`H&^Dd7JF96Z$pS6uKvM9XnTgLEv|A+q2q~g}|rI4**{8QKMV*Y?Hyg=?G6yA@S5Tiz@|#(8--p}kU>3fp2J1g-uiv9HSYwEDvR zHPw~6g+`NBo96mhy`OkokbEZ0{j$)j@%Gtj+I4SPVghy~O=YkjE5zpP&%lhS{{>W5 zWRPeQyHtf&4bCn%t?S=$yLrF=?>-AF__hJRwxyq^Oh-6G(JHbcIbMtdTz0i|mh0^{ z>=l;y0yF>aBzgRI5AL;VF6i(3Hmb09B=Euhbky^4_T={Yu(k8~l=2Bf`nfWEy@UEe z<@2y77^3i5FukoI`2N{j;&YR7eG~J+{}veIlX&$8@o{hNi`?^G^U3LRCHThhR^t2l zIneX&>y4B1yuiFeN%{u=4>-K>)sTe)hcvs;bNO)s2$6FC62ucTN@2sPF2C_Wz5GRa zlcbJsKFSc{x@e*?gSSAVrOuTUF8T7PuDNyY5jSV58piLA5Gumj9utGB0_+tdv7Z(o z#s;45{SqQYz}4Gp+_3Jt4o<#-gksV+G_(OMGQy@9c%{~Z%S{hWDjwPp6^?Xr{)y5m z*{%?x2SLaFwE83z{L3VNs)`!N6sFZ^{KGvIwxF|{Hq|VWbcAW(D|~n>MM}02Qs+Cu zYuK8Fln%(jYM>WR(w>%ggR2$@`N} zD;9zl9uabsNRPRZ%DF9h5FP2vdF`gq0(|!B(PC5BTw-+ZiK5XfD{SY6-Qj{~8-;v$ z$Uy!Yk!Vu@j($;O{WNX!R|(z7sp&q)mIl zkbXI@uKzhr@8J6E;9eVip80al$ezHjD0_TvlfFT|eZAVcv#dPdo*aHaiCTJ{s^qX; zedjC}V#?aC@V?l5zjV`SOT3y_5VXy9<#iT0IqS->>&E{t7wo~j5kWXjtQgH&`G)gx zXs^iN;%MtM-)$sN{$t+vMFGG(_}TLH_gPau*m|kjViTSd@Lp?CO>qC}L7yv6 z_c0%{f^2@OC8!Jl@PP6L$@>NPZteUEOuzeT6an@}(JlK)5^E8yf1xGq*# zHJ>jvgm$0*!Go#8F3KHz=?*lb7kdg`(?Tl`%_yLG$a3hRu&sQY(yH!-c6Fj4`xTw| z4N#-1pv}SoD?#%|S7W1vVSA!#QND;VADSZnqIFACjes8J+%(Br))bZ?aHDZP#-+@4 zM#b(cW%6~o4BECv;6%a4=^|5q0~wV5vXVXWelaP{=-XS7p|%3?dGxd&m+i@?n020T ze_yRN-MiM+l-oj?=(HE{zh9Qm`}3TI-zHI3C$g&;`H);U)?nng+{Ww5)o-u4XE`BX zp-$n?D-taxmNO3Hu$==^4Kjzz8faL0SOSp=#lL!coFtW2AWM36U})#+&&`vn!8wd% zjpDl2)<0dIc8%13rF973f5f4&l`-ig6FT0>ulNam=&=9&^$-4ho;%H)s;z+IU~7jM z@O?uK0Vt6_Sz|e+)3R5D&=>(tSqXzOYBZe*!2i;e{!xG z<(!n;rVSnu;!`PQ9-bG*rSLb(?rDUDtQPZfQbt_SsW@e@K99m6Y09p= zF((FY=g{Qv_U_B+8T@Z)3UqRX#7VH}YF#hPGm~6jRN)tYDHXImRT}`JXtc7S=-aV&dX4cQZY?vbCh6D+xzpC@6ZE^iov@ zb%BfT|AA&hu0VyumlBeYa9#2FOO#ifmU>v%^h%I=V<93U@&G~mGB++B)txT`kc)aZ zgNQd0K-%N(UV49}|HxO9o^siNyni3|NIzQ-Xkxf87HLp_`iBcXItfaU5dZzcf?l|l zKKky``1A=e9gydWf*quE-%;yauB{y@0Jpw;Ju6MtDf}t8* zWa^%%V+{%?n_pYKrD6!^q?|dd(%VTppZU|OR*=nq)V@h;1(FC1HlXBbEJ_h2D*uhr zPudyKzcweB^fAJ#k}rzjq_xr(^cx+iKmXe8RmkF=!Fz?T>8AIXOm2lFaBNTc4cAWR zch!G=0^qOuFw!_UhjJv`5U`et~hWAB% z5_Ax;t8KsYGK)YJ?P zk()3Y(qR(?J_Orj{hINDkfjtE$}!|sa)KtSC6HvX>Ielw5QV_qPK@zube8qFe;20K z(NP18sI%LBK~D>5SZQ(z&@jhOfEms#b}MX*yf)^!a@C?-1~F8{gMwAkFEC)3(GI^_ z(m~CfRrOIv=u3e0pc!e#t9S(rwSXaQsARX7xO+%!OCe%eCP|!hrDJJq=L^-S^MZ;o z;fdydRCz<|oP=R%2!lntm((BY1ucg!(^vR$4l4QNt7qrmA7I9)Hs6C~oK`(qiF6fk zP}7|4S#Am32%B0YY4vWSR)kzK=pE!W>YSW(?9$5(T6Z1I$H>Dx@PF{z;92^6#|w^* zpFzLwD`y&eO=G5X1%EcJQTnw#f6wG~L1@|VAT$;69y+sY^1zO}9=H9_F4V8=wkau# zr>X+@9WK^*=MYs6CTOg`ytdC3EM?s<*N_R+Qa#n)M;{lpp8eL`biMF5ui~lkk!UZ! z@5Ejj#!WlFy**&_WYtF>4gl%;J|>-Xih>-i)$w)ODlQ7QROL@4_?xZcyjI;O-|lmh zdMX~5F&=jDe?1|9HZ7CP0c=u!)hDl{qSDuDmOXC|3iL(Xn@n>a#+Tiu9SXj;HlSc2{nMx`*|FQGi^{9CMeUmsUhgOv=1| z#q!!G^WbdBV%YSkU(LI`_){)G4{bCclT@jTV>I!@~?Q@ zEzg2`cmAq~7w`Q+yRN40ZowzO!7{yGL}K5|k1AewFOTod%eZ%w9ka(3;g4X_&h%W!LugW1zGW%Xi@F_953#Fb<&>S98i9hYjd%H^#(L$%(YlwEqn96IW#QTw0r{xUmCX!0xNtTK?@ zT;AkpDvRiMX9LckxRL1P4CNe#m&e`+w=+wS%_NPixiy^&!aMWK}|Mdui$0T3%Dp zo4Ne%QpRr+BD?2m0KDyzJ3ueDyUa7u)^k@bi8HB&wJKky-+1H}?zc4JHW7Ovzc8%*%B&PkYugP5wvp>j3GdiO% z9HLS&nh85zv`SN}Vbza}+ix0m)*4c#j9wkWcz?vz3wREsc%cHF6tDg6;m%AZiI@TbOM;%IA~8+BImz5iU}c z;7?%hWmcA{t->^9jWb!r+>ePG>r^K^QA8ir4z=WM-*(zy;MO8(22K>|)-pCpLa1HT z=#)`T)DK`di2s7X`qJ>|Sg@rO`v4RK_No{lfA}z$iCX2#I0;6R)-e10tQ%^li3F+Z z9Hl640}z&Rk0lHN?Q(dj1Z_XKV-bm1=t-(%Es(x-nvNyaA_Pk}GxTq>FKXx0^q7fE(KO|yrh;f}IMWEHOA z7S5e^5qBj9vomG8f$Z!MFc})F@H=;8VH_?&LI~r)D$cRe{8WLF;f!Q~WDvqP%_o7E z0#CMX%ve^iHiIvj`%voVtPBb1ugWUoxE4g&$yeu2)m`v=& z^m1Qk1=(D8!Ir{WN16lSoD524?Q*YLv;STCp8KuHBxjfHw*{^_yQOh=gO}*CTD~!j zoAWHN^}wy7hSPKQJU^ez>NS!j0@R)Qg+YqF;O~z~HYa)t`p#9`2YS?>juVu@wQEWD zRjm9`1}LT!7ANKWh>7HE%3!$9Kg;$si`m$-i75&nDsod9$dhDs^t$-?Hbbq>ow4vm zO_-z)AoAl)HcP9oR*B@;GIz0F*s)SM z(Lbp|_zWN@(DX@K95LZnM~GNBGCDm(>Y-D3kR(hWX6_9U&)?)Fv!b;IcU4h)-l5 zlZ*Y@B46o@-k*`(81UWyANjuo$8y3p3V97jhJmRGqCRMH)qNj%omuI);fp_Mcbb3D zbl)!t8T+lU%DStivGLbyWfR||DvTUP;?DjZM|1<3w$<}LMCRRvllf*Z7aJR-)pPo9 zL&GfW>`^BtQnK=q>-B4^Y2Pb)C(aqal@zlMa!403nwgtVbjW3(Sk*ULFaN%KM9u|l z4EbvYQ_&1tC!pBD%p&A3l}M!Hbm5$A;9&aB2)zbg_N418bS`HkOFArv;8cBfe|&g& z+`p4z)rg(VznvY!4tm~qX$@EGgK?sF~aw%2rkq+0=3F>T3)&mBPFxl zcYC^9s+Wfoa4iv`utu+uEc&Kvf_{>)JF(-fvWQap5lBggJ#xk+vr-2tC0OELRPk9c zs)%fvPR!M|e-6;;=Kr?mnZ1g11XbTW%i({Q79y5yOKP2sL zn{U*}Vqs`*7mSHE`}X%KNPN#}aWu>(Dl|*v;Aye^G)rdb56cf$);P>8Gaju}*huPpew;^hK~k zQeB}-Ws-ZAs56qAsl3V6+|et4JtO=G2sc{hd!Y@m1bYz}mR#snY*4e-^6zkZ!gYiC ziC0I(g}?o9+)60dDC65Kza=NkK9HM3+|kSouzAqPNEMBKD0egLJ0~Hb&)_)AFp6OP z4zf&HIa(yuF0i@t4Gj#92k}*1&!)N;B82n^{k1pQoJE5FKHT&0m0gS;**aBy!c7F7&$%!zxYdaku zGM`%OtMb>x_CQ=bWiRI7%64pi5i63l$jX07zSY6Kl#uevMX;j0QoNS??Py1}m*@)r zX`ggX*&Vr6%^3?!nkCJMcSPJjn-F>9CMk{iGH<7iMN$eehMT3m^VLd!G+fY=V~xv3 zz)R|@b!r#L#+mnMc#rRsJ=5!rz)cJs{F5ztT&~K2N-ciUG9{heV}WL&Z8Q1FWXdoS z9M5C{0yfQw3aA!-AW@P(7rz}IYFsaVUTMrO*@)<95Rzpw>>eDP{B#WNq?LN)a9S#X zyob|mj(d?%GbH>})sRXaskmn-li#R%sArW%?xS5q2#O6#K|!xo&C|7%L8zIoM~eLA zsf2iUlr!pQO0K1Ny_s;CLW0F({VqzKH`VCTFkNORgUSY-EV7{}=#nsAceQmvy*mblcdXG(Psl$cz~#@x?%FV>BM6+RY?gQ8^?R2tz&n3d2r$ zLke2N)eJXV^wYx0oW$quR{d_KE)&`Rcq+UI==I58q*?Hm&D9?LQkrIiR}1nCOf_ld zo8pB(P9(w=Mc)?P$;h%yFkOD};F*>M(5e)dBPpG-`hU518D|V%7TQd^GfYTTNnZRJ z8Q)8~slcY!lCNm>tO>6qX6Rp|*8ag$Z6|BN=<674m#h`+YLZ!dFHl_yfjOZe%SJzr`_YTqLG$ZS>3QVVn7SvR zEl4XEpRtP3FdOCr1fuv>0(BZiVK8C@v_zMQKz!5fI~`*~%-$o@ZG3dem0(0KnoZf? zzRXp=K@wdpRU^P4IKN0~5B(;(IdBn@fVYpDFYtWFEtSq=&1O@aieINMn5cCKpiZQZ zHz1_xKi^^#3dkg~mS(2hfZ)`W%*IKgmzUH3>ku5YTMjfGd0MV{T=3%prhxB^#G1^h zwS+r11Y5;o`VJvlA6BzjC{O8DaKr|%scZ303mv>BGb76oup;|=aA+3lW@MJZx&xSn z{Jo4ebqf9Mw@yr2ug;xX6ZM3`!VoNS=XRmTb&BeAt7dq%J0P@ELr56lN`E+=U_{60 z_x`ek786L=Q7$qnpa#h;eT~aJ9B4n0c8d8PEzaX4g3_O*j-fZrf&b7lVUWr75%2QU z|MYwAHnFHVXxYvvg5M2F;_Al=?=EKECt}!*&BTL?nFIxK#zw)zqamiZNW_X~`w?^t z3&6!DNfg0Q;O~RX9GmVI=SoHDT18e>)ja9fJ=PG0Lq8WwZ?Mn^scDz+0Aa(xS z+*A;GJ>%G&l2Cls$1f@gFO6aAQtLiTR)SYoa-&7`YML@`_*N>uTRIB4k@ua{2tTzL z-z5EQT9n;HloaNIO-u@nK$!`1uS*a!1V^gdZ$G-p+ zew640{(H8)|Dh24B2f_$jxmlg!P5#;kKNaSU$+E&qYx>GK2oFB6P+Lz<|9rbmfX&^ zzqdK|$UDl^%Ia1}Je&mp%=o7Hfj0hCKFWyi_Q(xzub_LE^$ovwEB@ThL(6YGqGSv% z5Req!p(gkX6KK6n8h-wxQlar@8g^<)y`+Bmd0^rp3y6`1G<4*nyz|emTtq{BZvS(i0cF3$q=4Fc0i{>o8yWZ zM9_ZH0jMGWMP=yVL?@4^EJyN^?5g>ji$15_i&U;rg7;Kae4dSnSc02sE54XzUgYdi zG3&b~`eH=ls7d0{6<6mRRGjn9Uj$*^tzR};74n`w|~{c9~Y(F7vb>t#24W!JhJf;h(|095=+iA z_HtYoWEa1?tX_sem93OhHv2Ee-UF_JjT+(s^U8i_N_};UTI#neXkBpM2^z9k1tEL` zmafvxD(5Fb;3;@=cC~YA4(6-^EUOBv!XtiBVruxpMqIrP+7vD$XFG4KS^zwYpL?iA0ki?Jc9yvrB^9~ zwHV~%{a*B5y!nFKTCi|$7Dw#1W0Vc(7}LG{CSW7%oi>zb9GjY%qgYB=nmSnmLSB!U z#Z1|ai?-+cL(8q&evG4yd-nT>G+vP>-49g7?BUY>)vbkP=wIRH(m&InN=>K{)8{z~ zorpl3RI4gbhjd1F|2_L#OIpvTTMK1o>Jj&Z2Niv9Y$~aBT6QH`@g=TaZ^89^yllS8 z;EFPQtjR(@wLS};I}v4pe>qZ10F5%^ZNXfaUtYnE0gz|@ThcmS`u*qsRh z`jSkT@XbJG@oJ3y>b$G$zw!G^hg!>9I>i%F6I%HA*6aOkss~<*e@*61xyBwP`Y8b= z;G0=vy4JMIwA_Gt<$72O;(6f|WYS=oBYDDdL!76H%<8%_Wj|e#Rb+E_N)8fQ`#4An zp}ZoX@sac1;E>yC!b|nb^4ju?@=Q61n*Ec_C04}`sU`N9gWR$Pkw3 zxVr zxi+Hx=gWy(#&U)T)>O4+0}u9)j3IoOtNp+#*9ErMZwN?QV_s?gVWJ~;0ud4k=jM!J zLDYp_!;o}iZnFG1O|dUbug8I3wZ3wgwf<2TnPE#?_`n%m^Opf`>B#=V71EOy{?>&pGNqHfwSL1H=LYQqmf*S2(@tTc#!1Q9|m~bZL$5g6L#@YW{#0!3daH zydv)1qp#o!_N+ni2;_eq+1U>P3f~m;sZbaE<2dXPQ}ee=D;X1g664$wPF2->%@JhB zDl&Na!Zt{y?q7Mw8Q&$}No9&0j;jf0x|<-Qs7&$CJr~9yTC~`#ecjrl`&g|euGF@( zO{B8TdOj;KAY#}j=}j|1WZO$?YVT8i0ylpQWD%Yw2+XT+_A_WOZPs($HA_&)Q7Hn# z3Nunmu=Q=pwv(5c!Gq?$2g~26#4TJxWIQqLwK&I-NDbqYj!+I_ThI_wUoQ=bgM?1` zvLhXB;%y37z7paWisFd@@HiG0n?ba3TM%kRh|ogToyxvA7*}x>ws3_ErTy4jwm7o5 z>BF4!HeBPCPHB%TG-dr?-#IKN_q3x_52EJX*j8@*JnH3`U<<}k)KmIz75P)tO;p@O zn_^*d%r!#n<3YH&`}3MQp#Vo*5i;D~(`Qpg(hQLkXPi{i=}(B)cel|%rKOM>3*G48 zQiF;Yqb;habMp>_kqTaz#^nGnJj69FC>NMjC{Uv-?Yh-G*gD5hG-*PoxJK2;Sj^8QgJ4MesU<@7%Dz-rkM`=e*X~u-^=^=nuKD29gpSHT+Iyb zv_T`LO%st~2JEK&ajn0b+ol7}rt8sq?N@9?=fOkE_SaecFo1i@m}Y^5&mPUw^{N@u zoEyomcV;Fx>$Kf^ndG12UKNk~2Kn71sy!7b&( zjU($q!GTEd%k0HZ3t&u}^YuTpnM&sjXXrKtz3^9mq6b-FyU72`(!{bnO{Ax}zFlQB ztFQ3V7phk7mt_h$@LGD1pQq{?81)zkW78wJI3U20Hl5hls8wa87(PRoN6V#gYNRvi z5WkJLJQ>*j5^7c*p&24uQpb%4QDVdDCKy%PCSHMzzbqSB^y5wvEr}|VrBikFOw*EU zbs1;w;3+U<$SV1F<5{t+T^J9Y)DNG%NGd=9QiW(TbFEJM>#8MLvyBHzKUp*^E*O&{ zG%c`dR~U_wu5ePEEY$U=C_q#2D}TA17#L`0Tgwoywb|t@UGt9B?M;m*_?UHzuhQh^ zX4p9P(etfqTqeeaQm7W|bgPYlZL<0-pMwzfoz*yyv^bh5pn?HMOJI?{vd%!GbB4M} znWi}rK%#;yXOFXIoI%!0odhr~gKyqOa$pR#lq~~^T%L2Uz=lPFW1wK{~c8|X*_OW2hILw z>0W?2vIyF&H+MwSrkily&!e#~*UDe*RF$yI3nmcpj$><+rDS4YNfm!0NX;uMsz79* zWt@d$>}byU#WK}tS~@-Ju4_f1T7*tyJQjx+-$l&PO1utP(=96E?0--*N2Y74y$S;? z;D^)_01zkRszurMVEH#-iN~F&9a)z$;;w4c(^{ojtKIzE_lB;x9k-8w(1*eZ9l-^a z#+Cf-)C}(D4I+$(7KqzsJlv7TR>Ye&cAzMUM4Nl$zdP{ z%;ck=ao4vYlh%iJ;z+fmTpq~)@f{@ecM0`>3)}|0!}nTF@DP^V&KI-)B;Zf?2LkG)N%c!gQ7oFI{6wT}Pf)W==@fUs$_F%Xm#T`@1%p?yt% z$-h$uce({KHZU$?0QE8L0kKhhUz4Y_#L~e#t_XdcnXloeVa8a8;yZhJm-KigIZ)m; zqta@$I61T9CKwsQ?3%DqOU*HrXL7Z?XF?P4fv!sf2OMu^VeVnrCekd zW$R5DTZXYs)s;+mly1-?t(7qMIY@al{M;xk+L$v?d#KV)_Kw+-p|-mZ6Z+xr85de& z)by+AF9(9G!h2qyfXY}$A|?trhGA4BlsC>6&kT1KK#bSJR~MeNN_4g;s$8#Q43RiV@G9bOB4k=O;J`BU zHZQ5z-LL#cMDtJ03iGtNeUxULk69>ApPKu36fpY4OaT+^4PN>Jt=fEkxA z2gmWXg_V2XEe`vWd_IZ{mIHg~osQ3MCpYr+NG{IT~(5Uye* zRh`Q)%G<(aR%=Jka_3WeX616JLN_a~e~n&yz`s)jD|VCPk;1xyD#R8PyMe6N8j&wg zF(0-F_3rkXA7u2yD_PVd`pK2YTT3^pQ4=XxCWcL7PbF`Z&R`_c3+;^PN&HOeo0 z2V<k0{nKBZ5y?wq zlDI58;B{_D$~h7zVnd1*R% z{8A1G65pfYbn@zP6^ph4rJ(^WR=HIXFalQ^XJSyKvD_+{6^o4Rf*XM*20rmHHkdk_ z?Z|2T@1-u5f@Lf}=PN()2|8&ST0#4b_}6*AAa7g=IuiY_r-x&Y<;j#I3tR>Kf`)H3 zCk|_Gib)!YL-W?u#C>>U9(q8XXX`((_G~(!~(Lc-?zsP)$Av4(K*eUV`qMamv zhx#+(UhD~4v%EvRkrNeE11!7(O|G~tXBtp-)7$_2q7Pfxgj41oo^_}ty>}*-9}>RT zQR5)_S~Yx~j@hM&c`D!7Iy!W0eKBAn|Irb?9|p!H>uh3p21gQ>BVJrXoTgK7 z+)3%bQ`K)lW$~m!l7R|7XqUn=EZ_n(1J63A^oEpjz6_=CAw00QhLpJ#tNa*+$h=yb z-P(#Nx)Ezce$3z08kTm7gK@^T4QbB5FoP}nz*gPEb}qp6|D1Rr)C+LZ!5aL)koUP1 zv*|X+d_hGe=Zei0l7yA|Q`ECGjZzrLkuxje!=e~O-huEo*tPX{Ui|bA4&r&MFh>i&jw~55Z3!|yC`JpV@g_(tY2t<6w=A-L|5na)Syej7 z26R3AV27LjzVfk|Rm!jO^EnS9`08Ky%k;vEYhr;h-t!HDcuw|Ub^_t7F71x=9!2pU z`|1rv88~OT?x>_`q+Z-nK!pTl<~zBuk**CjN-zj0Uv3Z&+DQ)zNFSY`if@c{h(~ZsDiN^pgweirx}GtNx0~Fm{nz*FR$rost7ys_Fy*6B^f( zJsu>ir{4h`DMt=EItZ9f3Y6KQxfLd8I$SZ^GkN;Ou)3`*QS-w`EaJrKOt?zra(Pw! zQQ6hxHe1c;-2d2m=MJ^ZaB-)&uHV=hQqxI>g8}dpQ}X0J7$V>5;D5OO9;mWA%1mx= zp}&%a+ppLq^!0(SsDS7PReGXjaM|WZ6ZZK-(%m8bnTS{>m_gp@*y{F^!NoyNO#BF3 zHzIpV&Qj8KD@2Bb%u0kurOh<$5p*)frcuZm{C5taoEh^GBN9{p0I^*Y(SmA338j$(^y()2?_Xt`ep!I26ZHhJgcyl@U@+<_2 zHc`X$R?P5&5RvEcP3zzF=W4`CF&B#Agma`8pT_V$yT;d_DI>hGAjc`#-3}?vAeusm z(}d*l98x%%?h==Wp>5=WC}Zeg@w@1}lIU*~qb)%Hf(#Mh$B1(BkJgd8iw~y}JuR_E zLs+hVjrf?K0Z#F@kZ`^%(Jxg@!-BY5-yivKzJm>7bRC~<1yrL%b%TL@-bslHtLAm1 zIJ5nzqL8fXinlTHX%cwXE^Lc`sr&CoSr^Xe)bA+FGt}d++bGukCLf-5GgxN}VO#!+21IFz=xc~-U}+>)NP6Nv;|?aB2i!17y?699pss;rNuU$PZ(Fa* zhIi(kwrU+HJ?3mVxK_TsVfHN2t}RRY1I#zEK-#Q!bW0AFtRcfB0ufP^{rjf4>dE6@ z5c$7{wt7p=G7;1kg8f~uuPc|}va*cZeQjF%>KQAYK=$Aek63vZqa@QQvy|C+_CL<) zFKJrS(=%`a3Fdmqb>>&{L!`nKn?ZxmY}NW*0UtZx+B?>5W$6^Z$!BHuB}IrWu?EFO zZQyOrM7i@t)!mG%Z$ho>)VuDyQGF%kAY}s&iL$*P7_)sZiTbJe9*#|kJY6SHLdMK2 zp^Di;si}L^poT0x!lEuA3VZLfR(~`d@-4$qLjrdQ) zx^U|*+0zD@I*rD}E!%8{kfRR9MHjJsvPkMMRWk3y1%XDoq+W`U z6-!X`U)O;S%FxCip%35Cj};LUmYKPfe2e`-j?i^2b*D8bzwj! z30%Hrd4X$XB-}bxR|-guUzvm%FEnxO9}Er2;H~*oo-JAQ1R3D5HR61Xez#=TB00nU zg_z?za@j=UYBezz>8`;CF=?j?pvv zo)TxD9;!q$+R@&r-3Q`v2_cj?Qlz1{J%^MbA6Rj{6qte^Z840x=zCG<5^WV-x6zV&A3-Mf0Wm}efIlY#7zkeTrtgf$ll3EQ*8IM#3AgFv=4Un?OsCuU zhvP9^%Z+jBwC(SIF~kWq}>Q(t;!qsvBte<-N$kj3M6 z+$pvAJd^j<8VXX==#2~w(Ow_!RazbkROyW!t)_gC@QKD&95=F@}J{*Z+%u=Nd7ndrMLW95uv~rp+u?VJ*EJ_vaY)xNnekIf?ShiiwJ!3OYPUH=7 zo#W#R(Zx#0Z8qZ4;?nS_FJ>Hj*r5iTz)2sCrFs}59A)!MeEb%YaBDeH1kzPU*HCO} z2=Pb}44cJ zPkz(ey|jVj{n>khTJ&|gu;vWt?S^R_?fi>6a&()TnKm>emw^T!qO||MstuijV|IO9 zbnEWmh?P=8qm;Xoi6mZIQh*Bk%5CKN2*x{ac^mpVwLn$;}IzaDMH9yP*Xv; zhdb3zcgQj`8%=VW%6RK}cutqu!xkNFfpNvxNGiemqqd=n7?FX2r>!y6gEuJ9yKBKz zO}F~+x(}2H@a}T>yyxMohwfkTKc2h7E>iMm6&slw;w&yX7QdK*LJ)--^ zG41bG%LZUbWhIscn8J3n+j*|Mf{CjKb_IhNK~%=dQtu5e^Spc3Pd5Ca9m;h}Fq)ZF z%%QPUghR92=9cA|U>@79b|9S|pUzK4g8oUn)!x<%$$F+X8}L2My_M#M2ov!ZfA{fm zEh6tjzJJ;n>n(1^>(N|DC?zw=@&SxL%h2+KWgsgz{#tRxgm-pZ^3cSy61Zk7VJj3_ zPz}dQa?Wn<)CzQke%lW&Jh0+8RpxkW;f1p`-biqv5|5XRDMTBKjZ-4Xz9$PP5XOo* zVD?XFoyb>a&Nuh`u?vY_yw^9mIQ!O& zNJAig%nR#JOwNHQ`wIT>gCOCun)jz$NNhK2;EE)Y%Bt7|6XVKUQ+A$DGqR#Ct;xYN zqg4d9^X5$06S9M7O9K-nQ+lA>AI}1~Lda5>w9U>4&;dcQ_|dGB0(Ya9ui^L5xikbE zQUlR-0Nqz6p+q5NfJb-mzeTlulJZ zgdIfV>Fr4b`#rEl3d???kms7NlYD0& zj^t3nJXk~h>Prq9oD%9_h*rz0shisD)HI&=Lx&w;qvh26)Kc4GsfyXTMVvZu8Q0&q z8N(ZTY+3Hru`%5F_C1`Fc2Unz*DZOvE-I&B_JKq^f$pv}=dcU|?BnqiR+e1kRsv|x zbfK-wrG86bcGAVw%S9VkZ+q)zY#8CZnEF#5Yp2hiL%IW|Pm?M3uQmB>Pj^vXs4qv- z=)J6il%ecKaq-ewgrY_2)eg>~ z+e|;7czzC(6FH=r&vgfev3+M7$H*i~#Q+xP$!DgseOHEgQQj|Paq(Q%mVSHsnf;bB zCH*`$K7kFC*S`J^^U^Mr${f>k)U^)NrVij3tyMNUOeWKwLT6_;>(GvLJ7qB@uSB*4 zb!s>hK%_TB-N<%MvF~n=Bf$RKY*(AJwb_3P#Uk8^DiRBEbPV;NNuCDFA>@x0P`*SP zV|OP)dngxCzIT-E`Mrkyu-Jcafc>BHIYc>T^Ow(;G25I)x|cdT6-A?1M(N@TYTt8^ zA&=8Hv)t69$qi9Ny8}oJP(NpalpX5qYQy$ytwyQ{cR5H6CK2nVz9Fl^d#GQwF#ooo zdBw-cuZ;G}4aN|whtSXuo7p8yP={a2ui|pEhGmXPJ(NYAny;~`HAh-cuXWESpM3Jk zC!b_zx5kLq3-;N3=vt8WTG3uy+OtNRx7+5^oLFfYpe9QCMK8=rNhBP#$w`nCL~UxU zB587GzyaUYX72(NY(rzduV<>HewDR+IDr!2D327Iz^x;(!XGky^677v*lXE$IpfxP z7?PIJ)B^0&ysooD&zFmMer5vSJb4V?xq1qZE{q~uDPv#PAU=HSUHIfJ_u$7i?8By* zmJDfUFY1;^93dJ$+q!%3)}9_5qEWjyoy2w;xLRyFQ>j3QNv=|#oS$Fhq`Qc{)HMe= z(Y_&(!maHc=;wRW$rz5#&!Ezv9&uR47*%2{YGY11xjOSH2rU_ICNZL|N@69nN=9ZL)4=dKsm1fyQVMJ^d`#05G_zfj%1L z0mLvqHjgUxzreobC6Br#EwUgXK&0fq+f<|@~F}B;#a3^lQZ5!V9&V6{tyN=-P@4Nwb+;tFlzxNQ{ z`JU@=`yG3+d*2r3mxLSCrt?uWH-u5Yl?LWpgK#$hnVY+jzMcc;Fz+WTc6zL%12t+i z&`K;{E`f!GEM>HUl2byZy#S1`s2YEoWIiqSpp01>t$}ErgAkd`e{w2wngSp7Mcn0o zlnoo(cq>Wyi3qTGE8p4A$NMP1yUHk2cAT>bV4nAsh8yLWT1U2!rF`a*i{?=7D4{x1 zM}0HLSsKKZ5al;qMPP{a+7!Sl<)|=Tg1g9E$AXBS388yFP6IcM;06b*B4sdy1m!mo zYey{yxPy5xkW=dMA0iKS<--LHiwDh|$;&bcR57+3H4t03P5Y zId26%u+G4|0}Y0{1)Uh)U?ruW>bAv5390r9a6U63Cd$71r$J}g| zX_FV_3a(ya+ZNg(6}q%KhD(<(BS&3QU8*5JUqxZ5B*$m8w*vA*A$IyWxbWjeQ4dmScw(;9zOo-Lf z0YdsC)448+^bmCdc^GG(71-BI@||AlptdGDoERbn&I73oH4oIP4Zh75;yv~& zr(m^_rw;`tTn-gZc zH_K@2puXyNsN2{l+#0&N(io;riN|$1a~(-`!wB=xGVM68YxTTMv}dL#v9!31{A!&c zh}ESAVi9>jCz(FyxFPndc!s(PTAlkcYYx+5Y zSi(WMs{IsQ11Ps_8^o<|IgEF`>n7a&&g=1xw^R1sei-k2&&~MBkGzxeco>_v_aTzf zY2Qpd(m--!3~dJ!NbcjjpW}q{?hXXs9z}zCR0qyzj?iS2rS>Gzo$NyUg$x2`1Blk* za2Hs<@fEC8R`{GUAV-CA1Hp*^QkT*gDUTpKvy6qgSuD*gqA*v4n`Qk^&7(430jN77 z)dadi)bp{V?Khb%X0Z}lK{;7O`^F|B?2~~gWl!C_MtvHqnO;dANg}?VW!TU_^!*8h z?&27-f%5^=B?0sW;<#aWJKlTfdi?nHhjHtc&4?zGm{?uL1EW{*Cl@c_E2E=WDHhEZ zYW;ZXlTSYRse0@>!47jX9c8NBfP^NMIo^xt;J zZMfyge)RUVA*fSd6CoN)5kx~_+W0jZI}J128H`H4aAi`L%CYD zrA!+J25i?R&vb;vjmDzT0TB|>wW9@J{ zhF~;=Qmtf%IIy>x63u3_=CLS?T!y!uTFJECK+t~Z(6YeQrDbGNDfD*t(P(T(I2faG z5Ta3AromS-0Egx*+L++-{2VS{zJzCv9mW6ni!b26fA!1w)31C1Pdxr87RM)1UCF{> zx=a@ROlwP9Kd*xro18$cRK)z;G%k-_rf#I++8#$J5<<0JwT;!KEu~^wYQ}4s0QqL@ zmlMg_wLnU8l}4z}3rQsOV@=f#D3A{JPUdwC@X%I{m5Rsb;bHPwjAmVwMm!NWUKff5 zl*u>U^9~ZIpno83y1mMDop8o~S?JnP9XagI3+UfW5$yy-HXW;+nu6hp^IH%>o2j?qV!F!$IzR$rV&eUS{{3*lhc z4x*@+8Z^*ZSN-u_4*(1!A`diPO9Ta&@iD^M&`FIzA&Djuh;y71C?vt^Xm7V=V>->2 z)j~7YM6xT2RDS{uTB4KBO=05xQOrL&i~L#2@gi$f4j@PCHqs1xdr%EW(B9jIsbf

AtBZHzbAjaXj{*@*%sAD_k5`=>De*bMTccTj%_v#vV+mW^kp2(#QP*=1b0auKJVcoFCR$0?lrFK2M>-piOc zc@?uOS23NNL?hts+v~dIrQR1llTeCmJC54fEKo#hW(u=m!$H?_UX@)*DM$v>|R^_^wZywFh}bx~(O&_N>FPV^-l-Al!S zN3FKi7^D$^?=U~@PnV`nZ|~?ZT_?az#ZX&li-dB;OHC)I&Ee{mdA7?q z9(dp+{_5*T@z`U>aQ^%SOwY_>Za(Lwr5sIHM$y(rMy-`jIw3$A_kP4CZ`nr#BI_WQ zxrHoFpB}|`zI_Z|{K5nH+Sl&GiIc~%xVXeIrbt~U{Roza?4urSYYR|ENLN!1LUnZa zHqq6cL@ds6k?gFp-{zNqiLq%+OpM~f#Y_15Hy+0y|Iv5xxz9a_C!cyA;}aJ#I(Ct1 zR;_H+s>Y)-^|O}2IyQEge%4k5>R;4RI#8(28tR9FNQ!fG*0ENX$05gJ%2tG9U?Ie4e^ZJW?N$YSBKs)kf9ERC%4Y4X15?%5s+D#d$XHEHHn8 zI(KD-osKQJk+R$49iWk8pPRchkNKkuxbTfLIQ>74K)HB zRpW%6Kf+Mtmm1sDVZZC`Z^!-vyYP-%Z^6ws-iV%#PRw2%!_h|`N2%y}dcGFe=gpOT z^2sNkeDdjyp=-2fpEp=(!ekbUhDToOt|q!+hX#?rJQ|Dk%eo%4#y5_Ab~-0Ve@*Z- z0nltf6KhRUHPC7~pc$^3M9@G|P+0^lsyueeq$c`IL*HLoW~ij}AFu3i5Z&XA)>1@K ziw~{3wch<W}a)8D|zITF&c~lHiEL{JcFR& z;^@UOI5ihXdb+WXfz~1l)D0(3ofl3>`?Qzw!Qy z%S(7}VV*^E@xHB_5oKB1n;{&?^x&P_4M=s zYSD-;QrA3s<~cYtWcPLtV?%En0yL7PU!-)Dmrld|v!|ZI)~#D{sJ9QpIvJOS%_QG_ z?$SjXsV)Y3x{zUzH{J86E?}s)8y|kh+wn8+_%VE7->vxAEw|yPufG`|*?%K`V(*RU zjiitY1#Evlo&Hy)u`^XzrGYkp)8ygVsd3zV!;LsJun`p+l;L_1?K1dj;F=mFX$h|m z=}Lqg1MO`+;YoKQ}gG} zjhpc*1F?TFgPM52cAK55U~;U$>n3*W)pD=|PQEaO(JPBMcyKei`l;6)*Oo!$Rw}r7 zaREcaeH>HTkxGWKx?I4;3(L6m)*JDzd$!|k@7jRd@7jpl-?9z+k8EXmwqV^>;Gu7y!11GJkey#fd!mO1ew^(WGSE!m zMyMG?D91rjDw_s+lg~f{ugp4Bc-d?N!n?wV-O-Nzp&pb%t8gxoulGkV_0Tlh?`%g< zNN&0A;5bnt@4qXMlVi?78i4yN zG$uJ#P{Nv*3Mh`|(PSBVc6XxISwTR?{oW*!Z_S{(h2z{z4l6VBs665#`cxY-=@cT} zG)fy`AW8Y?s$(@+K=qLV>)ej=L=L6NECvq^BXAQd7H2hs9P8Ky!HxiuH^&jZndMD8 zDA2f`p>gegJBw5~VasCCM_xoYo&`z<_^musLH>LNJ^NFrAEuG4gHfuB2-fB}-m|aG z#xeECC3J1;z;(ZP1cN^{g5G~PjE)a?BlY1Pq<^X(9X~OEa6;abWvrZVBmo8sLe}69q2iuWIk*`cnAgA8s)Wp6ALse88Ih=Vh zkFA>qusEN`=+$NH+_fEBwo{K<`a$GY3pjI12fT#%Oxu$r*$3Gt8zIb$H!(Rmfx+Q0 zHf>8{HMfj2#|t=eWEbv!?_S*f?md+I>u~$+8*$rPHsBV9U3lJvAayi|E zb6te@v}rWQ6>Xgt37|olkjBx;-2u+sDrJreoU4U+t^M=Pe&HOt_fjqnq!H|8*`)L} zr!#Jg)m~c)%GI8G`q~h1#y%VgFi&z??O$VPFz7w;UV(4er%o+*f@}L0maDeM-Ki6! zICx|W+S}SO&Hi!mBK7C3TMWRxd}+@1r@Q`!EhtoKIQHE0)Jap6|2FEd{Wy627F>5= z3w2u?>QolVl*9H*Aj&VrFs$ z-}&Zo%+D=j$Bx}NbYzE>Y2V)6SeljBZ6|vAX2+QWW8091~Zj(Y7apmG3N~94#XD?kKu;^ianMY^$O4R2jkX5c+?Ty!==F zX#Z#rx<9~iS$!jXm-}ocDMtLoehLD5rme)m_0Rz>R%R- zsAdrCrXJ`iqDj3NS=uM=uy-rmvBR(7w_492sdush-BRRVe+Lp5^4RlPd@qNlTSXqnq+6U#)`kyWxTa& z4KHxA@yt^%;L%5(!2S2#kNfU_01rO=Fdlj25epCT`hMPfK=IWhen9(>>- zJouo(gLvQph6nlldk^BV#~;J@zV|)*{@we&i~H`o56>ObFRACTxVXs53ozqUT9d!E z$>8@92V?#-!$8Y`1yqP>BK<81RaVGjVrB+f*wmEE0OJUJSD%+Rn!K&Tp$K9$dXo&jsdnri8N|B|?8O~79>K_t zZ74Mxn3odT3M#412bIq-zNKYlqyADdR1-O6%-gMP_&88jt*2xWfZe3 zQj+$r07@zbq|zyLvmAY)FpZu7jm4Cdkm-_irVWFg8T7Zep(mC>S2)A+We|&ymwKIb z?1(3@VQ|P!4?aCRjq#Ob^NQZPZ6A&3Fru`kqBNe&xOmm?I?c7emOhk9v>#j6ek{>F zsnrPclQF4ydQ18&644SlZ)uzM56otB7@wGgPUdxF%(L#&c?!v73W;cfIuoSUEH`wjCZlp1v12-0a9tmPu2@gR-j6uLSx2G;fV(%|cgnV~Cy zS^K%(c*9}bbI+Z4-}`RI-S0etx7>ahANcW`@e}WV3*LSA9XNdW5DnNA^{(Io0S|Rl zXe?^!wpnZ}lgg#@KN~ES%0QbPVb(Ed`%F1VZHQp_)*kfUu7fEO(B_L%SLUG;=Y;~x z1F!DZQOM`fC{wSy8cWE;fcHa=$b%smBzvg<%8fE|r5wuDGQzF{%muJGK96#-gjh0$ zK!QAtatx`GpX8y_7DL~Gb`0IU30tn;P5J3ZaGq&j6_7%1%EOLr9j2k0=}jZQm`Ck& z4Yg5@btuCb3bj@pdu7rE%NObhAhsoquKn%Ud_xcR9NCE6^a`r;93SN>D>>m9Utu|G zE{%BY2NzGG$!8k>BoG109-IgQl>!UnqOCmxmxlaubrn&zS*Vx&tS^dqZv>65AabdS zd85f=D4vW{w$w+N7t@N$!=J(+_UZKDFfZisR%HIUV!_iVEmC8>j&`dMo?D&G^6vj^O?8zX9*z^__QKZzo(24G$qj zd7_k~(x4>NsRya6yewfM`%p=n#t#bxSiueN$hXcX?=#P#m~5 zc18hyuOi*CyZlefuC4F3?y-eiZOl=D|E~|^7YkFWu?$3QI?S}gc|7V%Ghb=@kEI7)1*$0Aujz) zJtS}N!NCFh;a%l7{oIs(F0<})x`X_4c>m(80afvU$k#Hirx`#o9zgeEHK;KLTF^E_ESwoZ3_l1lq^3maKbwcRoXn zJ*nlxNpy~Mn&+?NO|6|wC(s#hM{pv9T6+V@-Xt=EDa3}^mwUn-FXIT+sq@&^;Bb^> zS)7!E_CsyxzM~6+H}~6NT!q;JYAba#Dh-Z*O^#zRW}!c}8(;M_gja5HM;E%UaWYpc z;n>+z`15an9iO}R&+)`F&mcnCy7l(k@xdQ^H-6%-TXEaot#&|;=@kivPd@qNlTSWf zL(;VD4JR3hlT%YzUCnY*$B85jf~pLe$}*M<^H|EyVyQTX<CiTUw#9 zv&v6~qQVNpGFA&qSj{h@Sjo}Q$s=FRB3oERj)wol)C3k57i<#bfqk0#b0F3L%z@M< zj;#SzlOF|MYNFOMZe{4Sf=C97>d^ZB!$@fez-ztJdQbOtPey{!SMw(1vljTL=gP}7 zp1tz=1&qMRTf zPEF5YY<>mB)jXnQ8d*#?!TURVyU~n>@!0GP?w=aNXrPV@^%9P*%;2$^D|mK!8cXFO ziVYVLR(z9A3f{C4GbrG}u~WFbFoh15bwjG1MiX_7EGZf8QcX0rRccS&h-+SuKMzfG z>LmeuY#Zk3Huc>zR^>q*4##O&Iv$wkI5d!HqzXJ_x_tu~J8)xp!NvRxFh3Q<#HAo+ zr)gXUDt3adwq}q8DnqlaJ!X6HWmlRg`DzQOEv1%CI$w&u^|_VeG$c zBlcdu2_u{P4IFeiCIy>S152Xb1f_E3j; z@fA7>XvR4PQ{F19TRm8Z(-uO<#x#bb!@!9qR_C+GkI|<3C)+0#ucJ7TwK{cF zB8*Bgg>v2j)dwo_Qe$MrZ_OjLRDEzjr@3x0xtyI{M5eQb!4dTvmW{lVYzyGizH8ZK z2NR)6*4`Q?tdH?pa-h~#dx`YJ6H{c+Y>{=NTm_mj zbsRXn9pWG*@Kv~w0e`GQINCrYAw3vGL#~Og+K2s4%})G z*(G57asyW`P$*d^GQ>_6Zp(Dy@}*JADNthE#X0KdSqIHpiWRPt-!9T|2OBqJaOCg? zTz?Do?k(LocvB}1+|r4|xAox0w@~LE>R}&9B9#(Ps-|;8)CT%lORYc^Z~Ni}s+>Qw zuL)?52=wD$I7A+I^LjfaYhw(nPtPH-9718$^9ViN zMD!V^^gQ!?jxGNTN4GJyXO?4MiDP#*gyIA>(#a+wuJ=TV7PXX1Q8j$eh zOY(2~I7q%s=Dhr3Y(J_!)`e{utZ~e$*biNEoTmgURg6#{92p+LUE6oyt{t4qY}ts7 z{heq_ur87bpM3JkC!c)!8=z~lj91GCPM{ZErxVQ z5}6JyC5iKS9H}-=1~N%rCy+=d>{mKnC)0eM(%v2sJJGA1>F6Z6SR#t{OuL=*s9*iN zySmL7HSYn=@_L-rbMQ-Ggx2;xty+$Ky%##?uJLBDinj46@}c_A<`?o)PBjU^W!M z(=<*>{OjvWp|7VO9n3GzGR>CixLj(O7v8bOQJh$u!}81wHU&a>$KLC(A(7#vJ&1{7 z0mo;japKBlOswc6y)3RQ&SPqc6L?yF?>=}VGD((!ab>fK8&&F)JiUTrk2wL4svu`AhCtk$ZSV!t0-T*1ltRh+*%j-F;62YNcpJI`?gNVEG- zp^E3q1xyrjG$8Z1vNVU&v*Wm&n@1%Y!f+&GdzUH?c{}R*-kH-FEi59!g6-`c#=*@y zY@b5yohGbLPDr#mX(Y)6R%=NAsBNhCXvBExGC&z3i4-`P#P8mM2bBXBB!+{Fr#9ceh%h0)kTLoruJ^dSeixf~5h8l9_21-ak&Mcg_M9_V>2hC{BJhlQQw%J?_#mfZ*o(-UOrHaaU1?9;qO0y*t zgLU-m=|X%f^+Bg>U-CC3j~9+-6D~rdWt3KODAV9AT*;wQcj0o}4hB+idRXUF72)v^ zI$q#dJ{LuvWv*Rh9v5nMT5qNvx5Gs0S=Q;~GL7*%`VX)&72wLTDTFpet-Nj_gp1#v zL*K3*WOil{>?98xtRve_Z6)cF4C^|mC4qL&(>v=we=tdp2@lt9WUOZ+0C}5gue3_8 z-{fX#c@?>Q9&rkj6W0Eq@&T1>#mJKcmSRZcFJzCz&6bXu|&NwHdbX{sUTNi zg6#kDnvKRN%areM%w=fGv68afK#b``<1YIM!nl9&~K$Y?w z@(wBq7J{fwP=B2+qx*6S9i=wZSF6ae4U6YXSgNg}ZEq*y`(i-1_G4pTNd>VSnn7|V zfv%I3yDKFW=BjM-1}bIBW!m%3QeLZ*D<~Y#qxwtL(%m_WM0^OPhhczra8*^*;8}a zvs++o8gp~2ICXjqH{Z5{^V%reYm#zMz>PQUu=n-6R!P9E#Zbrli2`OOgBYb=Z0m5ad0U&! znImkPYCVhG3gu@u&QH!gIWL``rQBUDVt$5YsypZ()}Gj8NHfAQCqOx#v3wR6ikO>r znKrPpT*1aoF$@lLnTKeUa+XXqkx11Mj#ntR4MdYo#Ms8Ewjg3j>WZ+&8Lgbrr^Xre zQ;q#3Q@X=|f@@#~%KZxCI;fK~W{86iL3Aufx8nb9z92xlY|YS+CA7BSvO#@9W16 zo44V>wrv>d>a;n4<{^&D`cw9z94x)LkxxGPjH{FF%C^)cLSsD^*OHVlC%>FnXfV|zQ-#+*&FNGR&F1uzE(cg`x-9@f zag?XZppEVq78W?sNTR>D&)zYROyjX$UjzD7PLbZV`~1}V3ffd(L20+jp!)*kUN7k# zTV|vJkOY8e`JK`c_@`(3)nC^-3_$mlmqH6<)H|w|%KD?^?UPS$G`-PapYCZ{lSp8E zxm3dH;vxn%ZPbmI?%VrO&)63H)DRF8c987oKwn=khP!&uok-CTZ{WFwN%rp$_I31Q zN2&*P7HcXDOe`!S#>sbIM*`iPWOpKlwoC@AG_J;JBu`C@;r!*3sM2s7q=9_HrmeVX z%XUu0SqA2pYt%5kyo{@tFXQx?b2xwQJeG5-=pE?A9XH>C>pHp-W4$6&G7+j8?L#M} zQfayvpP$3&bLVjB>^aO`9H$a4p^L`QU`HAeCy8c98&Sc8H4k1Mz z@9Jv9uHG(mb<=33Q9Ho6&Grlp<^UFFXK?EAN3bwGg(`X1mP}&%$VOb()`g@KvJLOc z)h6 zfO#@rK0l8=2Zj-grZ6>0qjO=F2Ji^mD}+%Rg5@F&*aKS;qp_FCBvGpdY4jE_IXQ(3 z7p7<+>x9cVcI@uKu6=3pOITwA>2#3eLlmpaF3z1ljmwuuQ7ScPe2cSH8nh7%4AS6A z#0|jf>*+m$@TCBX(?zUI&ZAPuBE_nPb?R!ZTu0<0$Dhk9NbHEC@6LWW9W?Bduf|a;6mkB{CFEvv$XAP~r(LvfN&%bY=|jHH@!duQZ5unS zT&oiWL`D)wZco#&Z({X@0(v)fB03U9CB?Cp_o87IME^8rXc51Aq>I)U@AIWc-$&m2 z(rY@HKub(DS4qS>Fm@%2OP6NtFrmfe8FY5`qpOQ$VBeC?3I;TvAs>qg%I6e%`ja?# zcoX$c%sj*dN@_~t@=JErMWx6&kQ21DE?z!kd-WZ;{vi8P7yFF*BM{S}7$X_ROH(fD zxN>D2$4^{j-;{S~)j+)VjxgHW6V$oXo&1wZ)n0be%e8vdKs;@iV4h|yfMlAI$8u{DgY>Au-)*_O|0 z*={PSgN!s+?8K=PmoPoev4Z(7uPm7iY72$Ip?(YuB`6=%FAm4J4u`rugyrQrWo{nh zW3$+{ZP-q6j;HGe-VG0T7*D#pIliR{vNb#YeTpvYBHxf1UVj9m84&UQ1e<#zV#^^MRKRT2s^v`F+^Krh=x}; z?H|qM3$%eMtb3ScOogIIP@Zf*U!Q#P$tRzD`VmQU7#mwjuNUkKEG;c*=D|T)zq+iK zhGR5>H3QRrAR+U~CvgI5hQF4R=}(i-iY5h`AaMf9zYQBU;$t8AS@WpbyYo8Sc=#r7 z0Op@;S@A*hSEVEHQB3d{q3SAa#$+VbL2+7uh2n}-_7*c=FD)-G8*fsnl%?CE*0s{q zbt@n7QXrs6-^DjkI-RyppM3h;Aj#q5k3Y^nw!yq}C9?{VNaUsd(E{=W47RRY@ze(- z!-}u_)|dH?^(KB_T%0$9`GzAm*@=DXv+9pJ=~X>IhdcyAo(HU!_#^3gb_}pp^fIX7)qW-g<2hTt)pvdIeh?88uMB% z6bv&Tm4cJ_N~4al);%>?H}+538>rMOW|9arOfg+86bq0KfvOe?^Ewf;<-FRDPp8W2 zV264!k9abTL_C3rT*;VziS;V5%vJ6A$2@cjst$jU9!RmiZT#-AEaHcFRU$7J%T(mK zC3MF!h%-);b zs3Dz@2b7D|Y}U#r?+M971JvyTWEeNHHZG}5o&m})G_(XAd%24A$twwr5uexDy7jdm zSYmv@N;I{`&Q<%87y2QHG88H?UY=uKp@BxYVV+Yha4FyfQIR(x%Mos{F65<#v*s$L zrP_q+P~V3IFfAvMynFjwVx}9#gtMZ7V->)*?l$PSw_-@cZ zODju_{i5KaSfEj#(ho+e7u%E72&JN|CZDSg0(WbaziHn3Y5+?Qj3ab6koviH1Y?vx zE!*RpYIgyk0k(UvM1EB&sB0Xy19U`K1!tGfYoyrOo_rRg%!3%9sPZM<+lKxjZ8qR}JVheZ3zVxO+n`u-5v2}oOQ$H))JNob zrB-4(dFJc&nMkD0IP$U#vnL4nsjwV6xQy*ZeG=Bbnaor9x9VmxPtB<1_)Rxr9=SF1 zWNk79#-!|4YrJMT*n9|N1IV7t;_|mIV`aIBeRuCd$KDJYDW*fVs{iSP@;cLW$%{xd z4wq%uzKZfh)j229r3LmGowzKWAB$=)V*!bj;n@Y|Pd%JW*}khG@_u!h{GdD}m`5;{ zLWJ!tKu0AfD@dw~Hx^lcSZiOUFH&y^3Z%Dq(&D_5tXdmuywSYNGy$(EkK}EZ?V3)u zQD0M^Y41Ji@I*p;y(K7*F6-c$US+XSDCO+C7@&76)f{zH2H_C(BE>=av%OtEAd{Dr z*>YJw|_<<%TnQc~p@lWZ_ zUP;%)vrk?;7p`33a~Bt{p2z8P$5G(Ksoc~FcXD^HaN<(tfJt#Y>=HJQhF-0 zCM$}g{rr@t){fcHk9#;aelwW+se@-m`k=4LQ5w84NNO^yW$*@@kJCn6)3*YZXY@K<9W zuj$u&>M!*X0V_HfLi@i-)Mb?0x#Uo*~v+pO_ z&*HRl9p)v_%c+#in3Msndn^W;j2nuDk!Ji38VT~~qee2%5<6%_^)o9&W#zjv2Kg|} zx+jxS#C75-6waUxay}el0V|N$UJtejt(;MIE~hX0bp{Nkj){(x=w>jCRM}^!;rqB&SyP> z0t7sbY3-#eyJjBAM1)3^+J;7_Q$;vNLq>**Etiu~=XobdA4^ZGks>twWw^%Wjlydk z>>r(D)O8%dta-YK=xEqnk|H03?G@o+iS*Y#j9RHQ8louw4Yw z$SXa_`Ug|&2P}t^^g<{}Ibb~ndKhS_@kcltlMbZSN9qRjHB$jZIYtK4jIX>REOCSe zDCJUK?ur{~CvPa!GAbkKB(IrXi~~Cx(N3CG+bJJf>meP?kf1E+B*{3(5E<%Bj}g(x zWf{*wqeMe_wu0Uls!%qfT81hwO)Zy|9JDcQmQN;hNPU~1O1Cw=AqUnzCFLXo zeb%nkrmv#bd)}3FnC1+gPSc)eQI<`*#BGEq(>lz7x`Q$((5hHjwl>%4)hpQ?=I57j z@w7nQFg9+@U}!|(5E;j8tL9}FL@ryx$rD#7j#cd5+mG(v44aHxCBGsud7Da~L~M>C zuupsE87Sju8E#^EV`X;`2!+fhxSyhAa{e;fIf38TsW6k&RbA7cnbsk53DzT>T& z(CDMGdZce2jBj$x`vRCm^7hl8N9xmF+p6BunU$K_g3qbP>YR@@7*fd;%PuKpsrgK& zhU-TTc~Gi@sIHpR35>Nl9OYW_p?C?lM~pfl%r>$5Q8py^%*NzNGL(?4TR#w=8hj2L zBN8zkT)}o={S;T9`q4*vEs>zUV_(wy6a~tIe$?RD8sHb!Jf8=~C#IRA4&2aEZs7FE z37kKF1w+H_*t2gl<(B1Q`xs!Sw$NUe)Dz8MiG52lt>x!o1J+x5(bLZ%q|?&>%1U6r z^s@L+N6d*aE+1$+#Rs0`BQY9lSfcQWJ;bxMSF)fOSzQBc64$rB68X7q^PHG(SWOEAz{ z&0|t-Ra3vUF;3Oe0KoC5!gA@y4eNXS)A(uf>XT1C`Q(#NKO$)kZ>*xX*{0)#Esk3c7RHzV%(CI!4r(czNQq4!XW8sl;ya6B^mN8cUW>jK zL%mtH9f6xgyPJ%=7WZP;=&%=~`|ZEB*eOP^jKDq31!KPUS2^*z5_P5t_XAK`Xy*(W zGMkUZ>FCkjs)CzCq9d_(dX*rHn7v4G-($b}Li5JeT@#?%R(iln@Cy^=WrbXxSWw;I zT{byXbMnpDHEC16~=vM5Yn8WLFDVu}~cfNHgy= zOg^~L#!(AjMdui2O7Zm2zr^OrA+k7Q18a0Cx12vmC|hmHs@kh+3>goRwY@z7@<0N- zl7xS2vBTRi|9rWc%yaHFFf+u}<c;6OorLao!Yroa?dvcxh6*zLcCkPlDSZ^JZ`ep=KI@| zZ)vU?K~5bhHjU-_DWmdEZ(ENeh!d;W6av;KY(MKydm9oPIK zQ^gD19ESm!Q$jOt5*}&Bi|$yRR-#{u&w0kEsVIma!J{zHNowRnleWWzx`th;P2sJ5 ze;+r_I)Y(WmPH4$=(G`jZlC%1ON!*>$rb|hM`bOTJ;rUGmTOY-9{LyZnJQ_5l2$9s zK^bdCtSLXs*ou9PNG-FFPit{=-qVdJ@UM+L{;^e4us>65*iH%r`@*-_zt7%Z(pU^? z-_)xOGtAO^@5ZwsXV-qAnDm8`F#6?y7u}8dQ{1qO2q)^6(lTlibZE^Ey zsdl-CX~+i66>q&^SQy#L?8duBx)Uc)TC|RG#mdX2O$8C74+BPX?+P{+gdW&{BBK>} zdl@6h`;pJ3U%9kJ8h?XFG=~;J2i%KY{Z^;0B&7dqW$8YVhxU3f?mr3B;hc+)G%#nRn%R}K#b7mp_&9rEdFhp2u(JR4TeC@yl?q7CXG87c*tpy2C?kF^ z9e0)ciR9R7n4DwTm3;zqj?E(Gv{xzeEr;L zgCrLCIqu(inCpf1xc-auqT|spo7btg9qfx^%ifjJmCj(4)0|h(Xw<4KZ6P*cirnSY>BC*GCwI_H%Q3 zFbI-{0WyYcD1uy}kY)RGo$DTU5rJOy{Wp?ZMyi#Li48e@wL3@8%d`rn%1|=JAkWlS zWfD&)>}P)`AzykEW)p13lW0aVWpFCD8p?qB+rEeO7}t5k{O3?QK>_T0Efni8Dr4H8fA6q<2=odU15HaBBz%*FNY%}W-7xN9jgO0Si2Bm zZ2`sdM$SX3mtkEuKV2(H&@tYqiBmfuVq|OHu#zEb9SG++6wTqGmiioXZzT{4<#ZF> z9!-N0S&T9BDQNLGFN_d%oPTV8whlXeYZd_;V`)#QF@J1pn2ob9mI5e=`U#O|b{H*HcR#{~xn1{md7hC!V`!<&L(}=a)Bh@B9K!rMZ0h-w8 zvH-D75d-sd=u8nW1r7Uq0BULj%JwnPlG8l%{Y~p8u8+3%X;fd0AbrSFGu#wt z%$A!vNcEcU$r9$(lv5W?P+|CM;*M*`J&2I4lkDweYyI0*RB|Qu_`2-qyF?Lg_jWtj zPMLlbXo`*?CK>xh*Bjj!Ovb?EuEdouwgUR)U^fz*=6gh)z15(gFkpcM+SNq^J|+#9 zXA*~Q4%uUff+jmRVhwYC1pN%f6`fRwSWgt_x&r#Ne*|H9ap`z8VRWkf2?n9K?+c?` zQRcAYpr9JH4>RxJ<#CZte-v}BrR%Vvro^;*1wie_1EIjKMm=aJD(0tjP&F&G%R_cA z0lj--@-k)3!}5(W!7S!PLPwoX4Cj(L-!c+o6F259Jo12=mdO`jx2G1 zJGzx7#L$n+);V$^EIgBp4g9%-PJ$4KVTC+_YQ|(=Z%R#1S%O|`=+B9$f^||i20@6$ z@XQW+hUln$a4Y88p1G)F-=fG{U%LL-nx;WPePoOEmwLb+l^6H_{A_z_05_Rn@lJ$4 zSQwYg0v2989X|Ma*Oc3N&1bs3@*8r+f?wz#U>(}kD{c2Q756m-%4C_2LDE^9mD@GE%2`+AgvO_#ZfCF?vv5OuKDYjNBhnlr>Uti`>?cU5tY)X6I^9@BJrI}p zxOrT8!(l*e0kW{JFn@{I6}N4WnUm2o|M7X_NoKz#JJb5cR|`My{YUKJjz@CIe5zBe zHM|RaafrDMv+XtYFDwl7%rXlggpJUQb>Y0R?&h6=P%Gp7l=uEsS)jEuuO@QB;iI_4 zdAummha&Jv8g$5>W^>H3*IwA5)C3P&eQZv1YzPTd_1(ieHhcqOTc#~0yxmj-nC9Zi znG5y0>I_>wF*lzEd9DXUUO9Ykc|N$lx9;OXKh&)-b%OUkXIoRJw*oWo~G+jh;t=C}&Buo7Lda>hvLXE2|R7fXM3heW9YKHaz8f zg9hQa`}t8vp|`vi5u^^SHfGQJdQzOng?;yq%jdg;e7jmJkU!2n{KP<7DxTBH8T~)c zK4yer`OwEb3-VdcFNxak&Q3^k+- z!qwJa*)ib&jurmCBx=u+eXJ1%y7vAU5B8|2l24a*udwH1!wR7WT3pgvv}nd^yG5WV2q-@SIcKW3bRsE1Nnb2U znfchzuc%R2kaigAPF61ik&ed0CLVt*Z<$_%NxCA$eJf8D#{k*CAsk2o(s*2SuFcG9 zz6s^kLV!r@$*RCpWNDNx+>*%%-}_x|R`igkQ<3k3A3=Jf5ivhZikpJ2o2RNtB17eI zKorWOx^@LL@AsMjPZw9&>4%Dr@@T|maWV>(z3L|MkZ@nU{kte#UR}% zh1}4Is>_efllr%u`Bj%1T1_)d;6M*)tmF9W6mJ`vy$(llwu$NJ?NGY?p%TY`CM{oU zA#@uidb6al5+&^5u!P00gHagRg(Zr;WKd6y-nCWFHRRV!vN`NycT`rC!mKVxhP*Cl zkCzO;A)1u%=n5|K&bYVK9|T4rYSS%gGv31N{lYW2!4JnDfp3h2sOEj9-R&5EDObQu zWghKZXn=NCc#Qv*h|a8&5>23!FJx=r$xH)xl=*zEKBYUQcV1ayXnzylCYGxUrCwkw z(WKC~FjAQ~(Y9WqCR*FCW~1}l6=sD>A!!b0(ue9VRE=QGZ?!3oKAL^%$xfEM2=4*} z`dssM_u)sFXxX2z8|5(+H0laTf{7RN*@I{=#-y``g*R0#Ia>Su{B8n> zsAH7z(|XQ~>k?lisOQxzP=DHcZ7wp)5w<^v3wDW7BaJ{gs{1DAq@p3ipoGwAz;2ic zokUYS8FKu7|82gohO@`jlnwgEpNlz96Of-?_I*6+KDF7Nl8!f)W5nxI(VJDVB0*`6kV_rdwlP#urE^3;lyE_H#LMn}LP}!O;S;Kt5tmh&=+X<@aNi zAt^B^ppb9HpHX%1>p?7RnNDfGL-Fm79h1PbPY1?TLebl3>vq%tTrIhmilW0%9oCV~ zBJARIQ5|`9p;TFY^%~-ovc`m^O$U4I^idJW$d?a`I)d@$K%_(AYjb#y>WAO<)-{JN z&I6`!ZoQu3&#HyNI&{a(O=W?_D6p|%xuD-e_A!w86omSB&TC_Isi`F7|`=NaGFEu;QS*V?GcRmhqNkSw%p+eV*rgPF~`Jv z$@M9>;v87$Q`XR|8B*_GB!`Gw5?xLcL@YDm5`tY~UXU{=E4$Hz~YnhECRn1HaZ{$TXJ5;!{q(m-XX@^>eNDOF9VDOx#V4~)LPz9vq!eO2!P%$kyh-YI7@1C=7TC1K_zm`fC#OnT`*a>EX$r%vmzM&$ZbfamBRO~EypWY$1R?%9}l43Dw);5*rIV9MA)h2wk)x0 zLWnuU8<|s;H|)Sm848b9$~3l*Tf}O?D~ni!_Ms>O%;c)3f_%3SiMGtV$k{(+*idQX zcN%Y02qAw{BK8My8hfP468f1(B7cvMU^H|udX!N5W=H@vz8ACTIn9ArNFJ+)qC3gR z*1w(JtHzEnNr6rPBuFUgyBRPgv`R+KN@u65B=!T8Ia#it5@m)LcboUxF z?gATvM$Z7d<+Ze^m3f3m5rVD5A`Xj};kSuc51+oSjC&5S0czWft6_?wAQ=a23X$hb zx33dCFxCudrG2d`;bwk$un--@%1bX)@+ezswlCL|#5X&U^M~_31Ni&29T+vjd)H#9 zrydvAvy8FQp5~j+q|bQ9BDJBv^~*lOOPK2{mYM1w1MjI)DF=BKBqsR!$$nBjCp!~i zIpS!NaTL-qjx_4()>MH}i)JKeXTFsAE(SS_bRws2?(#&aemeY;^fsrklGYatYF!68 zi`mrXiI_Ev6g^hmWiNHwJ!)XwlUEd`l`jPvI_YUXGzFRXSr~52YNd)CIABuxPzmkc zJ9y;ZB-J&M9AQUn5_>*Hb6HS&-imlIsDG$fI(+as1x)Sb4v}7^PjoLjHv?#@cTP2<1buK#iqzoqLmyb7e6R)`6p` z{IJOtuk$E&mIuX@Qd|bLU#Kj#%2EQe*Oeh`v+1ky)}e+AX{{SO7W<@_Lu% ztDp2@3pKI1g+Th4D#`nu8xvR>6Q6IEUQ6b=Qc;t5_}9_$At_uHdAEa$#x&&y?wk#k z4Q;5Wpk7e+%r<8?I!KNUl%Xp9wDCZ$=A|rpD!z2E{AMQ-`67t_9=3G z9(WH1@kC!&{QW20@$n9xs4+!f$zpOPh_S8exznXHWjQ0W{h~Mx&~&V4T+Y`O2;Tem(;PpwmhCQRQujnxNnEx+do&HbULhYw_o~Pex0JdHgj^q?7AlE3h=sUg=b`B zJisk_^?EpCukOsq0PIc-#t=C;_I=7Dfw+D++1sNi3*F;X>9?95{dqvdA>Dj`Xf7-% z8M=8H>jKLQet>V?hXghP!bp9WANakdyf%XJ^U1*vjvw52?e})7)$||d zsb^Xmn)sHZx25e^fr}c?Qz7Og-@OEBd&I`k zR^FX{eg1Fn#&5%~cHY?37~me*>3*KZ zq!k}{8$TwKjtzNwo@AUYACnYl6O8oGdBTIRkR+1(6_u_V=d-)5C}w zrAXsYdi1ydxtsIp)%8D48}6T6<+_xQpF*$cs%4plGul!)gZU3MGpuc#`;}}t27rEb zv|`Jl_+jg*9EjbS^yFVsR8oD}p{~5aRU1^?_LF4m+hkZNABwOGo;~fhP^-w)`uzw){T0vtUbk7f!7)dQT%mq@`qq+23y zB_F}Jhf_pb1=M{|cN%e@;$Z|=G~ZO;l1sWJeQa5KM4ppGY7gZO|FycnX_RM=DvK6f zRCq%{X*#A{o~_ht39kc%aFva7W%A-SJg$1+V?^ZipvvlUg?&~pkipo-pZ#=o+I(SS zas_UK%8=fl+w#2d17EVdxHf-M&5bnL+(2LVn*H2^It(#!kXqvRApT-G2layQwpV<+ zoJ6HKw{Y$wS@N%zn(Jh?ieZ~Vd92-cIIhEiO68nDgkshyoO8qyYS*UhrbT~sJ26Jj z0xo>jb499Y)n{AT@>Ze6{62;9c@UD~$($wM*rPm?3pT&Kgs`>=d6+klC7e#5|-O?b!FhlJjW?{?CXCa2mk%JJ0R07bQByB|NU<|BIt$@F;A=Ln@p5SsHvCN zsuF09-lt?TTf8y1zJtk}I4qk^Y#U*;k)Pjdm5p~cplX8v%jW!%$}bBE^{xOm*p^eb z5!^dgUA5n_$`<$Ww~#8FM-u|6W8NaE!O z9`ds{UXA;BirdBiyg}D zP+#@iYW21{=p{KIY4h#k7peq~`-)hsw3O7<#$)Hw0~>fcOxThXXH;SWzJ9kP({?%#&@5ui^2{4ZZ1{h z1UmB20bE(s=&3A;qoQp#UHqUKoDA=#q)!qOT_vcY%q)O%`p46X@!Kh?#-9xn-c`JO%sQXY@xy9Boe)aLrbb|?sx*+G{M(mw zif(n0{-^)GYyWk+v&^@Hda#*7_QiS99%Xu z7_7+&k;(CvB0_Lc(C!QB>%XkhN_J=Y)1H?kDwUk6lc@=VzGD2*tynR1TGiLd>g!V2 zw0a>z9M8QNxmr_(lZ* zMdlATZOu8&cX*NJ(#DtVb@Acjc5&RfZja#)k5cdY;B!C8=f-fh+Gqzn_hGPJT;16z z$ha=1F5yrbd7|wDp?GHU(L$$=mgxjIbCe6c92Htxf*zogRj1dbmFxexvi7A$|B9>z z`~rdZXk$zi#+)C|*Q|CJ+oL0Ng)3a|FG~DgN+#?Di^>4595*9BN_cg@)B6GbHNa>; zU2b-5j_2uhcsA;=RJ1qtYs0}#w5z+PsE!4ufTE*&6~AKRD%BHN9Ie$`#nD%9j3F?H z7xk8+=q@r54sX$m7i};)49s<$NU1B}O5IKWk$BW`EBB07Rp-=v^W5qYfNi|FG&gqM zB!VV<{-aw`7-xAyEzVvami0hbu9*WJSeK~oeR&%;p6hPnN3d#zP?HbM3blI6Tp9D% zfpU(Mh>iHp?v&)#-txlcfy{!mo-?mWZ>D%(w_%LaRVE&7cDmEYaxmWM zsUH)K&kG$%Oo4IGd*>YjDnSdz^c;nVYkwx6fd8)7Kiai@iAhL8{c7M?3{xE9wkPO3 z3JRKg?Jr7>MVt44sZPTO$$k~uq;V=&=%T6NxqLXm?y;+s)dKgWrtp5gGRq{A)h&Hk zixiO;r^i+R{O4fme?P8*AUAQ%K*{S^DgE}fkw0d`P}lgc0?^OrAt@Lq!8o7TuiaYD273qK%xUdsn(d|AOrZSy@S*#-J-SF|c+Isg{ zu600O)l=7qw+p8glc6aX^e0#t$72|dO;84&Q&Q7XgBO`Z9I}~%<{jS_9W$uQR+E(L zr*b_{=)vHrk9NM_;)CpfB6EaSo{xjg^p0!Jpo=iSr?Ah0@S124W63)`G+LkIXyk~F zGJ6N{Uxwc&`}pg+K6JX};D~>5$2pENnOPRtyfn!~6;5$r2FrK?LW59Fo-D-3<47k| zq$vN9Lym=&HOt#PUY)QiK*a<{`jU%Lbw4d%{p*A8bDH1E?=pa|ob;ihtX19M|j(a2UeX#Pw`?wZ)mo}T87jV=9 z!C%7lGvoSwvEZ*X<@!3AnGGJbKcQ~R-n9Dk?5NtY+_jgiTAbJ}%1k(ngTaiGo;g6y zi*cqGD*$K9ZH%a3K3F{)@tHa z^&Eq#1lJe;bE1$b&%Xeo02tp|>R>bSz192ymFcSmGjdZDY+ibp1K;amiY>P#?V;+p zE)$4*C$3g7S;!OcV^6*xt=p(Ub(kYo={E|EahR@b81EcvUNP!W5cv*f9!hB;Nm)K& zO-3gglu2tJ*Kb)thcS{;otkt%_WE$oaCfYI+s07++F&Sy>2mS`sA6r-95f;dl34Tm zZfJDDtN(s%!9#X34hom;YE_0h7e(4Ph0oLih<$Hi3_bSQ&`5MIDR}h5>bLs%ym~^ z|C3$WHbEc&B9vfc6+{|@k&(77)&S-`TGn2lj7(L^DWmJW0@w=~nPs!M( z%TA7)#m?>3dz6!&%FLn7PDw+F;E)(q&U^GMsTq|XR0vTgDJV+i{J6M$ugYKRdj0i* z=640}=f4S_*yJ2k|3#AoPtvZgLiBSzXZma^RqXI^itk8Fir=pr{xQoRt-HpfQjgj? zqEZ;iI>NC^&6v;|I5#c|fH<8lOAZiS&irX;*S!fJRAM877}T!a~Wm7%C_ zI+?v{Q;f@Tjg(Xu?hyafPfr>yzVc8gTEfAG<0_btjS_pU0)gnaya(J@PG{^Ov{7lK zQ+`60C_tc145b-WdBu~WH0eJofSjc_9$Q>z?ESwlOn1;FvNtf=XSHXfJQV{? z{Tm9kD&qjVfMIV@(pLgu(6^tdt2PT#MaE`MmWU(Pgk?X_k)l-fyV&9q#&n_$Cdivr zzEz41c(sO;rOdTv!0LB$MkgfLF!W?D91_2%R%zuFxvhetreZhfahKdC@`O5Fdw)+6 zOqp9JCSfWLvMi|_5dMYXfDyi%7I*)wd?wTk&SOgEoEq)5KIi3h2a`KV*GlsG_h*EC z=B{%B&o+2@vuWQHUcrdT&9l>*@^%(5^C$vKrL@ytOKdbEMcb+0raV~-bE}|IWJ9?D z@URSNmMFH|xm8||7hD#z^u4-O6`VLyyeH)i|L$NX24#xM>gatYUgkzRZ-CkZOPW(0 zc?_rqjoDA)q%ZOL+mvyFrb-{0oh0s8-@h@>xfS>D@GTjI3g?KO)>FqVb2C|`W)7rz za7Qqv5F2KH8Qjztj;Rfi)OZq}N>4-f5cJx|>hyj=d4JvRdpffVyDH+@=x=TqD&?N3 zsObQ%kbog^R?s%|(+AV0a=Sra_`^j6e?a(wZy<@Yb9zwM#1c#zHVs;J2g}N2i3vMn z?V^zB>U{S4AQhg*+sDdxiy%RKQ=wxLC1q{|#0f$gp9e zQx(kqv?;wGTe{vvBOL${3r4dNiv*=f5B=!S_%?{&jnxPGbX(=2sopBuL8;1*#v0Qt zs)!He!yXVO)1IHd3w&x^oHSAk z!e0txhA8N)PPMZqD74Y@6VXjxHlsqKd{pY=ZNl1;Wq_p*dNITKwVzoPBhAA{i6s&8OzQMx)yDx&LZffchmUJl#elqIO z&c+|@7!$k0c*wdXU!t@#yd!vE*VH2+Z^b%3xupnr#lutK9E*BOzt?nfKjB9+?C_7m zP8w$4C){1`H;d-jE-FI=KNG=UPcY(NkKPd&g1!vpm#Mb%3k6oJNSdGEr9!*uGg@4a zHp)Lm)ze-zCNT1{aRrKF7rJ|x695NGROxKOgqrMj1i5+QSdH3)tBnAIV}pFNPOO9xBs1UIprZT8jV-^!5iM2s(F)&C`R-wp4)vi6hr5YaU^E0?g)Bex zLMkMc7zzD^!B+xF8hOzQKOC7ol_|e{3T*xmLF;;^QPS^+<$bw#j$yTw?d zu&t)HQh^-+hSH z^}m_E&t!&FgyqdOdT9GaUaQBh>;J6(KSQ60=v7sW565>vp-$fqYUm_8d*^S1S-J~S zDzpk5VV=}My*nsi3uIX%m`;6!fn$hj%qJIFeIenX)@_U6bBHrBTy_b}$`DLCiFgo- z@slguy&4v`fen55KX251dk7OhsGnS-AxR#?+%i7u^7oukL&`BmU%kuw6>UqeSa6Z( z@SLsf6vp+0Vm-sR2mT&(IM z;3BJ6Y&pv5g=Qq+SODXhF_uV-(^#vZmogBCe1P9Q&<_{%epeE+t^5lAzUKGps7MSL z4UV!q!=B)>Q@Ydi>;<~F-IJGCbZ3q%fel0Xp6O91=;m*9av(n2k7tpg|DYuV(1lk^ zHm#$|pLSq25pZq5T+hXhD>e7^vG_-9k!$RO=R@C%!|}Q;qt`~GuXlWY@9l_%7QL%P zDlJi=1tmu>cXP@_W~EoR-9f+LGePntO?W@D6D(bAZQ2%uJV6qeXiyXWXkg^pk-2+Esq6upmy{E_Q_$d8;k8(r4^EITK_@ z#KIS;E2BDi?i_qr5ZmO1RmZU3rEwBICwL6I=q3BP_3QX^Q@Z0 z8zt#;VuQ>95J)K4;V-x8$HRp7`buo941wN@=k#A6)M99Aro*q+mCv>0{{lZYo8q<} zN#10^bF{w1r`(r4A+>y9KO->zdm6M+pC8{l+@un7FIrJ}HRaO(*-!jeJbe#xG%@K> z7>Or8%<+%btTK^>qXxKe7RrBZ=gd-kAIqQyAA|hhj#D{fv;6?@%7St3f}v_AfXr)r zlwiQ*%1}9BdB)oVmxjplK9(zr?x566{IsuZt+tpY$}~2WHeZ z95p^oJ$E?Vyel&n}~4$$`HZIu0E1XghlvQHEwVag9O=V&mRWzIU;{O&k2znilcW}ZmFRJnFr z$o0uYEYb1tObt*TGvwNx;+i+;@LbrdY{}Ft;k1aKGxRG_%IS^$BJlkphgQ6>7nw9C zmhs=ex{ilcJMBi>6dkcY{BvjgMHMa1NR(zpmc*moC6A-x_YB?^Tk3*2Vt=rIy75L> zQ?_$qt9-rJKnv)#{aJvLA-BfmZObagtcvH$AsqQoke2$~CZ&Sj`H@GxpJ~BBfxUML8KIFbRFM2W+ZrI2&-G=doT{@6W?*^aM_Q%*hEW?WDlQd4HXL5 zl-+nzH_y|qzv|u7B8$@*pTi!DT3Prwt}=`P@$X%L5Vg#{(M^6)yv zuH>4B^&R!%R5vD9E7b11d-Ov14~>xLn<`AiF4dgKk6psfmouKL{>Xdx51&WVyb5eg zntFwox1F9ewv>r2Kiw`wk$T^+2g_2v_eVqo8CiXCjvIlXjn=X31_7zZjIL*jHABGP z=Cl5{zfJid&X~mFDJz#J_0dJ-q1YW|-?KjLmBoNykx-fS=Pj(OXVpnP_wUDWoj-PY zX{z*xJ_i<;`CUb*+))~pODN1c_jwu2LKn@~XDKdEeIg%@HFIAG^lPzHUzY9yD2RJw zz=9D+eN*=9aiEn>(rw-~9&9=syL767!!scMF!&3A`VSs>SJCE#s_zItG1_vGa!ib? z&AgvnD@SHFdHfZBt5A)<``zM_Pxo4HPRj`oPbDphf^ptNYORRQHeLBMlx(yUvIQ6#J1^GHp>T}V>2b!3S} z>-T{%WjM}-kiYuCOLK2x`()tpk&YC1ilogL1f6)%g=Sc<@_rJ#dLAc^@|`MQaD~9D zvU{79-18WH?hZ0(2mWNQwEM_bska-h!QSQUSP&@+2IyGB@*F@<;uaG2q|{EP(ow!~ zcecS3mUO+s1%;G(sx4)MNO%34=<)mZ2GT!_RT?G5^?y9sc98QOKSp#k_YTtH$MB6+ zwWz@4r77@gPW@t-!Qq9tH{=ps?dQY=bC|IOl^4=p@fNJ9?3&tx17N1_Oq0fLZBf@^ zrP!Bc8T8qoMTdY^^#D>6h;uC^+J`>^Jbi|en)Y>hV>NbNsDp0ivU?n+T*A}q7dm%? z8-=__6VBc4(SS>-2J@e@i-D63lHU={ljxnQ47+D%Ob0`>SJ++;WPq5g41W_Oz6t#* z9gcA=r^1%#u?!m=ywt10wFhzl`36|_02ZKCk}+1(vQ_V7VM2&M38GdKUtZ{fdXsH3 z*y(60#Mb*XLUs@#lnIDIfZ zY=f`0n(JF`I)Jp-^~-DM@$ieS4K3h5)yq#G~-pzo#0fzehj6f-hn&hpH-H zYg}$SmRm!Q7Zc_II3g?3}J}`5vusJs|E~4*mie{Me=Q8%VewP#p8A zK0E;bbHyEE&I0dc?yh@W7FYb>I6KnuXxnyW{kML_qBOBpO!?X9BkJ?FndsBT``y<6 z2d|lC5`OeY3!6hg8{_wNB{zb|;(x96n?pNWt3hR)Jx`z8NMB;vX^gVE8jL~aWfuyJ zLUtK`iloaAq}m@AIz|Ied6D$Qoq55aJ1hrWY(Sy_qn>VRfo!e3cB_TLcCOl7{my*z zPm@)8{Ux#StB7x^pWYQKI5!nv>fh8uB}_44dQk?7%lkwbwAPs_8|Q_=w$=PX>H9Yo z9oHEqVq8Suit+OWQ8Vj902vk)>JK(uf~5NfoTCkZlH`d(-O_X&?^Sd*jS~GDe~vYv zB5lfP9E|J`rw{%1?({{xo(ua5l6C-ic4ytDpr zGAqNthw<+xpNrYXUPfE0U6_kz6TC z@Y@)`hMTIu|qo?#16mQ+dCY*-+D#fDJ0#?@PP^o zLR9Rr*mC*Se1QA$JQC*c)}j5^1Dm@_FAnSZ5xnBkA`0N*5RSF44X@JpM?8;-7_8bf zoWvw8m?BYE2wm1mzUT8ToBtOv`lF+Skvvg>J${sY6z{Klq`+FbHn^8~rD9R`pt60z z&YvfA1<7HLcWB1yb*R*{cLUd_1Uc#5D&??iaBvt=PwYAsJ9i6FHR0+a`^%%9!u_(Q zIb?!8ztZXf<_DJ7W#11V3FOS)9$Kfv_4?4 z=AR}*KX9-#-f$>pn~|XN(@D5Yh1BmBh9&oJG!gK-v9l?b{B2ZvMtYz@yxqDy&D6hM|e=Vf3Pj6e$=_6hu8%HK_s_%MKS_s!?$7PvI% zW950CTaSmg7VW#$2yd1&-r3t`~rI02BiaL zldCt#Y=G30no`JXcy)sq4?ah2uMxZ@%XUp);ztN zg*ZH!Z7rkMo-WqTTey`i3|rc;CpN~e#_`@39gP$JZ6Y&$6KTd)@P057+bA3<^ zQ5hr<(l?hh{MC>bt51VG&ZR`j>~WC!N#Ic=P}#8)e~r?{_2uNqqBCckOf7+|7ln1! z>%WI)|06nW1l`#Kfxab!!kfQ#{bSY2!lTkAGb{Eh@(-45J-gnA22Q@W!z? z8CAJnPo7Urp3X5+q!==_^bh!!1CVx+SZywRt?@{|in3vt^P42QP@B2y@^OX-49eCa z=~kRhqvJ(ALK;CpuK$CZtVdCX8o=J*&~K4qULTt-;%JjXZvN-VszQ*4#u4ALy;og9 zq{NvBLWyQ+amt#T?U&;Ep=P5ue)>-()E|GNv0L;mjRxpW2-S=};W)AGa+ z9qlA14=f`012_i`d zlxbPiJ&TaUIgHbvt^I^YFno{@UZCl;HOKcj(x=R8n;apy$DA$bgT5xs5s(n|6^I1e zclygsOR;?!GPc5he_mBQiX6Z|c}`$VakBJA`nb5Mpi@m97$t{s9$g+Z2ExRB-;C)n zcq2dv4zG?853vxu?{yirM3$k|HP&RLtCrb48Ako{Abq{p=CIs#D(qT{VSx;vW_7)k zHnlaHQU8?G_4+tXqQJK(4~&*6PBer);OT6Mo&&P%R;(cI{XqhUFeD=6R~KTxs^u(q zTH<=@s!3fKW~O%ok+@Klr^xErk$y@M}%pin5c8ac>JVTyun)4)1A(c zp%`(tW`toD*YeS{)6{NyI7cDFicWkt)cb$nKKJTx|A|0j(~nAMM^9A*uX|||mg$zb zP24dB&J557Dx++;;#o+AmU|1~KlpSx28F*IY!~9uQyTS00tRXaTOzBy12s4;v}^2A zd%j5pdWJlD<0}_q$WxWRE@r<~JutNtMAe}2;Rp1!c{q7y&GFEPd&ttIeRdgPh)fCnBIlPeH$Eb4tu9JD|dhIl4kG31C zo_+upeCqKhbu|NZo2nsISMK~f-_gIAOJQF8hahrpLR+(99{R@(x_0rK*8YYl(%mGutW_HWCy9b%F zZgRRZZ*U`{S50vt5&h_2(RHzKhhGU6f8Fnq@bJ@2G`YSuMc%34b39YbCI@`-a@dyz z{2LTAF-8*PAgvCp*aY}ONY^@$*;Tm1wL&d@)zKXE7;H52^~d*ac~dh6#At$SX~JsG zi7_~F6isXXVn?_~moC+$Pnqx%_AMk}GASu`58Y7BG0TL2kG?zhl|Qh!_G8h_iiov0 z;FIeyp3)ibwOH%y3jkgv_I{|muPR^VEad!OQ}@pRa!IWJ8ng9k`{@%qR3%GWrHaYi z_ppR<7lu(McAyfYA|pkyp~sCBAZTyfG*6EzV}hcG>C*V-);WjFa1XAAV#q`BUL7|K zbDS^RdnvV+gREL%fKIVvDB_P0uYcwIq$pXVR3y>Ax$Us-rywFJFHLW4Ao~Fz-4v(H>i(5 z$bXFj0G%l&Ip$CB_MtsM-_D&8_ro{OkjwknPNEB2bV6CR#$;ZgWYmf?ZtB}LjZl3@ z>uYYzGR_EZjXPm06cDw=nee2;ZS17-i%Ha&001ERO@dST&xo5U`)C_gJ=a1og<|=5 zv%*StbRu`;?>Gu$z*Fa7gFZ(%jBje8)H59x+!ICjp=WoTDU0KY7)RZtyM0Kh3JuY7 z&t2giv;`g0BJCiGhxnWAA#Rj}=GuiP`PXGTiTe@0@Jg%v# z0xQM%pmIIu^sQsHsefB?;l+{=1bk!2kl@e#%n4$gBt;|&7HOmT#bLw7_^6VY=tG!x zE$&p`l`2t&`obmp!lj>cmEd|2s%x#GX6SIO@E9;=Nql0jl}2kfNhMYo07W^rob@wj zSs4Ex0Ixt$zd}@ULsU|Y;YoHt=HFZf8L`}1elkAjFbp{t6i@zg0@v`RH1y6jYUo~u zymUHk;wy|B&UM-cO~w$V;qF~0P<%Osk>8!^yLl*G8Te#ik-<-XQrhJdUjDA}MQO-i z_X0Ovfoo)upPcehddf$*x#=~BSGod)RTOD^S1YL6YLj(TE6gXX z8FB_CBqC8HLn(y(lxfV*Hf^VFmL4Z54`t}{F@lfpjSqe4}B?$w&gZ*_aLn9peCO^a+OS8IjdxDY z4hG*D+{74GG`Z<>E4KbScj`ZRr!RWuxf$G0j=w`+bL#-_9eOLSlk(kCU=MlocPIX8 zi{?rHZhGII-mTEL?rsP9J8{gD!Ko?iU8A{i?tFRYvzzYQg>ymv-^sllIN{uqyVSc4 z`07?Y+zQ`JUDvt)n|*GDm`-7V+**no}4Eq@2x_j-1C@>UwUYeZ`}_pTY7I8Ho1sh%2P-0M4m=CDo* zx6)`%Q$Dxco#6UAxV$yoEseYR{f~g+J2Z5`!M!Qn#;wECcL}Gzo8Aq-?<%-?G`ln5luIqOv zy!9IoJn-QA1@HQR4haDnf@G+X!A3?h%{J@q?!Gz8O#T(C;_SsIBOJ9!DVK`l4^gw_ zpJwyG#&Ps7Oor)%>&7hI#2;Ldo>@SVclnWpZW4PJMKa}9RQX$stT z&EOi)+_YTKv;5sO^z1qp+~?+e6vxf~?ZUc#&G@?S@4Vg(^sYG%55E5(Bi0Z4X}kmQC0?75BeaY*?Q|il(fcZ|4hN~E?jX_)FK|+E;@kl@Un#8X+`83EH$C_2IvZB4V(L&%bg9|e zmsEZ#r+Am4#LIkdpgDUbA-D}C;q0dGzED~34D5z$#;LiyPJY6vTrTlWTR;d$znN&M zRMa(VrhLGh&Qhdn>|<-VP~hgFi<7SouW@qIf5%K!;D2%jJ!=S(7wWv0h=!RdUtPrLW`ACK3JY^pMw1Fv;a93R^VPm-#i^@PU%wB z*okz}xJSV$jt#x#L{Gi*`4qmf8bi~A1#F*A`no7>5kj~)dlP?qI1#)N{w6MZr|`PE z&JCxl_*IBByyV=dhFi2FYKVM=i4mnG|x6k zsc2}Y-Brb_;!mY$o((7+#dR1YdT0$7;(dY6<-ON`ZA)vmsl^%z#6!TXiYm zJA;Z;f31dMj@Y9W=q7&Drp>Q$F%B zalD<$9l25DtxBv9aO&So|EeSjCYZrR}vap^VAdKJD0HiYBpXa3J4GZcqR!5z4C) za}du}SW^+RKd^W}eSMYqaKzrA%H65_&b2mSSG7dD-k(cXk@`TGXw(5>2_pn=NU!=3mMZhA36 zgTJURdb(}FaqJm*ZN$+l?I&XfOL|-5PxVZ`vRGEtf#J#jXZ)0hdfB9k_We$$j8OosO;N=Xb-VHRD z=j}jY-%0rI6z=Wby%pT;-CL#gR{n1X-znZ(<>P?|Kag-Y-n_$4PdDX7(>t~T^WOzM zyFO-2$e-ks^RDrq@wKlW1(azQ8pkMYBf<21E0_WIs_+gtVcda`>-?0Z64%wLjZ&#f z`Kb_6Qu)?RkIH~l3=K#%Lr{iJ8U6B%qj1K^$zZ-J2X9K0U@r7DI5(c1S0-fBvxb>c zR^8|3umVR)yZ^u4-2%s_+3(J9&F}64hE{imZI0uOiCdQjnv-+krk{IrXCOVhPVi2{ss0>Z(zQnXils&Rsy}lSZPSOF;LV&NXu39d zsQFDZoy7$XCpd2e!A712rFqwcL;YA?!131_jYhAoMfZuAmvlWU)6bq zimd8fyv(7VQ(V@ph^7#tbAoa!N;+!Bvg%z0mI~>ph%WEC;O2LWqsxtQnvXfUFBYh{ zW=S)f_2V#WMIS#tV!c^L&WN!ZFK)uOc{IHA&nfdQaGxDN<);-i!4UH|9g-t2 znn8w#s^yHQZ-IN?xHf3!fbeGvW*amHOw!&2yAu@`Ox(LW{jaj@j$=w{)&^9cN?VBv zgf#Ol7f>x!AUlb?#3H7~L{rvHjr1!OS$`F#z^JDEQV99{L9!5eO?rruWL!-K&}0GW z1TON;*xdy2gzAlPgq2Ku;$5>M$n?3&zRu^EF^W`W)MMB*kuyBX_eO@&nOX1REB+b! ziFO42H~Fe9@JZ=^XJ}x^i*zd)`7|S6x|R8W%AdAHl71vPLF$-3O@&v=4v#W-`l-KD zJ<3=U@I_3r4m9#;hTH1Wg}3r+zTOGn{$Ar38SS0+;Zj6Ap?%oNZzLUuC4}aFu!>@F ziGHdRZO^1W+dMXaHwI_t_ML+pxGB<2uRGuA+Hk_X%6cU;*Ey!55TGAvtvC&*x-lOU z82-KFXBWi94Qv|!2z;+9mkcnxz&7u$ZZoXlg6UTb1_ZqjUyvV7e_~TU(}%Y{;$ZZ0 z#t#M(eD9P;btf67ag=S7uUL2LOVu~&oxsJQzilO_^Ce>V&l~*x2|O zE?&H7_J4Diww}Lm9#^hhF=;Jn-P1f!g5@{LH@h5S+m`B<`)oc;!fDv8 zul};C1vsG6w`|RaGme2->YG0kk>R9}Do4#x32>CB!J&rs3M)lUGkq$y4^_qw1vTJT zG-Am3W}9yY4sp>M6XRN@`uQKByVg2EoRUFKy;zaC3ciM>*fX z*P#vNt7ZYgZjXRZ?R5eb13{IzaPF(q#u*BW**HvKzqP+Sf=W1%g z3_1y+q{G2m~C1m7~}?#wiW7O~DkvBx(LN`8DUucc$L- z+)#>d;;VSgcqyzy1IM}dx+uKTXRVuAOwK#?$062{`ZswQ(hJ-#>Lz`0h(PfPjX=)e zT{L5Q;tleQ(liDShSe}$W#_1#cS=Vf{!I^B!!H48FOj~WG7NjQ1(9dVViWt4ChQ!5BG3y)+r;R0+!qJKW0rOcBO z&vKG49B>UM!hDq|dnHjEZ7{0DXz_L=Bk`vX|#i5cJyftV_im0v} z-ZAwj(&%1j2rsoQKB>eq{TRv#4?VG;kSwXQEjc_)#5Eh!I6j(GR2O_Uh2np;ySiqZ z5oRqH|t!^@wyEfaSe|f$2w$Nab^!XWvdcr8E`ct-O!Fs zKQq&(fuEtjuF8)8R3~aK680;`IDcIygR=B9IL${Q>PqTDGLQMX|hKPl#K3E0DQ~fi2 z^}FGEm`I7Y$zSH#MkTk1deu2FPalpP2Z+0j%?j)MM5ng_B}06S zjNya$M+QFKt9tbLQSV7cQOnYJK8i(2S5}n!?-)jpHS5}Dl2E|4#h)(7zn#fQ2 z-N`S)SMMFV-Es=waEdR!HM}d()k)LoZJakZU={FojzK9r=C?7H=NUEHRH^*M|wLvt@m=m)4^Hs zoWmaJPiJJ3dF7;`(wdK^{8KD*^B(XsZ=JOCpM2+Md-wBNV_G%k$9GPNlpgUHIT+jT zuNc`fBTk(N2fZ`0TR{khrn=HeBFMklxnSt(fd?MEEl9q*?d6@$?0eV7J|(8G?!7Xd zo}M<#cy*wK5#p2xnz<*KoSekzGpBL(>{-mt%~EJ+u;9bu;v$Y8KaOkHu3>&*9*IN( z76h2u46l%8@U7fg-NIW>&VZkHMqx6;biU)j zRfT5~%HN6Ygp=8lr7(Ujh^~4v@f!rxp&?Fqhjx8btxK#S9KK*^edCP zwu~U80dY3Q483O@&D!((DQjd*B7M#58DkV<7>jCSiAogd90+UQUMjqXcZrTjKqtda zv6waKNntn2bj?P!&bC5)G`lRwXJ4g?N|DMt6{CQZPvJab`(uUwx8^bzAJGOe3?bhrIy%JvH3DHC2F3paU!B4Xh8?-j*3KZ`a z-JN$zSK)4@X|yqc;YWv-x2GY$JDr=Zc!cqdp^v;M9yXvfUB_@*4$Yr2^x4dxv%gKiS;ygqgh}Ijn7GRAyuJF%psC zrDUtUi1*26AMC1}B1(&doG*-|%nTir)-YhDVf! zXu$A%=gh_<|M9*;9Az*QEezi>fZDf_(Z+}%e{z5f;CiR*3kzf2Wil|wz?j=t|6Riey41L6h`RJV?{o8;kM6Df}bU1yr zfQ8|+Vk(*z46FWF0$ye4R^BF6I+nRn#||&>V(tY>!%63Meo@~ej&=QXbvW3h<+_`o zd%YLF;$hPYh?~<6#m_2}(;u^pvb6fxSD0m^Y`3ge$#*T=RV&i$MKdek7(5&JHQJ0L zhupr$1?BJ5qjPofcgH?@@8GN1+4?xEF`M2gN9AeoHF!6i7)DovFk`d4nn~xheMf#& zH7k#N@t*Q?<0nI3kYOc3O5`t%Glh{Mohb_WTg4O&14@%nqf8|4nGaEo`jO#IqfCu6 zBuAzGhh=0XeZ*77(8zH5=v=qdUKy5U39~)OKn^VjX6QQmLTxwWr!#n}9ho=ELp~10 zjd-HdI9TtTGC9svzPZt*apQpp9=s(;&vyHU@5(Y>WgtV}&Djv@+)O9LVGT2J_j)s( zkvhY2$;`@IT3Ryk6kfCVG;2>rKAj+<8G$n9UA=Y{*(k( zv;XGi<}f-oid-S*;J{y%(oK5=N+fyUfd?LZFF?a4XXxaD@HDm`hBN&w})zWZwD?$cL!&Aod)V0%&I@)&+WUUjOsM^+sgco z;7%M}_fEmVS;hq!i-eEj8-&Mq_NW_0UFmCyoAftftq+D; z_Nse3;zOi8g;*-ZeAq6OF2_QC85a$LybpxX+TMp`x)Z@rlx4PIX_-9JF#84z??4Q` zR2vfQF$7agK)R+1BW*~dj+%@u%#(gnnv4=IjZ0Gq<*D^i3UA;h!JEMKyY)`xGVR*j zJ5Ic76fr4a=FOcTU%H$1-2}$3!4n3U*R9I`4$w&RW;}u7eFw*&niXYY@E=KoD)mld z1CneaJ!~BSMrBt^*^KeW)QydB%1=7CA&^M(naVrEo8{ei#lZoGqn&nb_&{+q{aNpn zr~Fwbn$c%5pCC13XFSSxbh7o3oe-i@9tqNggU}&xu?Xv(ZJ<^vkuSvCGMmI0HbfQ> z6KL6R1v#sLqOAjs3`3aw%at-|z=AUq8OUWoYz7Bs{hIIHC$)tJEsb+84TPKOS9~G^ zw8q9VOiFyHxRr|9g)=^O1a!mQR_BfMZUZgNlTpRl)6OlEWT#oSr{&JZ*r3#IVSO7k zc_@GFr7b7!)4qjeGJ~rR$S7JY7m+PxQ6?Q7BWW1XVAAX>TR{;O(pJkRE2Sb9bF;{l zGsf5&3dLF82y;-~s_i-Dk*UT#2hgRY!9=8J6xrZUS_DE7BXg=1M{bxl60RfOK^$5r zc!ExUYN}htTE@{MkwdDp5AoJ6_$fE4q*sY`UMt&X>9*Kec}bR9TINW6)ozuCvlNnP zybXwUN>|SG=ZsWkn<0D5_hM}s-37SK>AUsT4AX4U@ge_N{6=*I$tG~{@$NYR7E}{or zqsKN*0o2U3)q|ONufBtQMM%A?5mm%ijX@tT^nuEx?SS&pUVL@R5f;wyXW68~m&_;- zR-YR(?McsdeL$lgUn`f{@3TFV;ncQorFc8sjgHNrc&__S>3HCQ2mj;nt`F!CxPwr4 z=8lYYy}iB8>^yZw92{wXy&EGpaP;U=T)uqSgv(~LX4$R`eeROp-Me?=)mL9NOLt|R z92*-mM+j@#@5soAnPH@N%2Nk{C=KO1Ffd?FzBPOOQAkNiGJ=&?clhykLdF9RJot7{ z8~)oiv(J$hjiHml;g)mn1X5YdAg}S$oim4xPG#rd&n&HF1P#)te(`5C1v(92jrQp# zZI(Scx|s5D-#3M8K$RmcDRmZ@yB&jfDlo zqA{4IY8+5%#*6k{y9G{ZG{7>mVTg}5gD&|vIQNd2zAAzDfBy{e|_2jFw-W#)S-1^Wz9drFS6>1kNDdPqFSgSXXU8OGmiLP*2AhFl~A5x$cI$U#otvb*h{`FUn@N- zGFBPoa)D)tuziq23l@BG#7UW#%FD>+sfbxVBvVm@soZ4k42m+cmh0w}dWBPa6|0%5 z3liZ#mWq%L857s?Cz&S&+aKe5#$kDlQ#_|K8m8r*` zn(E*e)um>RXjZflnxuz7bPq6tKt0U*VfzVCnWh38%mh)NV}J}%Vp64038k=RT~hsK z{rH)WO^vpuO|vMoYxN@eK(jo_M>3GGAM~-_{iL}LS8@)2Vg8owj5}FI&C-doz9V!Y z@|~H@#|rcXq&#b;j#`7-mt&xDWE*iN{z2n}JKGhVW`U_3emc!KbZE)YEW2q)TB>Yj zS1AR34Dg|<18g)ir$QRB7-Bh9eU;;RXX!X)Im;Ie=+t%98=az)mJY9|j-B*$HHJpw zY!1I8eVl#X)W`50(=)Sw89Zdi&C*teVoXPBJ2G6rE5oV3(JUubrTP}VRkdk`ZHV-g ze9(+Hor)g|b1ZMukLA#EZ*@^2_Cw`ro<-IX7cm4AVI-m{#KSV2L|FtTOdc`V6MeHZ zn-Zi}uR61*n3>n>m@W!`G`sMA+5f0kFq{;*^p)6scO+SRC9@lh8FIR+V)!^ga2%c<5%Mkg|9FplJ- zIef|VrP6H33oUupdlQGxOiR3^af-@e`XbE) zub0@4G(u3@qrB%Y;Ny5xGKDyliayNF@+l&|le}je3zAP*K8HHOL3xUgmA~4I%H@_% zPJZHTU0r$Q;(lw)pc>bQD_ymsLNWYm>kj&+EaK;~J_w4pnS@48#z+?+UVimE8tDZP zWt)jbh^N~0d&JLS$yC*k%%IJ-Th&2AVN}S2GN_m8 zc~k;9#wt?&5mVA$GZ~2f(NN$#k-DOtd1kqopEI6xJX9ooMo~?u7TE$dYWX&Et_L1?a0@gRbamzL z$}--&ft3A;i3#-d^q6J5%2>v^Lx&IH)X7tro1ZhY_wK#-UJMNlp|!OYeSLjcwQ7}l zmJ`ytu3NXxyj!_)r5RkewY8b$!7}X0FwoxKZt~MiK^Y5WG}N+Q85UBhlu5_g^T+{D zSa{&Uy9euUrLoTeGoZNx1wG#KQixl_N%>3SIAJsdb>2HmQ{|^&*sTzby9I7u&YR{i zjTZ#nX<%02!2Q(l4V)exL6G$Aqrw!@XIo8UAl*oMCp_<~RIbJs7I5U`3A1llJQhPd zqD{wTijy(M!A}EL7lf%m-)Uw1kx!VTJ%g7SNt)LLCmhvT4tCA08Q|&Gb+^M?``!+> z!re;k+wr@d`eroR@N;m!<##KtW%+RS%w_f)GvrG@`<8^pum;Rmd7FcF($W|p4d`Wa ze$CLF2c6;BJKf#jPJbyHPQR;VR+TChAbL$rF5u9iBk1l)8RboeEg2}v)gtdn8Iz1~ zYwpEl8D+x7fzNtt41FrUyy>Uf*77tn;3v;%pkAmJs0foj;TV+w z@|&`V@dfc>5ToRXJ+z^C(b6mh8PR0)Xbjm&dl_wHu&zoO<)hw6m5~CY6|Ul4ohK=y znU>NS$`dacDTGH|MvZC^K`JxBFtOxgt;#YmZL|F(6|h3RU}h;u(J{E2V#}C9nWVh@ zF)BS$8W~5%3(-TyBqAe4G!P0KV-s(Ysb$Raz*dUQ5o3R#AaX`~6NWL|#nI%nDN3L|5@-saVCOi#ISfHqJ6yNVg`K zmXy|JaMP2zq*R!|ZM-26ugNKFLm9sf?!-yyxXuNIQ9ND0vomF7)Ka;M68S>Ll2UEK zEL(FX-31Y$!da+@r$dh6PqbI^vZ$(`F|5pV7+1?&ohR|eow(Cq+45b*Cj%SXjK#F) z%D8rR0&_FVh@~TDR%R_I!vZU?KKM;$(3%Aq#FkV?lGy_uIuQ{`!ZL#5`Q<7=ELkene_<0=Gn zRr#r4m&$nN=~r;-)I}UUa+UIJ2DzLMi8TA)sQL!SAnq9M#1Bpmw?O=GD?Ne1gL%t| z&Z>rm1sTg&Mj5Qw(9N5=j17&S&^57};HDpOQ2CvZjt?VPrme82scEKV09Lu|7^K*a zd?gvsk1itpZp-s=qqhsQIMj zYwRhm_riwvDoe3eV%sgUIg=ie3o_IjMsx}hl&~t6dBZ%y7#wwEMEmQ^Ka+hw*o_JLq9i;QxByQaP8_OPM^Ar zt5-*G@YUm(7|&4V_z;bS5tV_O@yrJz(yLaJp-{zAI})8#7jgopZQbGnr`+ETiYL&^ zJIkOn z0O_W90hC$o8uqB3oqkSyERbAt5qHbjs7C|3x=wJbzZ9L#8@6TVM=+gY1I(@Stb-?s zsT;~X3tx+MqW(y>IIHp=^Rq@+#7C{KQE=X|pOuf|*H~}tFY2a^(&@X7K5PmH^_k?k zkQ`-@$)D-V$XckDjeHK1cYOu6bETmfn5;Lg64BvBRsO^~#(2(p68~xYiW2KGubIy* zzYN;~wJTz3?p=C`!Xl-f?k9h&k-`{VHc`^#fh$n-dkd=W!hCVIZm7!1Z6-RLu zw#-3+X837|uL>zpe1#WCNV?z&3lBVa_h7N}|J!ElbH6^iLPHnhSRRh;Z#~oLzjN;l z5#=-xYW7iL(oao!8fDT@PQy~=Wd=69x`P}yw!nSwcq$>%TZ6NTjYHS2;o!tDqHV2M z!}>9brSf#eH-l>yMuB-=nVrGGH{V1&l0-*WJJRu(*?Wz`o+vrPE;GZ9k#6R%?2Mnj zcC&+M82eN_1;jfQL{M8hdHz11CoUS@B+o64*6PH7V}bAP8Q zAs?4M4(MLa?c5ASom=^t(Pbk%@5Q!nym1P%RQ%KF7WDPEIGf=4W#_fBpv6Z|X5hyE%1{K`T}P z;b4^aDxcyt^1Yd~(iD(ZQZmetIn3f~Mxg3Ivo#$GJAA4be51srT=42EGtF11gk7c* zwj4xx&ced6B_wPr3jV0E+gU*wZ^#!*RBA8TC?8{+e!Y(T@iM|w#CJBtyyy~AHkG!? zh>8gR1)Rfbf+V_l#O$x8y5h4Ppwhwc!7OnZ+>8ret|7HTtT3fTAh~u)kx<; z5S3XQi`NS%(J?0c7g_E%iA#p*_7F!^vr7#3)?CFWsuxomMp;vph%bxz8qS;=$Mj?d z$z&WoJ?)gMH#J>?uI_z9S#(;iJP3`v%vJm%e}~_0#nU^@%}{*dBIky8!^!Dc?rnh4 zeM`jsdIkm3qCOAAEb`$?H3Y^<3ktY^CO$GP%_-K&;i2IWtb=td9LS<{Vi~wVUR|I( zkg>a!ILnB~(2gu3KNB6FvA*(;il%1%$xlsBOT)ESVt!&7qsK0zK53(^qXn_Hs59%T zCMq)wdBQ1`erzujjB_amoBUim2+SRvM>dx;$}t-rtgs5kI3^xrULi(@nfGP3mzW>R zXBSYJu8`+Krc7pURN`7CQu=@*Qy}cVltQSNA zC^3C457&OTzCg&7!F;mPvku6hJW^dyY)AwVE%H&8*MmOf50z0k#vI9W7^Qq=+f^Og zL6(BD&Kyp{Y;E#K@l***hZZq^cn*P+6@=#F2u5QFq$mT*lHrsK%JUj=dy{!x^&u9G zAb)xprOAS!b~q${5^d-jw7VTdIfs5SidHHW(|@U_m7i>diwjj;x-fx};W^fI3h`tB zUER`ys4JzT&_@yNo30NzsxKL&B;TFcp29-;x_PVC1oRhgnOEW!#TB^W?u?_f+-DaW zX*9)=v9VnAb@G=JxM2m2vWZbmKPq{mkqnah}n(@|7<*|(5s{Nx4Aa@VFAKKAW}8#&}}IPy7RMNxZ|@252Lp12Wb3P`6f z(riQ0N*@h0LsuV?GZhPGs3V*hyN(iR(i_!D^+t*^4iOXaL@I||5^2VnW$?yZy`6K_ z7(i@kW>oT}_8$~28szuD0}t+i?}@R`RpzCn3#9DpfDRe(G_y~8>YX@w0^x|W&R?_o zR;*ZI403{aERL4e7H0`A(@}ad`nf0h%19twWH{8lTOiA_Orc~<)KXsgD~!_U?CLa5 zB3eqZ3!bp>z=L-WYHNSnjD3=R9BUYG22s4SEbhI~a$ZZyI6sz7LoG89GQXP!7!ANk z7ma4<&qA7kjaENunCW_plFq=#Nn2^@Ubx8Mr_qryit%1bQh)>CLt{7a`r zrcy)&miICmDM8W3J-Ea;J}aEm4?_d~HyJCvMyZ_ANNLfXLXu_;TA5NBgPFoO`O4g( zIfnu_HRr$MoM#sl;#Q~z)tuV~&ITG4r>8gHJcC@ehTh&j46JCOQo$Sz4fr#IePU>$ z>7SgNzI)%Cr#_u4Pu(lOW{@V}8cv#L8@SNG)1)OOMR_~>8HuYjTS&9kRh{%dj_-7cB*>5i|29%sLCwIf{(sEAgIXW<6_(k*OOUdZAfACJf^h z80(7>_|H%wsQOXLlra6$ED}-rrxK_oIM|QY5Rx(Rk`HqSsKma?)Xzjv8MP3kavPqC zq8ulMsANQ=j)GvyY7Wj|7(NAT<|f)3f~y9YMu3VnAAMtfRNu6)@Jb%Vmq^DEswY>; zL6@tjPFpB%rb3-yDr_Vcl@4o({=dg^9SNd-gUZlw74~_yflAISMG7W~L!2DS@WJsq z77orL|Jnj7XL87m77>1<1^zM8y`_rU$|@C&pfU6nmkX#ID53JzDhkI6ut)s}osS?l zKZkr|4ymqImWi}eV=V?yoUk!xr~jT?l$!Ix0!c_rVZLTp!CFdx{5!0l|yF@>~hoPz5br3HV}V@{0_5lse>TVM$gZx@;?c8#QT(gpihk z%CJ%Lp*X&bg)`TXo^L^0Z#yDAQ3T^uNU0oaVvRnEI7{`&_o~Co%=5$&aESHxk{|O2 z7m$tRk?Kgn*AYTFSu<5_mb+0F`;IchZ?fge2)){mxi@A}m?$C`4I$3@uTyE4(MbFH zRq8Tib3A*KzNC>OY@d&I>M(y=fW zet#O#QUcW#eq;yPo+CDrb{v(t3aYOVr>8B{PFpBkuONCUjM_5!B~(QuK|#~1Sw>91 zO1Y$u2qa8Rho)#5b&dGyfC68La!$PqOG(*UE0Eh~Y?MAzLj4$@M}YDuUtREF$zDJt zPPvg{dE&&KtW}>3WA@+@vajY)Jyk+!gtCS5#y?K^%5Zj{%rT;+jGOEc_%8Bv7}4P{ z{3}`BSEf*!$|95wBG#S)Zre6jH#Z*$stG#&SsY(!&pz?Fj4cAm$nmi(u3nyjwky5o z-mTbm&q@rf>OyO4+|<45uv*n#f8-O6KQwz%2|AM@fv5=#@k<5{!e9JP7Hab=sH-Qgj%C?0{i36);%q?cHWEEg>ARngOlMK}k zX&Mn}51}k&?6OtDY9JH`e^5fPpcyezY8=o(D8)h**$n%Hf^GIt(^c{ExeD{3 z0!YPJ%7}0v8JMTuk-mJFSuUDuKCgWPwS3J9ClKDEccoNg8WqG75$xW*0jWfr*$dBN zKU5|k#KWzOLB**|B_Urm`4)($$%8oOvSr41`d&BfW|G}9$kNeI`D&4li~>eAuQv?T z+GJihQp>G{h!g80TO{uVseHtO?5{&Cn}zzhI_56UGL0-c_Jq;%u>@u_Q^<@gAxC92 zzmR7+nI36TE|gFq?T};cRFcflqaL9CL0V;Nuyc;_$KWzkL-u4IS5FS3cY6o=o@htQ zz7E*2FeXl1#l)%WC@mJK6a-+GY)oF9!RW>rc%p3Wp;c>%^#RQ3+H#oFk8tm&A|aQJ3%47R%%f`HTM*-+9M+F4xE> zRo11#%hPdM_AFi(UuURr>YB?|$ybidGKwv4+;nb3`ObUyS!Ja{pDAUq$nuw}CDg(- z%uG+Al&99u@g7ofHb>HqIFFLfH_nclz2ycU=tBPwuR&@{8|gfc8>DmnB!jW;IJ||; z==KtMg?wZ-CgYzBZ^-kJ%5Je-HMA4Y1xWaKG>Jqofj}_`d%lM1ES2gd3#Iw8;T_H5 zG_#*H>of_(yW)thj3PD^p%U9}K3Ypru7}w^>uh(mqB;CSD?T-=R5L(r8POOwSIU`j zW1&(o#vCo@GfFE#Bzfy$xC1VWh}PwDo%N-9DYITEKQcwr&Sm^?LOM<}6eFQ1v>Z~0 zKNQ){WFV4LI+{h6Otg@%oW4Ta8MyLmc}a4qZ0g@)S=9cV{l_@2W|1`Z&17(Ji?7k|~MoPK5*quKb2gwT{U86Bcve~`y=W)b;H7DzL=w0zdB zMgFx(+=Q*eQGApybC^0if&6j~iFGYlakC9hW8@9C!LvkwVkHqF9VAbd_&isIpXJo7 zbiWKbq-B-7T_RuW&>@@8!n}M$aQ{XurP&MSlS&51qu)*<~~PNwa}{ z6+f)k>bUaCRpbjftlGZ{+kSdGdbX`Vejrwb@65EzRt;RZbc)~g8LI-*fH&cJ?=NV7^iIEZP)5LEM&s8Wp#B&Zf z@_<*?udeEU93FL?WQTJA3CqqtP4Xz0%d-zFk>4`Zr53R;$2P+Ls-Uz@8zi>mPvNYC zTq-5vK>XsQkXZ z{bip0Xo>x<+OdgX%I3rq7#S+p;Ax$UHpOOr?W5@)kz}8u#*EVp-@iiUX=LKFH`N*>Dp?+Qzln+XD|gc<A%_lhS& znGAor$~dP%m6ir?-n`i?36_CSMnj!eCG47oNwaZMX)8aat@~4_PT>n*_yYdukNyb1 z^E{6%cou))wo?_5i)8_uPN@N34~0}sAe zpf>ooZW%A7hLIcmXI3aVLd1YtW`%e)c1QvBBI6n0AXU#=)==iiW`?Ml7jXP; zX5YQKJdOYJ%xn0Kul^-oId>c<7RGToPlYQGMMt^~kF|9n?GIo&=)=Ln68_@Y>-aBU z`7BMQu% zQ_tYQg$pQ9A!X$xh4u67$S}Tm`j|=ouTC7rU%qw-<5b+@u{64a8Wd_jIel#g zUR#{Q?=kFu`sOqEi_>TD7bj2Q*yYPqdMX%N$@-?^z$7t4ypK?k{Mxy*_zctg(uosz z>Ww3K?bI2JQNC|qF<>@sR~0x^7H|MbWl0=pz6Ce&N|j2xPM8Z)S(X2R7cb#kFC4=w zuO7m|S5L#rZo>HWAp52;`c_2I*VoEHzm~fBF)>od8?P?mh3C%VThATA#dC9*pDpuU z5Z&EcZUjtDF5tyyuj8d>W-vb$!!u94f#rob#)d7tap*c;eEu|*20z-{VpLKh$Z$k_ z@%%KN{l*m>Ja7rGy?P19kB(sUMji|Ed9~Dk2pY=wKBEoy^3H0XS8+h%N8#r}*4mYmlaN^ii96ES`*AXf)7S?a* z6GIy(MMVR2wleA}N-K+s;dQ6#jAnrQ=yd_oNY!f^wja=yO5zw)P;Woj z(wY9Sj*iJLv_IYr|4&9x9bg~a>&MhE6?v8^8f!;u0J%U$ziR?mq=Nq33<~6f;Pxn5 zKGuT5eGK~`74S*s|6&hPRvOqxIz~x5DuRVf36rDKu&5+=Z|O#UqK@pSh3-wQzz&9^ zvL()SoS;!jwMj;? zhdgN!PKM3Qy6oWv#QRe4ZwsS50rI89GVNLr`8I7X< zd@mx`qnJA|jq6{$h?%cVB1s;I#>4Qp5n1xMIeem>Wmr=|U}XRWe;GHfUq_|0j<(e; z2zFDhL<2~W-uiG-)rXYP3aY~#i`M3l*qO%Mcz?NvL} zGDTIpnU$j|H;TL?<#<84Qpd^H{FFD>@bZi2@%q6_IKno6?b;-Iy4%?=rx-3^;!}Fi z*%%+LVrtBfWFn28{s?8WW^*y|dVu3eq9LJ+AV#nH@!E?^c=o9?IPfjXj3bkvzK=vK zj&v$$_75u-Yk2vk6Zp~>o~InThS{-pEYA9I>Cz3n{`zqoK6H^d1P~5sKTF5ZSl5(C zu`*pvGmYqEyg`j=R`^~s>@*8j)6()lW?~K%j^SfTA6kamaO366NcFU#WlI{#UbgHE z`;%cm77t`GyO>33r;W}ZZ^QC^^YA@DYEH8`9f%@TOvCp+w!MIb{4-gU|0D4g0tWx*aB9h7rzpG%@` zrVUn)g%bH8Qe+wbE6_XMiO$E{fS(MaJV0u$_T%E@Bns3^S}3>T9cfs}DvJIh?5nKJ zYa!I4egt*^BiAQTr&wuUuT#Cr8)4SD8j|y`0lIge$qNiptQ6S~X~t9llT&l}<~Ltq z-*W_)E{-SY8PncheEt~Qylo6*Z5>h6Dn7jOd=5tr&2ns#M`y1OQOQ=;$#c&g z#OV`LsFf1v>STK)pB#Pj244Esb(}s*eqM;;t6zSB^?U`_F6kudI0jcG3}zLpY<4cF z+aV(tp8Co#UV8o#UU}gf-Z(gk%NH`}=nj()!=^7fapF4u>aU)|x1O1$tdUX9!qqD? zc;&!3eDj&t$R{auw3DY>V{8|GEH9QB{%@Rq@cFq$Ke&SZVA#k-_3aT&4snP?e&(3t zg&g_$0{QtY`FVs_q3}3c!>`(=V4~}AnvgV~1PT<%Z6S#5J!t8Vv zU4u3DujJ2~^ioHA)U02tKv$uzNI%;Ou-!=}%aF?cPVGzTTaNrZzJTgN*{o1YuWZGQ zSKdQ@KF`t4JmU*x5^~>t_Zg$e7r*#LQ?{KucN$}!z{Ot%%5&$= znUjZ~c;X4HUcLGzPBH}QUi5U0AG&(r!S@5i!+)EMeUdh=1kwQKX3J}!BFU(kXnG9M zX@haI{7Xs{omNC@!o|gzNj!V<4Sele&*JFhbrczMd-nkL^seM!#)mhjt|L>*qkrXU zY~OwlRS9mg{lPU5<+j6IYudp2xBl}h5J@hQyZ zN?5yc5bsO1B0}x({Nfy*y>J>wPo2PHk35L}!4;?^Vi;Lk!u3m+u)Vbp>nX1~5^;ln zfePtKyNZ{O9zl+pUTmu@_Q%IJp80hOVhQ7(kX*_%O6iyMJo%h^>EgQC?cW4m3DJr1xB$WUeCn=xV zH_ZBM9G@D;fkUrhXvS`ilmfTeY}xjLbV_{E*_EcqYpuv z0mFf8rDpg>2W)5-!pkpS!TEFI-xPN2+>Etr*I{|41y?SQBOIw<)!H(>4^fiZW%H&XXfXrC3Cdj z(caB=636J+HTDbbXl>&lmu+Zisl>YMz?#*)ShKd@^iw(wcw%apxOE_%)+~3HtZq|* z39z$n#m4o6Xl0*yjrgoy*Min|DehXD=r=PlbiH_C0>d}vNsbWKuUmtG6>Vs5>mn^| z_K_vTB2*}2AxweU>UG6huPb@UE8OUy36cmdUX z4V{~MkldL-xy6Pi%G6I%(L1J1@Hw!0e;J4~zcQ7;=PVY-_jY2wJzTOdCfXKcgg+8<3buFjuaz!=+FEFQ$l!Cv(C^^WecZHUc*(L- zaN+zcviU5+vAUUw zTXrT@%C8#hT1K0?vAoe+V^j%ERk?t)u=vyLz_GDOy!`AiI$2j6HuhuvhF%N~^%3tB z>!%%|2>F8bt3zO_wLH@bVrrbcJ?-PTIE8`1AbDK!QT;RXVm{id?fUiWIQr%YW+tjk zcZjm1i*l>QtX!F$pGHS#KWWW6pUyzIw4_x57{gZY%FWi!c7#WKNK#&e2DQH>$q^#qGCoWloi-nFTGzFqWlbEVjsn6m;+<#PJ4qf1 zl8>GY!58-dLOBj4|2<^RXX1Qo#I)`!cZL){* z`<`C-S4Lq`L}?~s{V&UiT9gsnn7}sTnyn!Fr^}>G1b)hm(pI)}cKA`&d**5hh4~Ep z>{r@Xb)(RhhZV38919{m$+A%8@o!?07s`mYM$x(=g-DW~-?ogm8OU4&GcNK6*=H0e z2h>M!e9+m30gfjk5%nEO46f|M2Oi&o9Xkf85A~tDH-%8t&w452!s!A^q;cOs3@Zi` zQkcV>gdK=j!DX8@+D>V zrmg*W^zjYYwR;Ww#(uV$uraoWLpoH(Z_2oF-4I%n>IauO=14dCLCSIRvi9s#9bUga zLVg}2KbKkd0m?6zpBFGYH_LQ-5KGpL(R-e4TFlBW_>ry*vGOD7$e?=3s=y; zn*5iDu)JlQK6#zI{2E?6aKn6{JAGyxXU>h|*wL#vbofn{eHztb)f{*h6F(Oy&&ba{ zvUGxDhcx@~@c4Vk&le{!-@8oOgv{QirK4rckIiH7lLKh&N;@a@)0q>l!_=*{WdB?c z!OIc&y6Px>fc=`;27x?jwgF-KOMc3$DuNVJF^)sh9VtYIqli;J+d~{vb?WdLzcVAA zLd4-kA4=sK&XzMcc>E{^+S>8Zu6x)nqUz$2vq~7tEa3d;FixF4gUh7r#QZ$!jFa+( z%`9mq$#fW)P9H!jnvtl^oo&F3t0fZeYz`iH;K6OsY}5L#E#p-t&7ce=6a-TIWr)+@ zVUo&{lzPp|lb^0K9H_HbICm+l;tD|{Sl+@3G=os!8W3chlp#(=iF}TNmVOGO;v2za zLOY=Rb{{3Jf~p${2j-2VOh&FnlG{N&cJUyl!b z-~;&J2S139ee7fS=tnqf*#i$e z`0j&RtISGr+MlILXPNbk4#@Zxk}g=~;H0cW6l^Rr(&c&?2N%ci`QxwS4_|#2-#UH- zOY_U<>*&Ep2iD_9`nF(K`v4WMDCL$P9pNN)3~j;#8@J=$4I8mz{YLEDvYl$6jq6&n zR>)z2`AxC;ADkS)_{0=8`cnAR`rUYRTv%9uc$DP(bc_%dcIIVysk*tvQm_R_tNVOl8@r!(`ox-g9lNma0Fm|0rJ!t@+c zq|^J?Z^4JwY`_!iSl)G;v950g;>1CQhf+t|-^3aye zxOdHJL?{nc0~!d*7^Z=tYRbo9C3^+RPxeI(Mb>P>&S#ZNU#~A+oaLY~iB+oxv1Q9z z%I-B-xhjNcG6!EU&y*~bWH@7r+TeVAaul)DGWP82q>?j;UH1-9x$Qt^A&5)oGiGV8 zPT34{Af0Nluxf1>+jp!$p*T<6ma%SAE4JTefy#%eG!@ z+ul!wtRL&vbyJ~=WAsKD6BAkDp%eO8A5`E6SM_1jmX%mJ*uttRqo$2`$Q$avWfYX* zMpV#jE4_~=ZLC?B!iG&9*s{G3UEOK3glTeu3a4fZQF)3`foY}U)!p5Kl|3n}=;D1> z5NRp_EuD1?4)s&%OC!LrTIpPkke&~bGE_LQD}dm-5YmseqkCNkf-%kNXJ4(^ft47l z#oYa1FiG3gaBy7iLScbWj5VI@b1}dvgy0#CiNQ`^zI#D4s6C zD%Ie7i2dJgDtsa4P33F3n8$*>XqNYBll#Iqswf|`5H5zumnkgHS*XamE|xC&5XwZ* zyP_YU0<-kW95UZz{oe?nP5bx~$67THJIJw!oin^AeEf4%cAllu{*@}~lkAJNypS~Y z6{xXEb0OaK5*49f5*z^_doqXR&rhQES_#ow0HH1|(WW9JYriyz&^}(}O40C)b9yt!Agr^x#mcI&@GPKBB`=?=VR~i`Q#04G zZhb4Z@9M$!-Cb-0;-?nFZ~Ax|jRlZy2~yb)A=ws2Ye$^&pq>2Ii=N&dboQi?ZfE<7 zQ|SzZ*dCchT(g+c=-l0g#3m}3te(h6IX-$nW!H)-LTrPEBxG?NexN=5WSGq#r(F2b zD9UdX5f+`>Ij&pInZ2uYl6Yh&gunwF;I3YR^&-o3Fo2oUq(N>R$(=C_uJ0kfn)yR{ zPI`nW$7d3=C~YVs@^B34PqZWXLn?Z>c{3S{aCZM50RK=12YZ9*A+TYf@5mp zS<~Kb%ufb8;ioixpx4qySm`(@gwr#c32MvQ0TKz)>P3T z%g0N8v4~)~#`SVuI&R+)^Wo+EFUNfGvAp;eSik1>yJWkg>}AoO$u zvj=A|IX{l>oxSMaO5SLRnEKQnX;t!7SN|Xal=ImG3&=e)kKnKmvF-$XHfxJCQ(Z|% z$d$=&t)!>E?>tX=^wlNeVs>jU$szMnMhw~@Xt&9b&3 z+Iy#=?>mFGCQ?|*0U6E)2m4LEtXSEDmUIg3?I~>8JV1G{3WIA}**^stM}}ohsHl)v z^r4KT)K(Q`E&J3gUD8N=Onx_p7xSL&C14@hQbq4Ti2SyK`K)BUC(VALvr{Z%)maK$ z%(G2RRB`=E7Hc;quw^Ivxm_LDzK3Ikd)v^{7e`pM2-nfsmBiL}XxO_3oa!LkjEKxNdMkM3=SH!V-TOa#o$*(AK`3xq9%P8g8 z#wovA5(%`YJJ8?T!M-wy)>f^&2%)tlik24kZSe$xQMSWyP^-03BmT+tZRote3oZ9b z7QDy&ytRbDbL7Nl{a83UkGb3|`MDF_1MFwwVb-FyiYbsT5?1U-i&54y$3mJy$9(G* z_TyC<)!d-yb59eKq#+(v{gqQizs_f0ayLX9tsG z=km(wBPdZu4D|J2h<$yU@;k+>R&acKPsd7pkYm+{SLi^_7EE%S_tMq#_}qmf_~NDG zcztRZS1Va$#CvQv+OJRR7DOJi{SBQoeThX6Jn-PXgNCSo=O6=>j8QVW$zUa;oeWq4 zJ!_Vqj9;1+p=TNRr1+@AlyePSuEEPSkhxCj-f8?(7#YG8SMOvnlyOYUhxN5`dS=?Z zH+bA3CER@ET#)hcsi&SYhka-k-bX(25$xKv%UDKrs;_4G$zQYYbP}-ARXLTv%Hqx{ z)XXHMD}$d)13|N~PtcsV-pNoYC&QkWIv+cB3@^U;BA$EhIb(oyjfWn1;KBC>WayJJ zCS#ureHO27tJRpvG4@gZX}Kr|LKBq&UYj4oSFfDF7tS8V^CMTVTrFeU$~E}pV;{ml z-1``Qa@8(uPIj3zWMo^@F@TI1O&bGfWKZjhUge4AWXQ;8wFzf`S@Hgyd+`>7CHiwYmdaTs4-#C)}cE%)rfS}M)kLan%`wIAC$ zR-iXdwKhiWO7dEG&}o@)HD5+Ln!u*ce%#yEhxLIt`fGlyk0h{y;bXx7Woj80$0=Wn z%UH^0Fh4qhsVmnpMZEKwWy}{d7%MJep15oOJ_=qH9&j~d{~AXh3CQ6dfNbq>O3=TMlOhPAMSbls2D9o^8Cp3$NgVKuj9V?=%i^^SpHxdaBaXH3ayD@;S-WtL&D)3Z(WgwKXaCCTv z{Zau#YpF=B2_VKn@X(qhRGs(*cM@#5i z5kOCW!0adF_HWwjDV0nyzY3P9xKB(@U~X=P?@K5b!kC|_VR=b?T8+wx%Ea>2sUWHh zRNAYxEMjr>g?cB$MP1u3nxcmze#8!$zaFGA*@f08I#BOrnnA!*~EV%c8sqxKCZGD-a9$A%GAzpL)M#CBrE5eu;ntuQKQDk%N+Jp6|&L<=0J zv{F&C7R{_)!;+*!sLPM`jZ_Zzq!H{7BCw50)8pic9ppKt?VI3_aq?41$<7LVTsq0aW1vL{dd`}4N zPqZR&svG661M`=cknPPNx<7~@+k%9QIpl)mq&^O7h|k+d%N@Mm$@jYhSkc)FdyH*u zx(1yx9VWl3E=Wdg;b~^&Rbs@Cl=GGOpQwlVBH?2mntc#U>i(wtV^EL)NgI#FJm9yKpWl5$xJ61rg}35Mc$KR5UGdFOjDh9TQqwf&eE}u z;C*Xb1fj4qK9jl5GTFC!GGI?u4_M|u)v87Vb`c=ccp9(m|K z{P4&31sX*p@UZ zoi^*+qm$ zeTcUuNQWRS8C_UL?a@}UGN{Fg3uRs4Ko#|;v%od-F!K$ouc9FEg?)(W$NoAvNPOSV zI1et9H*6$!$C2EYLa3X<)iQss1H^BD=*UzlE-fKgIdPRdN=%B;&TmyYlG+_&I$I-E7yph zJIK$i?AzHNQGRN_!YV60K>6O)okH)57+Tu=ND!xFiq}L8OVh-!5JYP-i4AK8@zDPJ z@zIangD0Ncj)&jB5f8m@9Ugmp3;W9laPPg_*oOMZCvhVu3_q`7-?pa((d|K0drRbJ z%GURYpWB{nNAD$e^iS2Xban}qjuLtwQf9ZZ2OUQaYT0}5!xIx;`H_`;=Q zc;@U0oL-p05*a{;wdpfUm@+BI`wHtlM+Rkbj(qXJ0}tLakOX~KAS0A(l#>xlhBI|I zdX{jI0ZU75WhmRUX_GOWX{oIQxTYG)7^rwM0=jY3@##yYldhAtmMFN!KJ8niS#$?&IT$!>ZoyUODl?_5RiPEfvbGEnN|Vddr0<8z<;9Dd;!egXgVpZ-()+OPeZ z**j26ixb~o7hlAS-})B*^r^4l{H2S?E@x1r ztkLjH;>)gB=%IWXT(cT^VxuK2k*Ko_O7btnYk>nMv*gd9#z9jkj6jn3CBsH33{%Ev zIjLs6896H;9(J*UY&OF|Zk%$o4}HB|lMcKjj!P|U;Q#poIHv7$w}n0S(K@eq$5ELP)4ul??)k@#b12+OZe;OU&5g`PvXYt z7?!Ds>Wi&tl5CBmyR!|wt!+q9b9w5e7xDbd2k`Qd<2ZET1}@EJP~u=%V^VF&5YkDC zMnM+PQv~H-{2}(w^zzGEYx=JmGE2*-E|FuY@aRy3 z-t|Zk!yh#&Tbfqp?;vk=ktb3tJ9$HOZjKAgF<+)FV|TGk1u9rW`(O)NhuRTJP$4Ul z*jdsx$V9|Wb*e;WRu$7^8x4^!RF^s&Ax=fvZ1UgW-gg4xCo_wa#87(_A9Q#3(6y1r zsHih5p}Jy4Kb7hj+p@FFSx!8fWPP=@wwoC^#R|*Ix{|R#%Mis|hK8))_O=AaP(5ZW zb>P4meEn-D@Y1)=uW|ag`#APQSLQcT4OBAMj1}igWYH&ox_ICjFn^g z8+&(}aTBlcbP_|`)+0qZGIMkW;l&76Z(WW08q$kevKdQ~KXg)aKA%IJ3Vbk4C7k>t zV|r9Zyk1~FKL;HmCwi#ShkfEJXQ`u%X=%nwXjV4U628V@%l4*|%xjd?nmI)dG=^UF z4;mZBC<9bJo9$INC|%)gnlD?IGt*M7*x_~4ZYh)Xtm!S0q>QGTF)>LQSK+v~jXXsbFNyY) zC$R*1CS}H)`T!$jWN=nqGB#-}?iiP9sHi_Nc*?k?T9)yT`KoT}Wy+S4$-~mt3dAWw z8Kn<5OpWq|ZN}g$qYcZm%>0Z3o(! z8u71|YsixaeG#V1CLdtKh>(doSFL6}v8|6~(dKW(n$2t2SePyGEfhHZR{@hwn%S9U zi8&-XS=W>unsrytQvMbwuWFPV;j%8k1E zSMnI&u>GglUfJ=m!jozT;fOPicw2A+i9Qv|HbXx;8QTga9WtU>ZuD#P4Wg4}ORih2 ze<7ll=u9fG^zniaAzjtiNVb@OfUKsOX6?VHTy^HGilld z!{|VcnVFpNPbQoL7enk!)z>I{8KBkP9Rnlf1lzW7)(R}i4XtD{ZGhEac$7p5l9|E* z(x3N6zhVIKbJePt^j%|L-53OwMxFdTZe&MS7v(bBrVb~maNH23Oq2}N2Svlf;w2&$ zWgSS4CP@bsyh_=sC9T@-Lq=`3ATv%Yv46-@=POtx4VQ=yWjbZKwiAeH)+}iyehxSD zGx61O*Rnoh7M94*q~m+U&t0q6Al8|}{PlT+ib1T{KxSUYY_(klt2a`QBN9xZQme3> z;!)u?<&wfo>Af$W!bdyBN~da9`uow?}=-jIICT&xu`y~ z#S(6eOd`*Ie@E{ctV#D%h%g-6gvPu6kn-`P!Z=0Bl~lR~+o@mgz2_dRV;OAn=kV2Q zSejp8ov;%lK6Ul;`eXgrczh9G z_~KvUx#z!$YuB%^J#$znSziMXnX>U9&x0Rx|oL#;tn0R44{bza}J#yp-{_M~G%q$<)p&wuR(wFcr{>8sAvkKkxJn+DS?*V8eRHrgV zkr<)V7e~$Tu;{8(Ld#U>$`K9(`CO(f8;Ym#;lB0w$iPOlOlNWQ%U{K3zw!nA#kn_e zsXUJ}{yeTlYRE8di6fyavw8gHb5G&v$?It8?7*jY-H(6#_{Z>n{_H=*KX~8c*wizC zl0T%e6A~N;B^RhLEa~LOJW@i9C{}sZX2u#=X^&j@=CG&!oKGVMpn&QmeVBfvpcjHA{h6}S)wYc(p&C0_xymb`?5cw)y+ z{OXT>8lV4P{|P?xKmUFF&Zm9`|NZa%BmBWX`Dge?pZFoH2}Y6U7}H02`QY}g_~)Pc zA^fvXeG(7u-h}1p>-hXLe}(__;!`-wf%*~${nD9Ma47myLmTmbv;6<)vBz*Yy?Pe?D_ZeGpW2Uq^z%>R@BU8@W8=n6RC;_^X8%{HW=Id> z%|WhaT=_WI)<*WBV4O-Ym4v(x`GRGXysBtyaQ3l`I!#gAS8%W&v#6x$aFQS@R0ft7 zswh)Ay?XHyUikV^eEz?_j^F=3p2dIuzh1!a|Ng7^gWu=#?;ghQ{pVNl-#>Ezr;p8{ z%<@Dv3#_+r|e7XAtQ z=8v=8by%2N%puv$B>7r~mvXv_nU*CiN9LG!4e_5zAp8##sNcuFK9^zkK`KJh;I-T_ zjrNW>+erZSMtc2Yw)20@M0Nuef7U1|wI~x-*{SdbJK1)b?s9Srz9;;M{ZbbKe>Z|+ zR}kfyG}7S}1-e>{Me|4$b}KORA~hi#cMFrG+K7L%Tuxny{fX%%eb?L}D8 zEcw2UP=OlfJjt$Qe_GB)dT8Hh8LEnu+ZJC}!$lNB8FK=)_Qpb_h+42@W^v0rL)Djh zauR`Z1obTGzL+I{F2PsQj9bzzP-eI~7Ri`PYy-tG%NNy5VYW@y18d%?QPNw+&p<5> zzs8H?qGxpn{=q+c1W$Z?1y-$Vp*)D0eHg#= z7q8>w(Rs|zbNom_FJq#N5sDX!I|sFB#-RjQx!^;kWK8!;)nTKm8W`{WY?oSls?&G^ zGE%c)o6RWo0X{}vBC+I4Iz~b@w6>Bi5tfHE3$iWOiFcMIUaFUIb#@-ly><$}`zK$* z|M4H6#((|YSMX)}@{N}gtJHP%VeD)8Y#;XTTVRCxjP{{U^Z%H$qB%4gJ zRZ^A&bQ(A142eS(!GgbvntwAVcpBnBN<<${-FADTC3dMK)mAg1+g&sk)4v!xtK;Y8ZvvMTD2m|l>eHg z9wQB9n2&_xMh8Qq)$W*fj(|l(ybUq1|NTb;Tm0`nN`klK$YVN^WB0-%24Yg z&zGumC=_Rzj6RyEf1=!G`xTF?Uk)=p=}a=2&HQznwU-}xqf`+t8L|NF0g1OL~re-WSmqi^8Qfs>e?(wc7Rd*n^x5M)0S zOLlP_)kA(38t)B1tN$&sQmX1p$x8w8@;DuvtokWXzUgCVB#u$eN2;t}ANfxoIn}jjhO1@}N)&3mgGz?-S!LqbhvSc0CB!T^9%esXVn1?vZWdpC z{dN4r)_d?!TL(HQ3+x!>R+#A%RZ3$_*)?-uTqfzqa6N->TsVSneCt`9J8%g8`7G`q zT!%f~E7228ke_XoIaU+jSnM~GRU6&xo7Anlcb5(eKiIkJDF*6+a6DX{l`O3qN zL|>7Vx6+YuZ^w=u&>nhw@7s&Pm4i+m@dSmQ`7S{j?4;8S%0qc7ZyE74TTq7($cU%B zl!y6B)o{vN2EXa)X(~y-k01TfAH}CX{b~H%&;1;J{nvlp9J=x7qmLScqr$m)7{S>9 zgFl@>os7~|$7sH06A#_rbI&~{4V6K&3{^JI*yn)<-v^-BFylt&)_@Ej#vVkcr6pm? zie4(9TL*{m(7yZeu_vFzM|9}U+BLW_K}Gp5{~XV}@+~}f>}8x7zK&d}h7<>`S9o6! zhp>8RCAO~{!m9o*w8UdroSwlfU6nFPd;RMGwcd^np&Far0095=NklM)ENfpwgznHRY`g zg3MFS*s~OpBS9&Wu}Bn&Xbc6)IgZHCo=T%T-GcQiy3y4_+68f7$<;a{ldtF2Q@M*?dt^kW=PPDR4mwlLDv8;IT*V!~OfVViS3-tE-#59C0W?p4Prk z;!{V#WQIgio4$uMw&K`FhpDvpbU-tSrzWOZH>7hL+X(3_V_{W_w3c6L#-2Yzd|(_1xAWqS@Hl}fVx=3r&`T%nvKq4Kpn$`cuM zq+8M6)@tyoQVCXFN$&Y6VCt3)OUkl@gb2kl$Rt=n8RdF~YB+h6?O7*PIzmbZY)Jl+ z$EDn>+-BUzR8_skdhUQ`{h2nzcUt1Bw#yN;>+_PsJ z7Uq|o_Q?blUvgAVZB6oC@`(uZo!Ss6Gc-$w<8}IJ)^dN}O8Q02;RTWrn(ZY+P05VO z^N3M~$!Jt&yD4c#DBF;hVv5HW7mEz%7!@>L7oW#{A@sDTv1!X{?0@imc;u12c;EYW z;DHA=WB&tN@&1SI!3Q6I7!Tb4fZ6l9wKYxrBqL=EE22^_vR+H16KTwLWWID!{u#cP zuwfbEEpapJb83>j!C$yH=?ak)9y4g(cexUNU@!<{-t-c4fY|F z^P#$2L@h>{-w}o{v=o~AK_w!KhaD6aNv z_8aAaP6w9pQnbxv*tfG!W|K2zQ7fsjUt@AchAA;k{L;s-WGZ23rbAQ9cHRCv!%dk(@M-~;!p})%K&TJN_c?JS()oic%B6ZRlWke!u z+T8;W?kDf=!;??mhYx>Pj`vSIfG0lq2>a9fIks6#er8|6v@`>}#6GQTmyHqM*>Qyb z-Xng#cKimG7njkorww*{4P&QAVI5_s%d|zj<=hhHDDQ*5IAR=sX{MqO%Nr7xSn^{H_@_1cU0FQ5AiKKmD+$Mw-+tmIhmqmMs{pZL%Z;Q@~G2HV;Z z6JN6;G=|ccN^L_%XtRPtNSYK+YE#g;xisw1Zw`1hMPAF;O!!9J z?*w}9;w(_Q8tf|{y>pFix@vHu*?rpAZ`-zQ*tl^c9)J9C{KQZEggMMZ9gfPPc*4<5 zvl-knN|Di3bZn<0seJ;qU!R=HqNQOnK)MD(4?OVTyAL&r3r&772U#>gEl`FJjX&i= z8X#%r4wW4Zy3-u+4E6M2@0zuEVB;n{ymk|w7~F_Q`c}iHQZO|-iMi<+SRCBNIap0m zfeYJI7kR&&ies6|__5hZTrTHO;h?yUd8a5p@(kb7nL?G}Mlu;JGVevAb$)IF!;3R0 zX|{~Md~?7Qq8u4&@5W#_jp1WwF{7mtR7h{+moP~gnJeT`J^HbVXe*Y z_4S>DW2IH(Ah=lI0F#5Sx>@fLLM)uZ#?9N2EmZJ275|qf$8nu_WFtYGnxDlRH?HB* z)D&{0lx7@G((lyhD9+C>!qO%^=@t|tNh-j8M3`lJG=V5bg<767L7YyNayVJcU?vvB zJY`-zl{QOXf>jLwb)vhp0e#`7V!~mxZN5)ypvplh^HWv1eTIw+GM>rc*U{EzPSu;A z&pS#3%XaNb33D?RDkUkv5u;nGf>#c9kp?l`xLn4_6)HkmVB}f`*RPGDV$EayMxC@P zLtBLDgw5;#j-%PvI5J&nhgq+Qco5ZEj$>)^hB3NmFl}=H9W~3Rq*NDjHl`=Fr(lWk z^bsgPg^S8G>sy0cEoahrTeA#9ffTaKtT8H0GH6O^*RnF@tr>BFfa;Fg1M`=%VrfZX zJwzI$7&nd@3mBtvJ+Q6=Tkq|{t_KFNdGAWByNB03z1T&!=e|Ddymt`Yy)njPzgg4F zPUoXsxkQDG3c013LM)GF0GJ6=9E{h=bCn`qha>PE4I*|Sfc|nE$syKu3oABZAvh30Vsj^Of|?6)@E<1)-e4d5G_&;0p|FX{ z&H(G&u(S`+O;IewW{|zIh|tTV>KlCVD$q7IfOK02{2N#it#lof2WuIZihKAX75Nh! z`y64Ld)|-b@mWj{jR3n?Ud>u$JJw7YXC4)KQM0&e&Qi{rls{GOP0+omL^_s1{4C-c zo>t{^JVZs$mXV9~;3vK9T7*1P!|ZesGn3@w=^93^TcGGbG!jEkf6|nX)q-@I%5fk@ z-tl8`j=5BV$SnG?FqcI(KTH11lkdb+b@E-EXV%^5xl1*7J4)HD^+_71jRD%(=R zBT?;1nZTxPy}0Lo^3vW8?AX(doqGnboBXq9Z$Gx}>cWaaDseF?fogQN_GFTAzHVBq znH8$?`89`*`Kt}daM8*VuVq+Y4AHY~z){gT06*!xI6jZ5RTHGMPSgy-Ut)Oet#!19 z%nL5maUXvdhrF@udJO0lvmA6tkWKTEEPg^D!GiK*?%>Pg8V$|Ws+m#QVP*f z3*ymM%9~bLwUROH>A;qHy-5C$fs)dZW7ZHA^LQeFLMclbQ$}W420o@oId9vfC*OBz z_7Hh;Cp}yCGFA?CnB~jjWXT(S6!H6`XlbP^%I2`VSU|19c4`L^ zwum>=ZlP>ky`l~G>|TX??_YuKyE?FWdm3B!v|`T#edLW**rLNRR;JCYG<~enl1=AO z6wRiVASG=y%SyA*WPp%?EF3kx<9>rvO8VkdV@Eu~iJkEEo0KLOq=uY+^vdUQkY-{Bb z$L_{}OMZ6@zml!IQq~8=D5mY}D)-i)eyv%IfFkln-fc3@)TEWxZ4nwKwS~dMN{0PzFr; zYtOty(ubA~&19X&@i!JRu7f6)fpXr5rFkEQM<$TV6&a6MQHI!TD{2eTxQyy0EH61r z#f7u_GqQvMH5X=K%4+&SE6OoS53-AKra}DJv1(7!I{PV|wA|H8f5siTUNGg#F9k75 zT*gK?{$N8I9PBoIu`*LxeSXWxGNoT(o2hD?fiU8cPP7HP5V0djk~dbZXu-bw*5O|E zm)mt%(e4&(ySEj)Df@Qs9l}=e^FW&8BhsFto4qypnK;Q%#T1Q%doTIaO6V3 zw6?wH6w>Z58MF=gry?r6T2hj{T*UePP9qFI++CLH)DCo$T8`3lpvkS}*VL+M-QB?o zWpq+hJ+F@zBSHYaWKC|zyxgaOAv>p+<$jCZ@ybeaDT(FEk{(lB8#tuwD+hDc?gy29 z72k#^;*;$K+>X*yL<&XFNDO)S7Z-F+2@QR?&HbSJ3tJN>5H z6dmU3X7=(l!2%ur`N3TiTqP+d^@$8glWSuL670g&tbUht_HKIbSNUvxN5@!Yk5%L* zw3JyJY^@Ev3}lb1P=IzZSk$_@uq%tA8(utEk)wO0~X+{DOY#pX+#G zrPFiilx&puP_LZQ*T42^-~M$j9o?+(XpGBOzKl?ZH`du-Jv==d{8z`mfGm&O6$V(i zX-n03sz!Lx_!EJFAUW2pq8)jb77#{tw}qo8MnPZSE}u-}h9W2>0m8D<-0e>Aif85OFAUC>Pkre8^H{Ota zi5y%lw2yTy5#->A<>G$xKz09(R6@lK zAiDBnFTOFTP)&ACO;oEO?^6VomYk?)`=A8)EQZUz8ZdJ;w5H zcv!AI3()OMlx61YqXSifuqDmb3u1l@%c2bh{-kPfT?*;Bc~uDH(!*QNjUa2yzlf)sFH7aH)K{tjK6Fk{?) zi8?^Y62pR~`(6%KMO61@u($*{i}}|E8bJ;DIV|@%viA4>IEweV@5Tt}iV(@i-c+WH z^Bv#cfJgi_rQ4_K>*cW|z>E)5qoU~c)@YrRWbRMYAN1|de|!mh%d9gJ*OCH3MmbskjvCkGGLjx?e7=wbOCWr&bP@cUBO&{B_bsoOia(47w9w{?i z^lBW#d&c!HV0vzQ`r*Ns#PzUv9yxdZs6JJ0dO4G-;iYmrJynDE95&Ost~Y1ux=F(NItjk-{R|}QPd~no61Ouhvx%5M3r{;u zK4FSBgx)lm!#Q{?0kJAs}m}!gOyLuU&gfsy7FiN)ev& zvnFgxm)ry=mf_BKpJH8t6HjaQ?gd~|jT;jW)Nn*4UJ^XPE!X*cw~?n!nKMmY=)rmk z8}M$veKxjs%C}&t*#y1ym)?6VYYy|nQ>aVGRB()3()%x0UrU&1lHU6kfhKf;^Rr#? zNp00;;KYEduXT;C#y3^9q=D=A{_4>)kW0NqMjcaL#46#SH$AfMD=XmCo5-}L<`i%u zHC&aYG@gqSOWY^Lw0&0IOwu@QS$Rg75w5PdhZd>bDaPGW$ZL%IZvL>YWcg%_EZfAz zQ1`wYr>WTaS!?Tpb@9?Z669wud608G*paGr7<4_LsRhIN_|YwfE&m#n6}NF`f-Ajg zndXHkB9iFED0pj)JNrt0Ai|Q-sYsQI*yk>Suu78U*Wuwqf8U2`-keNFMny&=%7A-i zQ?JTDi=N~ZsMy}&)GHL&B2mR@9`K{Og`vQ#@~sD-sG(>ZXj{>SYEtXh)bFXINsxJ> zLxoj(-}e1NS&=ZyI^-7bUbw}KDS9X(YY*i$A372BN4VjJlLMJ&QG?@J=6|Vt}V60Y>XNW?%bRsPjRhU&j=Kv>M;# zQp#9aeEANYe?twqK|bG%)PhcqapCO`XuKadL5Di?(OJz#GR*w`K{@9 zk4az$i8?)*g1(*jUt(@|WB@fWR;|La0ztWqq!&i>ZqwG|ButnghjeaBriS%Ts7)5u zX@~D{;q@C6{gw$lE88*SW0ZPOZm<~c#+?aVr*t}w7w6?M%KaW<;!HK__(?sj}Uftoq zUnx1O9K+9~53I4*2eYw$m z*N_Zh$MNOIrCPgk5&PvusCa#N+&+W+ON)IHA>Yk7YB9%2d}{Xz1-TSRGAr~8+BDkr zf(>A)d5oa$ih%RQ2kcNjOUgveJR>d%I7d-%h)%lIgf&7oMBfYJ!yFhMBu9FZR6g7<4pLqXIvQE4F{b)uNnY*bbfKCRr;YNCSZ`R@{UK?PPwQbHYpuYDGy zq?GB*;KITWHXskk-bm6E{-^h)eC)76?bxX}yitC4U*5~~39D(pjG97Knuqz_JYqH> zL(s#3=#oLm@uEttC=oKy8V5FMW!DUEir<_l&; z8L7X_O7^&oOXxkNUY8_uGj5lNg=`1 zf$B6^V-nC*+}xPME1l979cEnNRr289!{Qp+!}->mV5@A~zL=~Ur){91zPFmmXV0Ge zAtET46k{-$zLmWmd%_=3!oB~ z@*{@VE5AxTpNkyqBOVOpl7bP+(~x8m#or91t4)igkHrbYHu5Bh$HGpM0=a%p*;hLUcQEt#2tp;8gnS8}Qw~N>L znKI>~fOWuLvNwHB?Ko7wH{>fuDHk1$h=9#hP_g`yka<@Y74$FL9;uXB-JbjW!6u-! zSormOA04h@VOH`=6PKyYyekr2GToZern=CONiu6es5@5*V6BVp1=IJsH!krXcv+nm z`GUR&CQ42ct><7%zFcPp1bsBW>Q)Yr_~~9;_FDGD-lTx??S2UWCoVS*_~S z-+4g_)|hd%g61X{9roh$*x!uv6_#4{-QNQ6-<&2IA2I~I44kPQ4CZQz=tapd9{a3; z__GCnnWKDmA@|PeVN<{zqJA7&br0>iJ{b!=+md!{*GKVI8U1$J2BU5oyBMu+w5FT8 zBqizkRe34slub^5UA=7!@}R5_zvl+57n8EmswW$gz)wX!8r~mGQ#R;I&SpJzT9xX~ zOPD~QyTZYrWf=K*X}q&Zw+TaXwvcq)sx7@9iiXC;+Z#A|MWRb117!VP($*^NATum z=>>mWgnRcgGD;szw|JFS30w9l>Qx1VM9=S5ThL?fx%MsU`tVpT^)Xz0I!&--O_Y|> z?H$f)vL0!xST=A!^%x@W>y}$@m38oFFdh+e`?$V>{9>RLl^G_P#TDCZSZ?yc25e%j zplZ?O+O|)iOc+CMJ=M?ku>WV5>&2|qlM$KebpXMVv2LB*Rw#O`T=fJ`RGPWEpYDZx z`0L$Iaq4n%<}ettGpPa3b!sl=9fwaxyVn#oYwZpwQ+odih>zv$ua=@rDSu7!6h7Ev zcyM9#Mk)qZ`s_M3%WsR9@}3;VG7JwarN^h_ruI5V5W*{b7J}>WwS{J&n}e*bh&qC~ zwOuKj;hOiQoRPp0K)RS&i4$wmVh(RT8`CXAa!<0Y@avunktYt!#5}GKlrA6_@2^&q3)*^$T?Ve-kJF%^q|@@eD&-C-0^FpyDX#WU|PQkeBS+MQ|&nYsbBgP zZrN^}yXIwkxJ_}Fq5V!Yur~zAS<(`SBVo7$@Y#_7*goIcQ-(IHni^XB`WnQ2=|n@n zmO&vmdeQ#E|5?~-R3yiV@uaCafT%y&z!MgoCH(Ci2gmk#w|x+;{g?2Km6g~&)J>5gy=+e#04~xsSG60`R{_Eoz-@_Si^tT?Sq3y9@l|Rmm+wTsSAHjE>q^(9z zJAene7$Urku?q3LYU~qM!<%qFXSA*rv4?`E$@6Cb2aXGeVF2FQ_0!{Wo!4>8jDwQK zNhRA2QVrMk#OtVG`#|w=Lvi8dT!`qs2U>IFSjtop?h4t3Jz3WJ-c0{>e}Fq+z49_c zRKfc`*JurU)Nl1NgvPVqBdg+fcNtDa9a=@<^^Hg`{S$)jWwZAn8UDxO)ua?zMLYVTx!3{MT+PEg@zDTOAckjZhHQezSoHWOE;6VvU1`TI0YI_PvrFij0h z)g2n{QbyNvt<%-Gc7JClE}BF3D(g^Q<*6k)Z7sfWEk~`By8?stwRdgk zTG9t?ZA}CRXV88ljNSlEaw+^?wVXjl?za`TqxSHm4YS_$O^rKom&*#@YyA&tr|+O{IvIXi&kE(T9Ys*!!G~91}T@PoNV{bUDXyW zUFRQcn&3$&^O6;t^EU#v{PMj) zdE$Q3bfVL_3hslm9@l0BUm|^s@+-_{6Ry5RWuYHGs21x3IaT6m6>(x|>&zLHZ;%_z zeAZe}X}hlSJ^!Q+E3T+Fvu{7M`n2t%o#br^wo9l;*wsVLwP}F3Kw8vpfIxJ#0op@2iuy)gslI zTo5$QQv-sXQq_HVdc0xD3Bk!GWC?ShzWPEO+gz#C9+=+K^okFM$-H~sZr?mSl8w(XitelN+>%5o znh~JzYBlfsZ>MLI@}=|6O9i5_l$p{u#l6#OsB0MLr_D#CYe8oWF*c(_`#Ol~6K+$Q zeprkru~@YfmGAI|x{`S??)E;Sm{DMLU4rS%F5k*-iV@!U)8p)dcPxg>L7H1J5`ofN z4-NSP%F~ASWxD?@`#LgvwCHtcLGuW~V{Tr0t*{Ea!AVzQok8=Z&3Yb}7OP9}63c$X z=F{zHi0A?enO_d5bF|M}ko~1ZX14>Bbg8zIKs=I`eJM9Eli#5ePYwI(wvmmd$}H0 zR#qDMOFD$P+cNTo_F*?Ytv?~eB1v4_7G09Z*JDb9!(AL5J*|{rB*}_(34tOEFE?|YZ@~rcEj}4U58SA_^Y&K zdpM(7K8^PVyI9lEXDZ6u8;rR2d%6};Kw<#fcw)@GqGe>a70mp<2OY$<;~oR9FMCBR zLW-7aW9eP|sH}c;(k;q4eaAhwL5Gmv5x1ka*JDbW0<8A5RV31gb2#>6RiqYS$>n9? z6cv1%eLNEbT2MA@7W$QYRfj^yZE4Z2I@i z7q3ydNey&yhz$KU_I5k+o?H@a^aZ8`iziZ95Q%EE441Dgb6!i}@1x{@j;N=Q{MK?`@*Vk z3+y`5Dx{xm>#Q?wl-HC0C)ZV=;8g|``L|tJnD^R|KF7CzTMg*Ho~1yXmh^Z^6uZ~ay zdke?F-7?&sdld~2l-M)+LSd)7GROAm@!{%rzuZd{BHQe^8Lp4qL|;eM>vXktcCh1) z@td+eH00+?2^tJxjmyf^SnkJvBeNa**teW4j>F6GilSDS_*7aEA%yX4U86Jq$>MhS3O|A)#4z3h1pZaEa6c7bddQfqn4Otl z(zgWziy|>k$a<+n?1w3>k|qP}20g>V(}4WPUWZ&vvcH z6f5KVa3AqeFZM$XbZj4%Xg(to45l*X74dKS0vBK7jenU&oF%d8ri)re zW5m1=j{oNEy&!s7q&NqBIs@KWi%0$@!<*~8hbj7w|A%>u7fObno#m)Pqrp;j8{^e_ zb(aV9u^O_UMSAKg7{z0Ix}y7wS)ok>Uo;*&KVL!-{MJp^Tn@ef%oyzr5*ZF_ zax$g_+Aqzk5b+njg;E8J_E>Ocn=}PHe`tvC7wdUz{^6puAv1Ker|&hz^Z);zA^(w< z5XljpiJpxguA~<{GxB*^;EY+LCSGuD*iYr6o`a>ad%a4JpYkkQRW|+q;%PH*RUf>L z&S7l~iU<~kghOOac_`opbDg(Cc0LnTH;{7sE|gwg^%K+du9F@XF@J^-lo1H_M)=QI zbvFl`H@!TO-ysLhQV86QhCET6#8*X; zhRS{czI?Tb5*{H@;fDTMICm_vp7R|OMHN;?>!(fT_(8EpMa4~VB$r%{i3aE7YB~?u*U~RrdWx=TPoLVxyk5)Ye~p#S(Ou2 z3W4OCmb_@V?w#MEPJ^*ew-6je&jAEBsi zRie-l{&ZLvbckTa*XwcZ3onf}e;J3}?yf#1uZ@4n2=R&UjE{RCdp_lC!|q?6Z&LtS z36bl=oiY9XoqK&#T_IB)nIEJ0{Dfg{7A3p9FRY=&_$So1} zL0GHGXz~&RJbd#<0}!RY601=^_C?FjR}ugeaC%O-#M-9+KzJkcJa7Lg>W@vJh)#mK%L&(s^TUOQt@&22G^ z2&aj3^ERWSxY5Gv!(>MMIA$$xUISdVlYJJ3B>pkqd^0jO8vOI+I)HWO>R@hN*Ov2F zi>ki{t{vGt$F!6wweZt7VlQcHxh7i`9bViSp@N}?ugX`O}%n`?he)tQcH=@b`sh87OxFj3 zAsf-ZH!#t*WZk=gs?O7W;8&#k`f{!4Q{I_uM@zHgYLXSbwJ#Lw6>S}E+D_7iD7~K@ z*bC=LT|Hm+t~af1X3>PsHE0jMJM3Khp`Vcn-A%D-vOoR7=&!HH-Bz&@T_MoZ^Dg*j zB)FCAX13MqSY(xs+S|1ykqo|s<&W7#F1{#%x*`4GYSq@#Cv;8JKdnbjtc6rm+tn~4l$|Mkg} zbyT&|ouy}COy}eLfWwTtXEajRd|#LUr63>lzpy0|qu5S+5 zpd+I8PnZiUe7vHzbh&iC)D$$((=eS%ODX;`KFGh5hx<29@C1vt1_z8!R6~U@Paqf86}Py^qeQ$?GVPiHuln zVcyQBuW7(RcQw`4B}>T9zH2mbwEXxU)BPT&!2m$m?k1w#Pmb%{0b#PLy>jBt@;lWG z+JEtr3Z8Zz9<$y>#~@#!RW!%6ag-Q69y z>PfPnx{YWjVd}xn_%Qr4WpK@1I_BHb;=hA`z&jq8B^B~mn3lDOfqUd;N8EkOwMudB z<+Ym!HMt(10q3e(+)PY5v$QY!jDP6Xfm~KAUG_D+PC76TO#yemmWg7{&s{;wM~0m} z#w*BzRGLSW%Km!M%}Yn)#WaPBp+UNZl>Vpvm>)~(&MOBzjXw#br8G8`mn`d>XiU7R z5TQ8sBUk!+^c1Kagbsa;5g6SF=<}yXO?Ra4xM7q<6n4pE{F%A@-Y8LGq=xJ$;2XoJ!C6xeIhhi(I;_lyTiGq6&e`wJKS zmN6IXoZY*jgMfh5t5P^*tgxK?mKQX;0ytbri7_$yC}+X-So_-zdz8Gb8Al0f&>l;M z=f83sC8N{nK(>c+|28_b=~^H^=A+>KzpK~mwcQhoTFaqsYJ)JtFAv5;}* z6K4l-?CXw?H}P90#p#^dJvJLYcR3tBdV0 zg9X!KrAFHEv1=&tP0cA$cjwZ^Sgwq(*uArDIUfYC6xx=|~`fWOuQdZdC5` zz&|=x^`|4d5WZ5?qPyZo+Y+$82I^bYKtYB|)p0h%siEu)X2=pqyP@cQLq=pWad^nS z6{Jk=!j+_v;$F<6BKN5MFpjq=R0rC{FToGea`+uY@cpw=QJJczf_5EqUwEhbSRq#^ zz0KZ0s$^@XuBGMIc2(`Qe#~vt@0P08tOaS%R9kKRz#5-RBTWZ{d+46ZwYs^XX3n3m ze-+^WD<4m2P`3oE5`3fDe!DQ>2a#;+a&82@O%BZCwi1gQ6ekty0}B5qXJμ;}o^|@P;2}N(Q!n z_Yea446tg2CL-({Io(Um_un2O>a<>RJ09W#HO7-PTA+Ky7U)~p#Unq?_DvF0GdI{a zQW7l3x@23nMkwiJ=3XvXohs)+!~c$9q3eR72^KThD##%Tc=3$H9T)yeGC={(v{qiJ zAS~QcbPoI=e<3sABtEk08d(2*tBs1_JBdG}RAW-B3oZ%gxQ3@9Xj zbRQffP;uChT4ZVS_`l&00mL*=M%&R+9>Lau#{|bZK44!A`TT3Q0URUIPMgT#%`C-g z-6R1jr&9izqaZ*;B>@^r0l$=q*6K_lpZBX&ATbqYH}u0Qs7|-z-S02lM12}2U)zn^ z`-D*VI5@_LZSBptV97QvfZg_4QZY=LY@jdF)7%J--An&pJ!bh$+n@6c? z(D0J5&pdnbHaJ@yI(LJfwopHn&kL#$p^JyMaBQ$*v$FJ@a$A&67sGcDh4N7f^MyNh z`SVv>dD=okZ6><;j0fMNDBqV?*>5dcmzpq@>(%gm+Xk&e!PecyZ2FCswTUE@YV%FU z&0WwayDF1>%b`!xNrL)9o~!BlDJ|Buo^yD?L;(`K0;0BcUqLlP0dyK*ms0jGzT(aX z>_GU`rc_$1<5aLLI=s~Ln1)jsn5QR@yt**{RP>Y(N0rRl^av@ zzn}8F-QVr;-s7o?m(?owBvjhdUo@m*Rbc75*R;v+!?S0n*cFbXsI|y#H?=i<;clw2 z_bYHh?n_9G*K()E3gztu5(PX5IpsSa$OgV}SD(Ya^z@MLc%oy{^tY;BA6-b_Ok=g5+)D zHc&tj1gt5HW#i?06-k}Z0@_a6I&}1ydd;FfSSzW`Xpv$&zyBV|Xe&Z4EVc&1yv#&P zSShWaY*)TJ2cP^mAt0@G&yS+tpb0-h>MwpAX{GUWoBU7Ddin!C8`t32pm%liq3K(S z(=u*ZT}%tzo7^<<^!oG4ND!}SirJL>^T_b@ReE}xwd}z_MXS(}ZmN}g5&TEVP!2*{ zm-%T>9i)i$y0XGS%Fu8UnT7luP_QP|3yu>dTe#XE7`(Vl(WYaYf0d{mbe_d!GS}9OmU~~u9A4Zwx!x_cTqXNGMqCXVBH|YP`Z9QRs7}Cj=@TJRmqn&Bhnrn? zEF+CR2WWt*r@rM|ygNC=`>O5B`R@T&y+2KpSk1u8MebsA1(gcf)1<7=QB6$>!2a8cmGTzp|}nt=qMOY*6DuGxUv@?tz4+OJ79p$08JOW?IE+ui5WL^VxXQ z6|~t0aY7S#bnt}gge?@aM6BHwTuexRfEJLzSe zrl5op94^>oWzJf;;eMT^ri9Lo=b`HwXw}%VgNfj7w;ryVuam(JzOdeE4=)rty=t3) zoTHV$5NaLmO9E06VS(&={lRbzQGe&hy~sl`FPQIEd{!lJ`5e{D-g#U)!3XRxCS8qA z`3`?hw#7C?A-zI%!s)6q#`~{mxub|0MXk?V+@eT*#}WCmB^)#OeR;TkvHGE(-z_0! zgK3P<$s@tMh$SLG%vM+63m}vsN6@ zuK>`9n28DdbuU@|o$J)<;N8Hn2Fy2^dRG;e2-AaqG-zh7GpKgJNe7-q#%ndK6(uPQ^ttPF22EDi(Q7R=#Hyz`O{{n9ew;?Q?dl&R0XnXftre(39K*R1{| zgxC5M*d?2Hz-4Q#xi?8z-E?`m1lP7>MV|(6uCf3(nIg9fa;~=*0?8KWu;Mr>>F#{9 z*_**+2>%C5Azs@n?o&tRR)l^<6p^Y%|7K8~iM+Cb#^X6FD?6rK&-art0%#pnaJq03 zn*wsASo(_kYM@jY_HQwP-Vmp=_q&*5a7>$Hg1G>*GTglD=geOq=> z&K&w$n#$}>%_()8x}X(^)03Ce{~F&m@JGQDR7|R8pwbzlH+@Y}5aIL11bOpt=omKd zWP)DNhtkG+ljJn@v|ZlA@VEN?X@+UjmHGFG8YR`UZw&_3dJn`(X zqQ5QJ`uC9aKt~)NKCTXP+9^Tk$?3~I#(4R=a7%*(XB}{f#-IkUhAlB!Z#aE#lu973?JXykeBxCEM3hAY1RT!1Zmv)J4q|2j1!fV!AGU z&#JT?Fugqs9?e@RH?_Mpx_U_{7!P)H!d7*T?Y@ z?5-}TR>|$a$5w?jQqdid`rfp4P2hs!b7xZkfW3HgoYxfZNzg8t`>QHgY$o9xUAsdj z36PRhQ&$>pxjG{wtl7P zxBgt4JiP(^?8O=pO1{}-08*>#A?u;}VXqGWHS{>4y^B(ONO;)V7PV_T`Az_rPBi`E zuAu(J?qA7HJ}$3B5gwLOU#vtI=7WJP)WI26eNV;@c!$P(_eN6W}fV!9W znFFiL4-GG~RIbiqam^w~kxk%%3L5t*2r(ks7$=c+`0% zq!IqyG)qwr-H!0#a%R7uh zTR1?bQiL69tYX(q`l1uO^r}bEZ~Ae5+poiSOEn8xX#$G|2%E)-J^FZUz>eWKm%sK0z4sc&HJ~Ex+D(l_DGy!_E|8bo zVTGdKg*cMsBx-jkJ*Zh4R2~!0YpTl&uF+`#oe#=E=fS!>@LK5-D{h?gRuvCt`~FQ0 z^=S1$*1R6cgR9DhF?^lfD__&->7_V&<5Cid$H!j3MptO9(;1i zvRM%fw5Be>8o3H&6CTjXR)nD!7=*fF>zm(2v7yX&Btrqmw4fw z!N|FgIPD9qX}_6IFIoheyS^?t)sttm97&(#c0S3i^nE|~+{t_Lyf%Ec|F9E0J1uNt zS$;=oMfAdW#!2g|!@Al&ZOiEo#lRIOkuA!eP0?{rx_Z&a+7pfWWoF4{yX4#33P9r4 zq5UG2XQedhiZpG9zSp(uA(hQeDJ`nW?9fdfpR5YVv%J_i+x4;ar};vlt~AlCLgOxxx#-Rqw2=o+vbn4w$Xm)oJwemdLx*(>Z;9V^*Q~r{-+jRp zQ@_VG{>lAM8-b5MaA^dvW8dC6-O~>kKZ|HtRPJ~9!Y7oZ+5TIY?_IfmLynKxpsN=G zDs*r)lm_~}ST%hB0cR~_SK;e2#8+#w8&-(<-rim1=^j_dEIX#j6Nh)w3nzq#0y7^ghqyL?JA5}$aN)@0%}|*53b2-I2OIzR1^kf%{9QUlFoSN!1EPn z0;)t1h0V%yG>To)&;yag_L@`sM<`k7C@)E`gR_v0X>Nia* zgs$1)YPn|)n>EtVxcEA0ECRf8XaWP)n=c*I9^%-!5Y=p9F8NE^AaQY3Q&ug+r22ar z?e)6fLQxChX;t1ofP#*Yu+L=_JQs#VJBAnBZyMY=%bZ#&mT;D_TddxKeM#dXURNl@ zm(u=Fr2_m(<~KfTTcIb412tQrPCm_lxE2;ef9<#hxPOoZT4U0Lc0DO^bCMj=AoB7o zbclS%}_B|iFaW!{ee`$73c8_bbc2{Wv7{13=UY51~t2gudHkoGfJke|Dn2Y*S zIPv=(bng}WE9nOR{+k)Q>3wn^?3Fx;wOV}k!153tdYHOB8qnMm?J3p{p`3hQJ{aSa zWxUlty2C-MTv)f{jtIHck2%+v!Livq3hIs*jAVx*F~QJ^x^})FWWXf65aApe&p|J+ z#}f9;R-dZqv;w-m_MH1;#mHXa~3|Aort34ZvqWxrWTb~)P`1xw8?*fpGhi; z5kXFtMOBe^`@QCccLP7ecd~2$6friud9dW(!QvD?h7Jr$wP85D5HX`CUezyPmD0tL z(B)b(PtH{REVAmTO)J(+-A;b?;^kgk!AImCmTabX1OyMtANO`ld{B*^OLSK)#dOZ* zcDI};esctaOo9C)1C(TjDGdYYv z&CYLs)oCd=)VkoNuhGpJCroB&V?b9LXgChe8<*G5Cdo#ZbB*fjkTIy~D1M|X=y%9a zlwc;2xs-uXuCdnRwVf34bjAyAzD{9&w6VdTqb0b>BUAeBk`r$OjMe;GKGRc=^oC4I zae|>Er1Do~zKhp+;EHPN$$W2+Q0n!VO* z(oWlz*mM5yXZ)F?Tl;6>^qCG|z!wh>#&u}xdSuW64XBbMF*B9S~aM9Al; ze(D3^Q4M)q&(*8X8A)U@;sQa0vYDUoJJQ*DtVM?tWOrNV5_V=bQWM#V)THCh@NiS# z@$T^qGAjDE=r}&% zed$5e8gb1@bC2goD0ZgV{0vDgcRg7V_u9!0nZ}*{nbqJYGb)6^@a5SR3Ly^Q*;#V~ z=_7Q<#(av8-s$z8dRkI?vDzTit;>~l1IM+v*-|mXd&0o$Q2j3Pj;6}%`mp+Rl@@R{ z(O(wKVXg{gA$N8#M|Og40|r}kB#Bk(4D0q8-v*D;=gkTHRz#bexnZr&h&Ut_d44^y*nH6ofB=fDVHxkP`ml4QB|s3 zonU4mTLYBqef&QFus~10r%Utp`kB+VG&f^eWD*}h>a=X%t$Xd$d-mDAqZ_T<=-9#5 zsy%=Fs2#n0!R9KfE|-zjrJ<7DR~om^Y}{%0-Lcm$FDzTF-cWm*4Gk1jXCqcrD>OQF z)wOs;k~yEjXnLdeWt5=(2n+HowAl5~+vp))Ona^0^_)ODCL6kTk;l*>SR;kWAnPgd zh%^R;&2e@xo9c=p?UP9`B?d!5M1xWZq&njJ4*6)z%P#rVx+RTo10Ie|22ks)U74nE zEyxw+z}oc<-NG(1+~B7_^n0#Mb5S#yFByU*3xEt$E=Se3`Cduw&vWecL3T^d#*!#3V~Enqp^=d!(Y?L*1Jj3YiuUDt%ay zj_mTZo=IqfN*Z31B$FJ;M`K1BS&MIu5K{Ov5VD06G3&sa`11^AxmKv>X1XN@t&)G8 zoL`Lv25vAT9TZb4D@(6T7d$>}I*QUXciuohlUmF&B3~fWX zxx<)|*8f4jQW|;@E!ft$$yGBYxzLue&$wYEkryc+WAP~|&45#wYXyF(D*KuyZ}jXz z<-&FSwIBSs?ANZ+TEeH&;QSh7E-5(gH_(z5EjgzJLXtX#_xvTFXh;XSo7P~OjxUQi zS7i+xa}V^^eFW+()?19VCZ8$ea9Vl_?(-0Zk%iP!n-#1VSwWxdA8uDvwsGT> zoxga_9(nw`w$Pb%PC|rtB-Al-VK3uwdbXWqu*?ZMnqV@4^bY2$JH%9Cb_xT%DdNg? z8p@MNyEC{Hl{8(t;DvkO1D~)D-S;tH;o^fb>GUrs3P8Aqp(X7ri~i(mU$d)Mui9<5 z-DVFy_@G-wF=X@_`*4px{4Faml~9OI>ih!0v6kZRgaa zO=wb5YSk^3E83N8(av1CU^9!$wzN{SWnBlxhiv1fO}1}*)V7sM){#ujcN^AKpFFc% z6YgbOn4hk)iE-IDOA_aOU*d0_;XjUSaXF*pX5?LYumPLwSAMDY;!?sx8(L>wq+NW z7FEu9o1b5_l|{vso-;BwYCDF2_f|Uh&4sUTui4v$Ve-6r_8{HCp zJia_8e>e8&mlp$ovaJETvo>f4NGokx^ntsY!MM}4LObJWJP}Ji<;Q|mU$m;Hl9vf@ zVW(_(I^Iy`kOQ6()>|uFLyYTTJ?(n`bnrX~xsg#O@J7HKb5T5(B}q?r?})j#_eUTs zP8A{$C*q4hoER^u=xZRxzd=0bfh2s5p7nf#?4$l9=h(vTpfLQ`lJSfAuGc9LYo7iU z!W2(ld^H?iAa2fAILVu~N(5g^eUChtK4d&E40=rD<*l@&c5I39aD74p1X}P7^^GFf>6fwx*}G<5io|*QhTiaQ2tI; zj1~F}{`~OGNR~Qo$)GNJ5{VOtaY@??;mJ!AB1LcyXu&&9I+L;`sn#K25B@mrLtQ!l z(V5rqC4Sf7i0MOq>(4O`*U(1Sri(ya{J_S(xHJKgw0NTSy&wsT)p&-{Y|6STE!_X&V zzZ5`bWG2ND&7J&OR)P#+%6%NBbUz@6d+0WTL2aEI8yku~p@z?W7X({|9(N(?60#lkS7`jLJU^Yy$* zm!wVkpcTr*t0{b$p<&n}SGTW;zLsz&qpx$5Q;RH%V`EnGUi79|VXt9qvUzfO8$Y0aJLSW!3*4TB~EPXcM82}aXbh5 z!oZHxeL{114m7U=V|{c`roV8P+(q|xJd`@nmr>jKYG*yQ*7QW`D)bL_ zGUDeWtsC^H8zeNLjd#7f2CD$CKbM~wxp&Y_;^o(HAN~{P`WFIy z_^%7hW8+-;$7bxyWMC#Wy#6?gZ+Usy4jq2O?!W(j+p>9+9X|4={hR;rZ>+YpX~m(U zl`?sD>ALZ&CeC9r%qkK(7=tx~02AoCCb6kp+D3*)ELBxRg*T+6!Lr3d!RKSF#_LM4 zWTl9&*lFjj)9%>kKJjzzVE94rC@5bj7qI}LQ-scvvDw*K`#XQ<@7U|FziyxT%xCQH z{@uT8pZLTl+^`l6U>H2koH=9v^q>Ayd->&;?N@%~SNwhx=Uu5R7&HJ3Wk9UlXt>}C z4p#ON)@p^7f|MUenMqq>g74)D4HVPIJiOk72cuy$9D<804a+_0F`AGr<`WqKAa0BU z5Qn^>jkuJ%_qh*$vElu0>;p?Qw$S@x?+!95s1DBQijZz8lotyTukvuai-5&Vt_tWc z%L@j@{m$tF_JhmEZ0E);_P2L_*oN7{D@`9GN~6=ZT1yio405Tg736rqP*?g!!_`L1 zDJ(7dS?RR=rAD?{whT^nVYxBUq0Fg5RtTbq6jgW9DfmDL=KvN}cLlx*EP0%gGLlXY zs!l~8dKh$5Se%|?eNqO>giKtXU$WtmA<0J>H`5|05AdZ8MqO?8$poMVJw4|>mZ_u%!D$!HgQDO>&m#O z3uR;6@-nJks4<5mErvUkOHe{F*RH=!W`YcFCK21TQG(kWIK+&=L>Wl34UpVKzUPs# zG^C95t-n3aY1Wo=V)T?Phig-Lw&X}upq%0TbXYHtvVkrfkRaj;q(lAT<=zhK^%&bV ziG4%pO}joe1Eiuf$%;K+!4XzL_U5(*-i83Y8F*)%mcVV-0yvX8UjyQ=(;uKW-t}QX z^t(|!Xo~ld<`_@Uyh-3Hnc)ub30@NW1{L>qbs2Mq3llDtv- z(?yMG1SKrlkN|1GmU>KHYbY*3p*;RRKtS|lfw3qLp~Q=3l5A4kmX!6jlz!@tgB!Sp zt!~%LKXGhZv?w|AWUgrit<5`<=I}Mlsw41d6!L2zMvmJ=g!V#gQ*f#E{kf8Y2Ivla z3Z8Z)dHdQi0xYMgXQ-LF9;4gfhoJkRYWaHFtZJTW}+^Go9Qc4*gjAwc73 za*cSo1|xjn$5_)boPsA6n7rI^xIVua9aszN^$a`|@}^D+3-ufl;JA;pkXMhf&w(;v zb0F?ea>a%6AhA5Hco{v|0aWoC?-S^VF**njL2Sv46#Og(VoWKvUg zF*H|r90`ekT%nC@j`BHB9Owx?>p(8iGq!F7yn~eN!}|r8XUDnnJ2m#j_J7CFZifkd zyA@WHVIaZ?RmzpDkSV!N%fa#!V-vP*^ESI>*DZF(t+(5qx7}%X?z_Wo-+P9%uKCUAAY}ZrinEr)}J@ z(FTeoYt|dqt@Ca%D0FXyw|Y`ZtZW0`eDh5=Y+zKw;DzCfikhFFx9@%Ldv@T!0r$h8 z5i6Jq1u%Tz2*VG<1voI4v=W0HfboMm;!0eAxcC9Y!`Oje43CC4=$x3C@VMkp+=!2G z^6fR!fdiT%P5}C2T7Y|2H%1z{QvP0=`_PB?E5t#E>&;vM2|ak4w`)8xMrJf2maYzOtKdODXOYESnp$ zY+gzZGz-7e&M(Yfl!<6+N(M^~H<3Ax`kep0mmW&cT;MCm{f`~unK4R z0lcWWF;L6bJE!VxfbiZp@%d(ci39cZa3jB)0;&_Z5=s+Z>IJ+)oMiIh(LJ{RLgV0q zpI^L@b@hj9A*^hnR;kl~H;H^L^d#kW?a>EDT;dxD3{L+7m1`dB*Bg}W#{;n}z4a2| zCS?WI=)3-UBOt8T-3)$Ic!YNv4~X$@oMwHMO>~jRqPr2m3qg%v$J!e|_{P=^uzvSe zyx#MZ`v?*9y6cUx?%Nw4a||)L3o-Cd@(7-9fomxUY1V;|fPD=&sMK$K^{L;xw?j<5 zh9kMGr~b3Tk4hbL_!04Xi}p_9{}({4htRW1nO@V{`%&NOJUROcO#Ayge7cP5OD_9e{LeY1d+|0V$%35TAP_@7n(xf(J$1 zFvIq%g33tzNfkt?DcDvbn6SmX~bP)J7|ma(4FO zDSPtCCvDfRy*64Lx5@5^O|%BA)W}=DS+HU&Z>6dxwyWj~4S3U-uB5D8WBGN#CQFkx zUN6|F4O*e0$!{%d1MQ-9D_twJ^xVx_rYU+?>Q!Ph44(z|=CZ zI+M%wjuk|E+PBbR$lJARm!Cg)<;oTN@P|LVcEBM#$Nqbs#!BqMdhYtO^959({opyNXumvrYA5UIr*_tEGJNMK2#KQtG_YV-=v{ ziUgHIvZ>Lw<@sf6lou?US6cYUzKw?liM{jMW+(kpK;>lEJds6wSyJWP@jikuk5Ef@(#D zY$z|bHNy^V)n_WzP;koYGs<9La-2MMMNG3v9TQ5(x(i#VVv69_6(aa8%5lN~jI21q zNGG~f;_#*zf4Bf39sc+c7S^u$8(or!Ucav8BZz*!WdnbniO-d{tGvEtBL*u>lJ}iS z8ZKN}$;4JQ&J{}gsv?!xd4x5Td7;h8@xn+~$`1aR9YDhfS@dd)_GW5D?G)Zlvu%TX zF`|;CZ^!g)Nb8dqYKmIO`0HcUgRZF*Z^)G|Oh=>2wUjp-AzA6kN-D?dRmX~6UpMFe?tB%PBCcgYTdvB7 z^R3j*;wP^e=MT*7NT`s8?Utc+>38s){NZiq7h-@xeK{2lCWn6lSHXX*1q3s=s9)g_ z(PHPsH>|tMbkxg5V^ig;seEC5aNtGcP1aY6N>=i5HSrO=LJY3XTEf9=f{uqH>*y!K zKwb(XlrzStA1^yj@-WksOzS8nt2ZJW>DIfCe#oBiA)w&T&UI>)Xi4t63vuIpfY66} zuh%*d5r4IPL z$(1o%<}zLJxW;A1JGPettR4)qkQRhG@{m3Z4u#;H2a**!e(X;oLjzOdlyh<)`bl*3 z1v!`DlBtsGq1|FlB*1OYV{iKe2@O!-$P+QLt_TNzgr8!%B%F&o($eRVA7MNS{z^kx z7_)JrpFT^N@5)m9Q!i^JQWRE)I~RLA{qxFI4_ExI^afr|GRAwJo=E@nM}Kzwx^I(C z$$Z@NTIUz?%Bpy{w&K=B8|_N}rp-8OG0ir{wEB(X4Rm*Na;jEzAg>*QNrVjeM^ zYcDMkVtr8t`cF&)ILFA55Bw#%#3R0q*7;BEj1HqDfyZ@K7^5Co(6s_3;FEig`I(4K zg|Yn{@c&BF$!ZMEX`B*nWFIj-(D4cb46Fxf^m2@K;zpGNT8NEY%yu-UNg-!dqHkjt z-HgaxKYD|45gL-ZBM!HsJ^02s4rf58l|%4HGH~ajK+!!O^!B(wCb4yOyvY0D=)rw> ze*yFCI9GnBAL!^Y_C+DF4(m|_y3X;uIPZS53k$YfS+z#BZnfEETV7a{ z|D4Uu&fDVrqAkoX*z7Fnr)_a*-kOc34HO1#Vtm~0+IP3@+_p<)XjxXBC$KEN0M`rn zbIxGd{@@S(z;^H2{-MWX zrQqP;pc_-Z_O-9szy8<%+8%xMQQr;L+{obJ=7PdkoyCLr+IYp=N>_Tt5hcKhwO2cxa%1c=MzzHjjB z1N`*9{~E7=m^X6QfbjW%=86ZsUy6VRU1UV>FKn6Az$p1q%+mifO_of=@WJ_ctcJ_V zztz$NTQuvR)DTUNyXrG_)q%7F2KfXH-|(=4;^Z-~fukjbJRJ=CE$T_(d4(&9pY?~R z$|x=`Px#7tCLdPI)kH%7KiK;5~5f5?k-9; zK`6nbqQ!$67J0h|n?tx45fGxhiN6*@7sx`*!!Zrw$}iSvbPlwAxJewa-j<;*dgA{G zSdZtGB!^E=;n#q6B^>|j6`A$4)W_TBPFSzI8N6LsFI*oFIwD+Oh8Q>I9iG=QP`5a6 z#Or#zb$*VZa6e=o9t)Eq4~=J`DgQA2jYcwB7G6o(-x(7iRiL%4v>|1P$uAiCh_mka z4+NaS*BME7nDKmqq^lclxKkK17w9H=1fOf$(x)hrNu&ve^dJGe>$o3bk#0aSG!X*- z8tB${XeDXZTyld1h+G_b{ALitZXD;WzUw}*ErD2ny#PN0goQQ=GKFjOdl#^VzPE0A zAw&qRLo6naq3-=GKJfyRO0P>i*~=+EMiCsWl4ENqp~y5|8hFh0vW5Sl)KI~s(s_us z35UPAaiuHV#HH=QL5p^n04oy=$KJkfu?zg-M_@T}uo5oW#5n8eLKtnEmflZVpOX6+ zME=B&5bn~@?@#e3U{()M(Deik(^m10?x6Kg4k>)@d)U$V)uO zG!aM_(yq()P**@wLOreH3+1>L)!%sV>`Iz;l`ktSLoH~62N-hCph>I zTj`1+0+w*U1TlESR(~MKI&b&H1#?{>gLhQ;dDecbyzmo{E8c{`gATmxN+5~0R-@p< zc|AGpF-C;zpL>A}VkW)u-Y(6JAjXflfKV?thSFmF5v(2zx*#=SA207BOYm= zd+s?qb?TIF6DD86FurjU>Vcnld=dD6{a^oATU}lC zb|8Of;r{sX<2E@t=^qv$U&0=FfH^VRzf>LnEH|N%|LYX(-K3z~e?+FZMeaVukCt#@aVpkmWrXo#6_*`w54!>2ig=L3OUS|4}dTk zA|8gmbe)xAN*F|#av=^zJ$Z>JyeSmJbTGmx4P{E=`k&(Qlg zC{lzFIr4MG%S)L6VaRnknkN)Mj@FBQa=r0lnES0o=((;{ZsNs19BCpDhn5pArz@6H z)N4i)3~%xg4*Z)cPfL?(H^_!)K935H3GW)3Pz2h9EB@gkT&iC+7Au}Ie0h=;GtlI9 zLcGu?T}ezPX;E(tE2fyXi^n)jMseUPUxlM<%!4^mMy$x0O|0T1qz&z!J{W=G5B#1- z(jI}_-v2j(m{SsG?NK*wNW=qtTq72)axkh_4q?6F0QXQAXt_D`rilP9;Cg&FgBW%* zToGTSCB(xQlN)Uv>zoo_8;k#HQ@2*(ScMSoeN}+&!b=OuCAANI+j)=!%jyaj^12?I z@`pFVirPpOIKo<|z;pEGKtY-n55$o_b;uJdBs?W6x}2v(T{63Aq(kb!lga@{Mu~&R zoqEL$82Qf+(AAAAQ7-AHGV$|Jc-$!`&NZnNFTBJcZQy;v$VA+X&`wpme(yt~$|!)x}_waZFm=#Wst zKu0Zw)jD#{KY-G)+J`#wdOh>3cR(0jY@2u`QM^J^()Ru=x5k&fap4DCG`!$Xn&_aA8sQqs z86-Ki5w8J*=K@NHK$EWuQrTVJBVQDW>ZZww$}6Xkt6G8p`M2dm{eVBb93U5rukuOi znE&VojztfF3+2ei;|J;pVKp%{25(;=4zzd;B7VnAbw|3uyy@7k@d*e4}Izy=fyKkQRt-8~unI;{o92!asoU z80UHjcAbyb23O=NsBjeg2msb-NSufF;r#?Yze*bRof`Y#odif2-P?f} zPZL}wbzGY*wvHBTur%Zh+-|n5pl6oKuGXqHH$P{ySEp@$7K7h{8>r^z=50Z4X_3`? z)3&s-!?XNkm7hZV5 zxAcDYv!Atp@DKih{guD+SM0)t3%=?PfAaXsSH9xgi$C&_kJxYi=5N~H`+I-SKKQ{8 z`az7(KmWWN3t1ruKJM9G{ENT%i}pYMkN>0n@-P3gmxq;r(1B4Aplm$v-Me?K4saOy zc+bV(lL0_k1^VT zk=1hGWLs@s*CH768Q&$q_K;}kYH^;Pt|%qG+9i)n3>);?!p9b1HsvBmvI^(dZjn|I z$#&W&#pe<%&S(BaF& zzU1uK)3$ARmp%K+9<*OHwxxow&>XY)M zeexA(!4Et<2FG)9&n-9;dV(WBcwH$HJ${V|xJRbpuM6Pe$u?Cq6ymvGR8H9*%1Ph& zkq@pHCj)>u6GN)dA%DT&l^E6u3ns>9HGABW>hAy)Nlw3=BI>D`8sL0D8Gc4<>$?+sQ%?$ zHI&O44;MeP{HdlSbv_i7(QVeJ9n=VIL_DMvljyekBdcVX9^v2(1-g>YYC5(Ng?3dP ztFL1KhA;eJeCE|+oQC}C3iq%4LKHvL2$YA;xaf4(*u8^UK%8IY~G6Z`BGsM6+6Zs^RBgIP#MMbVNUBv9UMwhN& z4CRhF_W}`PWJ0fXA)TD_XS^mn$$j+a*&Ve*+tJx|@$kswNuTyXKm@y^^?`(jhz{pK zIAL64xw()22~e-9U*rb4@MY#uqv9tua#MAPi@=n+pfKO!NCS`vTH&|0#yd~1AGCUQ z5uY4`5i9xRTDhhiZK5V>tNk$`bF_gR<4fSgu;LSXyBuP#G?r~>nBpjVIpqO&`VkW$ znu-sX^I@2}^AAQ|xE%?NK`2Hf2VS<|ynZpVGFT&@;on%7-d4enRMaEJzLt6|E*RHn zkM%loAnh8EXRLc%Lae6=IM?rEI6RT`b*GPt#xD{U9r2+7TG&#F!w-IftAp8${>_Hy zu-Q5|{DMRONaRoCO-S&68n-kyVSJ^os82|VU7Vb1N{4HD8u{xQ4Wd5w$uHE-I`4Wv z^Z22>v`t?1j6-k11@j5zhF)X?Wk62Fb;*IR$S~m3KUOy5{OE$Mq$9ju^U*ae;;PL|gz~Ohj zi{&Kzod9%1-X-%L63%JY3KlEF2*QvBu(}SzRC#sP@|wsJwo+NN zqbCp9`ZI&l{Nk(~J$}Rv9zJNN&YZM`+L=F++#ocvp?(GYQOr`ulmY9w#k0_)1P)jSgWNWBQb?ovagVqgH*R#R?ce^}f7{O)JbLt~(?C27kPkonu>GaK^q2fhLGtApBi@MJFm+a`YCBIt! zF%Tb5nG|XCQ8El(~ugJk|lcBw%hC`Mm7ED zxB7G3j#v&^TNj=6mL^(mBxJZIwtmpbZJkIt)rHHMKm)hQogB1+6o2Y$JtQdQ zlz}k!c5wVjJRsl}P%ek?%>QslfF=>WNP0pY*3#Y#lK37^S7=D^QKdfhq}`-|F)nc> zn5URFD30M8O5h%j!^`e(l=GnI;-tg>Q^ zUIV9%k|gPG0HluK^mA33SQou@+>61tcoX02MUF`}kBRFAjtyD&3peD|8*M#2N~ zwdZwul6coe?0pRRUK@eOx1_n=qX%~k3Ytl*&){x^jHg~7zfU~C8rot#9z6u(AKrn} zLOep(@_1WNxTn>fmzO(Sl$mnTORncuVQ^sVqrGk&g3q<#F@1m#A35as{5YB5$8)S( zCUfCV5N{w%d=pk&=+^7gZv{QMtlc`s_;5p5OD^cW2cV_5Iz7AOIfTW)x1{WGHv@&O zk<}aTouuuBh~(NbVi0pIehv4&l0y^2xDtvqsNw{M97CS#}Z&K|+(? zuS4{`Hdb(++S@K*3jEp=lXk-Sy?Y8=!ssfUKz?2u-d7Vy_y>=qR+B3ba^*2n6hbIp zXhSy|;}lMFCn*%sef&9G6+beacmN+_K#?1TVwJf+9J0>Hp3EV+;o>x}<@TJ+9@em@dGzcR1l+%X=L{n}pct znCtk%mB;lwLfUKn0`%g!7DApeexNx#g!|B-k+M)*0m8HXi*D!Fg#I;LF%cK}{W$nJ z{1bhiKWTy-a*^fXfu6|eYb7BpK@)D0yd&{JP&x!%VNJs6zg!Y+9lU3_yCEdWa9$U3 z0pB_hyaozLqQAxU;U^nt_je2^zZl6d`0=3$j9NY+)V10uTcMbDV3S^NCo_!;}+uYBHq{%`)A?tjjH@i)F~Kl|5z+U|Pz zcH8&S-S*|b{qy$e|J%d%H~+z3x4-&#f60F4uYT6P_&0yne(vu)WS{vPAGXi_^^e-8 zf9XNH`?Gs(bnlp*SU6=@sjZk>2#ao5wdOHwLDPy%8hB}O8+qP|UV;LXf z;M_Y__5IXO{glUv<-~Y~fq<>F7y$pL|LK47m3g1~)Tcb|KmX_d+_%mW2S7k_4j%?I z0RL92Wm~pvaetOxa`ql|iLe1Dc|i|`K4@bF9<&h-9=0FfdFP#Ouw&I<#7SNNLXz~v zWknz~v%MHUwj#6T7y~6M0^@2v9C&cFCC`26!}}dVUylRnTL&Ya8P=;URwnVjPVptb zP?|9671DVtw6beor=DV|B?%+6wMySw%592wHe~BkKV5L79Sbu3%#|Xqyij_%^bZ@>Kn|E8?AoWhr|q zYfY88u4}0~WP|NNOEvjQI#^htA|vgi zWhcWbKv9`lwHCaR;n0gkYj;Y`M%w8Z1BddYI4BekqJt74Oi z+OpQMQj3)ptcXyXsw1ox>b8(qeft<7I)$T9X<5lQpO79Ro>v|nwnh3gV~3_ysza1> z>KDbB@slwK!woC$F?dt*P@if`)nB>3EZQk+#+qgEvs|;8v2ukpagryee+!FcTU})w zr7bwzS~#dPF-@FUH(D6ILoUuQ(3D_dMJs1n@*x?2x#+yX1Tm#&+K*H`t1o-JFrIQe zMzsoG2?A>Ip{iL96v?pbW#t1rp-$)!2~Da$_!KHw4^dR&e5)2*M>0V?+Z8?DXrz&= zD{ofNQj^H4x)_!(ZnMgczQy@zY^!8Nf|}B1#SgPZ@xgG{hSF*AnshQRAAPrMnbopo zRaZdFg+2riT1Rcu6_!lLSKKsJ$E_;V4yrmtaW-X@G#?XE${a5y%!bq;L!y^;$<=Jt zmQzbAe_66naXO)Lz~D?Pr`EJaN8!4f`nxjFBrANpCaX|B6qHptiQZDZEH6&0{yDsy zb3|LVDp`a9db+x774ceE{NnqOm|#_;udGUF)^loOwO2#(LvkYBQBA3y(m2T^%PD?7 z^zkM#P1HM)piw$V4gVl5LHu#%%W53oFK`N`2MG zEpl`aUDf7_#_AP4_?ja*)x9d7FZ{RKY8MgNqHLnA-4Wj-9~3+23G`#tpxU;ew8RP$ ziS_RMA%?&&!fcs~Qt)YUNg2`iAgA^=hT8@n30# zcS-rtC9*i-$cfI9Qsl)mIk^lHPvc%EU6am}lSCO*$p+OY3pO{WdJuhVk*wH~Ev2M= zrB=nKU0<#4G7#A$dVFQ6_$8$_VZTQh+h~Q@64WRK=m8!RFYkkL^t;g4R3G9Cc4nwq zJvn?Ok)V1eg`O3pZlQPz7L6UrrdA6l9ZYrC^fA>(35A8$q%YHgZOOV->0pdaEsa@? z2E4AZSdE7sL;bB*kU60oHI2FKxPv|pUt#QHEJ4R2e+q&;X(Rvj+)^91TP)%U>D!9m zl$_>#YNX353wNZyhI$EdO=?MyEg}@xR$nexB{LdOPAck0qfo7GHH~rXXrOwDi$8g_ z)i>)k>7y!dpcy(fR*O$7jRoCTe047?jMe*`H(0&q;~=`WWJ9*fhd(6)>f#Q@a`;&c z)M88%zJkWzg6gk>=vDgnf}h(8>#C0_lDe!$!aeWAzM|yuht|5Rkl@?ZAtVl zDkdxLSzWE6%|)~yRgh9YQ<{c!w{}~3$&VUt`PMg;8u zo0f1}kszt~%0Wb0I>&eCf`D0vUI8vh0-+ZL9S30i;#@pd(qUk`>#n=p#Z`lu~ywe~fr> zOD({PK#YADL9D26B1- z$Of^3vZR5Zx5|vgdhVc5DSn|eU>Q~dCiqq>tM% zQtHc^AXlYuU#ia9lP^4Nk3aiEJ1IHGb|uaROfk8W0>HLg&f=&@nX8HJjxxuv7Pc~I zpq6sP>N($L%ql!&i{y7xkIf%%5{kx{@(%$tvnL$l6{Rveps_@GQk6 z!wVG2Rw-DG=Bkt<%1JpGkl1P+K7N7RL^k11@%)1$DJebD-t&bp?$jGnqo7wZj|o>t zief&G0@?M66iO?~YgKholWR1YpauDt<3lXEhMyEPwR@$(WUK6kKt)&juH-WlTntde zqV!@J(Y>lkSzTHr^ySiOW4W~20JTx-p?F0tM0*L3A1tqktO9R|Z-|#so465FZB$X+ zmOWo7bgGL&v1CI7BVx;rtyEW}NSED^oJ%on!+=c=;sVO4`o|!|cDuBi*~?ZC4`tL& zsu#+*w4_OrCaceU_gOpm+?H zqzMhUaX|df&#&$u+KqOm4H%DD9omxOT~k|CSn01>P(fp1tx>Vp_n)-yKYGMYpTs!T zwUX$jt5W|BrGaCHhZ!?bpewdgS@!-#o>|XFeL&rnGkM4#URo_zd}}(?RBD>EXc7=U z+@-iak!gBc;{gwlj!AQz6p^y2G+|86`!NiS(j;qEd!QvV{aw3s=ic1){oG3e9xno{g*nozOBo_e7CH96{# z5BHF+S=Cf|RTEy(R#xB5NVbh+OX|NRtEyjAR*^|cPdQZAE!DL;gByd}4e_&Z@zEb} zx&8+KiDBC6BUo|@MNM4PuNsmw^Rr8u7{Ysm`QA*H)Rat+eQ)!kbapz{nV)SRkEJb($6b@P*aM!h~co zTarCUbsz^}1+^W98}bU=tis!#CB%@xI=ZPQk%E?8}ipGA2nKq$xBGv%S! zya;XT9$wbuQtj1fr0me46ZY)0uiBx5rz9VEd7SsKbQ(FNJTc^Q;~+lOEEJxD|DvuI zjLx(~l)a&Uu_C&u`XD{`8R}T|NLc7wDn9Z_ycbr$NGickiVGfiw~nk+o4a99WKfn+ zE~JKJAT#G7c{C*3)%EB@{(-15Y$RNY|B*t75$283-jvRiaVfw%8GNU!`obwe5${rYiB|2R}&4Qkuot~;3q5)9?0); zls!85g>=*<YB4%zH< z#pMKN`_?rp=fgV1+@Nq{tifQYa*99ls=K`UbW7vSN=1E84FgZp?*1iRQAB=~>Zn{}15E;%AR^ue_FG@S`@JuQAE1-lZc7s{Dx%BiksZ?V10 zF7jn$VB|v=L3iSX2ro_~q{J_xjq{B4!VvinFSsgRZu$-{3NBq- zu^&9K-+t%ceAT}4l|Qs^ee-e2h%?qi2T(T9f+)qvhc2gNqCencYxTOJFT`u8{&~qz zl#KVjuRvZ(G1j1~QO<(O&p3wRba@p+YN%Htvun=BK1J-f(HMC_I_f;@G~kD7g;1@k z0h-bgvtj_+pe4c!g_0bG$(miha@l_CUw%d7;F}r;>tP&}BfFx;ZY*-u#%hptDge7& z@}e#9bD*u+WJj9h5Js(}-s!9GHFU8nq?i}K`Iya*SkS9FQJ3YN0KY&$zY=;CawQ{O zFD?BvBOXMTa(>p&cPLOl=tZi(_mZE};%9V9&eEk0T}M?L^0A?mXy<%x^buqq?IY<0 zr#t`o0s%d_oR+Q%4+b0|E_J~8>E~rrZx})A(is((@~3<-R{o-+rLuS#!7X2@m`W?1 zTDo2f`A1CRBY%^4@rO=`J{?9MkBM*cQ<(Z3eUf`Z`|y(wH~1Rwdeg#I0Rn@;jN$C6 zgelcQ*lM)2vS?G2Q<|I%+2u=@?WyN}V53`VHZ`8H@vH<@yDY|%giWogla*~a->@OM zRCn2=+gZ1{C`Gerg9GXk)p@&RY|t93)9NKF>h$V-on;#wL}Ay&Q9kHwl zTzS=Q-*cDUv+sj}N&UZ5>(>BDnJ9pustEue+2ErWY_a9sJ`^?1y~D`IHdnUXGKip5 zef{fS_pP)T@>&`+`7$o&?EUml|FrGfx6ile#w$4K{H%cdo!|K#`^;xP<2c!_i-Py^ z%P;$xe8eSh@<3_(<~P6T#=!gUzu%34Y$yKYCqL`2YkXCo@wXgdeSHTiC75<`FErGyk7S!W_^2dZ6TF?<#WrCtYxlr748vUUhj?Ap4+;wkISveb?86BY zZvOxYl^xs9FUdVukP8%BJR*h2DOWpGy7v0ZGge)dQpgJN;&=wDRxZIPRg3&oRw-*Z zH(YpKt3R+}PrMB8(7sH}0?$(Kst;BYX=L+*=cU9mb!w-@S2gj_lk$5*$jh1}R+={b z`mAV?zbn%yh7?Q`Ns+`*kdtE)(a=n~nBrxV0;LpluzjADN}`_?Z7Gy^%0nErJ>xQA z@OZ5u#W1hBQTvB&CgK;B89q6F=%gE*H}2SISzR#*QpYLskTRUDh&!GU&Bv7UviM@5 zVOL+Bv3jvLNIjj)qTe1tQ2~H?Y=ZR#@+a}X%$)JCqEQr@wf73k^wNufCthk zO~jj;{E1gmyg^qyL|IU>HPO<8RBWN+t-acU38qkxmiAyJ{_C$Ewq3h++onyE3WJ9< zfmdFc416ZKsAm!5Mo7^@iD_$a@a$(v$*1FQ^65INOSKtwM;WL;A@p>hqb}8|mRU*L z@`XiPIk{x*xsDYzQ7(Ub!Ae(|ciX zEn2=V{t=~3mC?WGT~&WsRU2H%T3vDGADg$?r{`_p^oXV9hq1jmAX%*Z+}Kzb-jk9q z&zdbgUA38Ku2|-n+IW!<(tum_qe)mseJj&cADr)4^-$SXM90c26)W$rTlY-L2D?Q| zNwpE%D2;fo+p>DOYMGZs`(f3i@U`Z;HuuWBHHxi-pM{?a@8xG4ZRS{l%!Eh$yjK0B ze%Zk{j@r4iGfH2uy?ZzNgguNm-q@-~+C98$uJ!#UQ*x#S&f zed484;)i)Fi-wtJXRPwtywyZY=OxknYTGgwa#pxi<)^QTKiikOwsN*=3#S%bUfL<; z^M+(dtzfMU;&)z%(5I44=c^OpN92P9AZu$F-Ka0;iNs+%#n`vrHtI{rLbYE!fD}uS#5P(Wf@A@;-Mw0USw5WTJx=8#SL=7SoJrpbSFDjdu-XtFRxnruq42_ zqWY}*{lyG3 zu;N+CwdtaDT3Krsl@7xW2JpK0+u*^XqvvRzSJJ0Y34DA!h732&z$p5Pov4JobJ zwk;iBvD#V5L*)hityj%*^LevPDw9uby%!}{?_^$o$v^jHtZ{ko&yo$|Gy2-fQrAu% zo440rJ7OERj%qGAViS{NHo0NchDQdy?Wu24;`sPXv!gk&`Vr}AKQOk9J}52G$4u%+ z{fQT@7?{L6=plS)sGC-y)Fxh!@= z(axM)Q7(g~>AUMT)LS+q-4NapgqIk|u$IQbs`RxBwoAJErp;s0As7e4OUkV3#rat{ za*~UT8E%ZGQwyLSK6e==)LstN3Bx7huj?y{8p?xlRsrJm@Zkuxr3Ab0I#9iZkM}6Q zazpphiNL|~bo|h;(FISz&k8%%vCLrRz2j#-IHVe@)N9I1GDLEqi_RiC{DXE0Z>aNG z_Pli5H)bvUqImzAlr6omWRFs zG;ra=4+)4<7TztmCZ+1b`vB-m^l9MF@PQ6*?r?h(06E4aU@bE_e>>tyQLxn6n++S! zuh@?9mhITkv8_{W+r2eod$wk6=O)ut_uDeIbwkT;+bLy9_xJ1`un+9kpg7X7yKd1{ z1M{xUDZ68L$@Xr~+K$a#+r2$wySC+ZU$8AxIVrsoZfyw}2?`eK^G1gP1(pr;1Q*LD z7fALkw1B_)6mVwV=Rf~>`_h-bNtcIWkvaAWgLbzw(Md|qoNty`A7!$ zF&Jzu_(LZ68%_G6cs!@R+0L;A zRtmS|Mnkf(Rm9+>KBRtF&;ZLvH!zSPF2b8+4e%I3qC2OW_wDCcP!mQ(`3~!>k+rR37=~R8Hx~nxtbK4TC&X`s%eV zEB};%%7!U&s8sT;z`O3;WgB;E>|i|N4K@>Eg{Or#tLRyF>pUeK-_xW+G*^x^?c5{t zHgj>_8fpX2gSrn@;tDrEfO{q+v>VDSS2wDOCz-%9sY{DjFYMh!D3-<`*Jhhh7(;1mcyORN;j0X~>X6a!5h@*HUQOxfD@EZ~f2#7VdO{i3M1^hR zZH-ntT@-|3w*$B9OP^rHznC7zsxgTbPp5J$9JQ(@IF0|@w%RwER{mba>VHzVz7@^f>RnGr8mL>J40=Gv-* zWxM#;d8>TAX4UW2t@)?o^sjbp<=a(jEVIfLi6;J0S=`w7LrKW*sc*enu*RXb&3Qk&rqAy}73_v7|0->J< zgHGD@H&$tohx@^k;(?4$w)5gym7Ru?U&w>ZnUE_rlWXUpgfM6nUGzQlP4r563O^r{ zQlCL~9M>54?VRPlsXQ-s?dq%3cJ=kER=Fg;s;T`nsinQsGfMk%)>gh*vGO-+w)#lT zD&Otc>Ytb`eMj=CQkN))7Uf+P{)YHZWoD~9@)S1iYTV>yT0{7F@l?x(7h$Q(9b0^9 z&Q_i%Tl4X>Ri9|vfbciJwrJfq8yaJ!%hNP9O=QkX_kF%B{;Al~v!dbAnzg?#T)($u zwSRZjhWD!?6xZcUMR{Fl+T5#4cIAa>tDmo1{a{`A)khDA9&oCT{LAL5ai^WCl|yFM zgh(aBH-K(foQ=bH3m{Lj*}UqPKF(OAdUScm96=&g-DdaRt@hDR++!b;+qr9tzcg?| zhSC!U+0ybkfH$7}B%_`2;1}eSWK$<+rTmD-TVDP!K@V-q2Ub$;f*m<9V+Z$NvgxZU zPA4yp;2-rcRmQ5~mV+!|brGwSF#J&tRs+dfqo4Xa$7X1LqaWjcTm8HHL-oT))qgIl zt35xhK0Is9lX}K}NJhx#IkoG-wk`f}+3Js|-o7I_`E~iLf76HCOLF1|XUi;dw(T2K zr8KCWRF9&CT@+nC^HD|0=%k9i{Gw#tGc&gEt$E29wf&>3!r8LMpGX!RQ2o@z8>vC@ zi2BQt#_yLTv;MQ%O+32%SlKGyXp3hXR(`H!`PpHWu4%OttDKeoYRhKdShmYAm92J0 zHSluE>faJQ&#CT%C=fvO4V*Wg>KP-Nc#W5!tf0j^cE4)5V>4H4wm4t2kAC#S_ESIo zuzmDn_t}n}8ztW9*ZMJFW2lmX$yn6xh9eHVZ@ccMI2wN#gA^)$(|l6xM@Ecq_~+E7 z`2zf?GPKoi*aE8iVnKZAhQ2WNLKylWFFo2yCjqK_gU$mlvDy%0-|H`4vGb?cnk?SV z42YMIJ@l!de=$b*7%5r?hXyqcep2J$HjTp>_4Tl#)VKT!FF+bzz(O=bp!2iJS1P~- z;&(4UM&EExsWF&QR%mnj)V^)k_aqaW(hJ4&82VhL<3+IM{;YQEN?s|VR0?sj#&xAk z_*weFIhFqh(s3VA4(}m9`wMgFPL!RMd5qV{W#~zHxdLuJXjM3+B+o8d>f4faf6%t2 z|6kdzJaxrd;>C1B!lHv-E!ix-;&>A~qk4Wq{Qijg@OjC}XP0c@rCF<9UbR$1x@`-6 zK{d;ZImv%suqm~)(2mVkEB4&wOZJT;2khkXj4icTmxy_{V=EGVd=O4es&c4aSiqoq z5BjDzHjk_A0%x8;K& zFTM1VuPWTXf4{x@>Z`tr51Lq^cgG!fcpCC19XR4DtoV%4knn%?ul|)AFE?-Ayk_L= z1s@pt(1)LJV2BFCQ~>${N*UZ;kQUBrAJh@-85sQriyF!lcvZ8_a-B2R7_d9KZ`i`h zs?Dq}*d?|{ zGnN}2u+l`y&k8EH8@AM_+nL2>JKd~WSxOGuYf^HYNXB-6Oi7bPDS1ovx?Nmgi)Yi8 zs%4vB(cmw)B$_$9j%^9Nr1P(ANnu#&)@`o3Vk;FXw6&_OG-^@|qh=Uz_bduCHlQA-Z#=q7~%olIL?PQc{*yY+7-ztgP5&xfLeW%yi!c zAZuuoa#_l`Xjob*+wyYR>a~_tD^k*hztP~CNeJ8W3RabjzkF%VYSpgBi&4vG2OJN| zW=jfdc~ujaI)u~Gl#$g%O~=2uub1Cu&6IV+^dqz<#nLT zXT=9|O{-tk#CBHnR=PH%NjLoInw^q%sv3{4R;+roVmmi%wJjSrs{WLDm}9wf`tusw ziy5moq#&}jtSI~=ZMDm)zii@6od~LqH3*01s~(i6uWC{KyUM6?F(JYb%kal`r0JOz zyKr&VrsrmCeu1;2=prcfilbkFEr8h}>!>Ye7iQcbRaM(8%&*$~Vp;7>RY-wXLL5AE zb$Y>;S9!ZRr*>QMwxs>eoYUC6B6?cFoXV@sH6Dw%mX}wpQf^qI(zaE#G40%}OMj5! zomT_05(2{@#=_NAUbI!jV^!OC+g=+T9a7mtJ(P+gwxYhzmZYX81FI`do1NB#Q@RiQ zTo+$aj}_4q3{av&GyiNpXF0YVG7(o3NsaOI=qkSQOKpITpl9Y04iZzmMCG`TYFW~Y zx@ShAPT#Jhty*?x&i4NL9X4=B${v~V{A99CRUn!85R-pQYFvD${;(=pj$DD((Mjp(BPB~;N!jZ2 z%XaSMIoo>sq}~4WJM5NEZdTvScs(y(Uh*Rtvw60JX03LqV$Fr74WG0s@iE~rDhAFb#_iV z$GqlI>KoW=;X6$sT&~T{E!)zf^asW?xmBfEU8#zPl!wO54hCaZ>`T_P#AD=1Nz{d{ zE+L;8A5+_~;*V8jqDepGE~|(`?FhK2UoThHPt;dsvUpsyp_+FEvtTkL6lX9tyPtIzF|Ysw~BJ?LSRdNO=DeqPUF1l zRVtFdyi6T8J1M!b(wei~zqrHh{rh)XeyU)rmlv!yjln~-VW?3Tn`InPf2SR{er~JX z|C_hj|U!#uPc|uyPV;f#$czi(F`|)9v(;_>XfTx z=|AN~W9uh71j?)E7@YCS7osy7i{@STs8x9RKpkt$Rvp)Cw6k=+3fsV?!{j*LrKI*x ziJ#D8#A~Tx4aI7Jj_OEb_o~J)42g_Y&U>qR7QbYb4g)^*&Q1knQKc&Wls>ewEIkwb zL9%>tX;HMUIPc}NMbS|d{pg1r9Z{D&EQ=1sln71Q8^YR2nR_wGG>%HXXa@!ix z(T1gq4&;Zt%xTr@vUHoqqUufKGA{+tYf=&m!&9SHx_!vfx5;lHtvatscB_2qoETEW zk8X!CUs6aqJ$gFdJ*bF=73tm$baO>&OHUy#`jj*Ubp!P+$sN*Ok^Xb&*a^vyMccl0 zn{68!_k|0~n%i7jp0@?*BAh|q5nbtG&Qb%sU{t>9dnj)#MDUk2!i)3a&IwE8LLYvj zp=TR!=-lqm+T|Gh{5}ENoGk+=F|b@l13!LOr?1%Nja%G!#h(5HZ@eKcU9|xlk>aIJ zHY0^s1OD>76t$~WtLeV1NzCQ*3$BFCUoKm9LBdQ*;KF%nW-B?na(2;H7bKjQQ?__j zg66ygtCXsh>6%sMq`)iMmD7vXR>!-2&po#9mb+ZB;~Xj9x&(i&@o3P3LW!irNYMT6 zzyDp|n#%_-SOJF-4IzZ{$yDO-XAA7)y*a^UyVPe1K>@C9Jb)uU6ht@n{f9&tk)+j#k6FRKeNKw?1TE5ZNi zKm8|P&G+*^|MQ*}Ismr+^2};Owq3d)6&=tH9ZbIXILEL4>aW^^4?egShkV2G>@X1G zfOzf0UtFMnzu&jzQ>`^Iap*yGcu?NwX0D}yP!dtk)soO7h~ zgHl2|QvCj3FFk6%_2jqh_nvveo;!EariXI&^wl}*PEOgK8#dV;#UVel=4B1b|LLU{ z?OU(E?(UDDd)gK@kJ#bm1sf}l*_|7A+T_HT6*^21Q}*2KynW@+LHolOp0UEzfc@bQ zzGJ`p$hYjfuf1R==dRkO(T$cHDA}cK*A8js_2$y7{hM!n+y3L@kJ)z*zhTc`K5K_( zuGl@>!Z8J^Xf z9)L1fL z6JtX*I-ZeIEX7t6H_qXjyOOv4FG`+2dek2K-T~iMy1dv@{u$diHDqkl_AN0LDOYuP zTk``=UUOn7>Qq6R(5M=uLo$zNLeGk1OdAvtT-blq34qUZ~+XiiT%cx~X zq^=dD#I0&n`rp((-;&b$x|H*m8g}dt&fEN-EZETDAuEnbiQb{2q3~Dd-B839lhorS zYk$39r~hcqj{f^|cJ|4ORy|X*iRuQ+?UW+hl%{zxV+&7SvDbd2k39CPSd0cIY3)5L1rZ!#*O1gFRHx{+QB!@*pb(BHgj>*%F9`ednMbt zZIf->QMF2a&JAzng}fcwU$Z~`+97-D2dC`d>kD@IoF+@<5!G|XK#G`N8Bi}h>PkjG? zUAfS*mBo^u6Erx;`MPR5jjxSrQG9gLzW%izh&Nuf=bo6esWDA-d<+d+noA|ggIdl` z9;w-DFD%)KBMY{uiTYDdzF{vud(v0noIkf}8+T2KZ!id?G?9{u!-5>mmAtp8`skfS z_(t-IRei`TRuo|jvw2NyPtRLJ6XT(gK^wV!+-6@Ae@>RHbkBh0_6}H@6%a?VcJ=!e zo0~3M=RZt^?||k$ zk`A+3Ydll8+26Trr|PfR_^!=1)Sj@)!Ma_1W!hSs$~N`6N#Q73=Rq<2or|{d*-bV) zH)*v=P1bg`{NVKTA4oo4oU(zBickMi!}5DemL4-Zy?n$L=FVGIle;0wI=iYO?q9U! zi&v~Vx@sFfG-dW_)yYRSnJo|5$g7*IG@G;PT`S^O)y?w-8-8@mwr6)*{@3%C`h;q5 zkE$p)Z3i#CVw($-Hk6;VoZ{MG)zYn|WuBJISXLk0sd?Dgsx6#2Z|U1Jwqg5}_*s1m z;ly}C^9Wp41=iN&r{!Y@Jp>QN5UIe@bC>Px={dV}VaevCk3Rm`5AEpT^LF7htKQfF zGOTzZ-O+>d_QWHn?7-{i?6x~L$wxdUA$;llvSib7P2@8+JfMDFLZ?gF>SD@%_~@+d ze|c8CdcvN4>VSCZm>c~@hev#M4f5R2@?%Aq_^2g(Z6=xObJ&5~%>CPG=h31jb?Ho9 zVKqyiSKPB{K9gj{4;5@lv{|NYL$^p<-#Bc|+Okiu)6eCs_}GALC~mg$KkQoeLGkA1 zl&!R8?euAlWa7(_l*TH_gBguQFZ|InR{6Fj{`o;G4yqjL!;O1bIhWwk7jg1)nc;Sk|1Wsb$(eMuxeM{?Jd5>AAGM^xP?X?x~Y%i&OUW4-VS? z*G|~sLl-6Q^R{X8r0Q64g`4f#oDViXyKKia`5qb`v|ILVkv^_|%9x=B!ob1#Y^-cs znrqq*pEz$%KY7@m{ozqN_}aY9T-HQf{jMN!i(~qI!EpD&-gY>)o?@0mSXx4|*!%3E`}W2u#nUHsqAS@GbYpZjT}9qZ0% zO!-5#+xaoeJTz$KUlu>@F8FrytFsqudTv>KCt5bBKVMXNUT?Vn{1(kWKQd*hhtwxN zkg}1*f=#|E-kce<&c}0Z*vQti)|shW`Ru&p?vhR1I%avcji1-}v{e=SAytxhKgKeb z0LhhS6r8Zgg?S%yLS7u98}owr<>yb@_aAviZFj^@9$&DP<+`0Yf5HwOJYlELTvnR4 zO^k1Jp4NlI&Ei8mhJf;UR3`- zZf8!FmF|+_T@=qmsfj6#gX3_Hc)n{(je9^YE*h13dHe^Y-*(SM2nOioJPY)?Rq#l)e1iX;nR|_%$Y%vX+$` zfRE=T<663gZOmAy@|J6=&y~f`E7DuU8xtkrSO3jQPs=-xGfR z{=a?13g2zo;P9Ar)P4>1rHpi{OaFY{hW4x9X`G+k+_JKGX>gnR)vYDV+*z>7hE=5+Emfa{lzr?z`wrCdfu~Ph$%5zLSCmE9e@PN&|S+#a1WrKT$ZFJL^WSnG| ziq48YjsjA3=$@m2%$`4Y#@?8pu{-a(+rF^tRvS~r9bcNUr!F41Lzm9k{CLqSdFhNo zKz*|7r^v9Td*oJ-i%7n}|7l*pA~^hkoci#70b~(v2C$&(oowUnG4=&3y+hKj>&?qu zc)}WuL&;#^BGXwh*p`i3q$m$qxw($Dr+3}w4^?Q$aqLY`G~oPWo; zeP8&(7wqSL?&sV-+yPEz{I_X>H*6*J4b|xWr6qgf`idHZ(F~w@yykuAvcM5$NLEtG~DQ1B9mXK$qPZ*v3WMj_q6Ic35>~O0sFnZoO^N z?)%UV`^d-l*tRX3l!wNNe9BgrSEaD>ePgq&+b6`AMViMa;x9gX&R0(D+C6T&ZW)!# zvoKkjUh?x~HgB1*-MjYK=*T9!dUe58tC|Rw>NYtsAv!X4>eN|#LkjfF6_s7(y8G@a z$?grdvdRjJE5?e74I2k+d38~W`e`Z9`|XLxUbZ*)U$jF9F4!AV-d}zBu)X@~5xaQi zye8dU%N4Vl1ZyH8h0~rFZ&&2DYSO7jZe6L{$ZZ-QrleF3sh&h5ldsg*GIsS1Da?hA zjovNA?#>~#?U*%Y+cs|tRvoX{(54|xd~%w^w`}2wd7J;!1v4p`BfAG}{I+4+c*}%M zOpIDCQ?Rxs=IIg&9EPNWzmN2-eBWfG(nQW8@i~4^U%GNF0Y{!<}#s~h+p1ojm(jTU#CT!Op3~pIln$OsgHy7>ALnrL6J8pG8VY0;P zxwB`_TXC>1zF-EBvr8A|?bTO~+8uY^Dp^(11iNA@OAWQT_`fh{SFX%RDev0O-J@d5 zjvHuLRk~%L1{s&5N2WjG+{3GK%~f9VsdQ&9-gpc02IeRoi## zcH6pRMD45c>jzDoCpbUbu;WK&?8J#vnk4Zdtc^}LTbIwCy=Xgco$wW6GqYt)0?*lV z&%SD}zj8)=bIM+M<%GQ^-g@JWbM~EY|3DMQtJbY%B#%ZcKP(=hFY=KAwRK7Icu14^ z#TTpUA4MzNt~%aH95uvr$}a4mwpOEK<0|j??ZcKDH%&h3mOG_%$47&%^*Oc3peDv; zO;j#4?8>2Oo8G-*r3v9Xp0(QJ(&5g}+Q!dGZtUJ-*>=emPpc2UcGf1h4BCeKC#-^gVus?{Xd^23^N z-8Lm9&+N!k`>i@(v8mfP+sH$s)*90UhqIcGE8XFeik7m%=fuauS*yR;vgYe;E8SAG z)X%8Qd}fe|kNQMelh#hNVWZnb-zL#AP`5N69C&pgTsNzlO6r4iRV&=1KC@-g^3p$@ zg$2wX11PTYbdDBoR*A6HmKCZQH{kIRh6{G$_(kz>%XaSADgAA`zg%5iZb|=bNWUFb z`>2kyDaqiPUAd^b63#pC-sqEERh!MtE@@0TAvwe@jBy(pF8XTF^Jg`#Jaf{fHcV># z*=+mVP_)g)CkCYxj%sYxgnFr_N&h)b>YuY0o_|w);JEF7{j?oAqzT{tQ+ht;llo&h9XY~seuh*fy$fQB_Y4u@Ne!bAL&TCD}-%_&F&!??Y ztosVjOtmc;wqnaxw+**W*?{-~d5~YqSo4JXjtyI3L*7<0i+1AK!`42Px6OMt*vOsI z*Eg%~BpOzJZ_!_16do#A_r8ubGpete`rMg@RhMhl9ctV7{iFUtXzW0m(`!Q8QvWPl zzLB>BFB}mMwQbiOyKUn=>es{GVhmW~qpD4xowd=wnHxdyDfc1ec(xxq= zHo0-oS6NB@QQJxrQ5JkG=+dP**M+ufJS&z8#ui0%9(aJ&qVuyIJND+BWaBB9i?>Q1 zO^$D|)s?QDJA2OVzHh>sE%Bdl4-A55i`{q6E?b!6Wz@Lcbb&L|BR0fJ%$CY} zR@HJr{Vb1xLURc5ZDB&=%dLYpAl|NZmyMlU?N`MI2gNhnC7*s;Z91yLw$rw9dCAVd zamEVyF&o__KA6f`Zb1D!J7$&r%Qlvtw1EkYrI{5w{L%?492D*y8dL5nnr&0v)jGEF zZ?CHU2dwzR9Jzbu9@8Um2foTYJ7?oOL|9Su$c)dc>^y#gus9gKBOTMAApp=dT60 zLFE*3T-b6dAK_f3&Xs^XtNvuqd6R&01=*mmEVQ`@HO>ZP(BJ95=#X6EePdv4QsGG!a4 zHi*a4cJbl`+p=@OhKB~l%6W}LDO=DO@!0o{+vQ7Djay@O#~nMxtDCK)dHH~J(6O-C&a&#x!mg#fLSYyN?bJ*p96m z#Pb{Nz#AtdllHhSR>07%y5Iwy*aYTPpT`bW?9DgNh*!6ZpEtX%wW|Jl>hu{KpDaqh zQlHFe%v@Lyt~c%KMbmh_OLBCxZJe4EPmbA=%z>k_!50l87~=2)CyhgTKBB=kUCs!k z9&vm(QP-;xc6qZrWsb3=Lbk zS+hSn@T@J=SM2VcyX-Uj?zG!9nZpRUaPE?A>SXL*_4U15HuHf=pJbk?RqT-Z?#$K8 z_Ssu@+x@rPYPWCQW!opESdNa_=G=glq#D06bJ-rBK4Vo4?w`8-=M-IBI{MZLRX!}&A9phs*IXq-LRIW*t-H`cMCs7KL)(9?M1$0=G)*0KfZPf0R(ti8hW4a#}-xVyciVzDKTg>L>7HwvB!8UE)q%=dS^R`d^ zUU~kaWa6aVvTwpRZp|rf+unFp6NBZd?btnH_kC!aZQCK`b!xz-XRg@%;;iJbNp^3Q zlAN|T4_wwnyrwZoaX+xx_S`ls{F32PfR-1#Qg+j}Q+U`e%2p}Pnj0M(^i?~oU@4TO zJj-F=8y+99iA_>gC#2Yy)E6}Vu!Y<9r~+s3sf*@yn$n3zphmvatuoEj1t-i<)SQ*Jy9w z@~HmUWtG70oRm$?PNp)Jn-u=-QvOvwKITAsdQ)fE`hlvcc9ox|Qo1lP?P!9abm~u> z!^05s;tL0T1p(vP{U6$+F_rCP$boq)FQ@dp&35iBDt^Y!oLsgeho@b(eE1_fY}cMK zwL4#ou4uA!%qI%tcQCMHXE`0z0+YC^JS z@0556dDW6mF<{%p6T5ebFDA#t4<$Q&`kYm&OEx-@w{4;gV+?0C4i0H@xlu}eaoApa z>72&fZMI4LIWVlcbN*H}WLth#)49`2wpw1cPkd^R?bd{8ViLZo+liB>Y=iiFKocfR zF0ESAw@(dfl0A^mSw71-nkK!cpg36@pBT1@4Wl-s31Ye={3>szsBwEVWBHAFD{UFD z)`-&Fp-IIa$ba^AtI_`d?EMLpomY0>hweL1H5Uq1m_ZUCL4x2slg%c39w@6@YFV=6#EM_CY%6|C zZ^@4J96u*^oH&u}SkI1Mz|b({>BBHl`=4R{iuz;XG}EI zue?*Xd}PTczk9}Z?2=q21#|iEye+@FXr;!8C7+ifH7~lkLA`&A2xF;j%fE9$@|u+P z@w`=Mns)B#8SB%ys?E2pI>p5%ZQF2z6!faui6;(OK2fyI*YA+>ld|JqJ8Rj+oE1(= z8It1DsxDgo$3)j9<@?>T)m~~?X`2u;tS0ZD_JjAcjl8wbCv8sjFmkQrunC3xvIQx^ z2`NO|G|qOlt+S>1HZ|A5_l-)_5h7Yco!Xj!yyu4=VfTS_ylQkr&QQgg?dAYb3OZ`3keW}$4@_IL2$ z38gDaiQXVuD>z-8JUJ;Pf6@&~d-jUoivJ9c3|L8hJ~UjAtS1^09hWQ1zO9ZA=C05e zOKwm~lfSgZyvG0VXi2mnIahSd*y;!+onf`>Iw|No($*RhP3@2ZE}oDX5En?-tetAu z^3xUJxn+gz>VtmK&GYK0NwcE(Vtcw~v-5QuyJ6fitg_09_YIURb0KfbCugiPuW`YO z;%DZpa7W%Y+%sgkEov}^*~OMk|Jyk$N=b$`8hp%nT8gvg&HDMWmBr^fnWk;nx6xVy zszayjb0IfacWNEWN-3ZE=DhRTtq-Z6H6HwcT~-;Zv)UYx-zPk6E~!l=OZ1DEji)Vh zy&68D{uixt9D)Y7WEIH_AVleWp^$i=Vu}n{U?6`eL1j0a9Xoln;U6DHHKuRBoey3E zwqblgGIh}n7{xy5C@5g?KUG@uN5u{uJnypk_8sHGk#GsW7xQduzGsd(d0hPKzy;A) z!|r}yr(L&eqmTb;ZOINDd0D)!CEla9lyXX+lMbMHTJztLBWEm|Z`)1#Hi(~2*oKV* z;$5s@)_f^hgWXHDTEX(^qV;i9k|w>J${s3Yoz8O7GX^C?Y?%<8}1+%k{Pg&{4k`3HDXvtkktM#|6QEl4%e>i9Pu|CU< zi^okh?7~x`74gC5;Cp3<^`8H*zj#upL zsg@hYwp|-O5}Z3zk^iFTuVME-bd7D>A>DCfNqneohYlT=oKjSLjQ<>U(y}wB=j}UB zzTorDO*e0nJhee}U?)RS`nUMI#&97UoQJx6^LAwPc$;K5~ul zB$*@3gLTz6tFnhQmlk|w;1`98bt)}6NNhEW^eRI>iToIUrwV>Uc6ZsX&W zVODE$c=EgEooPFx*&E(}=e zM9QX4E&9#QnR98I`R0rb-=I3(C)$O#mL$t5f6ms1kFemWp}giLBV^T}(d#5rjEi?k zPK0m3d6;Lo*sU=l9(A33B!49a+m_jxw(NGv!|DUYKtVH(+u`v^`=!v|XWOrRQ0@V{_L>K6*LCl*ZSvc+^={iK|7~0EwH=B_-0Qa8Z@YIsB&Md{ zyWVBjT>Fp|E# zJ35e}aOQa4^^68Jo*)e(j|K`H6EsFKR`?Nyi^dF$Au$cV$%iMu@h4B>@C0I9p5#Z~ z7zHU0f6@~cY1+35#9laOr)g-lTsecEPEPuf{cqO<3wBtds1I2Y{1%(P9{IG>Mg1rp}Z zpS9h6`v>|HnRvps_wOk%SYlWDuIG;HI2eU@w_ z?a1_uy{dLLw~X1T^HX-}#7QY9CnY>i+KiOJ@0>nrZ%8qm6>n&YZ>RetJq?v?N&?jE z^ptg^+!v&H?iw1hiEP>mAXCnb8B)G$QjjX;WviT-wB=K0EG@x%Yq8Hpi+RZaO-p6P zKU^4a#w!;A1sA7$ocS_KEK68jv{pwL6kam9mMvCJO4%w~p9HhgK;U71ptX|dpM<}H z?Ye%Ggy?yjJhvo8hFj)p{t<)oiV%QNxHvxgYVsA554bZd-wx)*ux*Z%Wi+yKHDM%xGjtee+)`V0Kh;$zfRTEcXeAHYFcxv(hg`$ zNZDHw@Aq?Fq^FmpuoNZsX6SDTj;);Pd%tMz(X6%iidL@6+t?%9ZFrNE8K=cAxJ;om zZO*aOQ@_~8gXquP6OQ{}E>gFvndi@6&|H(Uty{KBQ69FDiH==+qkeB6w7w#SNN%e~ zQI-%rH7_M(K?=oo8yHQf?>e?+hm^c+ZOivnMYHO!x@bv#*lKCM%;#*wSivS#$6`s! zeYznfAtBjA@2R?#z=K>bs#Ns$sgA3b`?78a+iQeCu4ZP9A=WncAFD2b;os}%0F%WJ-Oh}RGw{1J6&}`5+mEw?>FrQ3|=A__( zkCBm*-LQL$-GBG3_TESK+56vni@pCnH|zZ-d(V4rv`_ry_t+!vz14O|@ky({I0C$r zS3Wxn)_?ne759}au|;y$kL&rU`fH2os5xAf3PVo{RH4?gQnO&KBW;`feQ9#vRX@&V z{l?w-%7W@zRXa2ehiaC+J8K&r8?)_)$F2QGZCiL{UNqORu}3#sW>iWwO9?bi(#sl~ za~o^0nt)EUzC77XghYGpeAkOcv1AlcI~2_J3A@* z5S_NdS+Ja^#nxA6ks50pPA_rQ*=9=g!EXw{_@B!o1Exfm3wB2NOwZ0(HdnJjDVo`W zl(B5b`ufw7kvG^Q@4Lg^|KWS={U5yB-lO-2AGt+NJV~^5?}InkP5U<6#O9Lfl93bM zcc_p@8SnHF=BV=10}AeTtV?*}^rq2#2{v=`j+D3|PICvx0Fw zmv2pkAQZ$08+bs5c#;vy%T=46p0*2`bDP4^=vdZQa5C?h_yD(Y!t*eau$7nF<~gfT zVz*!3q3PFY8Ik}v*Nr16HgDb{J*Uq`#u7F*k+jX*(^eX6+6yln*4(=!oq?6cp&m|7 z$py>x%~_$p;zlIrQD8|8EA$m@_nzJM{)g_ik390Aedv+9?EMelXzzP?kG=0B_uHc% zxz`?g=pMUm_f58Opx+Xih9z?1Vb=-Q@2P04Xm;4qvEscY8`!(S^3v&4fa>07+0=m5 zmc=JH2a)OE^|Z}@zHH0;YpPpb<7%U|X2nA_M?*W{Hqn=~O?$Ulr_!|9C#J3bwYqIt z7`M_xgO<8kI@WN@DtwemHmuC0Aw?k&k?X5XoBzW_Ygg6w0Sq|Mj%Y)}ANonwH0}}` zY_K|Htrr@0^0TL`@|u{6=+fsBF7*-L#^}O*H>r)n>0a^wJEatBZfM@ww8lrP*1Au5 zvcK7Z=UOg6)UWsu01W}CQs~&C5b+&8PU0!_KvxK))LRc|f_cO*c2Gn>S~iR)%cj3owxbJ-h4!5AU-N zO9uP!qkHYcnm6D7m-g7BkL|IC-o3|e+q2C!O1}2hyIHkU{H^}5>dZWOHR-uOX=8Wn zuqDm;lTXfA`wK1GF}2wS?$26!uR3E`{8fA~BP{`WGa)&G+jRLT+&?m`@u@kmt->Tn zxIsN?>$|IHnTfnL`B2eBo^Gq&?0`&5j_F9YuiE$2 z_na$w-{uM1qzu!N<5}#lS}-{LvQIid0E@S_YOuOA%TP$WV^h-r!VFc@)wG(#Vw63cfd z#>aj28!JM1-?DkLuaLvB4RvH>)W1GsjU1IwIjZM|u`wxb8~yA5#HLMhdg^`S#xXI4 zg0H?{wali8O*Xb+%r=m2c*M_Cq8?p&tqTmH=(+$`^vCLrxxD}*Z}u4JXeX=YcoN2w zG}IZx8Gr!}!(Cjx2N1@psJIf4bdECt<;MJCIjrUj3R3VVO(=uuzY_s~NRT~rUk)&km1+K<2Iy>`p>*V{mUzqQ3QxahPkAzJglm8D3YoIP*Py|UlF z_|>o3fBeE9+vmUXReR>Cr|j^-H*9`k&c~PM->BQd(xUPmw@kj^X02R-ZU3V-Fu(!a z5;DXW8`6mCt-2ehoF~d5{8cp@59PDIz@{Nh&~G)<*j%VBT0;UM2cNS&qE>IXVmeeP zDSqBc@)JjJJTRfp&&}AeLx=3OmtVFoeD(|WTYvZ|`?r7eDf{}9Pg=WNwv6hVQDi>K zq{S<@ZrN-Fy&rnvMf>`O1ztYp>Yp=?gaBs7QPZ{Dp0pyEksN%|jb(=G=Mv z_T%5SKmPjT_SB2d+oamks7WzUJvn>mN`OP+tIY-D5{7;${Wb2Yy%`ClTHnNeR&i_`_kPI0tO|m4-#r`;sj!E?Gmux@{H4 zR6h8^Sf|NCk{;@f%2Yn3Y|cjWLoU1_%(fGvMG1#h6l#A4qdc_K5eTCN;jOZyK370r zspP8{nHn*6pbW8XyHPK@fsVPC^RD=M4R8?w$9L4!naNa3bG61RN}mu+vy&Js)ebIH z(NyVp_f_zop6zoTo0)Cc8wZZtlTW;6U-;Zh_Sw&VSABm_V{^u8Dw7pl*v(j-!@SEF zfM!@-0awZArKDgmP&r|~ltQT|U(mzc8qfn$STt5Bq*Uje)s=UL20G5#p{@+doamho zaT*T^^%dKuFfOx|FfYc$SfDQ{DJLF)!BLnF<6ra?3`q(E0sW$L)dxAux8kYn@m(7& zJ1oVP1G{QTW7}RTAq7GTWWKLx=~BY_2Zf*hj-7pN(sELuuiJOM**2*mRkaaXP%vkj zJ1-NDVufL0qG)+m&q;4;W4sXUNWD_t>ObMjZwBX3AQpb@%-V(>qf*Y3R@T^q-m*n8 zUAjf#4Ji|qDi=vemJ|*Lw@P-JfaruL^zf|8QJwuPGu7y^7<2xbO;YtPFGqtwU4d^FkY|`vwO4HNM0H6FH3~ z%_~i{s@u-k!aCzkV>+J+AMasC@Jud{K%VxG=7_oKQ^GVT# zXifFVsIL>t!d_><26Cbs#pRNVRz-8;OBE|He~k2L+=vMLgNeo_coCk_Lc;EyaDhN7 z+pt_oZSZbV{{AO9SjdJ?#eVO3xdf2G6Si%8)<#CN;uCZB?I&KhFMs(}`~LGM{Bo5B zbEXJ@)p;5sKF=n>OF;^`+RH}(zfo7?pBucn$-7YKi1(=vYFz42kxn!r^kjUF;Nt#Q8|MpL5#o`kHyjj7pWLHwCKS>q_4Ru$&Te3BAgRen=Y^|A=O&z;n*S>4JZP_i}q{*etfyQ2W@>W?YFY=9% z9V3589gjZIpiXfp&g*W-2Rx4Xebx9`MHtc*ueJb3iICoXJhu65i;}fm=KCd?A zi=4`YEzR_KHqKK z%IdjrlQ%q-IIQT)`Z%W!;^pYM!biQ{^bcR*c3kj6e(H5rL`!CEb;`bio})p+JgEHC z?l2ERH+8RnL3k||(xT^Zv6a(vI;T2o4iW7_&l8frIiG%MiAy}0&&en6bVt1~ti!_@ z1MorOlr=W0^s(fLlJf8G)6AGdzDX*a71zkv1+}Rtf3;a5Y;UE{;m?J<;tQX3@!_jV z&wE7AH*B%Qbkb&y&Di+6#%)NyQ{uN8-jK3D{U*J@7emczPCQ$;nVDH{BlUBt(vx|W zzKwdh)0SS_LO@#dkrls7i`NI9ue{{fR=be>mls-g{Pa1=#Z$I%!v@=X!;Or5=Yutk z`v!9jxTU`&6O4`x+rB+}?8EPR)E;~PhwaqSV|Mi5K|6outSul@QYEiN0LqK?T8AHP zfNeVX!M^Uq5TwxHSaslznbM=e=wN6f2@xLG#p7}s)EXX2D}|tw)c^`B$k^69&{y(( z_X952=iPB0U0X0k;ls|Hx%7kW zzp)OzWn7Abby$aY0+@t3R5mKED6E=A!cz{hg{`+d5l@iK{0ELia`<_e9226rk!OYi zf{TNnZLkA+9g`sZW83!H&+fk4c9%x&oK{kvzsWZ%1R*go;DciVsY*vIYv{S$xL{?;S!x7~w-s$*I4n*IfBNdj52 zuWcJ7fRv;-Qnv-QWm%K=;{3GuqQW)S*wU4iguv~+oJm&Fcq=G!b5TMXjltko6wk?M zVwNB^Tp0HFEVrLm9gD*Qsv*Le@)1Icor%NJN_BS5;i2c?t{1^7` ze&Ij2-~Rc(X8-8xTWv{=GwH-HAHHSbX!k@! zz6yTklcpIe;EEIVtXArf95Sosn2hbab%*`*&)skT&tH0v{hfdCQTs3dU+=SD{N)Gj zzy9yAnl+3g!5D_Jy&@>e523rDPG;P;n~Vn(Srt>PD*uWb4S^t8)ZRbu@+;Eao-wFk47iRb+)pJ=2!?NX*P0^LY zu*`xhN&o4Mqt@)}h(s<}>Yjvc{>Q_X_(}1O>l>CSXq;BFRu%4B9noS-g9A}c3U+yU z$;yd2TjsXtgG<)FARd-JXU&ubV@Z8EqUikzn`xGTI-D>xxbpWiT}FaQg=u>OV3zcQpj8YsO)0)Bw5$h; z7~?i8zK_k2Rm+@rS8XiW;_{RfWifYkT2b|FHnO(7#0{)MBI929LSKyhw)h|``7jQV zFk&CfT^jX$_nOw^O${dL6s?T-Zn14knPpqfEU8a5$Lv#|{^uFn@XN!N+MTpH zdChZ7f%vU)zM`_gpjnUxUaAb(;)$|-_so~AeA_w8-kP>fPIHK8wndKXAMgLwpS<`8q#Tu^<1L8||n6ll$$? zyKe9m#IL?MW&iQFpS9<{dsey+H;9X8DIbgx^s%pW);yip{9dn!bJXA)sdFAvB0xBA?ly`K+k?t8Qp#{E|nqow2#8rakq< zA^Uf~@eTWz|Lp&?U;XbtX}|hUzGT1lPrq*elW620|NT#j?|jxi``Ksh_!04@CLhM~ z>PO)WT4v@D^5Hw3Y)j)_d}l!XL*s@%W*hL}a9_aNN1%6VptE3a|U zHy=ebZdj2hLBVFvNz$q<+CWQ#{tdNRWi6$Uq}n#vk`7lZTBfnd60N)&Q!;;DbK!s6 zZ;4xkJLW;TVxGaLB+8;zTC}D7qAia%E&b8FUH1>iE&l=4cXL%!qxi1qfivH^p|~kN z2`RSx%HJhRXBLA|O|xwVX_3j%NrL}qSg&)WIPUo{36L+mKN<-&1+9ke}O7y9|d25Oekx@m?nv+#7!iBesd7kk`yK=%w zP2+(}yVMtw2^uPsV=xenI+|;mYF8#zQP_Efofn2RR+jRTtIpf6|EteS_WC{hC;y+{ zll=8*`(OTN(epp~mgxEOqUX=pXFu~DJAPPmW=o|BVQIA`ou&_E`ey>X>QDU%YKmh?~{~#q=jOXUqAz9eC+GHZhpCT^q)2s8v8@Qy$`*l82k3 zml?HdM)8lxeP`){eg4&F?B9L%ckMU-{lBprheqweZP(hZTejQw(NVtZIJHx4-a6gm z2G-$69|9Nq(?D17apO_<83CX0)j{-(K}ycQq$myy(P(~~nZ$d)(= z*n|-`{MKOAJHLYhb+PL$e-+-ENY_epIlr~@|H0wUlFwT8;Hw};KJbE}4`6j4s}Fzn zXMfgy@fUy5S5kt{wcu*CYaQ0%au7HL&+xbEBl`34-W#V2t5Mzr1icm25(Z;)*KFA> zyLQ`8J^DfW`NuwC?|t8UrS#_Q`!Bs{pZUsH?DxO&Rr|_w&-vJ4^}*zS2hh|*KHVp@W|{Q0_{ zkYI-4kO>)M{h;#YX4n%)k622I!}#E!Z5`ie6B|ZsR0=_|gYpvuiL`j7ulPz9?AGh9 zv3K3P&+fTlpKZVHTANy!xA|(->MaxyzU(J#SY`G}$Q&CPwtY8TZy$L0A-n%w4_a|( zz-Bo|XR&N0DK}TfN8n%ZYB_`$<q2s2THfqjD z*@+WPd+CL$U$}AezUysZh;wW(EJ>!4FqO|G{fl`^LMH|ub%kZsIp-(O%CBW(8^i z_ilUmp?mD12k*9rg^x$x{ixk@_x*m(*5CkVs?tW<>P_FwWm*?w#CIo?=ygv1$Ff&c~dux za;%P_yh@cLEwZ8yK6ZtVIFcAzFl{Sgq9*U#Zs{z6FZaG-2fzw5sqr%9`q{{-VtoKGO=% zN$J}-p0~X>ZL|CD7tK8KfIa%iy`r7F^?sMXzw4oQ+wHgQvum#zx8Y&xt>Pn!K~9%3 zvO3RKh=o0TLan1^b?%Y0EQZKcg}|oVK!f7u!v( zPkmpIQq2l7{Y~uLXvz7sElV-1rD~Q{+{9GE7AEI>+z*Y61bP-}jc=Y19_x1ez)?%i zC9E`-x7=Vx<3M?fKGG8!$8GV(cHIw1FXcj_YYu856(cSjm_daYN`ry0lgCD%t-uK?S>;oUT$3Fa_hs3Myu{-X# z#8%2{8bWcAsq#%$T?D49uF`9j4`pE>8N54n&J zfR~G1+wwDIYtFQ7=oay(y9ey-8>j5>YX@yveOR}KRT5QS%{q4NsIS~>E@>VssokR@ z0^x9edcns`-*CTmGzVZ1W94q6nGn5J?exhrHXx;(gYr3o0OVp^;0!)i1<@|=W01bm zvA76p`Mn)<=_bhsh=dIeCT;f(gZA)yuD84H+UtEhd467edRhEPF_ks(z-PNbpC-lA z#lN#n%Xm0?P0s5LTPyYT`{IYmsY$CfmVI1f9KN9Odg6FEH<=4uF#d6fKTpQZ(o)%G zW@c?e^n8exH5pW6;V2 zcITZpNHu+g9~aNP4z8VDOI%j12d9GYL?lM^>NZx z-#0{~V{Bz-B^X<_Au5bcFN)txE?eL3ejB`f*iIfkZZE#_lF$3_BUV{u#XmOh92F19 z+0s6W>^Ld1_>M*lnWljfu_Ul9_6D^o^r_<9%^-(6WPBt1Zvl;=;5g zHVrUP6gkuz7mOIHljsEq9HATMP0&r$1;!zKQ< ztjhIyP;!~-7Un_bId+pFFN#h&HOW*(U&u2#IVDj-GP8I?Q{w=7=2)php(x(Q#}4uj z?P0!l9-?;nR&BNgwVjs|A6TEJ@%77@gDo}zw|Eq@Q3fRkALg| z``C{^Y#;oTt;v3=_n+p%r4WEgmW`nVhnuT|mi=()4@$B%#0e&Y{*$3FF?FW8x>^LFE& z8|`O){AcXL?|Z-9yYCL$IlkG7l9|<|e)&g3gc#;HX$_kH98v4=BMq11a(`wBOeyIO zfSJKu;u9CQoI?0KV;DgCH~>}#!aBSOm=wStIEn^6exvN(bkj|KBXGp&I;_KA1jNzH zbLG8U!1X1%M$2Y z5{^5X$R*^ANupyta(u*RZDQ(A0!^)5LX_nQPeb z3-k6`yJ4qOEqi(9oSmvK*%G|56SgP7G1D5)80LIckMhb(U{LmYO-dNbN@&gc7rbI8 zW!r})Y+K)ioj-EUzIWlQy-=C6*BVRq^3t3=bN;j)ou0F~dQzHZV}Ko!1WAzS}q2o%d|Fdmh+f zx81$PF95my?(KHh{nyydw`@~>CClbn!O9o&NTs;dHopmkO9Zo}YSZd(h^Id(;p!!| z{dKd#;i71`7%t0@p5rGk-y{KiL({USL|;$NSo#_Leog7*5JD3#CoS8M@@G;^nra*$ zw{hpqHZ5$o@#%5Poss{f=XXwFPM57#>Dc7CvdzwDq_)F0p>8%dIGnNmp}L(s^NKA@ zCv5U`(oP*SJ9{!=m1T|RkqwsOmVf2Vmf9NIkW@wxO8KxD1YR6?WpzO+$?g48xVSl5 zf;X!YxIm<`EHs?&_~GAm@c;yHbOj7KDAp)~Y-CtoOxi$yNo`9@VWYpg@=bSjAF)|w z;ZEf$xz~w0H0l^wvA9Zc=}X(z9fNlJT|4d0d$)^*w#sd`+wa|Mciz9%?tN&d--x(* zJBO-E0pqMBI2o&blpm{;ygZd_wTy)MmMy0+qL@|2g!el#mf|39(SSXr_|K~ko=n-y z(Rn*NFlG6hrD&;NGQvgbq~bg-1?K6RE#9|eg^v|&@QyL-H2UrA3rp5~L-aG5mtHJN z+MKhYTgEJPN(DWye4i7ho>3!}{%q!iRd-dbJ+ARAn$JjyE!~~6vx7(c_TAKT!sF|T z`-~K;V?#ECfrCTwHwZt{$4752+DQL^We*k2o>jizSDnABF?i;@o!)rVQg^G>;=>$B zp4_T-?pd&-orBhSLdxWR`6+%{W%QrS+Q5zK*R7H%3sv#ciYuV(R^ddpr0C4r+2Dl5 z%L8G-@gbS>+%W1u6Z9uaF7(a$ffuGP*r7v{zBPU3eA~~Hgf2L)hV&fIz&7d8v4ky^ zFIahjvp~(3X3Z97lU6I|tzIqo3Lm!nap};RGn1lq(e;wXeOvrFJM1^K4h#$`@3is^ z=Rl256zztaN9?vcH`^WeY_;3(*<`oeHKzAXcJG5*?bf?C*w$-B>m^n@s^TK=bW8I_ zt7Ms0F`SW>tXYEpZCj!;j<=FDmN!l;!U1v*tD_ z^2)Y#LT!0kee&9I%Z}!3z^FY-qPn@^Gn}#g2ON#eEJ8a|3)HU9q zQ8`uBVz*}|i4+yMmnLd}&e2}%}d5s^9y$wU6W4WB@veB$qrN)X|^%-Y=>Itc^ zeSdUh)E4JEnny$*YVMNyJRx2SU#vIFZZPfd8?=)rB?lZe%{{El6YdnZUQOE#H|`Yu zi0}A0n~Q2z-RGBq!JN2LpH&u$)~skOwbg-=OW+Y0Q#NfLwB0vuu$ym(zlb*Oov_>P z+34=B2R8fMwR0Ccs4$4Ak3<{rm1IKmq1u;Bi7wjH##y$$V@qKTic!0=Tl3br&g|sE zF>61iK6*Z7t!I?S3#wPWWTopTEVY3?(U{Tv(tM+1hh92uiSYp&e01D0A1v6yEp;n@ zddZScXuKa6GBhSw-CDZ4X6J`qw%JoBE%k!N-ywB?XfAzD<99>9nRtb5l%M9o{@aSS zu{dVKFX+ePiu{r{?X{*m+CbCH|G=$Q7hPws+X2qH6Cvw4EDY-DV zqdv;h^;`W!j^QI(y|FC4w(K$vyxJMFTNs5w5k)>%G|u6xRy41jYuds>Q*&U&=I0iz zTxLsm*_xU!l(QSz_-ITyu}l_$va+|;$Y~B0&*1y0%Eov&HnzdC8FqC|NuHUtxoM15 z;?Jk&?cA9~TjJ;rW#e<3k0F#on5=Rr3JD#(N3!2-{4qE=Wm=r6YCsntFNF&e`5=d}>(ByrFho(R2 zCc?4CD)e@8Va}dic#67CMRv{_U)R~5tbCF$w{dd9>zM+ zJ!y}lWb%pJf+yee@$dD6li`>@+4bQnyCRkJ+?dMQ&Y@wuW@Ow(hSWX@Nqws4wxMw^ zd$Lux8BKWgvZSa=+ot9g?2Y*udqtA^nW+nQbKivR+P2k(3q>ocGR0KhUYVb=MX|(1 zjkC!Mv-ZY`6LwHa;DGSAXKdWA8I*9O`qU(hSLez$Q*YY|DR#%_X6?1P^LBLVtc~Uh zcGJXGo7lL?3TmXUxKRt6xfrovVmwX2rOSXlxL=Zjou zE!&Cmf*qfpwj<}x*|C`^iM4qf80z=!Maz|{9X~&5U%7C~4okQ^J2&TN3C>Q<*kChd zx9`5uHV=$Q?2@7?#r+!>PTPUWb9Q23-j2@C+e>FptG?%KsamyNLmO@H*k&6Zt6fHk z0095=Nklv7ABk%8OGrt#)(ihh`a@JilPa z4q;fT+Lo^dFsq;-+niu7?OExef!8ALc~_rAvAwUk6E^ssLd*O z=8YL^yi~Q)q!j+^bC!Nr&W5)PTBT67#)#T`u3?3PDtlJK=NT!P$Cj;jv}~=@ZObl8 zY`Y;P`tMlvWZf3tShV@$iArKO7Bs$4Czf)O4wZKqFEh2uWI9eHA# zaHeWNAGP{|#?l$l_eMK&s;YQW7$%W7l6LD|BerpDL}Mf)g=xVK9XeqTKYD|Zn6g6$ z&e`%}&2HJZOJ$nv-+$6L^YOatCd8AnHhsRTI&uD6&Ch;3f4-t;(oUQ>tLnFG%a#$l zR!ReWYwG-*ojHBMPMw^wV~1y@G%cx*QB*3nxOBnKxEUJcuIwAJ@7xzOL)o3u?2 zj9GhwSw@KNe6MBgFW0TOn6u8$CoQkBRFtx1GbKBJvg&8t6$T5I?a!)W>f1ubviq~v zH>Zk<=31}Stn*sa&J7>4(TAkO@5x!S#4ZEIO4E)n9J2Og%W@}5Rz6m?>}xs8sI0zQ z^Ok=oW2sT)tAELS-AZRm)_-usI@4xL$29(5YuMbwuwi!T5<%#E=s4u+jDk$ z_Jj={9FSrqJiV&+ypi>l38lX(Tx`?86i!)@$O_eLN`q61cd{iqXH^FHThZ_X-3EwH zAo|0@LYkl?O92vjpF77#ggGhbtjx<;b-5wlv?yM)DE^SKoAz$EL5!KIa9RVPQLEdj z)90P>vnCn zkvlEN86wYTeF1 zdETa%7i{ybn{42&erxP%Sb4Bw{h!`oeG5g)jHukPlw}*5!wLyIp?Re}rM@|pwdJEr zmfT;rfll5A?-gIYuWIc~(~=lQxQY6dS^aVGuqRL2=!dpi|II^|*pRTM=)RR|TCO7= z*cuN6IwC9Ln$X&aau zu*{(7X`}iprM@dH+3CfXt)aO%b$Y;-PLwTuNaL@PwDcYQmVUTog;CAR8sZJ{nvtVJ zHvau_YoBl1;;BXPjh4+WEn01NOMGM85)%!L0nN#x2TpZY3Ho1+(En<&$c0%>utF}x zzyZ;dXLt{?i+(Xb3TNCB4c(zWBVU|5J7e<;({|_G*GXQb9}uJJdeIrhw{=`ar-TClUH*oxbo2bY-#mwg@#+jm9Jl5bC+o^&Jl z{QR=`*{t$k5I!`9KKX1jtfNk13 zV51u)6S3trRn{02-AZOlvQkzwz=njJcwbIDC6#Ww!IjHSzc$PYi}2QZD?EJ^Bo(Q*IcuqVXbWy|5%?mo3)u2=56U&@qqnp%b&>Fa9J|# zVoUNaw~aR~-=~IX+VyEh5UDLtoSu4#D<2E`FDaRMw%M>3&Y!Z!zyGv7vU{KH+kTC0 zD5BwrR|-{px5Y@uS%i(6<&v?31D-?&-y+_0E}- ztQtkv;>KcOg3rc^5o@##f6)-_JiwLS#`}&L`%v6tp1en69|}-34)Ba;&f&t~gb~A3 z8Mq=dJ>RMU|96LIY@si}F~2eFVeA9P7+uy4ee3Xp0CaS%_veOWJj<3=e&aw5G8+xULH>T}{{V!N~xn(=H zZ?ju&-C;Yg8PdFy5N&5{cx1>*#W7d7j;Ie`e{IIjO2A%Nnpe4HDb58Ol%k4a?fS6l ztv4qyw&Jx|+M*xO77&yH(VgOPK2xAX06aw81R7s~iqidwSw4}q>{wcY+`PT|%&S&E zK5qlpj@pLzZnWXshpbg=*}3`|8^{dVXm-N-OJkO3=j_yhV|M<)8JoK>X&ov1Lp#RI zc6Ds2S+=v^o3bNcJZZgLDB7k14zkg7)Dg|a{UJA@P;dok#&VukRrLS+$w(lIa z;c<*wEy)p5`nfzuLi>yRS)D0bmhw_8^xNLOJMGpxN37JB@{25HXKFS#zi7ALxy`E8 zw(_2LI@x*6HXlQ$PR;slnG;(|YG>YdT{9thrED+1a?oCW`3<`;J@4C&xJ7ehL&-L8 zmGU?)$y*J1?ztE3sV85u??3m3>Tq0rowG`%VaHD%v{zo+FNN!j-MkkgV^;O3Yi!ru z(6eRxpk+m0tO!NvnwguoZQI9eVq#1*$GMRKSCXey$0+9nSqb*U8D1!O;oQSOb3qgj zy}&9Q=~BDO;2VwUD-BwSzWrUD1joc}ryzRv66MXnnvc z3sP34q*bKEaf9Q;{bSaD|A;k4T2^k9#Sfa6o9MG#U5el7rk#6b%BBuY+33Iq>)YLD z`Fl#1x>jY-mYl}66wkt<6u@bX?X&ZC^7Kh5!c`l&ZpgOYzts{uG2(M3o8-en+EU_U zxw6K|u?seTS_<48w;T>x|HB2#JtP@Z^$+t6063iJ(k;4uy&#C+wo|pe*}wDjoKStx(&Swzz3?Csk)szcg9|S{kT2* z+#&Ifqi$Td=Gq;0)6Ls$)0Te6vv1{xP#Cx1HC?o`XXeDC7W`O(si|2@W%@O5neeB% zMPsVe$1173ojT3@chviwt)P6&;$~I0|EE<}It5==%QWcPt&)F8IDF{1z4FQtJ9_MO ztJg5-^x42*(Lb87)srokR406Ng5QJ-6<8U}&8%d|DktVPr*B1LezpWFky~n$^3wPb z3N)$-7LQh!6g4jo+n}9!>8RDjPuZ$l*xYBM_Y7P9KJ^O5nFX`SLl^9s@9wvocJH&1 z`}_PX#+2sLQSsWrc@fH#$`US;Hz`w%k@n`i^4`7I+jYAnZ)xsEE~u1S&POro z?btCanNNMIxr|Gg-gs@&b&s))B^w%Kb)G0smEoukw!vfAWKKj^33^80e@@TpYw-R~ z(etxU@3%cS?6#2yMlHP|tiJ3U)3{q4v*c9GQd>2r?af(3^LT2hU@t!Lx}ACAj5Utc ztet6DGrg>NYtl~6pR@VFnoaDL9M`AHG8~~L)eE5d@wq~$3gmc;ql+{4($oo?o}9AB z9(W{NR4V?%On|6R5w0?7SU)$$OK%yMF2aST1Co(h!N;*bS>Y|kV$bfp=q1qwfnjLo zc44?O6~jeUuknHi@oR)1Xy!0edN02F=F9XfQ#Zoc_uFU&!8 zh&Rsgg^PI0(-sQs8`?C!c)Mm)t{3&>0RITUoW^6RecYmB}_3{&4UI&H{-> z<}l8qG0$xJ{G`45^2>JLgAYm(>PnfKs9HYbAXtU}!<)}<4utxgBuMkw$pd1jp!>?s zMV9DOR$hLT$~I3`&cl7XA3RF$4bA!pV?`9AM$;-1&ko%>`l0*GR>(i;YLdFGn70j{qiu01a1&()*DjhP!xp? z2@t7F+Uj#sWHS;j`ioW&Z^VcP)Kur?vc|N8kIj9mi}2Zzz{p|tm3GC3l6_L>#XD8^ zP8Q>b>ff$gwZqMQh_)TeNjMsk3wV}-B)-fFfSMY=RIaNHdQou*W22>l1WGCBZHx>p zD`oQj9&ga>2oLmSI73K6tkSjBM)Dp|>tlbd0#XE>vf7)GCjroouR6{YBph;_gX-ev zsi;0u=&6%fn&P3vq^u?x?7)F3d*S;>MKT@xOMmGe%k))UDdoH(l+~HpCHuqQdBe&Q zi0^rT?QIjvqoFzr|B8na#Q9FpDQ8i+V#`GrE%k$Fd%0Y71&MQCxCM_JgE0*9Wfftw zCC7?26o$|i9Ooy7cDg{M{##jrwF2}h4v5b#NQkt5X*PdA0_meF^D!yriH6ew3S=wQ zkdWB1tdzrQz3c|i)Uq0)=q)LL=TfSZ`T%x7iNB|$;tD8u>XW(FtdvR7jM~+%s=cD| zOil9u!$5@CP=4iBRd||_5IAZ@(U;emEnQ0*Tp9|;VBD28geT++R?03cE($dj z)gxtt#X*%-v8kCUvCVeB;4s=ht)l7_wfSQ7Rf4AQ4R(SIEMNpn9>D`f=#d`+O4#vtdJcN zouc?lW#MzaaoD-q)R5nz_`#xhQh(E$;^)$z%}NsVi#f|kxlF)I>dH)bWvf)JkXK%` zkTF@YxutolG}KQ+eNu$PKU5D4idM~7Q_4o8Lj5Jj^(kWwpQQK_d=$e7D;LYNRgE2u zH}T6>y{$fJNqH-<2+|FFiL7J5%Hm@g{_2V=RJ5h&)l@Ehlg>#gRIhNMPH996pJ2Azs#L)x>tim4$y=6k2>5s$?HgN#zEEI8jyiSKZ0 zdZ(K69SeN4%}CDasPFOx4z+B#fw|F8|1B?AUw%Yknxj)Q#w8Y+M4$YJZFX+KO2v$g z43?Cn?Ps)d#xdJQ+cniibe|DF#pc_|tD@pF(CnQ4Qeq~BYTj~{tnq`ZX8 z*09nL?HUyR(Ru5d8#r5WNGVe;oAB|f-l+PK706TNMah@tr1kYnmKqjMW^UBj;|K_L z3$Rj`{)Do*OluCj_2rc(x6QM<5Hjf4@*Z5+~SY?C}JFPM0#$wR`$E&a!KA-FFqDJ%JJRex87iSb=u+~6KHmxjq zl0v>&J>3K=?oK;`83TnH8^6H~bQFEU9b5^oQ zQSBV*A5t4NO;sfWXnaGpNtN4C`~|hMq_UfmKjD1vsNrON3 z&c}kowjL1}1lNrd{Q&YqXd(>%@I^bsU57UVc*FP-S4{Gbv5$9FS%S~lU&L95b+{7v zVK?@XM@q^~;0irVt^10h52u2yDFvMRW!*p{exkDYgD+vMYVlj~G7?4-syoLgV^-i| zkEDbkj6PLqh$zcNP5zR*sH%Upt7&YJcR@nY0@E&9%M4RXy9Xa*8uSYBrjY@F>BL zI0!D1C2R@hPIchSKiDc0c18^qOHef8N(58@+ac+ZC=wzNqhOJiwzU+eB;|@E?V7Mc z1`?vl0fm|EpA`kRGATbtWbN;M2&Vuc5?z!hv;|#&w@5F1;K7LDj@XLM_npP1MF}Uu zZ$2DgJTqnX^wTfd$rDqydDEzU_#?MS0hRKDa-eVyUq;yc@~6+*?98&=aS!KLO^9YC z;OCg*$XmjB8cJ+iL&_J2o>M=5Qx*ur=tYg4R^$*?eLh2aQ%nR`lK{^g6}~7dhVV`s z5ga{@;}~VDyP{^r0r452J=?K`|7OWrQyt5HJZJfzl!72yYF3vlSCrzea_CG}WH6_K zvt&);7YdP-fEfitDd<96-m)yAmA=c$K|2^>7?Z5>0k~zZAwk{e;(Ei4su&EsyD$JT zcn|_{>g-Na@le7quD0M}kKXx6ul^fl6QZe!=D`cNqdxU^)e2H(xk<9o#bqAjvJ^HK z%7sUiq_W0BQA&yudw#6& zAuPxi^bz&tY%TiA5s(zINKr^9a^h{&1zIh6+0-9{x?ct$MS_oOg#xPxXfO4neQbkE zNFk7dC4RvQXOyc(E$avLL*e8VcqVj++$H5sW&3DUor9scYwUCA<{!w<U%lz_!~dDb4CnF+I#haBs}|)T1p$6opOXt^8FEbg81S5y3R1F9n)Xy;{O2{oo%r z)X%QePzNP|uW$o3htqS`QZdhkDNWI9H=f!6&X?7WVln5+7b^{N`a=P!i2p(RIpLbJ zs^CC>xsH_Ews50z>*=zGMPpx<^C6Ws08etb96%TPUv-0Sp#fHx&>!$C0DjVJm5s}1 z(y6@0Q&=L)I0a}c6`;>q*-Re#CxDLYO4Bv;h^A2Ny|ThfFnTkus>f9}MqYRwlMqTB zV}!Wm2i=6?0&RJZC_g^ha7%WmiRVlT@6DRi9De}^&}DOJ*@|xX6+aVCV-*>l*{N1E zzliohou|d1sBzj9{WE##pY%zED1RnaDaHOZt)#lEzUAhE=&EM@=>bbs#m`wbo}x|a zEh)*Fe81)$wM)E*V+As~j?FGDxq%oK0?qnmL83_?9t<1ur#_+yjD0N8X^WpX6*sGQ z+In?BIKpsX72~pjR+H65!C>2LfFo#t(pYH-onXLc<($Hs4e>~VbHLJLZx-c=ZMu28izoy%Cgx@aAZ_oT}L8k^j1tvKF%0jt8r zP1OW0MnI0x_|EY0N^}b?+pOM~l(J~&s%a~VUzUUic!1NN#s_VqMhONO?Ztqp_Aw?f zPuGOsTDmR{sXR3gLT`m)))qudg#;@r`6!?{Mq_HICETfgjdatRY5GSz*^PbO3U?o1 zX!RN2*WwFK;c~(+1PG4Z;@lmBUS)XHthbw0wv2zqVk+4uybEhWGne`J7#8gkhwZ)W z5=d#zWfdnyV8ACo`ZcZgHW`nO?Qkxxuh^x#yt=dqd0c&ioT^0N5`$QwDCtuX`M~V5 z1M-COq!qkVUv8oA@9$IHs8>}1;?2qsVI}? zqBYSe{ZWASy~~sUx>tO)-Q@v+uPW78_B@138N~*aQ!=2Nw%L(v2 z+D}_GPp7i}B!5=(u~j(CtBf%r&s%wQ>6sgKtD;-)`HJQfZUis1?R@isbb-8BUeV9? z2HL1MdU|%yhaQXKd(2m88?1uQclhubBxvu-E-(gw@V{~Z_hr$Y+V}MnuiA5Sr|rh; zcH5(a+pHuR4w;>g@oP$%ikP z>+og(KXfjPAN(QY`K??M4hRE>L8%Btu@39-!vJphTZ1O?>2(p}J5Lyv%MVOSaxC!U zYzp)Xj9oZ{WhP2oTKojVmkW?`{5j~sJjZ;8Yu1-FhAP%+R4mCK_!i6RbQ+ix(qU zs&(C`C@xr4Xb!ln0D@5{f*(5Qpt` zE>!9N;37o);50v~PnSz`RvTNeR zgiVo=8!Usf|&vSQ`hte>@rLWi8-h6q2LpA`#%3jOv&y@Yr9p=3(xPDsIYqc!72p+Nvf ziS}Qfjx&BZ3$R|J5BL(!{3->rs&UOKIQ0RFH3}NadQf7iyB_+_>!g%?N$2+7m?xK& z#O+_+7Y;n37X&m6ju;A6N3J66;sJx3UsR*~Nroez3k3Vv{%2wA>~PpX9@DEe^UGnMX!UwxUIG*8s`|cYvLbGjUWBN|LB(#gr+)S38ZS+gy zUj5>2RNi2p^U%1tkeh#l@lRpu3+%BH3N z63Yv#zc8AjAj4BI7PC5#8o6-?>|2hgX}#DiqJL7jOrXR5uJxbQ>q%E|I~Z)zMU|JcC5;V?d`U43 zE|5}_)dt~0_#mmKJ71B+oVM6nQ2j8Tvl39eMs&`RVDsgcRT}WVfL9f!TB*K?h8N36 zvKAj6)OZwO;#9FDqlk)5Skc_fstnMnLd98}W*FMZfTXJi;Gb3BekqTSXZQ(5J=HBk zpQ$f|uVy_hzJctmcDg|sLkhH@xfjcY>ImN;4<{jS3q8RqO^lV4rZ)H=^OeSl6TNV% zam0AyV@Hzh%NlFGZP*Qgim&c+BS%-?3gcgSKr_NOG{#3Gcms2u*F*IVv;^(xp}q`! zRe6fOU=_Fo?G~b`RT?@FpJ5>}_Zcy^vgEJv$V%XHot{&tBot3&K=1Itrp7XLCiisJGileA&wDD4O!{AdhAuRa(B@J29-jFnJbX}j0Ciz{RTK9<50nQP!i zK6hzEhgN}K@gdc}-Ux#Tk+SC;LSS6{K`zyG|w^zut~ z^3+K&lDZu^dcBuImTBY(?G>{^}A<K4Z@SAb@E_K3zu#9>NE0_ zAeBm@8p$bc62VICPr*kBQ%Z2xkn+}$5-lB6f`HnuFz`h?P_Hsdo0W^gW&pw?IHJ2x z{javM!VG={&q64T{3}EaLND9dgn1W0*;dT9&&kOto1Z^#8^(%u!2qm0j z=yT&BO1YF*SCRG3lds-5ZiK{uf*)HfL;E8RqKgo4$)Szued67>>3ySctNL*;cD7Uy zT?kL~q2g5~Kue6Yy4uZ!q8(SVC7cTV%0lSC(ATy;DNHVyL#q;mGii(o^t&R-KP5pn z!)j5{iz3P~2840NZDxfN<;^(4h|aibp)jFvsO(VZ%L9EI=OSo0&P^JEu2g{gmXtfr zq(WIiN62OJ#%;XemRsRfI1DQvBtZK?;7X(Vv*oqfl8`MNhHaJPjRJ#U%1SSce!gp)Y(14fP6&4t>SS0qC-8tWmQ;WuVzmCI-qvzF+ue#0ACP zX-oh)=Q2g%o$@fI$ly{(4u zpuk$XV-*bK!bwKBWHl)r$~GMikPnKI6lskSRz1nXQc{!>tcaC*KpT?s=c|88<>r+) ztKh0`a-lScO>Jqaey}49p;U?$ZR*3S_yWfvU>~Yupcd`=_AQJV@fj$8 z)DX@|g)`Ha&)-wgCM@u(j%Rg|Tm1*c`vLs)^8Rey6H6pCt~ zc~Fh9g*@mxw&;R3it=zTG8l@30fzoyOCt2F{-;mW51|g=5t>k1wUK_q5EzYt&PP2y z{R@uaTNoPPc_@?cY&gEs$U*BIOrKF%!PqB0qGC$9g8Wr}L%r43SYibYPVH(+*=>lP zxrs9?eBzk4pgYHbcqOa!8XBwUi14R6AHLeGpw3z`HQ?-q4-0kipen05)xJztV_bch z;^=~`_*=GPi>hToV-wXMMEdq^aM9&KZD^0}y9y3%^4ZMklJ5depH~e6{{iQWFIEr1 zJGxbI-oT1KE-3KZOc_7I6U5IP5c-<>(2vYfnZT3X5F`YOH+1yGIK+n@ze!Z@j2n@h z(++d5Up_!x+R%~aN7bWP5Y53)s7Q}~ z=Fh>b_>RXF{qR(-Wt@0fQ*Fnh)lvGkaP2%>jvD6F4orPXRUjjtP{<*dRD4$j;*csM z8WvsYl^dPmQ*;;H1s>!?;~?Do9FMWD7r;e~b1?+6iv0-$eDO!1d&@cBO(01H{vGnv z3&qPL_6_4c%spzKcoleZT#2~&@X0DaR{y~Rg3LjA$ac_#x7=D>(Z4`+OfhB+nOogor$Q&^6!y z2J_Vb>K^mr9a!skZ@kOFrRgsRSCUsRtd(!CAB-3L!*FrD#~Y3#5dTW+7a_(6Vma~s zoq{W=BXQozI;=y0rY5tie?doh@_%``>`RZg_uR%CRwRf9y72-)6G6PW2)g76k6`j7 zZ{m0kDxVO~Mz7KnUEly>UE1A1{=zvIZC5x@GE;25(1h+vsiw*$Zn0Es&do4?GG*db zDmN+#n^f2*-Hv#ue;vmlhssKxYB($Te2b^*%?b$==uQS@170p1C=4SCX_>}dz`$6g zwjzw8(0jkgCB!>2xsodtVAEGxAz>((GQ*ietSmr;R_>feO+2h^eK+s2JMO&EZn|lw=7)aaQG$%x2mOSz(j?3a zzZexphKB6=JsWJ(mO(!}pZ=pJd^{o@LLkZ(aS@OxFRON>)DLNdN;d?!j9Hx&DCz@(wKvP&gE!qUPYGX&rRBfqj>73|B_)vi3qf@SWToH;76bz-_ zVl|+teEDNVM1ekYOaP>e4SRC~=~wCJg zQHknJ9|3^JC5YnTcU14vcL^D^k zoCclesvbGz*=R7nplf)8#)^19`T6P%IhF2hgT`3}#yG;r8IYlH<|r>wd0;f8Ds%$O zgaEo0b}Ig;iA)5dF)V+0UJHd?c~L_S}}{kzyfjrJ~;3Ro4?(U98+yl-evFTowMRiMoww3}Q^=#!3`tw#cOv=6y_i z^#}G;gD#&WL_R_c=LW9Poqs9-udJ*L{RW;jB9y=5Kt39Vjykq7`Wbsdo3HFtJ-9*D zFQ>~YE4B2)<2!z!SBx?ICXa};gmg4@pwcC>k{?9Nq$M4IK^AJb`qMH+=bTmA_-83kG)PCO+IUF5MKZcNSq_G|D&)M1JA^;*Inh zeHHowATRW6gn}SJNQVqIu9yO^-0hHxOp0SR^MsdQVx8)Qf-Ky(MiSAkmljy?eKqNY-9^?KOMux##Sce(9HN!^j5D8%D)BE*#v7F_E%q zXkajb@lq%JdW~Ff1;kxFTPVCYC6sZnHu>?}1sK&S#-psYfOtVc71MZ|c$7i?V<%BI z?$R{)5r^{@oarEdcOm*w7KT0EW7*)Ea==HNNchb=1QECR_CmxRc|<~q?I&;i=H}-7 zmM`$p3$!Pui+S_{e!!*v`zM z!idwFg{Wpiv zvakdlr#wHe2~~=t#y44TR*^&vpG*;~**4KYFw!6S5T;7AB471Vozv7p^=JMR>qD_l zh_^|nu@3dnq>m8Jc46v(bEQ>z1;robFYS{w3NywDjB4|hnhkQ3uIRj@L_w@!ZI73x zH&BA0hw`Vf-IUPglXe_$VQei+C}X>3p9Hog@UJ*r6yo)hE3;xk^`l*^jKOe(B1qnx zYXd(0oI$0jDLo<_d7zHc+th^ZaeO34iDtDIX?P%zBXokJRp@=cJj6DHWeBrMo-DsUQ22Sz_zX4t;F93OlH+*~*NDOk&55l{V0u^?l z8+D~hy>L+;U05p~T8;w=JU7vaXh3hUKa>TBc38>bV@M7mu_Yy#G#KU7gKCrT(CC++ z5~9d^z1452kXBUbN;bk=1)M58D|BVOX#Llf4Oqd+xG@N)l>#pU%0ui90H5hLekD42!*%dt?jZob;IrE9E<|sK7*GGx4qjY&31bzm6YAfK zKf+fscr&czAL~pRa`MNp>v+Ul|MAvlWuW>vVO2S{r%A}?RD1ccE(wl=U!WlMvxp&J zPrt>FBAxjM0F}YV3n1VXKMZtLRYCrQc{pKw_J(0DFZI2YEe{qz zn~9d{-6o#$$UueUsJvXM){jQOV@v#}?Yu_vfm-A-Fx`eU{%I5VO|dMm$yiM zR3mT$y?1G{-hpcbLt8dbaQ@@B0sxXQJ&ejb}cM9WnLZ%KG{;J5w{5690c&{sjubs;2z z;`+GN{2Jyh`j9vw@6f;G6*8k;w3{d3?~X*?f=@_!Wf1bG%Sz2IKv> zHY-$BSt79<)Pcdcn(EC!yl&=if?j{}!3ny3x{_~i_6mx9yyD;Oqwu>o&u)a3{JOBx znZbXhk3GKLV|3!hsNHr18iY=}b@d3L?oOvGc{4tv+!FnG#`h4P-=SQ8?fU7}nNsMd z5K3Tp!w-l!@Hie8LPD$%hF_=$E0=i>{T29NchHCIdO#kUx2`TdLrWO&GzZC{x3-P5 zT2&wM#6$<Z=cCIV5c>u61gH5_e!Npe=$lo^r#J07{BQw!b{w03p=b6v z!AL>pizJG-6Ok&0aSvnP&3kXQyY9HlwrtsA_uqfNukeYhh%hGLsI0HFkwYOb6vG#W zc;hUFKn#G*W*BGyVQV$=QlKsvlE{ZN7$-2Ok!QJFc0`hAuQ3fEZ;WHnKnBFTqQL`r zt9U%AGi@ZC_uhB_@6-){;`5vM+`P+g9AVU%&ZYj;m2lcXU1FZR^8~lFh4L`w;D`&v zIQYdhIO1ZI#mGwCsRQ`I;7D5P0Z+NMI8wpeTrHT#T!?0x!VumT0OJ(Ul0# za(pDvSbh78O89ce7sz@Y zzChYwa3>GGXwrOEk#HswPwJVIKiBAE^ecc5gkH6uGjJT2`W;#aURl9}Ax%}2K;WxC zGHEyLaK;GRtD0=9sfvG9n!+h{Qj-x%5FnH#=Y&=Cf_mu2M+w`18n}2Y)7Ib@y_rWY zisH5QCSByenx-4hbZ}9+03qJRd4^a1?{#DlGocn;2-&axzKVd->rz%!#A93h|8A{R zae8i5JQH8^rvNX}lGm1j5j=vU&FXi&qpt&qAS%v%$tyg=iia3G2;F}g=*4w}(C@UP z2e$$Z!*2XoZfG0r4l$$a(h7lHSk3A3HN7ZK&wR8B!SCW)MOY*LTG0fq*tl}j%}?x>n70+fP*Q){0 z<7)Y;W)e%6%pumGWlSAPcpKja_W z>i_Q~Tv~?!j+-@tR{gs5>OLY4!fLA+Df+3UJ(jkcBg%ufJNJ@Y(9OW>>!6=LhslLpd%!bOxI((r6Uy!SUj*GU10Etz z2r^A{Z-&(nat0t~9)xr~l-=r;JOoi2b?>Yo)!MpWI4*A&&R4zX3h* zE`pFK&_lO>9PnQaTZQN!!rlxapG(^k^2aaqQ|K?bw;126@6mrXEWC$$kZo{pRj&wF zlb$^$Z<4hYxdKQ*AYy zw&irn7FAAL{3M%H97RaBG+DC^pp&w8weH(ulcF&uPd99JSFv&VhtGrh&6c;YWeR!} zEpywiD#OJG&@e_bK2WesUopL%^<>L96N5!r$OUXgWc!LL8|d8Y(fwMd_>t~0U{+K{ zx!trSjX!SMs;XWK-Fk7}5jR+3m&vK1OtqoPw!>B@<>LlAJy{rGE!D45wPu-f3C$|{ z&5d>7)3+u#RuxyjS)oWmw#reTjPOesM`eLmzKaq^J>p;2J8o1}m;*5{1p-ypYNShm zF^?mh^pr>ZOVfm~;0JY(CdLCWzTzT;azkG5HTI`rqP#R;0m7L(h5PEc=wf_17vKcm zM9L?OXSXiCA}!Xbo1d3+sf+E14!V%K0>32KL1RHUZi>eF`icvy>^Pe)r+9f4Q&4t# zs?X?`PO9yi6SZF9WGOmIZJ9CYJAMpVBA`&hq4(%{)%LBg0bVwrD~81wMuAEeW2)Jt|^ffxpUiH2QJmO&x;k zw#J*G|3u5sD$Z94(gyV-#}oPyh0yCts^IWrp_wl>>-ITmJs!Flr=9roAGHj9 z0LNVId+NkE6jE2Rqn{b00ZMmX4v6RMR(N>iCu}Vf zw@{bisXB_5#My&m71c@>frsLJ>mu`p6AW`ND~Gd60d_+Tm`}5!$Bgph*6*77hsBj% zIO$UAkFJs|}0l4DG#81GHh6JF641vVAGp_FO| zbAlYa0S7PX_-a*lfe^RlE$~Wuzr+{+aS2efK!K~!8^1SQjN5(2vDoXVTkNN9+k;=X z|NNb>)&2=KHU)?e%BLcrF!HM(?w6=)cH-XlYh6k6&K@1w~{92L%!6V zyanE_Ze3RD+vB``gb~LJ=%yjOTjyT5=yxm=7b}RPY&|1R{WoRtDdK9{EM@%>p~lj+WFin7i%0_6`e!;p)|}SIG;lG z8xe>ZhSGh(d)F@}!2xl$q>zXnd|K8WA^m-p7!(ojvhTKSyaPBOde$# zLldjO$cJsID8?8Fa6ib(FAPVlc8hsouiw?X18fK7KqSF5GZ{AwuyZ@E+S2>(`5mh->o1FB${m3Ov%r z{7FliULzo3m)4^f@QV(hjOa)QP)2Moe$js&ezXC7J@({kkA~@Vg$~r$qDmLil8nim z#J-HCIn88DRw!$twxEPIME}Y(laxRxUcvDJ80I>elr4+HomrT-SI?ZXHzv^Mpr$Q@kJ!37oMkyIR3{ne#7vK@&!LVKK%#R@tMGcI&OtDH^BZ;;X(FUboKc0jz1FG{9O@k#ER>ulE-Mel2WKZ`n&L;QdtO=$#!9yRq5yOLRa5a; zafmxMGiCeF9J4c(d8>0sunMAh44xdf%{vA~YQy#_+L2JbIg^jWhQSYYV=FRZmQWy3 zOmX!aH80QD)|0kzmI+(^5V{$K^cyoq7K5qq=aYvDjAtU!hiii}a2MlU3e*Be9C9Te zrv|woWCa)h@QW%dj(GfGeuOgkBR=tBxhP9G=9x?C0zr8Kp2A9Wjv*XzNz;wz@8lC* zz@o=P*p2yJ7DDZM9S4=Wp_vX8@PA7ExA0dAEWgF(HMF+cPov_15*OfGZ17mGWE3 zHqrq<5ML=5TzVx}>Kek>4yin%V&rK;YXcB@Em$dR<#93eggcNA{f~=qxx80tchyB; zu0>3z&BVL3uF=t!Sl*@O#=0r38(@V8jI%8qMp@{C7?i<{t{G8p(v5xari{ioF0b)g zV7#|gR$Fw@LAgf}(W{)7qN<>TVDNK3scNJYAsG9D0V^2#cy`AdwChgUs(*um-pve! z3mO#~g0U~+pzGu(lI?dbB5-{ zJrzYXudtkWVZV57pBy(0hFr+w?I6Yp5sYpXBCGrUiQ zAJ0O3DezIH63y`R{FR0dO)5=BZN%UgLlg4P`iFyzT8e>Aa;=j1!FwylzF?>U2;x(O zf7lQmVCaKZpa)P!SH+fb+yDkY&YXhgF;GThqZ|7mTJL*ueJ9{@`Nn>Tj=s6HF9ig7 z9?(IYk0Nw&yHW;E+JYl)x$7>i1HZ`!M_lrWdA%JFC&ni&6oL6HEhkyOYaGE~k!@k9 zqt9uV_&_3M^)$wj?<@^6`>}zN~A&;VXkLH(%rXIl1*YKNQIgd;u z#}mUx(36ya-}F5+B*%OQO+(cf`|@&W^?%ql4!uXGw4A|`2xe)vRi@;>(p{@BYk(Kp zb-fT zc}9na>g;@w!4_#1zcujSeleW5r0M?lykdJOPY;JEKT-eQvT@W;Vea(P)AWYNI(OZh zAO`fdA4gh&2a`vT%da{;lSY$SL%gRd$(y^N;gS@~4Am43W9-Xg?32T8NKMdb{30uo zD=zE`QLLM_L&WGsAWujbmgrIh5%80X(oi(^xxfCRBphlb=qUpq&_m0+Y}WJT+07!R zT8ICO0Dbz700Ydoiq`7BUH7Lx@l$^O7X~(rOBjvz?c3+)^qoC_*0Q;*Z}DZj?(hA< z@7b^XvtO}a`PE;sPk!o?en@`O72vR4l?y>QM=l!DST)EQVZ2`nU`&{rnzHlf&)d0k z=j_CZ6MjL+*|TT;k`Pw9MWaLS>bkgsFm8j5E9!93F!grvsADuT@;latcZ_+gY{US- zmR!;iMw+G+edPMQ0AHt|B@6?0eyyM)6p*m1+9O;1QPg%sHemKgAcp**%(#5pYbsc_m;q4|5 z!l4buTsAGI$rnnX_c=?BTRg*_QYI-)X3TyK38&&Q$Q>vaXJ@8uYHq=54aHYFjY@m{ z$RYd6<6pOLe(y=U&?vj2&|teNMnUDvmfu{Ct-caKQ)$2TZ=qiH>+FM=8R)RCG3~3^ zy7b7N+(bGkv>a@oLO4k?kYnp zqPHkPZw*3FXv z_*(Vqb-l2fN0$opF6UYD4E@?W`Go$6V=ubZkr&HZrGgN?N(+J>8j3M`A%?DiBCM7W zVy}irzd#4!|BF%!R^wbE^QZZY5d+eYvgrl*1(#6Yk0oN z3y)sU`1|c#giC+F9lRADE{3a3&yi+>e=j8h{O*4Ghyh+kHqNZ*3gYZiM`OR z$yI^1s&OS%TZbPVctpm z-rowiAmnfV?Z0i`|Ni%VYjK3@uDj0ez4u=G$VWcn^`aag8s*jk+SiMd%Rz5E>W8rs z+>MQm`T2YpvSJwi{Eh}Y%BQWL{NyL?d*Az>z4X#cZtUB+bEnty$A0X`{E`#WPzF!( zp>5#AZ#PyR`0=RIynB@TI$mU_`&)Hl`{fAZiN>r6dj zS@_|ASRPM+IJA$vV%-7ayP-mH@#nb??>x{KSNnpFuX1$dn4U-x~IXC%$SM(?xsW z`di&V&Gz3v`PMh=)x!sE%fy8JwGTgL!z!y*Yx{zz9EZWT+E$slVA;Nsr9}Ib&vo|c zbj}TmNzTrb!=T8@5&>4Q@iwRW_*oVT(!Fg>s6q)>p(~al(3b;(bw@MKn+O}Fo5BQdn zSif#N=xkP@01yh0U<*~ukJUOnqw9rU{}@L3m&RKKU&X5D#j)ZW6bTIhu^p>1V{-$9 zU#rS+NDJx~yJc_*4I8C;OX5aTHViui30O{X7?*$xw-Hv;twIcsozx2#mlfi6%W{Ze zF%RP7_|1wIfWD11wn7L!&-fl!Ht@U9ea0~Ii*$uww`~{Y8|xLr@WT!mM>c+QwSYAdX!A-}NZ zc@+Zng^ID&n37qI!yE6hhiGxg4J=c>__Nx;>S}!J-L0WC5}r$1E~MefaSYyTdWw6w zUx1K;7Q7iE9RRB-0>pGY<2xmg4{;-p0Ad{Y2Tz~Ty6!}VlkcVZ^p+7J#_!F04e0V>U#0Bhmfw*MR1Y5n%HPKn z&|Bvi54f}r(H~f=erv&6Y2uSG9KR`_=UTYJKlED&kIuz#Ja!6>&^NehS3= z$RFB)o@@TR8uUzD77_4vQIE519Y9WW*;Bc+`;=5Ug`zxfcrZ_L;0Pcfikd)tjWpR? zF01!>5{rk0 z_57aK>;AKU@AvN9b3W(X_vdYh$Znj(%HfctimK{z*I5@K)f^Bv1ewP>pR;8q(v19; z&jF-H!itz>-(3K2I&KGXXHeX1&_^LYb2mpxKrc0} zJCN6M*KmnW6bdlwE1qdlCUBHb5ay$=fD3^fQlW={xZ1eb*k+Mah9DFJ-rEB6#pf7$ zDRB$$#PkWZ5+q^(m;HRdxIp?(bXfJ#s6*nCArNj0w6Lvk>vzjM$t3C4)p_1fHD6nl?xKgIS{hNo3tm=8qb(Ua;w zzpE1ff4NVI-IGeCCfC;fT|7Yzv!JbSwSVnrQ~RU+L`ng=e;AhH{TKZp)7<#W(~750 znDnRas#v!)26?hOXuM0|*{nhe46mOVBWejl(g(M|cfIny)ZL~MD~l;Z1>%@Ts@MNy z*j6}Bx5D2@I~4S>4Ox3=4!KMR%bxJ^y<^i5vA(Rk8Mr@Xz37;gttum`%HY+Gyv_@62at|ca9YInFQMjs#AUDp`LCAtAnq6B9|x-^ zKhe8OKI(F>Jj$JrSxkK4k@akD$KLdJ(48I)Wt-6qhKfq*Ujyn!mlgf74mX=0{rHL` z)$Xv{YZq#u6hCpV*RFOjn6sDDgNOMA8APPJoQ`b`^BTAxJ+MqRH;iHkrrePaI63#- zm`*{OD1vf24nX061xCK~pmzeTZY|iEc;-9)@z1OyVF7MsAOe3Rl5$0GWpl^Vp>=5b zrs5d-GN{}K^kb+f0hb|PbuV&}5qi`l=p%AkOZV5ifuqbFMjPchElGVs zTF)}+1+#p_N9}o4*H!dAe8K>=HyC+j)xNS_6t8+QPX;>Pi8|1#F7Ws6_^Q>bb`i01 zNq;zd#$bvq%T%7dLwidlxIywpR+UwVSlYw5N7a+33Z+1Gz-i=H`6WNpFth@vOmayk zyOOB`g}oXU-2Q^O=Z`dSyb-5fSN4%g6TnZmbu)QHPaBA=%tE~dXRDDiWJ0!GePFp~qD8^U7^jwZoe;w#Laa!l8OHE0q3|m3ppKz(DBKflee!P6 zVeT>QEPfjg12CwFJvbxiw0&-3=(mjiOAB&mwbz``71;t2GU6}9sW*Ia5#u!M%x(x~ zAKwmwo-fVOakWCG&8v0{mVer|a{|!I2SH6$@`arMRR?&&7DM_25b?}>Wo`a(-c#RwOr0g%g?aDxMx8z7pW%QPX@;I? zE&+%VsZcfiJ$Hf-7hb;Gcb0Nx{X90q(Q~7x%7x=|Ky3U$$5H-~_&LvQTX_|7Xviir z>qq_GVxvj>7p_7H6URDprPF*Q;O3FBmy8##JiArcA!f67EiA1)OD>{Y^D%}rao&Hr z)7K3;tT)u?6WF#aRkM7hj8LBr&z6;NDE=KIZt&iuuIke@5VCA8={H4bCJBle4yPa| z=TR89kltkYl=TSWi|Td1zUrRcJCZQ3+aArTULUU2BZ?MJA2F^Qk$Us{g^BIi#qht$ z^QqE;b955>`3+2UuP=qm*|EhkTmGmba3;B;QNt-P2-k3PK9x7S_b&zS|I<@DJew)* z7qnn5nDKI^wlr2?YUlhn9G8Mfn4gTGnlCHXnx8lBGtbO!C^a796nAcv ziQwX2M6D$?YH&{5O>cGHDc6AVCwI9T4bQ zMs)hX)<$kk{c*`G-u;M-OU_L1<7?iehRHh5b1d3OCRbaFBm?Zwrxi)G^eNi?Be+jjiim6GZ$34pw^bk+Wf(uPKZDg zcv?5Gv2B;+mC_Pr6cRm0+4Fc?#Fj4N1&AU*F7OVi61r)VX0eTxRP zgi=#^%ug;aZvIZwV8XWvF;JnCQ-1ae6oZ3H1*YR*FjJ+~fV^$0Px|yO^xv&(^93#C z*yQ4T*qm*#*wW%_tp%|!<{U(~h#z71ORf?a22(s&c0>qs$py=Ges)?EeZjTg1SY;U zjFTkYBIlPU3r!;~J=DmAK!B_N6;v{pGBz|V)}kU|U3r9cnJaL2nEK@4z8djNG;)^# zS<0me1$*7cd9W;jFJ$Rwn9JCLW=FGSkD}ePXHl(g|3F;^=I|@Tq3h%J5W81Y`t!s< zlm?0*Gg#{t@hPk1sXE2!OkC;GR8>3dq#vQANWR9kxF)zmjHt=lR!mF9OEdc30f?a^ zGQ+p;-;dnU&CGj0@~KdOd5hUsM?wJ+KjBA1(C|^GBeW55b-caurq%IP#HzkJ(2F3` zLNNMZ)!bsmK!oCqT-LiN*Pv(St2zwBd;+DjTv~bAj(FMOP_RouxR#P09<)|MbpZbQ zGe(f~@6d|A@UI62K)HB?-U=?%yyy2yvuKp=DX5=A+V`Dd+dn@q8x8sBok6hI z0LQ7~A6CdQUP&7bu?)moE(uC{5LAz6BcH|j8E(Mu;CfR|o{_{zCR`q)!NercdXQy1 zV7uaHF>b&0%!KxkD+u&t#Ylr;+kN3_pvrKqtAi) z1z~>8!QK~vlHAKbc7dC#6cS6TM54z;MpJ}@&*5+rP{;IcY*LK4j)MOD!w4kC3IJ2K znltGmEICfY(lt;ZZ4TFIG6+&U<1MX+$KjHdN(Gx>n^X*wtnK zn&4mjun9b{c0r_PdtaF%ol&TgbrYZ?-Hj(^`N5}2VKXU4KiNy`` z=Ipk^u8XItfOtu1QC8AgDFil?Az-W@J)=|P3Jt(=EGqadkbcjPdqlKX;(MN<33XNF zCDS*n{3f7Z!5PS@3+lfE`#{RZLZAkA_j~2mOVDAm*jjCgwle>xsV~?Cmu>3}VSTsU zr6Nz#_C+@7vt_4T6&sbWkbB#pAvrK=JCw)pXiQ1V$4uzrm+5hmr!ouvK-Hx0YM>vt zbJ)|FGQF@z+zo8K{7~!~tL=2J_BSqLm7D3)2w{)7>#OxQR>@hDJ&dO`vFwA&51XgI z`#LwNUs}xn@OfLtrDEoqPKK({{dBfBOs&=dcq{Oi!q`EzYPZkm9=6rDliRh5wtPcC z3ZC`^JNr{?lvwp{R?ovVt9G$nFNe!JTG&<#O~^w_!wa!r|Kezd%n4QkmKR_DOgAK4 z;vF8$Bk0dtIE=6y+8fcO9bBdIN6pxd_965q6fAY4iqX;fv|=^C4LB+QmHr6rE{o+8 zuBuR*UiGE1?G~CS-;D?l4d|-`()BN%hx;ll6J;vzJYI_(j64`zNehU6pwBKk$?CP0 zFdV2B5`W`lgniXM#?u9wKGV?ts&o^` z*s{~?5y0z0udr6~z@DxAFv6BlPb$n`NAB^#nn9j;Fr{Lp0j<9S`_>5U?B1mLb~BZj zSp~@n)1X{-x2$58`CmAcWDS&4)86n?!t776#PFsmXz7lH^B2%rv0JX&P(;oXyHg?E zN853N5+uv7GEo2I$LLSHKFUR8d}$X+Kb$!-Ip}r*Ts)^(bjjnmL8ggG3HBy@{D9B+ zZ!s0sUGr|Qe)5rn-^IVwjt7{0t*MeZsFbqt zTd4cqd9w<4{rEINK~HH83zsnr9319_Vk`93G>>9cyP@hirj3M&nzl}aI6j$+yyMDNl>Ha%BC^m zqvV6Rwrz?dTL-1UfEHnHi;Z=rR=ZsA48>c#Pb=CkXsgOlFw=|up~`QG!oOOYnwJ6{ z1&-w5FI-PJaKs9j*NYuaO^j8-G-8>S$7V2?3Be`=MFu<+=mwA(p;V1`VBdqwHQR7k zyI`u9mpM|N@Xnh(P{0l3x(Tu*O zDV53~@~Q%PTs;cj_cUi|{@VW+vT{41aLL7zAPc$hfy^>*T4ojHWD!Q75+os~s*nvE z>WCL>U9?1oozl@t&5s z-v~?V=IgeAAu;i>bee^?3)&)95t>=m2h38R?D)vM3ow0nh-<^J6{Aa#_sEFZx5nV8 zIPamqk((1d0Y|4i*E3~8^F|`0ZToGr@`qjQjb<+cYR)^T~a1EurB3ZDhBS#v9n2B*)yb1HhX1Sl>O?uFj{OmuKZlM!Eb z-Tk~xT-$zaE_gMlZG9^fty$FaT2@f#Wj086uO-90K4g2;LN9@ zIRRLKty?x@pEr|s*qm-$=)2wj5Jc_9CZJe+J;Lr^b?>5g@-NCjCt5=|^d8&rjj8tT zSF--QWAn;9jgP+2n{sRTssxKfcCu02Mg;16>;{`Fi0+4do85s~R5;I^W4pV;_SEh1 zddb9+vAa+Pj^;hmT3Y4f*A_C!_jXb!Dc{y`Gl{WQL zH3IyVjSky;G!}PI#fC^Ty!rh~E_^Z`TMj9V|pFsBXG>D?X z8$yi5+#g=A`dhOpfPQ|S#s=@(J`|W}462u_Gx+v>Q3nS^$IaI|()Q6F z*rQKamwk`WyjAKF=I_x&1FpXSCCvKp-#H8@q18>|y#~IB6?n(}Qxzk>;+&ewL;V~l zWz!*$K=XvJf@JyUh!t5o+zC$an>O_}!{MKkXEZ$p+wTlSMi1*40#I=D%HKb)Q1yME z=Ey8uZl9JTuRvlhzGP=)_pPR2+s~l;S*j~#^|UPAKbeEzD~1o+Ni5e%c4vP3I4l31 z;dvDZ9I9eyJzc#-^^6{(d8{f5ophuw2|~S2VyZS$MrHPyv2q3GH3b1fusBv(LpR_Q`}*H}F7D{T-|iD-Pu;_q z%QeOngsA>eZ0aNHv=v{xpqw|(r6UJ5SP!!U@j5m>dBS+9PhB+Vg|Q$wMVEkX!Wgd? zG<#`S+7|uP!;+W5_*WfWAyjiW7t5d)=ao)e1!#^~9`hRCIzw*bWU5-=2IX zes4zK-_7{?W5CA6(ck!w(PinMlSu|N-U#*lC>Qg9k(rF*ln_^>`-07sG~=rJD>?fU zxF|UoAZFMQK2EXD0OuTm!B~~xtk`bW=5BKK1qE!x?U+37D50%8k=_R|An8L5V^XPZ zibmKlB*N`Gw7B|40q{P8g>BsZg+biG&eQZsL!wdHk z*0bKj@Zi&5nXVpIEM9pU<=AUR$U0~aA%89+vGnPWHOI0RiH|cIr@&be1@wp!ih|3< z#+Q#3EFo9|{xlKajvQd)!RK|kUIj(=@Wt@g+hxF;-x(UX)jRB5L`s7r_EyZy?iGCV`&$FHEv{V~y3&2{5YfrcqMHiYmG)Xl-lJ zMO$*A&7vSXC#zAZXE^)XwI+awxcuj5q@+5A@*{miRLKa`w< zR>x_U%VOhXV_d8;u)}t`#prq`iM{b(t68JN=c$FWUmZa~gYq}0xpNgwOO`%nJ|S^- zF^#Xmw_U56*Zq~5dso?Q<-g^@n}-(H*x;M<>`4Ur)MI{umrp_38vGkb4Vt;DxTe;g zQ}d^6;=YBZ#VH>&AU~9~5ZYfy?BCW>?i5Qy-H)5F!?l)t78l zpD4Gc1^O@N66n|d{3B++r$J5orEb4W3G1G;&GF^dq+US!wu*rvqcts3LAW=S&CA#1 zX7|BvXD`h^^;id2`QSU2uV&qCrDl^iNq~`u{w?puDlcr(kR0h*w%5~Rfc5TLv_7}B|#_tVh$GoOQ{EniKm zLxDl4tKt?|f7|^qY{(P4tHR|4lJ%#S{!-N)p{Nj06@HH|9@(BK3IDg{GtHI`J0T4& zVNm9h3yhNAq453qrrQ|b0ZW5lcYm^Bs;K{tqo_(G6^KmTDmr0R5M1~5rNeout^{UT z?vtJ{Xbl$Lt`Y~6jPL2!C4ff`RxR0rLCwIz%{PWrgSnHYu?rfp3}Pg^wXHc$iv721 zH5?2nrm3g-tY(HoR9!+_%PGicXbxY{!+c@~!LYwtasfRC$4^J!LfU5_(4wB&`PhIi@gztN6X`QTp# zCPiYRWVM5RbO7YnHq61qnBFS5A^S*TRFCpWkN&M3DC=;(f30R}so4he=LIEwe4xol|RJB;92Pyy((Y0Wff2JhLl$>cSxphk9Yg?}gh4@?2@Y={CD9 z%e|~^HFcU386Tb$f__5!@9>lr(|AwrH7(+O+N*AQ!Bh*&i&Zt}klw7!xK5?Q@zPnkM3gt7 z*@%r-ApVbwSyuLag{4}TnCS^-#`kv4Z@gG^83DPmWSs-keDpHz=x-gzJG`H}ep4>ibPGi`7@V$1WxseYYRR$U2?2uV9ola08uP){68S zqdbNed3H#0j0$hFWZx!g(6Ud`sdvX;*}qNJ^vM$sQ@;*g>Jx08HKzC-gyRS)h*Z#M zdAO1#e$_7ZyyuC?O#EEj(H4t2pMRuhX#k&L+=;iyttH#cNkm&ZC_LwC=S*ucS^{w} z%+rk7dMd?@Gdwb(6;zuJYrx{;*?NVQ^4~bH4NoOG?&W2N9=2&vD7-_@|EGj`)%D4} z!I;|q-1{?X$n|1_AZ4G;YW--5Yvo+?CQ_$sa~DkA25w}=nHeX`*8kbyvGB&H&ue~| zehZ}N_53J!rjtk6*V}c*Ahv^FIZ2%(C;98>j<9~GMOtqO%zL@Cl!uQ#y3uCx5s*Gy zBnB(&M&NmJa$c&1?^)dR((34HwY>hT^LowW3mcggO*sBm^||JKso&@>LZZSIs{=Xa zf~*nAj^xJZe0 zcA$6iLGIc5oMVB!->dUJ&otVoHwt{LM=SO{u@jn$h2LHt=L>iN4jw2NN(>u59QsO{ zPUB+owh`Sm^*8{092>Vc=PUv0fD@hES1+zj*Kk;}m*wZ)S5yfsNUZ%VsC1NW-^gEu znsR7zp5K}27p?pQLRRn#Ew1$hHpl6^XL^lW)VYA8QYnhbW)gkOG*Bw0@V)rBOf=U) zVE_3eMEQrx`R1lQX452^bK&`R;xB~m+Am zl@4&kP0{Yf90Jt2?E06nTCu`bYC2%Izl0)|V!rf}%sVg$t~s5tm-hXm_j;GgV#;n) zoFkuU1j`QtoM%huu=Rp>V}~;_1J~Z^lu_9fpFW+us+c+P3kMS?(X^Do}HxJm8aITR|D`nt7y{YLVMlo;6E~4>NfON*6!E@zF1(BvJl7*i-4DQ3fJ(Tqf7yHZRyJsziaMfo zi~;b^Fc=UQBKZYNut2QMIuT4oUN#j$9|FN5AZQ|s#`xrP$*H8z?H|4QeVBtzzofYg z$N#Nv+KmNxceN(Uc4r{u^7_2TscLS1 zz*C>kKww_`N*|G)!kk&2vDfedvDf>@Z_)dvp|KP+clGh{P{|i-8Hc6ArX>-Rw@no} zDL_vCw_1E)O;|B=9Qa?nJ7FP+cieC%^88V+=fZGfMw;G|e2XHVMOaeM?ncihQ*2ka z=|GS(lKFft#{@lMCNYN@ahcf}y7UVnE*@FbDR7O_U#$H$Vkk(QqQ+H`re>VRPT8-J zq%%)y5YF5}cM1d<;%W4yMXhH(I*uuKe~1k|H%T9LbQa^&*O;pK!9Ypu+2~;1Yr)%77MY!H-EagPoz4iVHSyl7 zjq03etpPF5Sp2^W6+yD{(2A9)pd0rZ`g9H5QWxaVn{ZpidQp)L2y|;gdS^1u&dCq8 z_yM4jlN5whC}+72VL2OQ!2V<8qZopiY4IB{xKkkg#dh=0kTf4V&Fsd`yW96`x3>2P z2x(L}1__0Y{j!OPZ{EudaxxKeguh&QfRGG(9n<=0wdb2MfJiC`Y9>o=6*{TX zK12`zT7?Hcc$+duSkSh=DPX)zW@)#04jV&wf5P60wJJnBoIT=F7-WYuqG>WJS_?UH zS3UzS4YQ+S&Md-MB%j6lrm6zMJHHX*qo$3M;H$;A>df}6z63~xn@NSen_DY|bF0qe z<5|ef>`g7DK9i2T%f0*$sov`|C{~z+|*mvg7Q4Gq}q7_A~Qb{*kc|mQvGacu- zj^(#YTtR~@&DXAVwnLqlToB9KeX836@O~6%A(%a)LLB-jim0iA!{xZ=K61zW?XNw! zleZC9-(vyW(sX#Lsp88Yv;)Z%1X%xF+-EzHJv7=tgXYq>j#=TWW;=MoESOJIjsp)jJ3YSg)A<-uR z%xOu34E#s~olBlJe4#c|qPw>`PwV~ZDpg~2bRPc>zqV)f%hi2YAWVqoa;6T6AN}8~ zJ5Q?$njc|RuxLV7&*Dy4uODBxP~k$Cn-1`}eyujq$~MZ~>i@kU=7;fsYvDtFy>$K* z3YCfvJe3j$9rLbBo&`v^fZ1%Hg}tWb{cxFBO~6;-zpKKL6#bC(xo_YNcGj9(1mrw- zzS61cEV}l7gK_EHM=f~=BxGMoV#dPIa@y2H&CTqek<_X8m+=P$bY3xK)56b{mFVD{ zcZ#HQiX&c=6+E$lPuphcMbr>jJ;F;DF{(pJWkwh9_WJSs?R;|bcH^;W@V_~`20h#mr;fAq+ zQ*n%mT3M3g!^5&HaQ!v;<9>EmKp}bK4NWeVN8yQrh`!1?Cq5$&Hx=Anje30|bymjs zf-at&`G(fuK_x7K#qE)4Ws-46m{3~1rl3;5gd#wN4+z#@l}A@%b?lmvPbu~H4<*I3-$Yt61q-J6<{ zVnnt~ixS>)%68%0qtyzb(QHrrq=#kQd`sJxZ;hT`8FWv&p;?Mt-==6>UyfvZw{RCX@TKkXmK zi9sK?Ojlp_898N1+a1i;jp8zMF-F&{3a69jH59W8D=IzvI(ndCI~P?RP&q6}l`|Rd zFZRC|=3}=^eu0HBuXRWgm6g5`>7j?;e2(AN}0?$50yfqfvp3d3=_e!k;Z4oi3{oZeP8|6|7qstXKq zb`kjHk#3`GE^Wcn_wFT4x)z7?(~O2B#c_DJ3OanZvWa-VbJP`j=DjE>;+m#1OsU1% zG@HO&A1H4Z;0@GznBE_$d3sl)s3lP8j?Ak5N^ob;(6$GhgY`~({s z(JL?S&s2@r%h|1}Tb|vk4MD`u8$i!Au0uwH7UVDcgyv4FIxQQ#ht$4jna+&LIs4G) z>t=g{3`}0y6054B_lfsUUfeVJ4Xm5g$34DnK90J8%&~NAQZByeoEJZOsM5@&yLj>r z)sh#TF2-3_!ME1-|A*n-!)?M+AZ`H^`%JllLCsGhtD0amJv+`t*rk^M?$2 z+j3u+c1V&C*QV@~T-P~p$jvy{ZNI^_X`N?r)mV&)s>qdc0qtf@MQq)Tu$l#3&!+|l zJbT#wKSo99{R~d^1UFR;!6x^Q?ZlY2S5d-ni&cCl>3B#eAggV^bx8g+kE&%n&f9XS z=T)Eh>sKZ~!4+*SkH1~2)x^OD!H(GSDLq^03Z>BQT z1u${J$u5YEID!PsYEBH^@fcIUMEAd74T2s}k z@i~3W+Qe6l967sL=fz5Sc=f@D*pG)agJtYF6Rt0B%G*piZNT$c_}EZu0^{ybxQ}gZJC}%@LP5`+rbG3lHRqQ8*bn9vj(Wj!%qMx(9 zx|{l9(86P+hU;_ZQCYDjRL;lv#RSRJjNz5S!dlkH;cB0~ zvmmZ)c&=AC-Q7z=yu~4Om{Deil2u{+SLkRh2evI)^C(q4ov)~(D62sv4=QB^UzD2M zT1eIH*qjrQlx18oo6|P2D`5OlA*O!njxi5S5N1C1L=p0ZjXnhm8cS__6z{txf!)$7 zlCXtRE0P>xd+?-H7Y*%n5an);HH16{O&Izb!WNZPz%;RnieQaT@_5TmMOZAd`B}esvH<+tG3SNLO1y~DJn_8AOjMzyZrUo|iLyML(px8~3yNS}NTs8mh z=aWbEdp=)FY+*Yv^oY^6#=Y4a=d`g;@niCyvp8b0Nd&!pn;!^Z%%yl1S=;|qo-Bvn zDv#1iNt$Le@^rne=Va?-{W|#Gn|o}e4|Dz?w>@Tz?D|Hh3uL;X3V)(T4apT?Y}f59 z1|7?0jaEbYnWA3NdK2&70w7KNeRi|UY9d$@Ivw#POTq-xk~ef_QuTds060v(&9g6M zW@^cj>-`Uh2#JK+1?@1OOXcg(zeCmPE*PKUJxst&(Yo%NYTpZ0(ilQx8(o)Y-kM&8c zSdj~^s+w8O2+bT<#QW@rlS#D23!y)3orZ0C0_Us%2>u}$w9DBKI; z-uv2$S;(@BiisCA3~VK^f=*!2C^AlsI(FE%NAI&f(ZW0(Nk&6zgztO?T8X&O*2am; z{ehTY+h>{~9UDHJ8u_lIQrxD8Ha=FD-q?njUiXA3{hSHarM!JauN08>qu!<#+m;zP z^52J)EeD9n-h)<9>7N%#OG~d+xfBj|e{oa1OQ5h?8Zl>F{pmHD9MGRrDd&9x0>#rf ztStF?uN`qM2nAin0OG>`yw>@!jqlGztA8{KUz}Q^D}ZxqWLcN#SJYPS3ucOY{m6FT zdfe!_z28j!Z8Lv67ZODGt^%dXFG|*}PYVx44_zgkF(w`x-r- zAo1$8_?4`EMDJYlFGz0LMRy&F#l}7Z@ADF;z+UYCN1pN?$80_EijdVWa@DIm7S)`X zjQb0Plwi-%gsDwaa)m$~|G3KGans<*#c0$h1NKD)F0g|smll$ndVviDKC66aF5}}J z3L$>9=#I$!lt%tlC~#I3B||N5#TPm%ok77;a!p8JO zg!4!_4E=ZJfp{rXC&OEqS;wQM-xXxYlQCBjUBO45`ZCmB@Mms6$RX1+NF)P1rS5a( z%vvUPP`OO2NZ_NUNsD&Wnh3)}>1p@@_gXclOPEd{uN+w40BE(G)_xWtz2`NIE^sR9 zJd|LCXN~j$<@1@f;f5+8&(J3apQiUfp+}zNX7>hBo&v`pYmI^;MfOYK zUn|;GbPFsGdD$*}bMEK3_dBVDjb44ZT_gR^kjWN`u|i3RLD>n$5z)HmuvH0(%Fvgo z9Tsk*?AQ{JgJVr`P9QI7HjGk%Ul7T07wrNH$h@hNzzNCVn%Aug89sbQ+qOBZtL)Bo z!ZhVB>DsWhL+D|Ue@b@X+B=c2)_Xf*=Z8vEVXQqZn0t3P*~#wolfaRO9;TDPGr3h3E4GAyuB7uXRCJd; z4ci2F!_Onr**&O&RpeMRRn5A`S`-`R*mFk*s>&?lr#B_HU$7zBMz=Rqug;IBZGxQ^ zR$11MMN)ciwR`Nce^=FbxQIvBskK;dcCK*GIS!!$sBvb$pT_Cj=iAcJC%xt6aQ_l~ zD7Klx#VvqUHge*$`)Hyv&zK;O(0nX1uF}^QtcG!oE0HLpn*FC%qAz|*3uIhTB7QS8 z9hS=Tt6bj9uHm1l{R?ciyHIwmXp-Z<+9UsNQ;lbP6J8c~#NNKdQQftr&O0Oj3DgB33(U<+X?SqUCINU(D(xLO(W zSc3piCsq%*bGUIelvp4OMz>7CD2v(W`~vd5OF1uX&N{!8$S5Oj_F*(F1wg376+=mp zMX)_?*LSjE?)BAI#*{p084HNqB)VF?ZXTVOW-I;!8-2Xq$vz(gsk#JecIe6`40z(> zbmruS==Y_S-LBQZ2x?KR5tXTYoa8J4?~%fet{$nX8k4>2|AcpQK)CYx5usSd@O-=9 zNrf)&BD2qugU^y*vD{H_N@L(9X#n~Nv6pTFYG8E56fHw;31`N}>{VI8vCI(c196vy z&rTp#hZny_tLsLk5$2bJ2J`nBOaaJk6}qSDkooyXiQc86Xtp#(qZL!o z;%X>`bn=h#;sBl2KW+QT9xDnRMM}?hgw1&RdVev5jlCCno_}M>SVL5yzZRIBN*Psx z(02PYa;dv;Rd$@1y_v{hOgZ|rFwub0Z^g-j+y*8WfuO#o@I1W*IDKO(6xSCJb(Df4r6&^;|M$<_fv7UcWVd!m15m&`&PqIho>rwd!a_A^t7>w@jA`u z!kFq3ID@k6S+Kby^mP=P@yFav#;~9vyba3D!-kW2J(ib=1-%%Qk4FK2{$0KssD2R@ zyk5xb4sg@@aGGR!Xk;&&1bnBql9-YDzIVEFfN|;CAehuj$_W7EO^p;y_K>CdZgDk8=Tij zJ)jXHkMyAqj|wms*$R_alo{ZL*(y0rU9xoXVd1pDqzdlSi@WAGw(K{c4XP*YNQW?d|=*}M9u021d z5VZ(O;vUuZq2~hx+&iIcbACGjjnjXy7v`7I19d10z74te;ysR3iIKkEeDLaHi*Kx^ zV3tU@DS&J{Ot?9a>C&8g7|T8XLc4(T^dqgNq$(5Vb2)3_ZyZGd*>)1n;he5sD{6K5Fnb?Mg2IYso zr{k)K$QO`{6+BPH%Cg0wQ@eTv*J;=!Xplk@I!2ci(BdqP{q;U*_!*;mU%s9ip~8D& z&dr0e0G{rpq!JQ_=dnVYHW6m&N6)S>I&K|MDq!TTV6sgGmcqc2&O|=skD3u1cA4fo z94T$d^YOv0cc|{o>>mk<%kyLC0F8mJk?HZ1g>@uRa)vsMz^{BJcGT%x5t>8|Q6lld zpZvGU8!%-B=}IQtGEaC?>fEsSC2=xK)K9)pT@>;=2BZOau}(gaK{qKr{#hhnFs0T~ z{7>A8o6v`-H`5$m#NiBxP)_zE#k(|3U)sZkyRH)FR@fPg6>p7-WKNfz&hi?J{j{{dweP^M4F=>jp# z4mV-#^lI2QaSFu+9e-?zU?8H>mCtv_BbWk9yRi?8{w(`6o{^OA3`^r@xg#0qLSI1U zvN$8U3tnIZnElxk|F&=Abqf(U!sm~#yUetb*v$4kK_u4>uDP{JyLv!_S;hs*50N<% zKW7we98;3*I||#kDKs8bzNWgmq~LiZ1iMA=Y5iB#$<~|qr@CLh-?~gH#FEGFQ6TTrOJ8x^ zd&y+gLe{7NQ?et)VE(~k-kDs@{40$5R+Y?0O`}QxZU&YsV995umdeDF1&^$K{w%)t z)+5>UC_}T?G}e&WH^IC%FvFYJHN5@;825@>O@WXCu5=tPo4=U+ww3&Lq!1>lcub)X z`?n)_5m2a`Mzlx(%9f@rqcAuRO<3c@zoC)8eK*g3 z;>j%nPtQ~&s_~7@yygz!b06xco9-zw-mR$p>%`2QqOPEh(_=owucr*BvBW4 z;Q^?(1K;{dMpIPxlA>c@xK`36|5qxZ7^xJoVSp6YCWVrwuoe?4FcC(Ua2JRg4TXh{ ztt0$vdPh~&kGWw9?}J{2NAs(noefMsDA~QZ!4~1WZ~e(z!ge_X4^Z2)icGw3<{4;c zgm!)%+C9nW3XdiKj*{IRdJPK(2`Ta-Qccgw;6WU`e*qdKTgYAmD z%TMy@C6O^}sJH#YL=XPFxS_zyT!ML3MXfDfe4drq!>vR~W|3qyfN9=RSUB$WdUkDS zEA#Yv4WL*w=P;q^#k~5Bzh0x)g8va!eFk?grn75I)MlOT@**sX_qc@0vV z`yqIK45E{}iCAH6zN37el$^lM1G^>eX0rSq95KR2^<9EB!#elBcZSIyFIvPvnPF6i zSFO@zr4JOOV4&Ce=MyM|Skeou_-CGsp)Jb|mJjxCp^7yC3FK4WMA3hT1)2S?7y_qz zFD@whOohz=bu`@gNT$4sq3u1l#WZn;vf>P z5WA4xTUc-|6;n#xhU4wJMrCu|LC14M)dkB%I^cP{4U#56ox@Ja^CEfUI=jPQS_%cZ zsg;H(jQ)JG&-w4pCQ)r}L)QNs2ZmIOB=AwhQvz;O=zWLBe1Ig0G~BXn5ldNWK!vI8 zi+Huz#DSR^1JCa*DEx%L}ALuJ)o~r+3&v2`d!JY^o%Q_8@D=9 z6pj{y$xPL6Xm{W4u>oe3Hhs9N?~d&nZkikgE68GxhU|%{OEA*#Ne+ zVD~b?;-pdsNpunU3;5A`D6^=jn)`a@CZo4sWJzL%0bDPIa;&=HF01&pux{dR8@M43 zE>g()f9h3__n+75$$a5PA^G@&UJx6#?Cx(y_48IEy`lB%Kdv2q7O{8#neHr9NE6Pg zTfQ%oYX&}44^K3>hE>e_nezf%Z*z6OCKfEhsWK`woM>1b#_M(Kx2_$Hf&avZndQ*n z*RNmGOp46TM(OfRynm?Evm3lhCFd=N-Nb>A2| zMI%e=v6~3a(?JS+u7}Xw{NBJWCEpPDeEXP@CKp7QQ(iy(vPAVm90HRNBD+!1)pnCN zS-tRRj+FOMq-T70=7$0OmFF51JCU9pF@~XUGlA4xK3*-?RneZnw`r7-3G2m7Tzl2@ z?BwDp1BR)f5zl|B_q&nU%5b|^-!iEEJ3qzT{;#TDn08!YE0L|I9R z&mN{VFGBe?Ti|xw25N$M;U6|B`)Z3+`IRP(B8VgN(;4w5t zlJ3Ed>?tzQBM%d3D~PAQ0q^!~`P{~z6FC1J+ygYYA;EwDbf85uhFLlYm|XF=@<0?e zZvL_Oy3~Qb^{Mf6+=DB`bX-xQb$Q41^{QVPlJ}%|OgA@=?-nQ3CSjI{4+n^$?;iy1 zEBEvQt$2B~>6V1Xrp>BJlqWd5S(W76JGs+=z**2;b&n#>#WyZP;zUYYjn&Gz=uG@I zAcPH%y|I^=ONO5jGVKG@bMuP?^1~g!me%xhxTKexaqk-8Z}?aCx70YBrRTNOIErT? z)h};03;ccU1Hdizx>O~jhuQKjntPYN5h%0=28scJnvr}o8xI|X%h1`8Vx~04ey0K z7?n&%F7MjZn#`m(2FxOFbnqi?C@6GoCe(eYwC>UwXBO63zAl5M3VGnq@p}nY-1JOz zl|eZO&te(DpaxyU0SQ8?$I9<04}xK^XF!f84G!b>nuzev;$1)yd($KS&oM>|>d5O_ zhf<17(_B7One(wklS-a+pqW)HK7_L-{4TKHn=K$|EmqlonAFcSl6_=Z0>Rxcsk9p5X$G?MOl0PX zZQJ{%V5#0OL#*?$OP{VEB$&kF;?A0OWUKd6U+21-9ilL#5IZVhUXw*2&2C!U*a>%y#L_z4mVvrfo84)tE1a_l^= z+=<#;&>tTxsbn*d@-3dNv^s}oTTGGkh?aq;6VSGjeT#B6X?+g%4!mf05h$}@f(<_? zfyp#?ZSJCtIVi`b-y=^M^t1jgJx3%hSR>Sgfmvt#X%EeI*i@k^yk3d$IB)d{Cil;KiiwirlKYwb5q9L?3Hw zYuI#uf4`D@UNY)e=9_GFjy2TA7H>U`dz5Uf))|rYY2R6RvbZsSoSf)}V!IRVvD$W9 zUH&Xcq(ay8U^hb8ZL+p6H)!*=%1ZT*Ip3JT;~bLB|EUPWnJygmQ)Y2MYZ~A7j@*~U z5#qI_G7;i>Ke=J?LuvW^S>_klz9Z=*`b$O!1|Gw@;-b+mOd(@YC#E;76nb}etjYb; z?7{uk7hMu>TT6!u4)sOIWt#3`?Cw~7NK}w4DobG;zQ`uYjsvBZCCXUdcS>KGX;A@R zC`J_J87PLIf!*RGSsv`pS7~vcG5B(u=LRfkRNw4hLCWn00`Z0+4f*imZ2;Bh45GW zWC0^b=x6kCCbc5u$aV75W33b=)$=l2ZtGH<=eO*G7$JKolavbNLZoevR5 zK5P{@Mv$oA$QO?N_h)&?!z9&k@o8<6$FE+?IpQ8oAEo}y0D&GZ*7u&`WodhN zp_ZVy`-)RxJbhk0^yNiI$!&_2ol~8>qN1glR!+Q|3#Rt%lhWS#GSSIL@ePTUhO)zs z+*0YKRTFMGq{^O1?uI)G`buKj9Xv->H-^$b3h@Gsud)!6(rGeS#2GgH;T|Px_ONTU zX$_L47FZk4qX-@;G{g-Y?~K~H15}R8$B!TT=D%fL*5>vqD(k%!@^FU3lkryQn9xOr z+udG*jjNPN_d4rm4WemzD!!|a_$HuIGv1@NCauh%=YW>a5U#He1b#!js@Z}n&RQ2@ZH+VJ&_EB*1*Q?T?@=`W zTfgTeskvJdJ!g(M-)$i-s2>_=FD3|ZD+);4i|cPHmX_l5M-L|jQs zs&b^_n;yEjHV|u}uoQ>tOUJ zGOJ)u_$cIL_QVM=E4iga*_|AW!tCP0eD>^_Ty&e_uz4Usmkc_%C+d(wq3lHndGvAM zus;c$H^urFVx4OJx8{{vtMCw&d10A_WT$V}ck%cB+3}i(E+Ua+<)5bXF75t1@`CWB zt$CPP zw+|c?UAlU|tRkcKIPqpv61_~2@>LePe4P;fS31xf1KIGDSoMMpB{IK>3&%sh$E**e zyhUUBm&_;_EPpJ#YKl^nY!qX^c(fK0!s=r7&M1D9!6b^dFeHks!S8p2^}9l}YX>{w zodEII(eA83@x%!=1YoKG(01Z*e+$+z*+9;9ojsgaByjrqa+@F_ii?fQXS*~);*yIo z(7#xAkne-qd1fAk?A#Aw6XtB&Os(kfm+c1tl`rKjlm{y0Y%iKw#BTxtxn3weu|una zyXM|yz;!^%z8M!QEQbza84aR=GX8NA*VVPGz4hh|&oZa1he8^1j zUb{e-@3cwod*!tGG&v2A;F)X(Q3OtU6Ys#(T)F=-$HSE*Jg7vzL*mqsw}v}?FW^x2xHP1#Ds*y8aX(SeVQXD5bgx7O5rsF7Lbg$sA_L-t zLgRuRr2b(YV7=x<=!L}byl*G}%A@nUE)b3N)i-AoQ#mO#h-^mH_XEpS(YgIW*9M&IvRT%JB9UmUr@vxY{ z=+nUOq-Ow|UmWww#rV%)KWBQnnNl-Jpvft*XQ>N=<+k z<0TZ&@)gj#kmlk8Xr%;k@w9C&Ki-Z5*Ts`SvdS!1`nSqM_UP^nz?A9!4R%4Auu)tJ z-l-Q8z(l^$>Th_n6&(F-tM3|TI$iam2RegM4n+xyOZoMhsw$KALx?x?x}yYR93fgf z7paMsXx{}si2*C2yQBp+=%wD`bx?Rj%WK&ciFZy}&b(FG6MI$d_Kzt7UTeXYi z^#ZMH@5cM8F#WMqwU%bwAcG@w5%dPjFaVJa_(2%v`}$krUTsco5wp@hecSiSquSU! z2|C^<>w>!LOV$cdt+$6^{}ct`Ozrq<+LX-VtD@Eo_R;V)oV5?pP4b3O%>`^rzgXU( zSeDOM*D}OB*N6$XOQ+@lE~2?OHpe1)_F03fDP6SVh^-68wj_Ealiyw5JKjVL^qtAE z`@ME7d#B#@a>Rv&FB*R$5OwmS`~nX*u!WWcwIDr7g`Vdok$XZYVNa&$&P{6~&x+u& z&5$gE2zL_QDI`?u8?&MASw8P8_oWw{!z6C?FQ9}k+|usoK$QWjb{dJbl;>ueW}t6ObA@Ep5>mM#!{KC7@Pzb(Wg->J!p5w{|d*y zYyh3+!^wi*l>d9C?ydc44lA9=MnYc#q*HO0qNz1A#|6VTsx9BX9eBE!1#_@7cHazc zw3p@G@|R>kfN4HT)hB&7!r{{Bcf8X2mk~02n?1t#L__}R^XDZq-uJG4;nt;x0dP7eZ*qk3%4IrMzi1h9_2V z?WG)21qT{N-YnuH=BID?T28556t^ZUcEg;^sS-H?HX}6E7 z_s9MT%LASU-?$L?^VNA8vTk5pY=BJQc<=IjmV3g=AU}cZ4>ZdI;L&ec|2-8?_yJmu z%|O(;Cb&wz4uuIQXbWg8Q=D~eK?IgHIlbH26~-R)nd-G9CL%zf*ozU(sdOzCS9Yu# z2VJd}Fg73XrfJP*`h{1S^r3G-7-Y4UJ~}%LeaB7bmIHs&{}jct2lNBjNS>K-UV-&|;L||Lyk;blRfk6u zaXb36=VJYA>SZC1dca=7H$4vbHcTe=_ey7Z{N^*ACfM%VFsF-O-p5|QMUj277W@A5 z4&9MpZRl?a;fUV#F=3KxQj!2w=)}A_Qe7>az0GjRcj_gvFOu}^0S4A4# z_cuu%w`EOYq$ahirvbH;Y$!7`c{3sSm4fZ&etl3gt@c@owHr{*%EL5Y*2?YLh6d#{ z@Aq1HajE3}iP6>(aP#_CPBuNnY9JJ}%hd}UeVmG0mX!|CK#ztnxg*COB9{AsEdB2R z7nw)4rGWer4NRnS~Qh%yfe>{gB6!D(Wg3tqVJU)Mh01d*Z0t8J5GDvt3pWbL1>4PYw zKhT|P@Y@nC>%QJ)1la|QS1j6`rx`tTd8V{pgBV;~SEm|${%Efh)OSjZ$N%&OC^d>@ zP>A3f1JF)|a2C$LHOhI46$lBkxp`@5hd3HHJDY=5IUaK+_N%M`2GltYS!tX4@8H(G zxxE?nu3k&7>q6*8=!Sm^KUoiwD@zcSHBNw1HsUgRCuff-W9qQCsw?|W-3{uO&x);y zK)dB2UA@-#0v@@bK#ti z6`y^D^2qxX)YPJ)Lz#X-sh`ETjUoFEth%%cMZ;dNJbv&GJZ*l*2=ZMld8e~~-sZYY z2Rl4DF`8B$N( zA{7cYEa&xkh-Q(HjM;WZbzZzbo_`nlTC0G0A+j5GCk6Hp`U$kBr2ADiq+q4{9m9(7 zH=Ea$uV{9oUucEP`W^@^GW0~57aDWKPHi>zR9(S3tLC0F@qCyk*WEHusEZ9|elc}s zJ|=M+c}r+*>{2+XIVInHKJq!yHsm=F+kkE8e=RRr+q?f--gVCN@q}2Ovj)V>L1<k1q3&H1ZxRXOVKSr@FXA>m zeE}OdkEEz@{${*=3e4rq5v>Q=(t4C=bMZ7OBP{*bYa+=dR`_nD>uQ5wJ_0WcSW;Cd zJnPF>M^Q77y)E%Pg~O^6$ZlhbaGvXC1I^}PH%I3$2+8^Uemi2me*ilVH;D{i);ip8 z$w&M}$_ztg1!zppeb=b`n?DR!y7AMu_QV2WIlh~nzA^Yp9Lqxq3O=l>j{B6G`*5<- z(RI(GZ91Sa5VpA1YmEVPt6y&JzOeH3#XpL1)Z{a6Q9}g&_$+4%J!@dh93ypETMDQ& z1o^4iUdWh^-Z3WIm zrRPg?*gwSLTlnvmH|7!uLMAhvZ+p2X=J!pbR0YI!%QMe-%{2y;4&Mt2utXoe+Uu8XZ2yxOR(WnK^&dh#IN7`DTPN*dq~eKxXY8tsYqR%8R~FR!Isvu z53PzbLA5r=;qlZBFZVpx0~psed?KlN=rKdeR>PneVt>#vmk}TC1v$Ug6fi|?$DZ1j zW{_qb7ytWG0oJU_WQZi=lH5agq*w5v%!BIg1f@Bo1hqa2sl4}kC7;(X zYwr84|IgCn_ZcViZ4d;;$!NgP^%v(QQzWeB`!D{T9y0GO;_ip@O554Ns<{41jjMw3 zOs-%drCmyH^wZn7c))EypCSo8dnBl?^iEv{yET{Fdws$S)2U2S)laOf?y+&ngW9qj z-R;^4638O27$*!c__d{M_%}c|$lu14cN>TO`G0lqWk_xn(pMPpO^Bh0TZP&9X<>f8 zbj9jly0`&`qofaHAqGAX>fdvL)H)OFM0F=U4s;Um6A$K{P*OkM|AIPvY$d_-;<=GS zmiQ*qg<=1cTP%t7Ytf$2RTG*_paHkVf!=?J`10i^%v-suhK|mdf&A^9&5OUhfd?*W zMyuQX^s=c{zj6f-**0P1djc+b=C+ZPL&*M=z=Ij`7iBxzie6Ra&~y@Hrxc5m7nD#7 z80>w%z2T>>5nSW2gSoeB%(ZBaKjrt!Vd`cuf58!=+6QndLiA?I?!fss! ze@_gt<5}R(Z8c8{_3_na@q%5fxOK3T2sc-e-Dc99=$$j zl^U7#!ul;>i`k=t@?^{nR0Vi+amtYU330FFs?XU3TF;#KgW&rrY#Rp@@bL;)*1R5P zr+|_M^GZ)!nz~|KDLuAiG&IDbNaQj3nATK(Ecc z z>ALMtMOC)dMt)VB_-U!8&|ga89*rwJ;FhpfW9;V=hI=Nc7L~>7xN|f=^R}inI{3va zu|)7_rgy(Aas37Vf3J1&0?m?@QesAFnnKD(&QrBKdfvvSrt@1&YP4^TF61pj>hI^& z>zZ?B(LPUjsVRhxtlhFHNwYfN`+XB)q+T)KP}`#ZVNyWf`}lfl>D$Js?}RE7+uh7D zyN0ljlK;*ULoQj0%8-`Nf^x+R7oZ$-L{O{39;*c?nD>WB1tzbQ3Zv@3fm`6`br_@q zb$_>#9YgGU!fOdg+B40XY(l+yK6y`##qpH{#JzWN-RIxWhx?VtxuFr@-%JSpJ~DHV zjktk(GClZWM9|c?P{&q)S=#89vz&&DA{M!)_1HT*t_s(J!-ZN!2h|aKn3MC`jE=k zwD5Lz3`m`Qnxz1*&dpu@p3wPQV`5A6^R`wX_^6)!KSN%O`y@^1BUwJ_9h@Yp1!P|0 zan1mujfT^}wr+GB5Jw&bw@)U8I^$CfUMm0)T1h9<{QV)NGJV|2DaY~plRGGrpx$(u zD(Y$z-lF2+WyPcQ_ZnyO+O`;bbXf(NU$Oj`O#R>AiXRB-Qla21u5x@VE&XBf@!?~G zG^xt?ECo&jzjuaN&sKyd%Ra)x<9WxQ6=LHm0W`kz#iI1mOcy_47 zbKNH{)j)JE_(v%!*2F=h7W9zH>hTJOEBv&P>xU28IAG%l)aaQ23&{&|%O% z_-jRLYw+E_{@pyl{QR#79c`K4+RxZ<6CC&R<^L3(761%K-8nk;4$+1SY)cC()f2BH zzlV*Fe9T%Yi7>XZg?}{6Q5b%m^%_XinywAldFtxs7DI~dbRZ-&HZ~@XI@8m~V}fY% zIX2Bo$^ZmRp}IzX$`yG8qF;C^dyU;;_DiVX^2*AzlQ7eP_{~HL*LfifHxBcPHY@r@ zgz_iC^BwBx%*!kLO&5F4a3lt+*>@nvTGR6l7cuSEymrYdtOA6)?|U;2Z6mjhc~lv@ z^!U2w@zJhHx>~0F*fYFVa($J~A&LcLvl(o(n`p&BD=MN@^>lJ=NqdIsLDJ2c^JF+2 zzR_BzyGhnpW`R|J$#=Ud7LsMx$K`wu=Bw<>G_2m2%TOZ7zCaZ_N$g1`_@ifiGp9z&R2r#j^6&nWvueCNvI}T&3X(Jd51RXu!dg<&zCia3;`o@MjjytJPT_OX80h z9e-E*Ga*`9D6w{jUxT-@W&s1X8e8#Yd*_4g1s_uC{Rtqg^6ARTN*H5m$C;xkGK6Cm zvgo3qZG3`-c)SDFclsiVcy^XgqmN|G)R23Sz;P0H>a@yLNKC z=JUf)aU5vDo!t>j)+^Puw)qTzWz&jpiP}aNZq|#st>5n5xAm46*J}=YrL*U;MjnD$ ztz^i;XdIct1b24R;YX45wFs&Ok6J)nJm&zf_Gtqj|0p|!>M+&cQ$Jja?UfJL@mB8c zdBdaPD}pRcx3+1rk8lL>;q>AObNXs125+0=BQ!)ee{^cby8uGx++(_tpn zH)(6plYeIsUw5FqWlM5L;=7ktEB@A>0f*ze3EQ7Wdus7@bAMRlF8Z6_0;r8}azETc ziZWSd^`rf--jUH+p2E`2{m|m}&jEo4@$`YNPZz&xc#Vusx|h7@I-t^xjB@mwEl8Z; z@}K#-*U8sakJWnsVM=79Nr28})vRvs@IRUT^73cnO6_C;lU6I}Gty1#D|3X;kcn<* zV0BMB;S0-0-4I%Zf_bLfuA}9ADcmW_^BFLEm1T)IRcJ22oPy?sf7w)nu9Sp*(|Y;Q zt}XbQeuAOi?@{&cr?U5?b(lQ6<%Gr(;{eO953fpS{IN^}kwt|v1}h{M2y`iXFAgc` zuKr=C>m|Y3m9qrkmC5sY^hqm!)&gss zs@Z(6&291Q=bF6NO*j5PxTll%uiFtpgHaTwTw1j^A@c?uLM)6W`gTao%~wl{!rmEl zp-Il#UANnUW*&Cs35Gi@jQO!nijZlvDo&Wl5HYF;mINc$P63t9Vt=C)on4+$v*?E-PMtnF`+u}B3X zhiR-&&;_wwv>Wt*c^Da9G{qRQ*Usq(5Y)%uPEHlFDqzVx`ramEHa~uieZoi?E&YuB$`QvnSE>AcwRrVXmAXEU&8SwfOh|zBKBN z?DbQA$D0)O9#h<#HqoEcRKUBmxEiyzo~7bm4pa>C`WhWfESOrKHL=V1aDy8i+`VjQ z<85tL{(V}g(9~bD9s9UcNtl&`pywq__@UOw0Dn?n;jnY8SGp-gESz=4BvMs~z#q7b z;SBC!u|IexP%y=L27XuI7O8;28TR9?p$~PX4*On3h?$F#gnkki(Yg?(T8>Vl)!O6| zfHNK6ZbB-Z74v$m+RTH(Y&N4`ZyWPXJXs!cv=Qk)QuBwh37uwsU2QVVK0-(tJ5q&E@0dY`v=F}>YpfIQ)%TtG-}o>Rc!u6 zZ}JSW5o=ey7DQH&y-BZ3+C`(mV~M(MVwxR%YecR7J^(07h&L4cTx=%)je~1m2p4f# z^P)3jL)V~X$cez(SfR|p+j2r^Uj!RVLUG5?mh-JmW}6v~qiZyq$Tz~TQO)(+%GKJr zc(j+$WtVhn3MnTjE~<4QU9=WhfdX?XG-zMwnL&v6UYVuPH_LjR&4IVE>J!2^MovXJ zl&s(!Q?!&c&DpE?hy%A7*+9y&#KB8^L}lE$$5>Tmr*b{5&BWKL>JXm!TM%#Fd1L;& z_y0k5<~lpBMJn7Y*nUj5&DIx7hQ0?q>DH+3u=Q(dEL2ii{?zU^SQM>*I7pF5V!EX# z1T}QGvrEy>7IwYZ`SsB@hv_>Tqaw=-%zjLQn@D6JZ~^f+vNVk!x2qp&O+B(7D9|=} zIskz^x^jitPfwzs`9WNctZ=fQJHKu?r?-uqtWOaOqT$6E5%#1Qs#JuwTV<29eVk*( zH*}7_%Nf5X)$vG}>d-Dyd6U_hC=9%5jY#*hpd%Jz#tkt+EJLWxUV$ z`%o&l=Vj~X$wEF6eoy9lF_Q#gKU*1VHOT6K;=V|i@nl?IuQ1s%bV=nhHDm0>8RcHY zgeNAF)zQr+8Fg4>SkA?<`};>88|;l5?s*y4ctszV;;#E>1Bw|$B@AOl+`3;%O3Y}h{@bFk|K?Dr2088*WFZd+?13W3D#D-JBo z&MrXjk-+1%JXJFpKs|sPJJrFb>cK;@z4NANGTY&v=Ooi&0Ah=A6nNBo`%Jo}+c6rM zm;Y4eu`}2)T2AX)3rX4?1H6mZ=h5)H5CwT!H|1$y^EBp?rYFWc-4_e`YrJz8AQ5YU z(&j2onq|NrDGg0UMVa%dv12(CkTMBUpy16Oaf0P)D|NL#SsK0xQ6c8*a>^OG$A))k z-Vtj1jgJ5wopyXr*lD)I1&HuDzG;MuaBC(zX-+nuN&Ky?aO?-@lHh;jN^@ddPyr#) z%=7C($f80-ZHNz-ZXT~oR&ayZQr0iCPcj}*`4zC?GsmqD_Q+k4d%6xkzLQl9UI)h7ez(B^@!)O zIA*$ceezl1ZaR5Ig%1*MD|j!mE-eAlhbA9xgBMmY`XW3@G+!caXo?JQ42tL6tX~kM zDbaG^PP&Nk2Ke8w04|q2q&!lL-hwePKnLrD7cmFA}rpt}G5Jc*c7qb1K3G zX`=(Qlm-<^zp;+|qx=ZH0$yFn8JzIK*m{`i%&G3{wBVZ1p=8e zE7xW0LgOxp%Ca9g#!bDm6%zEfAcB||cmQ3*FW5t%1N0fZE&j(v$s{UJ=??Tn7zqCC zm*4gSdDmeMxu$&>YuTN4F-VuLh8Zgj>cqokd)cLPB(2K9%*g2G z+d{)g7FhI^EQC_Y8R(9(UaM)bTZFyri0&Iul1&jP#Ih2nLOxV`;YN8GF7jceY4+$@ z%5$l$7U0?5<>b7>BA~CquC|>VTaFa-Mtg-@?odxuU(+W(>28z>`b8&OiBcG|-33_k zm>i%(#0Z?_B6ua*%$)Cy_IIE8!+I4mFNAFfPfBPBp=7_yB=upnRZ2ke5g^H~Ukjh4aLSKNPC8}Y8PSR7bu844Pe{7p$H+?tgru0l zgqyfOVn6SlnpyS&u`lWm zH3+1UA4%!3v2=D-PPhZzvBw1&dnBm{`3C4FC3F%&JDdUqF~QM3Fc^KoG;&buVk1%r zl$>~nRuYDwxnGGjUYOiQ7hovSC+|+qjEZSL; z{m01EkiC3EnRf)`rq436ptPk$8cb(&d6c82@0et`9rTM`_w0A+HfoFX!)B$6GDIHx zIQpANC9OG(8{3zit-KRe@GG_BJp#CR((P=)ctGt1&^|B zNezk0A5y(mR+TZoH6 zv9E3KHl+JZ=ocq!C;PoFqU)!ZPmi7eR7Hqduk(g^AFJ%iaUo`p(9k#}VvnX0H}=gE zD2b8ZQkCTxS*`(YA;0&&zalugT0fAOqUBsi%A$oXV{}hav{pgJTrw*gclHV&fsZeB zw*LRhng8cuDBByvI-S`MJ$pL%w;2j~H7#$DYfmtMx@A$G0^eD@N1@K1%sf1q7<~|E z-Q>Nyt;2=fyT$6`?ft;mxQ;}J$hYW}8`n&Ed9**1hz%SzRe18P?EAB>TDKXuV*7yv z$f88njbKxoRs&W+lB;Vn-_v$l`;&xN$oq8x&C^$aQi8s`ggl2WJ~#Sp)bZ?=w$ds5 zOVyCy!;a11(S*7SdK#<3g^9K^6{t=UU(e(lyR;S|lMhRO?fy=$gn-u-g62Pn^+*b8HId1AE_){Cw30^z+ z=)=V|e!;Qvb(;O&S-#!9vgUt~6A;V)RxC5(F!#whv9^YNKHC+Q>=p$FPQ8fZ!L*ox zd%C4tyoDCJjh~LXL>JmN|_-xq9iE6Mg*7$k>pBcX#@o{z`f8k*UFF1gtUU zO0v_~YEF~eWSVpu@TTd={D{PXUq~L83Pvajc{S+ZEZ=?FE5}-@;ZbZA?aGV{(Z3jx zUJguxa*4XMOdT{V(NMJ_HfE=6FO*LryUqPNT?o8>O!Lhr)>O~sqF%&v|4#W8=6#*E z6sL1&P3{NwRv>X-9x=!v6xWKD@3zJb2<&Fa5O#fzGQV@c zpZBSu-k+HuaWxM8%C2Sht=5H*(1B2GPZW9SE=^BQGczjthxm8)9h*fDhvuPOOYgZ)Z10;^D3QOd(=`SQ@^ z+0)vzG=~e18*DJExw)LuG|6Iye{?qwZdM!86OHpfGn7UTm3ThF(;hVhd2WzS$9u7C z#;9^UA`V22ItI1&O7HP1HNFgWu<6;d=HoxkMH z)r$?LC67`)2PP&Tzz2&ijEu}YRk>L^dD$59taP-zHmAwgD&O*xO?emKAN*L-&?%S6 zy|l1ew=(H7aj=u+Ha$T^xJsSMRSGVMe=5K>+i<@`geO4I9XV$3j^)(erBlIScd;ZE zxRwyZ?z5C%n%yKm-SEs&uLbR#{Q8tafpnAVTmEnOG#osPHiPGazBa@v%2-c}!Jb>` zo``IgRxEIQAw|0iFqQ-u0h*c(e<}pb;@HC1fBRtW_f=oLkI=fX*+4z@+X}0_$yhT2C0Rxio`%?2C&2f{+ zT6Vxe@}xe!#(QaWV!n?D-ic$Y^_jjoS?OI0pKWcgg-<_O8iiNZxGIcA(q|Q}FD)!o z)s8SL_%rO0!N8tO$LdzBQKFWIT67;RRIofuD0Ix z#mZ`J{yd)DFIZrEwKV@{{v&?HE|(USrRR8?E&qJrG>w?Hu$z4()k{J=0}!M5!DlP4 z@O21XCQ-b?_@rKudDz;WUQN;wXUN^y&Oq= zw*Gc`5enm-qk~)qB6fQM4>{`v2;{Gf9>mZKiMp-HV8M4UAre^m0hex}^xNI)tAP9R z0VuVb42kx+f*BXeOTvQnG$Rb_f;%>8zJn=p%jPf4}*AIhU{QVor9=Ds$ zcL}QP@4z`=^0$vraCPQlH>XfNlT5K^k3O4^L*Ik^#M<(5NUvjzMY1E=Ds7I%@m{5- z@>GR`*7&vT7d@2=al_#^i|*WnFMJnW`|FOAsm1MkWv1npTD2f>PansD)?|+kV@=JO zrr|EMQeY<=DY=lzK4d+ivuUk$Qp`A)H{E{A*GMx_XyaAe%3_;We+%ykm=3@R6v7AhfD9*1J%Y|e_<*#+6KGhC{II3L`yb|`&O75b zd(A7;z)z%H8@{V1YqN$g-2qplhI+qC3UlgZN>EfI-U$x;BTu3QJxqDj9(m5eE@<|F zvO7s%_F}EuNLhI8CsOBc|htPZyyv%Y^LHr3ADB_*R^&a(u)|WB0uXoh!nWG)$W~jDlK#f zPP2F*z`%RNuKR(kyKE}jS9{sEtIYhe+*9d$LUJ<1)Ozf>fiO@p1>Ea+Z>_2ka#4@= z-{Tj_0Rpf%OPoQ>Ku~JZ@UWv)x`9=`dFAR%=nfk$jZXOeW8LX8tH_iKj{d*Y#hZyT zQ^K7~=I^Z~M6#lP9_f>w-&y86*%;LPr@sP*{llf+XU%#Idt$AgO7b%`_bc_^11<#> zB=87Wy8>`@?;i&*O|?jlTZyFTGe%Cao!_?Y!bxq=E1xs)5x7F4N@v}+whnA6rV|Bj z{i+Ruc!wc`(^_HaIH~h%&rB2lJ_k=sc=ne&AgKIA{6Ao<(hX$4+t0d9Iw03Jzdkt=?B(L!8so%Kh2E5Z3OW5 zE3BO)N6V{7`QMTyEym1BEhY+TO5HYIv(-A2SOWKYqLw$bQ4?t^JoGIYy#>2WQP8~= z+nKD4mP9`#SJ}$V|?tmDB@UF_?UbIct#GKEG=@ymqzqVIVPD+^zG& zaQ>k)kK(g};h~WSaD|k^DQw!vTO&ZYRRWr_37G zqNe7vJfJ%s4MhKq-=S||6cIW-9?wx%H@fOUeq=LyU7M>xd50PdsXn}3<005=v@=lB z8vNF$luDYaC|xp7_xJ!ry!uko$>w!fWf$ja3-9AE8dhU=A@+NB{|Vio^>F^5s5Iyk z8qqF9<0Fo7;+mHJpiqgu0h^_8Y zC|zEQfp+$L=;eQ~Fb4YnmHBq-j?D{qx-Pb~fOaz2Hq)-MN%j52vAMo4t!r$gnZy*f z2OJy_0{6KmbgZewg^`<2%Z6{%TGcwH6l>vXQB@<+KLEa3(fhzHOV%|ZK2NFBQ=UGp z4|F-v>zpLJmO$8?v%p6Vy?(ViUjDOdbWkynBojRtC}7<*=RESlo99jmA26}%$>I`6 zhlXxQNHaZ|o}R|=4u&bTl-KnQP|u{pz|}UM{(J`o%d68(1bKu9s`bNUkumPO3|Wt7 zlNn9)_VMXbmVGu+Xg9B8t)Xu7LtStuQ3Y_|#yP%fZRUr|(rgADZ}-XrDOF*0D|QeW z?3j&&obMa{y)l=<11`DpBSiGSj1u(gCnyELN+z3KBH)vi;{kyNBY;QS*uBEhYy57T z+tLX9QvzR8S+RHcbO1g~R%UeSX-T7wB&$K1v3xih6dYqw!rJcgyTT)FheO}OdI2+I zo8{KtI7~se1;$ESS9T|4T<-+mB&SIHKXjdSSkr6#_m2q*QlfywBn2q}0TCDqqeMhd z=?=+>bPkS6*C;`xlv0#%qiYD#lkOVA=FX5G@zsLgxzSMdu ztWFKI8^=wN%wXsIqG+M_($9UgbBgank9ud{P(y9PuL`1BrQI9ma6LWdV82|do-83)iZ|Z?Qjl-`Uv-rE zl*gr9W`}^5z!n(GBUePJ0Q~t5kb*Y9OB6D?W_3L_vUB5eW>i?tJCT@RshqJK_2emW z7oZqVP*1%p4&`i8cXm=vFV;Y8s>m1=yz=q&*Df%$l>VJ@A;iNbC-+5P$4D)R*!MxAFVNYwtNNXA0Nf>=YJ!X|G;rK0| zHMKN56I<%G4BpchqZK}hR(Y13*X;5no|Ni6X&#Pu;&j*ykVw9j(>kJXxwpA=q`3Q% zM1H_UI?z|> zpxl(nkt4=%mwoDZt0=t$%yNaI5K|4In!ngZJ^v<;;8sf*Li!(C0F}2|y!B7&jope) zgb+@du9AH6nqCk&tc(2{O=yc87YnWP_WiRT)TlN3JoHZcP5-FcXkuP@MaC8acNqG{ zUaD-+a_1~F-l+WLN0P-xbB~>gN!a2(Oe=_UU8;X@P`P4C>wI0kCli6R`$(2_jJmL( z40)3CFmxsfE9|!s&inT}=A%4dV(jn%LJiHEr(;_d4H^A#r z3mk`vwzpkoA12>2lTWFVgNXR9du=dre)5_xO9taTXX&-qiNjurhGug$%MYaSDHJQ)MIB{D9W^bFq?cr*H~NZ}`@}j*o@mI8 zUS;vhV|Cj`0a$Xd0UVyXTt@EkRt+u?a{} zCYJHmyr`JDT6Jk}50Fc-5;NFLvg`|M242)2?Y>iHYHITI2mjtUuN*)PC?RVm z^3xD6a)q>9?A-#yA?(@ldazuCC~d_-Cgi9Evy<;QOp*v0`1tswFn7O^v1U;JeYcyR z03sEh)wha5Q2TR|Wv#z?;?at=zpc*k3d72@B%-0i_b7NdZxGSd;rSzYC8f-_UdVU! z#>&o#XJpM}dI4(i_y@yEq|Q-d%?{+`2X3oQc=VE_tYtPPLfkSruhbs|yA_snM5;M- z&RS{ftQMCx<>{eD!}h0HomI-5llaUe^@Q3)SZ^dvKh0};pq+fv0H_j;T)P!Dd1;4( z)<=PN_`^4O0k5q0;Zj^zcH{4k&851*$^kP#orJxBne75O)2LASuh)zl?m#y%`qg;n zi)Z9F=Y^nh;P8)ek2cv34B*_14v~fpVU2f1lSM3)A_QRdDAz<@8IB3YPSW^2PFS4h z^*Wf4vKYQs^C?!>JJ|4g31i_9(~#MKRZYjLIRLMoi2lYqi>OfSb3O&5tm`XmK#Aq_ z2xeJxyiI0l*NuZ zT*AHQb~1GCR&voVo8;S8LC>^?_X;ov2j>C2q3F`mz{qUm5}Y{DJ}_R(-(usz?cNzXC!P7-fk@H29kQOxDz z#G=eJvlbFF4_P3@AblJBHsjJMcV87N{9=FdkF!o1w?RcLk|T-{=auEI1#%N?$tKA# zG309AB#|F4dlH#>wDsRISw_Drca%SHgl6Bg!fvmia~>300lR0%Tt23>W6?m_;K%0+ z+P!xNdfYhkQtV4<@t^dJh@M|rMLHIh9~EDV%Fq1id?}&LQXo!9FMG_MAx&h|_$2J| zvU;IxFK+LB(Q<~%Rb7$Vc02n6f#i0B-1^o0r1|~FX8n9xZ|mxf1Fpm_fE9EUtt*Md zLjM?pI;r$8dIlBKnH53MDm z*@w-uUyTX+US3mxeU>JI4Py{CcCb!$OCTXnq7tX;|3F|Yr<=thu6*Tp5AL%qdHcFQ zON`wr-$rUlFekq-r6f&jV0Ze-z-@2zPWB?)gWZA^!8cm`wbOlvKCdvo)VVRwaIwtc zi)phUqwOa)V5pRRk&)`U4NI2GY%Fy&c)_WZmFEWQc<|N_Yk9N(kz@4J@_V$;Zg5=| zy=dlEf8hp9Q~BvNHIo<(6`_01PU_AJ0WzErkw&U=*d*;MC3?ud5Si4C?gV8@jyyvk z4bZj-HgYfHj-~#n42TfH0<=nJIj%22M3hfhVyvkU*S9>^Sd*i!3no!#P}*#zxO9YO zM7oT8a6HSb>FKcQG=U&A64o|eKso->Q;J};tcD423`hSBJeXw`kTK=>*e}^1*6vcu zq?0z|BIsxjYEr02HYtfY9C%$}o7FVF#qEdh$U5V^dI?~EEM49!O!!?|Hi(g!Sl!x` zC!;!)C&{8;zviPx3o-!(Yz={Uh^q#S}RXi zpkXGrC6B}^FaRut^4xRDCf8GFo-JPtca?um3s;tp^>SrGcV+HG2Zit@+Q5cI*gE(W zp!J=cxE2LdTG>5P7+$HGaLw(D3X6|-i+dl1$2dEDsEcWlL&}^DV1@Ty-K6d&kaW1a zJB9wT7iDK>V_e7^Ymb*|qY&#D+0FUj3XvO|p^LeBO5fufJFJ z(UA|%05p$TYMCT- z@^pK&+~8OM2CzuJ5MzbC3ND3vUfWEkDOzV~ZO5z?C>oe(odtemQ(9FtcQ4)D4jz+l z>Xw+%OHix2r1vR_7s5b4>lLyL&*_G((+;}lP5#5t$dJ>5s@wS^#$B{Sq~8vo|g|HNYQGGaf9 zA|$gO!c(u@yuu$JHZb0A%bzG%qUbShg_K(-^8tGHm5pY$3k#Y)dl(=1aBO3#cmGWT zlxvcp_f)H=(N98S3+C9oyQdb$?X7_0N-U=qeR!fDvK<9^a>SA-^#<_<0@jL8ov4$}lk<(Zn*~y4v58XXsoZ9vbvF+3PH{UbxaU$5tYP-8&ETnf)BrG%*9ApI8)c;$tWj%%&~e* zN=x5JMUda+IL-H?+9th09Ptyd&W$;Hd>0&`5}}-$ZiPMFdw0 zoXoKL^vxxCPQ91P?J?jrG(ElFJ5Nm5)%k*bA z6P8wr502TS-3L5<#<7n|6VqJ+jGQ(mcdZj4R*rr%8@sdoQK&(~+nvT93#Wa?xr2w9 zh2?sV55p{0E+w+U{WlhiR(2B8?V5iw+DA5Gy0jDB946STx`4D34xA?1H?L$wn76`B zO#I{Jv$dRVnXj1MQ_*AtKtNld%yv3+(fy6$qi2a5cefj30_H)m38^yPp|KsC3z^TL z)C!jwnIs2835uk~ZtV5Hm^zn-i!2%@NfpyMW@f`^jr0~7%lBn#cr&RL5WvDVaWV(= zX0{WT5n>pU?Zm21pe);DF%J1|8+9PYBZ{<9ymvdz?_j|xNszE_Xvm{QJx}J;;%RiN z2YO2g`6Ww6Ijs(Oh1ez(TgGxU(%DAWy~h}SSF4=>#fy1(*D6l`T7I=la%3+5u&uH> zx)+&b&R}wDqC?s$Bh_WhMw$>H;=a0(xiTe|GbA~lNGPW2V*p)( zB%)WTr?0#YbB6V@<3ln?B6_^fPPBE4Ko;xiXn^^svx1z)c0zz`ww&)=Y@}~t@ZJ=Q zlyU}HpA~QnPF9G z*B-?Ce!y|e)C1@CY21sjvb&y1J%Y|wgZH<+-Q5G-%@2RimaEW;A;1?0`x=Cf;k8Pu zUT^-y%~I347zyXTJIvDVDJ3P5Aqa=@%Jz$a=JDzRW^g+Pw1b4+U83u5yTWZ0UVtNP z9@HYI2!mTN!`(>J45ZYW&Hl`4Sl6g1o=mmk$I+Dtk_7o|x1R{=rVaWav zF4AyKYaJ-J6?~~7($gFOSkkHiqK>-fgVkmBs^3wkUrOi0+Uq3{zn`IQ*k>2Q z4UV=Dy9|SL28+;M@WOe`HCCD&^dx3PV(zLlvS9a8RxBFdvJn(Y$yv-Hvq z$CT0ztCZ5ds}9K$w$ob}>VV7v=8k6cWTj4%+!FnWRm|`kc zn@2Juf4ptxK}1FsZ+w!3TadV9Nq|qp!o3XB317AjfT&}b@DJ{rQgMAxaj}8uFnz;q=jF zVr>M3o})AHy7@mOJo|!*CJbn)n{1>LD2i0icfN+&A`eM+q}_H$zh2Bnq~nmfZr6JIe6Jc9TtMSeW=_dKvU&{Qv)Ak}Mi-_gm$bBq<4rbE5gYJE?cD-+sYi1pU#0JD%i!Jlf1z zf*(XI3_!p1+b#BpoQ^mU5QKlWEtReEUM{FwL*t`l*h`K00E2@tG~#9t_aJoK8iDa# z2U630Qu4dg{tF?4%REUXI9%G$^PA$XZkw3Xub{&LZ(2lQkRI%P*!5pDJ}?H3foL5d z7B-WZ`as1Ip(55mE`Qec)qE!9jvkvD7t2}&X<|cfVEClq8jKn1Ny{H^_OMvhfc*A| zdG$2sXYGG!OH9wrGNiI^m?UOpFcmPp!(gHDwKScL9H7E%3JU*8!p1lwbZ`Uyc(F39}Tp71t`4e zTFZ9lP;FTX#lPnkN1g1&>j!(X>c6g$G$$*yC11fVRav>peXX>8v~(znDq(cjKaBxE6rO?GU#Y=Aw|PzKbtcEY_*JnqS9HvY{IX`15Cfu1|u_IfMdn1Ut z^}iBfs_XBG;bVh+Q>%$@4qYIba=6l`O18u`Ms{!L&F*}CFucF%SwcQ`xS(LX^u>?o z0M_~jB#<~8RJx`DQ|j9ruW8uxNRG zKX~{;GsICWN24rB?D;%m>&V%W(`}w-S&p_?^xmlZ7q^R=a%8tyN8C%^uIn*pNEstI5vAaqOD$WzZCZ(# z@8?*o7Q$y5*xt^(Ob1-@W^sIm;QB`Id8>E8r$bbE5M>B3wQHS7cSg>mZl#@+;f6Q% z=@8Z5l{d?~x$20)``@EKmyrM*5ey58xU>%0-F`=35(?B-Iq1;ILdhU^%yB#wp}E*+ z`99C^GYW^R1{b~o>T&7Q(h02!D3(ANjWRM=nVrBn^yYM;iaPFghKzm1O6XR^lZ!U6 zVcjq9#|f-)aZlyyE3p~UzZ+DRky0IPl{sSB^;mg2I1C}H)WFO+tqv++*~fPiP1M~C zI$j+FT343I`TF*E+VLu2mkW^}y$m7Cj1B79T^x8rxPptjYf2|Mmd~QGV@XN;uc1A=+iuzZHzb20pO;8vj6>FGt>QMC*jO;5f7;}N>z zE=6L(SN6qV73jsa$iE&G1c%*YZ-9hz`~k^$qrihk`h|HF9YfvqE-_VCUFv2y=1+`E z)y~YI7=HV*_UUywCPVHBxhrjhk`nsra3cqPy_%)VzZj1t4>e-T6Ak}6a7hM#;AMu? zqjEa(K8nCgcJ$PW!oR-Fd%RF)UTUMQgL2wL`JPecS{@~oIqwzH zhs7%xRCH-iZHD~9>Z0!}$NCmg-Q)PZl?OlC3jFBnhvxBSD%lUZ57fN)bsi+3@~pIy zc()E7AQB3<4KRHW&f}hsFL~aUj66)uR!GtcsqKET0liL)N5-*-SYnyo-MA*+Z_xloy)fgs3Y(QkF(a1=Vw5ys71w|ExW3MFN zPB!``y=&GQ_r=Ivyz4%Zm!DrmSGA?xwlVHA>>#w-p*sAO2cElquS%0e+kE&KXbM(W zOFIgL!1!B--me8Z@gdVYld9uuVd|NgA6-6VPgYIaz8@D*H`FEkx5W*hsrTwYQ=@`D zE37*iBQ6Q$92|IOri`k!iH6jkioGPErMdq0rWOBY2jX z!aFnzH-ImM&6K`uH8()nWsigLdZ`Ox^>K~^V*hwZa^`{AL-IynCd!p8z(%^b=( z<0$*s&OdlUb7y$)1sA&}UrFOlVoq8SZ*$a9%E zr7JjKgY>1FODhka$EQFU`Mp4@59(3_YX;T+96VZM7>E!(xhBrBD$lA)t$I}~+81ib z;UwH_gDa=ckl6fAn#kWq9CG$AObH&l?3FYwAh~b|5We6mBS3wzbJL*6Ju=^@wqqdA zz@A?xznJ99tbAExh7aldcqG3Asd?5QFxO{9aH6Sm6`x*P8(v!G za^tbpu#T`%2-v3^lm+^8AAv}2XGi5`s?lQG%qti7yF{5i=^co>@Y0u)PqYD4optiK z!>B6bFV0Hp;pW-vA`iZ$IE=lb*=iV~f%`pTswE2%-z;yIQeI`>UR+!}&$-%?$?O3~yO5ulsjGr<>=EisA_i;&6uj=ON3L{QCFAU$iRS_?5 zu1Tu9L6do!S={dFPvsU8ZUoqx`pU;jWv=Jw4upriaot05mmTVatu3i#4Dtr^y5t7! z0rG-!wFXq{Wnum42}`*N((`sZG{)Wo4f~RtVE<=4P3qjL{Xu z-BDKRxzW>3Ouyq~&Jsrg0WiL|UL*MF39Vgg0iKJLk@qK|aJk$$~XKBKL z#vTQD65>iDVCCQLT#*MKMoaBR!XAF=(WXKWMUAqUrzlSm0Df=)w;m)p#P=!1oaAhI zm}2s>?%ZyZB&M5$nUKauh%G}-HXs%g+o8YKBc*LVef-D{wVB-mPNxo1&fN&md@?mh zepYb5uF433T(nS5>pGStj9s34Gj=bw3tE4+Jm-;xg$0Y)Adr{tiG7g4e}wv_A){Pf&cihGCpR=oRO4ZU3ac`ybEw zk5u7ygrpDVGcD*Rd~jQ7?@n?x5N){a97#Q}MU8icLAu%IA})Lkw)v4!57ZYHu8^ab zNSG9i^EK6-0+U+kgS)Fw1~Rkm^sw#2*g3dgS+)0pH!9A^XQgBpbg^S3cSSQ^^T{7? z=$#*2#M7RipQ4MY%@pOU#U;g?vuoM~+d;)@DcCO9y7L_^Cyb<27^9775j9CTj%iV* ziDD7&xps{?gC+Bk0zx6)PhfoJy^Z`?h&iQd@MDE25{n%S^r{(KJ%+6khlj1x;UuTW zK5W~_xs}_=>sHJg6{W?lD7D%8(&P=?Nt z*T3c08Ocx#pF%%R42%C`Bv%hQv*6~N32K^sr%wOO9F!5m5pb_U)jJ z0swl}F~TO(j~>CPJOYp!8E0YMjr1L23zm8gc?@j_vRO%U-H^3fmkYsv!$h4eD+IhS z@d+ZA)m!Kc6PD0t_oAd%Lt=c#GSwLzLt01#crb~Os5;sm#Y>rYG=6{}eAAk>OcTU= zSM$qocKRe<%Wah4$X)FqIxxugFqW;vRBWrT)9*bK1a$6SRttmNQ-h5>mT1Y zWm2!n0L43@b>(XR>Ug$TTk+Gz`ey=7HOCk~Je^dz_sTT`jB_A-sJw2qV$B#R7|5oA zu7(s;Y|RcuF!ee#gk1fa+pb^0V=Z3sKFe>u5K{mxHA?;($r1VUArE|KCzJ-Ze2Q>5 z(%j%|w0Vc%OfGN9wzjA69YI2p2qPBM{z29-)b1%7l)p3+nA3dK2TWHcj{mS|NYre4 zH*k!ch%m;efK9v!PJ$5q+TrXb!T*G?i-(*H;WnZ?hT0`3ScrHdgn)RC|3S^(RQSq! z`vMf?XkK|2fN#Wh|G;4fh{>RZX~a!#T4kG~7tV%~OK!H3Jh8ZAW2~iEg77qa6qFI9 zTyQ)fUw7nBe1pyc$x=UttIMcK_+mB#WZw?P$)ZI~;9t5_r>GK~0}{i=WkG z(S$?Pa5IToRp>LmWZuW-VkRXFK*CMcfWhRmUPrb9reZE*SKjTDl2$R2(fRSA&X$z; zW+aj}62@~t_dJUoKa=!Fk$vjNb_eD`#4>+kY>0%(&2UD1tzAfiL8-6aSngH!7TRuQ z=4B3+Ra)OO|=P?Z-R zL(q=?#8Hv9HLzw(AlU`mU~OG;UBOrt)096()8IUhL%_aW2@9MbG5qW`bxkvn5)drV zpllkR6->_jsgLpXayTP3I~CD_DJ|XQ#eNR|OW}QXZ?OUH(+a|?a^6Me8qG&#bAe;3 zdkpbEPXW1uJ}!{}Ht$Ma%Cc?I^Sv|Il?!SFGuA^GVU)4Wu_hclykLAPzZvouS1eLp zvigW*AzamZ=mTX-OCq%1$yL+)<3%wAD;lgofWAZ=YrO4r(W@ZGw)?KQIEr*M4ujM6 z`w?_W1W68^9*F0AvK;^95h@|!D*ng^_~lsWjX%OU&hekYdZdT%nLqqq{y8{498Yo7 zBf;#Lr;DnG9&{u1fqZXF=1mI*$iwg`8LM215J@h!*PWTCXuVqQE^-LkCR)}0ykx5k ziMRH;#q6^#gdwf0z)sX*x6bDjtzUe$T-&Bhh=VI?Ui0-I_sI8em?G+EtrotQdci83 zVUGQa{ky?rCp3!88PpX)UHWKC)pÐw~|vjk+e@W@jJw8LCnImgeR@#ho7hcLV2CSLR<*%55T73sk|) zlj{33efP7Pfxu|sh08p;;G#4vP?K_J*|q*5wP$D)k6vb&54TtR9YZ1?Vi%;mn=St% ziOZbID-aewd?q%sfw(X6Vu)sF07Hx7P)m3Zk^Xp6eH1;8)NUH6pd^%Io z#GwywcCl|aKbXoeMq8;sr`dXtE%FEVJ^;i-w#nHsP*Y`88&o)3MO0e6q2l@&UJhbq zl|)(!DplmC$4PclPC9A;wJ5I(Qj^Nc8C0ZW15W7;1VX^%`9pGr5d}}t0$ycoDi#l6ch#dE=QdmTBtev9#=E_hOJdb5K$Y(i_(jA_|x-N7k_IVex%#lN3kc=qQ zm=ll^zllmeIh@m+7Li=^k)18F>%U1{(D1h-Z6lUP>i&@>2U^~Lb^_}NT+Uivg`6#g&0oZ8k+MK#|F^h=#4etaXxF2e{d|xCxH@7 zp<#x&x{!Ni@Mk6+z%E$HT$*RU}vIr&@prh z6c-n7oL9(Vt6Dbp-Dq^`r8&zcziB1xqh~ma)zu4oI1QOG-g-X>qBhB;GxmU{OQ(?& z>k5QETRfqF_eAm0ju_8nL3f^MJ-wMq+cRQx*L z`Gs9@L^hOJs&xE$XS$=<>`zqCtTunblYs%~NYowQSEP)0_kpBOqf*xCW}i~MGgw{5 zPAB{C2_(z7y^c)1)0Fe`ojHgADcQh(8k*O{=Y7O9ZtkOJ)D%aHK8;>H;>KFp34{{V zj@8ble?m;Mvvbi<%CH2a%n{GPOt1eHL(9eB+3{-A&^TH*D^jeAlF9)C(u^IfV01lHW}LYb?#EWtWVIjrn#t*}eSaEWaz?dy zQPiIj#KV@&wFcrIQ$799i~(FRXGo4tk!MeySf?6|Yu1b3zO8S+MQ(3zhtkZ4OPKv+ zOqPPbXLXyVE&a8APQs(ZFj>N?G7=))tyGFE-5t1SZzYr>X{_;#51Su)|WTJY}&HzZN?2bj|o6g(pKh?{%^oz3t&6kdK`ADdJ3(V4C_ zC?3lYJ}^4r*nY|FjMwymK0fU2@JmjmTC0evz}K~$!mO*$@`U@caX<~^05)^(J)t-B zpD1vSQ?MVO-R}6`YlewoNda`B+=QbC&LvsU7(Jw2sAq3b<_O{hz^It>v|TKD^{scj z7jCxZT#cwQ4;3)Y6(T&bP1xiS{ovwZEzn6^t@d|sNCk55RM#F=3Au;2uY6IxO1|-l z{eMB0Yd@f=Y5af@6i9S2cmDG0>-A@kSEgCVr)8H1%KHmwSekr^rNlKSJ;)YRw>05_ zzr8)`$R$cW`3=NNmY-^B*udwPWQ)%h`3PaXxhVto@c$1l!{gz(H$%PsOn1exfspO& zHeB6CZA%UBhCH(>?$~ddI{qP*Ht!WdR-q|0&b+P%Sk3wel!m`gGcs@91Br#4|Lapj z@|Kx%#yu~GpzZA(SPr{Qn4>_D@xWsf2)EJQh_~~YkxKoqx%n4DE#T!kiCOPovVqaN z3+ht9U$2py?dt8?6OSFUZ!MVfd8Mbr{tbi$UA`xum;<`-X#tno%U)H6J1q`(Ewat) z+Ons<6ZaoDw9=mNQ)hkJR}q!J_LKV3B}=AFx?zwYa4Tu(zvyak`2(90IK1m;D6)AE zX*>hgXYB0CEo5uP0o4wiRU?ab%=P(}H;g&9WE90z%w3aSzRwQOgRAHFPdlSz^lvE? zo5Ix%B|?@hLMM;w83@f?&Ye%GLp{I!c|UB(i#&`$T(PhpyUq8YB5cO?z{mX12Ks?0|dyxP~!T^7=4uKZ6oL3GSHt>=wetq-y-gFAfvkykd4N}YfhiKb^&wkltE z!g}UG<4u7X?#)7nf$_vLb6#Y$nzd*`$MgD{Ig>2f3m`0{C*}^p{@x84j@3y$989$T#_18*Yio&+&PN4IrMwKedt~X6&3rvc+)q%*f4>4c)`dapleNRe{M0&wst1(>(_9s zm#SeEp#utw6^9DT z>^s+(nekNzzM(<{zB#&R@9&5RegUEsYUe;^j>QKjRSC3H= zhmq9CKPSO3;h;X(+@;N)TbodL43f^U7@#p91^gW^Aix+U!Fo}c#B3JPu9**i{`I+l zz0Ev@nHIs$z2TWVT^)-SG$oh^)L^*h8vM&mS~Ofzg)k0T@B>R`vC zgPq+OVgKRdqluGolGS0f$hy*f2X~YGhh(H@rY{R7s_N;?34M*fI570s8sw&=w7-}7adE#IthVFa z*JKGJ1>;pyO-yeuQutm*C6JfS35|xd*uStt7ZeDQrU~=-(7Kr_j7%A*TuJpj?@`D+ zzGc%W81fgQ)eqWH`Qmt|y`W z-Awqo(MpS|qs`d~-6uqH>BV=P;5H2fS=H7_$17l&T@&djVXx{oobs|Ko^Q0AR^5Fs zrNaHt=?0kg6|E!~_*r9h1VE(XxV8ZMk!PZ@2K@__ySPFIBF@pAX#+=pvw0JEyPWAg z0R|@#j;of6FZNW|FMDBRxab40p~HI{cs$zZz%_SnZS~A%@B@hg+Sf439tN}vuW=`K z-|&OCap?3z9ILKTg1zIJYT>A~=9oWr6&q7GT>1gitNp^}yy-o*eX5d_CrI9M5sM~W z1qFql$|2cLTD69beQQ_7O0d9=#e|JulN}jRD%L|7rCzz{We*XlcNW*fT8Xh24!lpk zmS0dR@!;-A4)qBCG%;h0Bdj1dx3%r<=C7))dKL3pXo2+`UJUb#@;YF?cF0tU3_AjS zM=vQyVTf#P{hz_cYSmgPc-F^DEn8!G52N2K_o&H{d;z4E20U=r3#E| zM*NPq(wi}U+nJEz8Ul{F5jTXguo#ue+#4gqNvSxulCtF~Ri0Z)P8cPWZb{Dj6OL0i z*O&(VjE$n!xRg(D&-_pJh@+KmU)7UDt1mXcV8zQMa< zo?o9lSpn??K$BYT`__Wo&2FU#n-b@2OK=5~226tmsguqAWgCw$uJ7xLCKcJk?se#` zN}D&3C*N{1kBB1K!}6Y7fLo%6Z%w*>PV{P0U8Uw?o>~RIqR5_=oa7f_-3S@jG@F_y zgMhASGmFbZ0cEdy=Z}HTs+a^sx>y0a<>9{ECqVW2Y!~!>4Q$@?A9?3I_kc-7`KR@m z%B4*vIjeyJBj!S#JY8B&@&#rlU~L8T?6mMJtKBy;X!a$oJ5qS53aeJ{l3YufcU;Zn z<)Y^H6aSbkRF^YJ3@c9?2_=?vbjX&vSO-V4$=+1T=u|Stuyn&Z zCJbx%oCn;vbe#w*^ssBhd9Dm(NW~>)30U3SMT$B_Zz1fnn&k*4v5dYxiZ=dhKHG&zp-Z@DBpVhx^k?Hfr+Uh2{zbz^|=S>`NcCMLMAFiLAXyMFqz z;vNZ(P4Y^ThEEn;lOzc7Cp;w7$!>v14VHwe=G~8ukLb6`A~YWw zZpZvl5f~qWi{vX!YH%KZ;`z)GFa(5_UpTL6FL0mx0jW!r@W=Tk%lZ4!8-vcLcpzbb zHYj2~|wT-g$IjrpJ+6R3s*}ktzY22ykj?ey8=?8z{qw2Wbi!RrZmV1&OyW!&QhZ zwrA>^x9&BFJSrFkp(C;hH?~sl^y~strc`+V1AB`ShmNtA70a0zzwOq&5IW02)20O% z0`zYghF8D;ic^j4-xPP*m!cns2;e-Fs5OI+X z%(ihnkHTKR1WC7=n}M?#zoul(1I_CsUKm0YuO8j$SJO?uut;-%o{p++FQKfu@hUKn zkV38C%~7oGs}*xfn!(%n8Y^l*ODY_SqaF-sHNA$cyUAOTPBAbh)eDvW>mS)uo{h*cj_Mk#65ix)GC z_!;W{O5{l7e}rFkKGzj^eDg3-)!Knrx*{?~nM)QTlrg+$x0DB_)k^7_R5Z_I5whR{ zzVgJ;#4&H_!iHiNM9n~zfYy85E z6q*?pbkwmF*)$76Gx4)@)3j#;8zn2XvR3^2NvZPa%n1@KSW|*S(Gj!%+z0U59Z1Q( z5x+7i(X@4Yp&GvRP*PmjC%GeAk`u!8pNUTDny+j&RCj$}o$G7M3Xe!CZwcq=mOE5*Pm!TcGKq5>zbnP#IZ@D0*9Cl2_+^E*wb;7U z00Cp;Nu`hR$SLa3;4v^KV#_<8S<$@2q%9kQWNE>^adIErOe}U#K*j#9v<&QDh6XPv~?;KE27qZ1|YE zyHU$HVaVOXBTyoayEI;o5v52Jh}>zBHsuqBi=E#zN?{*s`6OdCpJ-DFlACV}Rm{lx zJEU!pY%~}hq}6xebTo_)h@ZoO^_3OBPCuG*(qD|Cw#@|{8kihh7QUb)gKi)|_@GyMtMXEkH%Ib5VLjf;w46xo@bXop`~ z-RG|3&lm7D%O-R=w~Ly0oB$_OV~5PvL(+n7MfiA1c&cXtF#j8@zLO%@9O6ok7t)XU z*yOg7I)}hs*&W!^mdwvWG`%CPNU6xW|8-2t;gy`&ORn8#CSAfcbFyop$?tBRj~cTG zT_YK8hdRHS9N7`~n&sFy&v_NdTyARFn&kccfEjydfi-sqT)~c&l3eO6b2;z-NVBJX zT2LT{Y*4M=u!Q2xqp|gwYkSQgXLpQ%{UMzTy0kLk9g%r zn!282w`#?+;UUuQ4iR*gRMT74jGQhT)?VK@o|0qYKs2Jq*tz62S$_&HjH7R1-HG;5 z)vUi57v{tl!nfuRCWFVbp(-;+amSKN6VLwdgtdn+8UKl{mz)njEOQqJjl+l9E_TPUeXsv$$Y)Ry58J=ug*3Tit@pCpr~vjV zXkKBpJz2U_3CVVuN2S;{sR%v(bf(^O}v`QlV_IusP5oLFx5 zMl)_NtaWS!t9TXSmoqYW40;C(IK z+B&1Ajq7W)8o+DI;I$=HQEez`F-S5hJcDr@<%=!}~D z@oU%Kn@_+_oFM6elv{U+o36u(UdSMq7^SHciIDgBmc7GN1XsZk^kw^j6Kz=Jbw!R0lw%-^lRTMFGyqdg1%_;!YE(}oi!f`fu217k5TG5gf)ZF{*{P6B`6DI?yFME*Q?Ut0$vmS8 z+x~IJQ|R%hyWMPvI8WV>OP$K~-B3L2<;xZ7_)VdOJ990v!gCNxIQ!w3!lU_@HMM7p z8{gg0BFmkE2*6|4j_$kt;ey_nvexAw(mY)Lpr#ik)M)5VD>Y3kFZ8*5R@Qq)v`2te zucSnLgVlei4U9y->JIlz8sAJsYsC;&Rw6H6tZ8zOcG8$zNQh&97JdtoIVU376i}@& z;xwW7LG50LQJ+`9I<$fDpPk>)maqyJH?+EfoZfYbOT zO#e5J=kT=hE{dgpMu+3Oot+*2*Wwa$UJW811mtvpiS^~oLoltO$=sa4X^Qm%qs ztDNSd>z-jc@}B7TGHwneI}&jpxt@*|g0i-qviS95yTXa8cww`5?!62iwi=^M6f*g{ z)KL5PYCVKO65RFCM%8M_c`=+@FqcIrzorb1J2ryV!~XoHYz7%h@xe6xKKYOLhNGcvd-PbnPMNtZt!p^{d%FOl;pmND4mSs}I84Qm z%W-tc{P}-r6vE$@zv_CKK8dJ7A&8d|vHri`K6x1uAI#TjRqRuIwx}7oZ4gQ_1L@;a zwlI|Zr{?0$E#W;;+|z#ldl4qNhG?x^OQ5Ya{zJygo|ek+c-PB!!q{;8ikc!nJ%zTw z)y#Ef{Z$R+m_PQ}Q;p3OT*O`BHBS$dctoT5Ly^kDLthwSU&~bEkB?Qox9123tksJa z;w{xcmi$x|28nfiyxrI?9eY9r6DUe|r#`#l?&>-OLW=~_JE^C607+@P!Kcc7<*sRR z|Gq*eL8o~usTVX`c~w=?QFR+$cNIF1fNQIepiGKzV}0K&Zee*00&a-;XUN+AkU$9w zfL}>Sx`mWbjQ33R0R=*rgP(p&bN-Z?tCx;R#I(i23k=OUL!LO4>s|Qp!vQ`lx3Blg-5aXx&t!R3O8B0bAS3DKWlIciivEi~9O+uUB-3}j8I9LJO?2aU7z&T35&X_~8Mn*g&ezuRUW-o>8Y<)% zevkf~Y((;~o?;6y@B24Xik>oD2)y0Ku%+6dzemj!WcejG7ZntA-rm8ccy@NyGwo|@ z6M$C(R~H~)U*tL67I;BfsENjzhaF z^2mkTdZ-A71uPB1- zS|oRd37nd|G0q`#(+m0mYT|=zi^~Cc1XsviC*L5rlmxEx-1K6rq~IUdy@qT6{?781 z_Pozo_$lgvYT~ndfLNT$y2?-tWs*G$QNX$3RuX48yJJNm-#fNsPqLTP*XhPEX_#(Q z2C8>dZ#TYqs3wNhkoLi-gMV?+)XTOZ&$*U#Uf7Iw{7&6W^PZUboCi(x4QUIqBxLC*1h=ID7Duqq!2~UUH6Z(pEzm8G8WpUnPmeN zH3Ri+$Ju5do)N^QPN{B3L3MwV$Qy)kxj}U&ceqId0i^7O zmYKGQOhB@ai9MqM%zHO>I5HaHLLw~ed$=YfcwWdA;pwe6=k)2PHJ^3=$Lb*o(K?wR zF*EY88gA3(-(Edl)nJhey!KyVYIVCl={p&1Tl}Zc$0xu41@U=~emD?(Abhc$Wg{Uf`+d_K){7x#t^{u^d84%2W$ zFoAt2ESx^xTCbVS%KFj+Ne5NHYGVOG)~v4@AL{P&ka1)EXcKyVw#E5rhD5eA1eDE6=LZnC2$C5UA)l2 zJIoD$za8TdLD3ALQwt-{m#{7NHqCB)GfprF#fYYxjwXzN{ht042zC>?kID4?q4Yk` z3g&>KiDZ@8)rKAq0@ipqJTY+}_7hDaYFc^guywv!fw=lTC6Sc-N2KkQ%8#bri&lQcjKupbVU6J-1Wccy((w@ z5BOE(4V9n>v$5de9OP{5+2k{AJq<$1J+^k>T}~kQ@Z|95@JL_JIc0ktMf$gPSNV2b zlS3hp*YnA}-$u3cq2yzu-{8I;pO~Oy5kOH4*ta2xZyjFnrqeJIn;t8?g#xilluyB1w6v7oQ#;}SmhyHz+F!OF3D8lnb zN}*)eAW&jY4l=pNBlkzpIEtz(z-fxe4OM6x3fOV03}D!j*o#pAfz}Aiz1|O|zkJXu z#DuqK(+`sKZeDg@`gfMJQjCv!nJ&zT;Wg`tnEdwsuyu7_?aODkGMCS)}v z_S#8ZdU$;fd-(o{z`Wt3Ozv*ArTKO<{j*^j^x&Ov$+@COHi5Z5R-R!B9mmA_w%c!^ z=ivgsvQ@(nKea1|N!=4JQIkslP^Y;@{w&5(w7I~~bGLFhugef<2&^z7-F`m!4_}Y2Ht2F1JL}XF3O@%^h%XM04fH=_oSwo(~lOA49}EX}t=i{pHpBX#e&2sbY6nU&UeF|`Rc&%2|3RL~liZGE1f>55 zGZSTIS^ z@*Ki27sUfOd|(lewjZ<;87BhA@nAE$p2>p~*Y5Wl-5{9@>cX(b=Gu>eeQXvb#Qu8m zyIw9A6#UtgWcP{v6Pa@n60wrkr>tB0@0BxwpuXdu=yV)WCOTy=Vpk{;!#$rn{lPZ< z`udT5BqmHOLm4iW(G+oR3KHa1W0q=SmP-qO;CG*yeo<;4;!PoF#*p&)+8D|AKQmJZ z(ZYADAN}~QMH(DVU^K<;3p}L)#b69-L8xOUf`o{wE3>cZ zGdAhHH54k=>zeIAvYKcfSR>WWI2?wqL|aaMYF_5tYw^3;=e{Ay!=(HaC2gkGWAph` z|Ggq@NS3@z{1DQi_aK_MvcGTFk8;E9-hb{v_=6Est@Cb3P`&wawzO!oax22$twbzyN`Y(4~RMrY~a_~FZ((;(Zf0-4RVYtkB2QC z2b<51e_N{mSlOnXkYzGbuexw%u8~HOw~x)XH(kXUz#lWwEZ`8Iy`5}kiR8DUrFMQ9 zYYZi*+8P&Ze!j)^g+i1BFBbp6ijPW++pCfx#Iw^K zMPwz+eKvN`EdoGb8#3*c(Wxi5_5&qmH|3u&S?17gTpW1VtuZ9R8llt-H~Q~Y5*aLi zNqCst*2?zmeh9z`+Mk&JQ?DbS4RfO8{)&L}n|98P{kSLi#v9DpmU^aVN==vhecPbZ z+|a_C;kIVG+iLILF5MG9UK%fa(%G&;pbgx#d%kM04Oa^5S#FbktQBS1#&IJy#3XU<6N@*!$jz!A^x z+xg3&YS3tA+v#qy-IF$Iv9BG-;_vH>rgNGyp6E@gDtH#rHY2#>I9Ix}LkIe@f%o&) zeqaW}Dxjs1`c&$#4!}>PIYG=BPIL&XK8y)j{29nGE*k~{)xy0lE7Q7xXgZUFdU{JG0WF|af@>EW@y&$ zvypDn$;I5W-G-~dOwpFdcr9MU$XqQ<=oXdH*C8m0We=gM!hti%RdJ;p;}?=gb--Qo zTq%wN8!Q_ z#g!M}K+3Vf+bzpiGrde_m_qjU8w{)rM$*>^|w3&8Z{`oV;%g~B`cZ<4*(01T_d}DU1AZwP+!*ly?F<)de zr;(t%%jEQ@XQCor+M#A*JE?JjKv@^FfN=F}Yid$%X%Ve;vr$(`)0P9ty01j6J+zl~ zDAW_N5|&KK_Ddpo7{3rX0OQbP$d-)j?<`R|y47x;mC!>*dV?;4Iyld4LwGwQRh;J< zc^u8%Pn#8O60PzGlb6JfpOKBw-C>DU=n^fM)J0Y^+;Y3~XtNld_A??bWttygH6Wgf zfmlC_ns-jWUcX!9dtiv*9h2OtU(D@i!a@20K452i%4_Zj&Slx=m)YF7!@;158HEgr zPOoDP|DKaIfLyhS@AR3UuNwOATLNxO7Bvu_20q#?-%>^&u*oVEa(><#4yM3Br?IR( z4k62w{YrtejAY%qa|cVCE6LK7Jxy02Atv39>D$wU;`}>UcPyc%^^s)6p?5TWBdl~V z9~`VbO8TJmF!`divSu~BTV^IrKaYMznYNsgaWWc_Cp!(Hr^Gs9 zwX6!25aP8SWK%b)-T(Uohikmo0R-#TnS2wVo-NzXb@sW*aKx`R|NhnV+4W@zpH|vX zW(YuBVjP<+h+41e#_kf#5%wmSU1VY=e%K#ZxWo_zA^#C`HU?oM3#|^3r8$^?$zHqRpnvns_+yL8} zZ-AH$rJf+LT#!ih&Y@fNKkW||5a#t<`69jo4HzVT5)wRuHJQkaXcu<6El(cspXUa? zn(BMiV;HYT37po=Qmxx^ooiSW`WJIPju!hs=EG|#mX50VKngm=90KTYsC$XFHXEr= z3SnUZD@o~X93I{DER}wcV^%|CbRQ<)rtr})M^5@}+T%Pt`M%Y+o_P7x=_W+j-XO?M z70&z4q`b`3qgh9&Fe5^4kaK~6Qr>6BKpb7RWLZ^6mTVwQ2T3&uf9j?lh5e#FVOW_2 z%$~8P{*MO{yC^1@c7F;hDN>uDnEf}Arh>>w{cqe8RvB%&-Ff9uUImHd_4f&BsXtyZ zCUu8M=@vXsR3)3e(W4{}hlSGkM#bp#c5Iv_K# zL%yg?5ck+D^uTz zu5aHy2b8GmZ4PQYIY6c32S3b-W2{oxDE-g-Jm`k7O9;68$(=XHo&ELS&jLTZ8>v@N zs>~Nm&}NDGDnq9;nm}X1nUu&5W4d=c64ufXSJL-J6*D=qU@t7|PH!^Z+{ZU=R(i_H z%3=(Swg^TLB3cI>HZT}`UB#4Owq@l9(zDlguXlEHh4Q#gisf(Tn>ayDX-?|3^9V+U z_6>X;CKY6IcN}RF`H|iH+qZlK)b{JuMLA^Ot8Hf}I^uneH}6z5o1j05%ZLu%XG-T@ z{lAxw47i&sZLYnH^A#J}(Mdz5qtx*-VXdhvZ%BIhAxXtqsV_zalsz#@iMg8n{v#?n z%9wzAQK9uSf|OocNh4{-wZ9X*n_CoEgE3|7kwStT^f|CImY*De{Rwg^AD_YG_lX@m zRSnv+rHp2<9Nb)u5(=nRp}&U(7x_L!4}2>ZuHeL)scyeQ?R!3{@1k%r{`UoD+wlm) z^Bh^IQ&9Wm5&~mSJ=5Zpm=oBSa5tu>9wslSy*jIg<{LKLdzP!EB{C)@)*+BLvzX}y z9fW)+Nq@z<*UtPweZ^&7?@+h2@`k-(X+ zfbteNij-P}O2liph>2-(HZct`H)X5RslL_MVrtvE3~e%y>c_`=fjKEYjSUv_pBn4-}sK&zg>2*;hOI zYI<5jw3Ir%=0X##sw2-&16%xC-}#$shwi5;v6pDDhaC6|g&pv&%91^rVM{xY3L06@ zuYxdlr6j-qO8kX9~rml=FA6%}dv+Fl4_|MSts9yFU9;!$e3xO3!$b*8Z=RiP>^HGYaSqGqQmb~3-A zrUdUla}F{U^>fe)l4`-|h*R32W-Sf+_~CWj9W3RXmge(wtgJZee%r}4Thk5;MId@- zPgYuFeEG-Mbjcv6AFYb^^0v}WF$*YL;WjSX4D_oG^qnHpY>&CGZBmz_((zb^;jilT z#to8IXuY)z(A(`abJVCZcRh<_ZX(0SCty#CbDfGdG<W zFs@sBtQU z&>Jnk>|7uDjKVy#y`aN=H_0Ll^&2-JBnx+MU*8>m6Kqhl*@XmCDyuF2v)qN}-9Xaay0vx)| zEyOaWQ+x8nd}^H%oF9Ab$Y32(u)mVluLnjlLHsfaT%wp3_%Vf#G%mv+D4 ztsmp#WSjBo($d6f?ZcQ9P9JM1iews$xdeyrU!gq^el-s1sW0i_-c^;iETSFGa|nB> z59LijLF2es{bp>Q>a0r~l_S|39}dssi16!Aca09d%juACrZsV1XwsVi$R3^MPA+Pa ziGTN^nuQJ9Ca|uCyFPz@N<;1=M*krDJP|VYpQ8XxSK8b4sqtI}GUYt>B%~d5giJ=B zxqt*#KuT5+&%~CXR@4f4bg5gJst1vz>P;Re+NVl)?VTvl#yJX?yYobgBF64XVqZ!W zyDoeh_ek2}TkM855v=FCcx5Qfwpb^XZl89X)8AJcDQzRnDaN_#0UOWGQ-aRO0{}pL zOt5ti|7z)>B-abP4n1T*#R$}|NK7ud z#So!S^tpgs=O^t7+4+ouq9U|f8UHkTfjn(3_G07IrBTE2Jq zjs99=($vq%*g`R2<9lmN59*_T?{~TYcP}rX2X)Sb8(WooX73;N{1Cpq{c^6{CpIyU zQ(4Zb?az5;@EM;IGN5B5wU>>Yl0$sIx{MU)~~>ojb00q zm_S9fITo_jrvsa_4b->HX&JUq$eGbEE|NCRr#_cr!Y0;E3b+fHI-%K`#V@lax)n?% zn-*|PJf~piKvz~~^rC?oM`HHbF;HOX0x)hZDCN(|5{D-;pas~C*XNsgc!0A1C4wAGs^G0{TRjV^q}&QQvpcQkL}d27@c z-WEv@esZ2G2MV-jDQ1B_8H=DpbAL7H8@^zB10%oFOkyG7N4O+4`AN5BtAYj-n-e1z z=l+n@yBLJeZ(|BVpBg7iCe18ci715{D7Ef?CL1umruY%*ALkzFXtrBZXFObFp)O{1 zt4dQzgx<|;bTsL&n4lonnKzXQ2uOUEh=pRi-^YGWnf+dzGfYQ%Gzmx3a$dpKjDOAP z4cp@9PQJdJ7GRTD1Boh=fP$9~&PUpFfFOEU7<699vkN4|F+R{~$R&|RUkW$EA}$5O zE4XiHXt+R(9r{4V8maGznw%u>Aa|1`(270NOjafQ@%b;d&iLp1PQ_(CoTP79+8wbU zJEg1e@#y1vc-M{Ds3aEBEvNRO5eCc;Es$qu1ME4FvX!G-SFt_4BLb`aNP#b)sJ?T{ zaz1S)=;jL5?BljIx|4pom?1zEfR?KrR0b-bb6%YGIT0bh7>7a_-CX}%O*{u`ek3S` z!WzhudfO)z)3l{M((*U>*!ZnRA>+k4p6$hjPDYJ;pROwIT1~Yyh(I-ZK8|fgmWBVH-p& z<#H?wJ=CPX)lC5!mc)G{Tp-Yg_wM766xqgGi|@I2Ab+u`i%v+M0WXeb)$sfA0@Zd} zi&*Afw8Yy3WVJh7U!DO!djwi*2y7@9K^_2|pq^L}^Vu>B-V89?BzxV}Ic=nBLQ;~~ zdkBwiNH`SP6R-gxRQ=8qdwlFU@Rt9P($hXmFc!H{`C?ZOldY5y2*8O4g| zJc^rk>;-GDr*#hswlaLX3#O`VcEm~zur5@a(WPn}DPwg<*pqve?jNud5ZYVWYE<}7eL>ZE zRNwDUD+@lJ(}NOGjq!3OQh2>(xs6l>CAR%-Fg!G$LgM3WqOlc2XBJvK6&_om&~)z# zLZM%;C?e#TsQXMS>KAYVRz>HA(LJYAnz*nr8@o-kb4dL1_qV+O&zT$0fGO*u^W(tb zLOp%Cm(}DPuxTU$nzyy?>mx#UvK}p7fSwm6sfRO28^M<9<~2TNj(@S+auVxidD$F= zXf1D8dCPhn2k+)1PlKjfG~2o`I4*`~bEERpq6JF6M;OQFXO*b#a(ngCzNTa|Q~+ zogJ4%aW!93DMS}#?lRo!S)ECfINcI_F+R%y*veCIn z+oN4<`UtRy5slB1-2V4{$RHP8f&LZVzH8@yzpcTm<4IYR3agn~sYXvp?guSM0jWnX zQ*|%Nmy}{XuIZx2xcDC5g3*}XasE<>yuD_!iC+iFV4_k^rmI4Pg>9|AW~esKDs(@l z@bRwD89R+n6n$|cj{-j-eE(5Mug+em`%{*^?^_TTAacLGPo!67NY)4716)d6?c@v5 z%q`6av4O;9Kwl>G*X!4qIe780_#E&B_RBeu((6mi4KI^0B20g zLerr)H{BaV9VOxJPs_84+HXII;?f12BDZ-S?ije!_`shqY*BneC;j{&xVq(5NU;2h z`I95+OEWSXx(!)$Oyg6|K{_Ay85_j+{~&s+XX2rgA&P`w?6$Q5g%x#|^o7{NkYzOI zV{_-zTp+32($#CYwjY;(VC>=Ye+oRYi(7B)kO4)t(!zNbp+%LBYfO3C40-MfX-sUT zDkZINd1HF5(i8gxN+oDEtLH^bxRZ~Q``E}L9iJezY*mu{h3OVIW|SB}G}=*1K&^Cz zuPM$I(EZ3z$2_WyEAcx8{(~jy2?&zoO@vs;Jxx_6hB5ft>gD&%7>J@LR%E%`zygqj zrZus@Pf!zKQxZopevMhK>MjTL6*V*=<7l2RNQlb^h>BQS0N(*O|vC^bpmkptz4kY4-KL`BpUlO_aQt75wYbmz& zF4#i6ouV(CO*xT9Gk`)H9OlLp-m%Y$A3vax!=0%rdSQ8AJtL;s6>!&rqV`1WchUu^hQf0V?{DdI**PJx$xHl}{J^`i5vs{IU| zfyIkFxrB%pD_g`iS0Jt_`G(RMkN)@N27hD}S{quKHE3zx{3zP*$&bW_j>37T;;GzZ zi+_}c+$m`Pcn9lKF;RJVSd(;_R6&9aYN|zzF?c$O^&~v(j!ySUsrH&))kmH65dk*( z5&e`65}BN`TS?zQ07a*Akuj_6u}@+Ai3g z#!&FUjYZLAc{m1?YIjkl8FB$10l$f+R4E#)VZ_0S@>aa^eR(sh5+}kSguyI|H=eca zQ_Hp!&jJAwYs`EndK~rZ?V4=XEjPKx@#?g(VtRvVlCDEp+>!hTDyK=ywK1>Y6fVI;YwsL;mL z`Kn&VAJPCH`NuIN*!vt04jN0^kQ4pm$d`-@2p-14waMOFlXds*L%L3hkj+hm=BFSs zN;EcGz<^36{{f6G{BkLyj$=Pgz?&dC#YSrSDP`|lF<2}M#cMyN6=iC);w3Q#E%2Zd zJblm`Hz0yGkp18aV0@Y;I27xK^8E8gzDJKlUQJyii>D3qoWARU+l{|eAFk8d(3S~i zH6vyN&!v2)?XbcQS}YgNRSUQB1hc9>&F)7Ad^^)>VK#1CPP*7sE!-jR-75W*=~6W7 zKe6X~mDaVMp+{RbswS+(9kl-1-eI7GKfkdtq&G<`F>3f&M~Ii7=^oO5o`}?kNno)^ z?g*0omw5-jyg=D=Qk>jL{Xk_(Xo4c=;MCtIK#MV&K6}WL=`mrTxln12)%T28i=`9p zTHVM}(k&BRnL!?BaQJ#g26O*ftn=yNgWU}@W~Pj9P{@)&B(4!Ik~~?3i|ZPv@D#j;f7Y&T*x;w)#qb5swAuZSZs=zki}aJ_&tSmi`BMI zSDakqE|wU{da?zbvjBWLWfQdMCjp#}Hjg~ww`BzH_6T`|OR55s@SkU+XMDhIofLOp zAn8ffviX4G$@mPVpOXkO^h<8LoCgzzTFK!5CzLSJ<$gH z(LzLbt5@?Pt)>?7czAvuCkADBh|G&&38b6C;}G&UcyJ5L;))hNM7PLO&@&ZBEsA^5 zsQw_<>GJz6Gz&Z(zF?tR5F&=L+;^h!w3>1Jz4t4au#>sn@riUIt+F+xXgnbZ6btpV zO>og9_|oO?EjMF1X~BR=Nh+Vk79Z=sGY8TEU3L!pF5yq}ITo!=p$vwU?+DMUY^704 zKGVi368|bnqVFo?|MPw!m8xtP(`y=ib3Ry_Hu0^ySMjQ3B1g&pwkpOP;9GPAT|^hD*&Pp%0^VAkA!p zHGLN7f;yK!LYpC3ak@oSwZUg)-de7VE04>Ern_&b|Pz&W`II|756`NKG348%0kXIm6Z!Df!c|r6HXc>f| zkONo(2`1|iI-ab<3kP+%9_2bCdVRW>=V-OgH#D+CngjR7%px6n!}EYd*z3<3!c#T3 zu<)TF#Hx4adEb*;uc6{lqTRqvkehvW_tSi3;1MByk+v1KU5i#v{zyc8BztKphkT!q zfY5v8*1hbvfYYl^HG1{S8j`^px$ zDl*Ds<~ZsO3~bwX*Xmx)0`-uI%i;pJ+*u6i3sUocP=bpOY@Xl)B^skr*^0rFNPj?=?LUwWRuTCYLazRXA54|3` zbLNC@?()px_9hIS490$LYXjwK)@$mgac)Doihn-cLY%hh?N7X~O)s-L=YOZC&kY|N zH6^bj1I+Ol3y2H8X6K*s&8M0+ z#ER~Izid!;A45KDKYqLMsB-srwpP!Fs`VPB4s&KpM}|J#(%n%RI_;_2Ez5rB-&I!k ze9Ws}`#@6lE&6z$ zJ;Q;c9j~Q~_r!5*wiZ3AT5YIYgMY-5T` zwY~%F>Em8S_t$}5b2XG<8o*wfpo-pHd4PC#=WPVg6j%m zS$gccdF`pe5s%w3a)bR>y2basQ)uFY-N3~CZ3z?hISY4k#>VHj&uV5NdK2aXBvO|j z#Ls^PSFM6wav&&P92;naiN!9tyl@S@j$pT9<5EUMFj$2@w<=I1z2N<8kUc}JF=cc(U(4$0XY%JMm}D!xtiu`ZsKdEM|Qxql|t+>hURMWuFWW4y(qX+o?{ixDGFDQUE~U?Wk{ z+G2XyRhq0ooBbgfSDyRx$Z^m~eve=rH3OYQR>E2J_I0@u;?>>F`;Dd^O zYI7Bq{a3s=@-g{0pO=n>CzLq%iOf8+p%MWqDKCNB$P{Hpxh%RC7BH4IX>s!BB${D@Ub%Px zzLPy+ps1L+(AZM9gXC+;yFuFNT8lGRr@2-cKklFM0HFI;)GJm*#Jae+*uaU;+IHSl zxyCB{>PcRnjEPBR<4)t8?dFOq6(yB+alg?GJ9~68AdBbyrNlzIm-$K^v(l=EqD@VV zPB`op167Qzz!K?g-1QEcI7^zTyTNxK_Wo4`&bvL2q3M}!h`Uc`-NN;&6@*FcX83q) za--{t`T~00vR#H*mTj^0HyI?DFZw%BGWSMWou1q_#(;55We9x| z>8G|0=g|PEAThEy@#-eMBZ7~)f4#k6+Dxbdgu1n^Su6$G702P%OKB7Z>IwhqhYpUZ zL~T7|$kFD?lg`aWT4?5V@WdpBk5A2&4VV~>PfZtw)0>nI1zg(3p0~+SGNqNcR zn|0Fc^XBXAKC{uetHp;$f~6&lT_*fGqMY?AJsr{*MF?oe=V zwfiy9n6MCkK|Qv=Sk8*+LJ2m=QkKf6b8uRx7$xHlr3-TJyisn&R@!)$W$9!RJAugL zQ^(AU$@Fhh_tQ=Bd8wKXTG1l{`ji_kL;DY9gsG_`T;YicGAiWt~o<(m@H`Q4BeLBDJ9iL6q<%Q@Z>ZW;Zb{yDS2*1qL zB2&jr>5l=OiZ75???`qauj3=MS<`SPb*8LD)_M%#uy;B>?=`MZ{*I46$>8C#x;F`J zsccU^$$~6$&-9sKo-XTUizRL#y_eu8=v6VkVx1pBK#$;YO=& zZSfWN&zT`tL{?B}Wg_{l@CSU0jfJW1i?#v!qBK)8w<{HDXos~AWhIssyG5ay<38-H zmN2_6hEw|sYl&v=mtLsz2)~tzf`FWDpG32LLCd!A(3IfkCnuY5Mv|j z4TgkXK$4VX*u$H{&VBjE_@QShO4d5%Ogc|#bo1S-*zZugZoO0vf)}B*asLjy86kw& zApKc7Ij9J3b+;gqNE?nG`{hjLavualYmniz`%^)(?yToPfCwvM$&#S z-ZBfR4kgW`5Rd(>&q7EmXe;CxzJjs;dDgs6^ubK3Q5j8sxj3OixZ7-{To6Zv?BV#t zJmqu~h8NZ&`I{PH(59WvqeZbGB>q);U!@zha$=uXmx9GqGKcR+#P83S7@J$hSP3Z? zXB6$U8qHIX$cHB0Wh>rVERNS<$afX4(dQ@;q7x2}mp(0{elA?Pg!#-LJG+hN|3LSBJ($}HwFPdHCXO#>il;qgM+j)#f*)^?! zQaxG}F#{yAjew|A>$z=sv@=s|k)ywTihVh6m5nVNLP6{gmuRQJwh#Te^Ze*})$d@` z#j)DcC7QF^DrfZ$nvoIIa{`X}+Byzr_0@$x>#&#Rsde40(gj5pp|+nwM&{L7@{7(% zmb95EMx0j%AxFpf4z~I4q$@wz`x}!TK2OuiQDaFOc^^aV4mxma7m}y1xmuS>&pQR% z3i_^BZ5nr9V?BBNm?-6D2Rk99N4iYER6K~CHP{v->OCXEYZ-F0vAMaZg(^-h32+!< zSm&sybKDk&DfEXIG~b6Q1d!;T6{9L0P#;fj3?!V8skjHZcqe6qMNa=7Qpc(Hr&8_? zOguD7V&_*XQ0Z6H$-lIfUDnb$E{@ka8Pp6|vYZ?5YI|ABb#avE>plAFT&6*ZLWd_G zvdphdrzsp;^#ODgbufZOJy3Rq8{9YhLq`CW!;Dk(*`G{5J|69=<1`?+Lny`lI<&b{&kuT(ti)Y zIw5jhKR9xWD*ZLjUP5j5W?Pi7Bf?ZlSd)ruz4MD^uZe%&{K`Au%{DEr2v~gBc-(1J zo7beb4waj7m$Sri(uv&ISl41titbZ%2EI>|fFtf)Z3}tBdlt6h$2U;>gOb%~!Y{I- zwYQ@f`^=pvIrgL58^-e_NmidP$63^v`(!YFVU=IVkWl@QMpW(jN`$HNsSirF#=wLc z*T-{Eyf066i4N9d^WuJS*Dxw?C?_Y!n<=|p_$V$jI;u(xtCvv{iRP}?DCg&j|6ZW`g%lN4|II=^^DZ91+gfxlcB(;(!sH!5+Vi$-g(TFT%qrJf^^go?3+lz-=U7CqVcf?~ zS1A4Yj$ui+MAMC>H8ot3qv=iv`2m00of#~kIuw7noU>=9 z^Qf1*jnbzie-q=i!K1d{gAvO_ni3r&Vf*c!*a-^)!F%YuJuGmho3*DA=C(6!ygjos z{ORBlv*8bQ(_Zz%y)@KG#%^f9tmlj6({g@6!LI$KgR4qLcTdmGlNVXu3lV!wU&Sx7 z+;;x+{;HAvv&hqmdbZDv0@?TT%YlJi+ke!+A!aeLcu1_~Em-p3JAJA^}5f-<$sBRym6nPA`+S{6ed`B{qo2KRSy(GbP z`L=pfJ+q+j7Y*B9N=#AqFYGib6~=z04hmm5X1b5`)jq$z{O%xKz)%&%hRiagN`3wp z9p2ZkR@9R;4m?Qno{+cLIQgN9V*7T}?G&(jbzY=A^`TvEyA`DeMfkFT5z8-*;XBJh zhQgcS0>Z0VKd}aTR&mX@rvlP3i2V-b=}v@fOqm9Yoln-n#Xd^drFhz=JN#>AB}C7x zP<8bowc45|4(lT=9^b!UZGb4#@5%^CwXU`$W|0N>H*{Zj{arADR@VmtX240CsgXiR zoX7cI_$4YSuGc(y{G(K-G#R+$cJ=)H(pQGEOIKD{V4GDNYpKPJj&cnuaxq83%T)$3 zOI@Cj)T~r_%H)FMqoWQ`MJXLf7wo#*0s3>D-7E<&YxnUQI!SbP65y^3*&&y_Upj~( za#a^g20qD+BuQ8--|eR@RF)Ftt32%p9s4;sIZ6AhDH(I|^&^c$@8sUt4ry^3KT8n+ zSWlAaJOZD)66s9?Df@s=^BpL2vdM>D0FO55P~ zTkjfq@hdPfxJ5NK)`is=WM)!-b;)b={bZZ}_}y%dDyshKmeY)H_Js3g=AAoqTIDa& zMXd2l%K|j=EJd|BP97!oJkrb7FySKZEvsnt<2p=JDIC)`tAM^?Q;proYhPTzHt*p5 z2C}+uM*b!@eg~*(@)dzcrAI(MlXBh_bSZD*vz(-9McK{8bYIgM4!}2^sgB_$OcWM) zUt|sTiOrY<>OS(;eL#BQX@!X~L2~g`@*#Vy^7LbZQu6d6j{k@nrrWNJfFm3-_$_(= zHq5}cM-+b-#8do`cwCMu65Y2$@VEp5W5b3=g+u#r^88LBmQP-SF}N z4&KdI7zFTm92c7r93rbF;(2Q=LY40>a}4uEUA4@lK^csvL_TAyZ}(^N$|Lxhs5N3+ z9o_KqX<$fuAm|v&5O%XLb;^f4eh&bOkU@lNO^>((eM9#!^uZNRY*tnlMUK4D=TjBv z2shEyDVhvb7j!r*zllMA`ZO^FdIJ2Z*`sKr(e!(9RORZw?vsk-D8mIiHXFSm9Xt8B zXrjiAMqY1(Pfqn!!N_Z+dxY8|G@B$?#gLZA#zPwB`9+C$`2DPP4V=cG4ql>;HR(9u(7rPmSW`nVYNaonrzi}DCFcjc0`beYZ@Y0W}5& zh6^5e+j=LGu~1Ayz|Wr=@jZ{gvidy1oD1j%+Czl$X|~64pg5^ zwozqTfNQkT15r}ZX0+^TT3n+&7Z)R4&R!j7g1zjjgMBPCe%bS&HZBBMwZ|dhG`Z+Z z?1GQLrT!h5Jl_L47B?57fD=k#5xZQC5Zi~pXgyo0UrJgohjZs*vIYNY1b+J?AWZ6l z3y_*X=YIkvP$OtZu|TPi@I;da*MdM$5Cs+Ra~&}fej81%Lsd~I3LO~J2<+SoBgUR4 z^8o=*+v%4%c;o6Qaks$ma1qo=#!c@BBm|SK6Es+|>Ht{kC(!i8?|hZ=?sH(bS_mxV zT!RUfBlIT})P3nbo*-FS_g)|ef0m%L>AfHYmEL^`X_Tdk`lZ;vYBIo;VnyHQO-&{A zh+R@wqw@2`imO&`uC$8^W1b8I|B)Gmj`N%vnew>TEa+nsQQIde**m4@oZ*t&O>Frn z6;_uxcX!~6-nnxw;|DuB!QY5{6j-j!OV$?~7KkG&Pz9IKu)ygNq?|qJ;?;{)s60MC zK5Z3?vXIGzkP-(>ngz9>X|yKjER8VH=B>P4qt=FGw+^f-$JtY-LU zi>VD*iphN#RTX}O=T>D`r-En*tu+2r_+>7gN8HQAho~5bb4)ABZBk+(w-uGEzP3x{ znXQGb>Y>=(rVCL1rx9F!#}#1e(m}QXx*%_eUY~#`;ctOKc%H{OXqSQ+{r4C7C3-go z2u;MAC=e$eB~|##qM@;eV1>{_@1E{ueikY|sr%HzIhbG*jCqLwN&?osIA2FDY00~) zMPzw*pRV-HINCh_|7<>5p>Hzvk@omXg}l9pN|3x&z1lyL(NJCUa@e2Kuvw$bX`q+9 zQ#y2G=i|3;dGq?ZEJlTRRjlARrLQT=Hjp}8QdtQ(B0PLVTu!}_jvj7^z~9028(*;K zDRw`d2R3$lhnSIZaj^B)Og69Zp98LCQA+D!fy<;J!_U_9@rq7Zy#jALc?7G#tgL({ zU_BhUr0oK~Kb=<&jPjAPv%&i&z;>m8C@{U_DlEj zEf~sto!9BH+D4|&N}9-g2x$#CrgpWn&~Qw{e7F{+*Icq~I^3jo_L}MZe}BPV$G)rZ z@B7zRBYPY}psX6>^p?-7e1#)5FIKp^cK7hU~XY;tuHMJh6*D)fmZfpp_c`gYbXb$LZ|a1LJrODbIa8&DHa5|{kCLKX41P29 z2@)*yXz^2lu>~xf8#8W0G5e9smCboO3sKy$B)DgxA1qT00HNHZ>*5OjyqZ-Ygp$~g zBXl`4sGB!;jmR`n%A5G8lRp>FmYg`8h)FA!agqb_o-^dZn^>#2Tc|Hukje8kg>T$5aJ-k!?%_x27CtiN&g&3P5Y#X8k1{looJd^*~)XCrv{ zfXi*0FD2TD(vo3La?gjE`|MW0Bfb;;8IdKag6_lYMm0ANQyNM2M+Mj;)c>y-v~ z(rq4;!y{$BLr|k%VhgCGY;O~o%L^xe zWx(!XWGCt_%ou+-OzzPwGqLAr4`W=2TNCYN5b`mtkrD1xX)t6R^0p-=Qi1r3TNGh3{B^ok2_5_=A2F2 zy5fzO#7;k3b*4o{Yb9E-UWDO<#-C%|(P6(o{{4HAHg8#HScst!pZ$wLwlpSJeBopb zh#8#3z9+ur@MZQDqMxfFH>n zm;|0J^EBcQhk=|t7gh27IK#VKJCIvbYZ%2{`t|YvRkyzc`Ap=^^2?SxvNvu>7#R>9 z4i_f%hXU-dHZ&hXkMymPs-YVWs_79@I8-;PF<+%&lQF=g*^ciPe2rBz6wuizj>-d$ z#sA0FTgFA%_3z#$p-74}2m^>ncOyBZNH<7Hi-5$C2GTWj3DVu&1`IHC!%)%$yJ$Si$`c6wMLW@V~SRIl@ci9*mU{$YBZ??->^zj1{(A{jvtVN zcZjZlMf{H%;PHMGK^J7lajx*E(4|Lfz#lc_*-6xw;#jG=(02&cGZhDm$dWJJIScGt zZuP0VU}p&9en`#?2_4Kfnt{`R6ywAAsH6h)X1gw8oAN#@d7b%w1z*O-w8?m7!) zU|^tl((6r;%1)OjiI5j&%#yn#3W4gQVFWKsO-*-FB~-WI8?m6)ODHK!Q56Y)+obUH zvl_9Z1X$p~MpL;Dio+mW`uCNIJPa~JB^*qQ*A|s?mbd*@CsoFcAk85yp4^Va*4;kw z*0mi+IV$BpI()(BdsuHzh`1n@zxD zZ?@ra7uYhr0R!z$KgBVtp8Xcs;hUw9M~Hq7mcgnC48#>#3sMAzn7zk-FE8AB*bhg> zyPam(E6tUzQo&egEy*Hl$^vSF2>!C%>bVmT_E}~?JrO-7(|3I`STcvA{t6cw&Jh!s^EG zzZ(Z`Ac4aRSd1mfHQEEaM8NsR3mCPe!7m%m?w#Du4AQmjM8|`H;DqP& z_RBt0!8`bj_A|W4lW{t}n$0OGWOgSzOm+?%&yDl6ryV|QZ5O9%=4gzcE*|`rkku;e zkSAaMkn|-$^SJv;qmVj#xUBuR7sfCGG{&|}%mz!{oW3twhfVSJIc&VxB-i(38cHDc z^DPF0-QaT^NcAe%YFf^tCx(gp4*n)oYi7{@_hAEL62v9kMu-^Qn>8n7K>H2fSw4>X z@F6|QBZ70|ClX_@W6@jXW<(gx_DPJO!m= z4;?&KvP6tc#c0blJ};NZK|HtC>X8N4&(q1|6Ub_1)j$fr@e#8o$bUOftmAz9rieK} z+t^qK!w}2vVAzVBbJwR0YZ1LV8eQINX?a8d1`nN;H`%{MG1cPm2sFRQzH^Ii*fkCV z=E)+L4JVQVUQRz6Kz=_64*vk!O?l++R+W$uI)J(3x}gBx^;-{DpGllqf|IvNrP|DK z;sG@l9@S?PBXd&YJY&AI6~4b!LRhIo>Nh53LZ8QGe(y1YzgMTxdVh&mv1|5m z@8x6btb32;DeR+Mk;C{S_$=1PdGFD&``pYZER?y}k)klV@TO7fjtupU)yE%#OFnXI zOdvWnOpBq?*EE7$kqpPt{wMl32SY?wh6O{Mr#-$ScEltXMJpYqs~Rq;Hv88d7AP15 z-t7xzG+$j*>^}_IzlKghugrMNw3WAr5<5q*ta~HGFC_kz3|%>&R#3L*gLmf3um9g$ zVq3|h7(bS>Hx@ZAsX5uiS;X;2j@|C8?U}?S{wHtrolnk}b0T)_PiiN__X}Imrbwr& z(53{_vNpFT@l-S#ztjpXi_zJo22DR1eNUI3yyAmFQG-GUcgB^wJ&8+v?ArcSI?^PoBPYH<_LComW_$p5AdR(R|8f1V&3-{E! zwk9LHRzf4fT&cpoQmbr?n_F-bddD$kxD$F$;6uwweUb1l=n{YA2;Jo0>-<3V_nM?+ zS)1D0+#D?z*WGAgQvQ>!*tN5`nVdRxY_mlAB0*+6UCCU9L?*sGQ<~L4&VxqgL zC6AjMj~sifo~XqfniIEGr7f(@vZBqQn`_jT3ToAQweu;o4v0RET#2srjY@eio~$#CmgN|Fhzdaex(PWK)Eu;z)}<>5!-2 zK}gy$`2!!syCQ|e`-!gJ=e~_^(LxK*dm%UD9&>2hdS(V*tlJehMTX|KYEQJKi0|X< z)}Cf&cFK=i8OEs!+I!lt?d9~MH50yjbRg<1?NR@!A+OX!71%HoMWk=4z=uxS2! z|Npn6GX$>iB-!w@xmPA@Pi|2!ZAn7HcNq}{UKGK2pD@&B?b1S8#31Yh7V?1Z zbxQ*ECC{#Rw<79KwC0V3S#h4bwl1#4oe%psBCx_K)X;+Cg0lg@c zr&X1s!jr4=D^^FTP)R)*5JHxcc<5KZ!RSrHm2a*OW|V&fGRj(?dwlOsn=+%BTgZz_ zy%ZjwfA#{-CU`1_EJks@l=r$SUu!?lB`&*5-`?%D9EyyOQ8Eg+)XK`KAx5$kH$}TF zHgQm(8%%MIm9zth<~Q+%Po3mroC-FAhCecoE7>pMybdRI=e?evA8qQ^yuVQP|cD=R;LzMj=57q*>hNykSHV;!z5h^`#42|da%R=GGo zPlx}kcDL`+^d1l=j#e`rGvip99O(Ibn+97%dxz{)2jG2#K-bAGkN`~HGbc{9H*P)=EMPzn^q&&NS#$bVghh~ zNqoggn;~oACJ_?4e>wV*wu*2FHUvID*HE_|c z_otF|F7zeyX5W{^fMhj|jg7Ye?mDEPY2??d<|1d$iAhWMcuCSYUa6i&T)y6VZ*PWm zybdZio6#*7lP6>u_*+D_dNcL4wvJ=C(e}?>EvgL+4KgZ9KqWlf7d&%g6T61XG4onkO=$@{HBJnUJk3vPHn8Ceror4< z8+hpG$OOCf6RqcwmcZb4H$22OHmv!k8;e>1qnkZbW<*dOxd*_psKi-63)Tp`mP14Q zZ&uah*G}aQZ{|yvF#D_tHc8@V%JDJC_@Wi%0Lh*MNp~kU$-J+bUmJW9yZHj*`8n;rBr2P#JjUJiFQ@eE4%6=- zM=>S3HhM9DMA^8XhOJ@Cq7(`lCHa|g@IH8#7r-fqL0a!oDCRSKtPKd}; z^4C`a^Bmg2I2_3eFEQ*3JO%rSl3_rN&H9frAA(j@RV5(`>vKc%x~3U_J^k2i(7~-< zU%O!2=J8_TQQ2-w9rZ#h>JQ21ljfjV=nCNAdS>lu{YBMS@5KL)Q@Jg8Vbme8*J;oA zAe`Z1qdOqhjAn-5w{S?fKma6~@mooT*f&Heu!filu#km>dR#fE|A|DU+$2>uCZ& zOqBW$S5-Z$F!Os5%agfUZ|yjF?L6do;|V z<@v17&Ex#vBXcrn0vQ`_oJ+Tp0jT+qpVCz9jcfLSKx%IMzQG57S`3G4rw9! zQ-JR)a@(Geg9Sy4fU)SCFW0eUd$8Hotu|SSUMq;NJgB;>|M9cT3DGK4RQ8g{7fNBc z*W5frpzg!zJ>7c*j@#-%H10{@2IYUi7g2D6^%`dJSp>a6@qNXj1}5)wM(0;t@-tFX zKPXxO*>@7mK=f&xJBXZ|k^HsOQIu0%7Pke+Ko(h+L95e&_rSnnAD@{xf%P?l9 zc1VG3RrtO7A63tUT5P$y&EP=mNz21KMOp@D(h~BY4rs-Q)om2oz}`e&@#8P)H}JRi zBw9EevG%gH@TeC%`}>oaZ?xTb(XTR*oQFKlYA<|b_vr5`x~ux98IDowWw6CwVTcZ` z7jyd)m_aqC|5>l|SxJv%BqpT4)O=Fbf>y%JYSht~w1;!y&rUK^>V=u0zU6be--~oh zSAs2z_s)yk&x@0SI#wixn3w6D2O1P=r}RQ1R~m^)aZg|>+JEo%yD*<(eS|1!p(QIv z{qEjoWDJ2>P=^I1O4xO+1Mn;8y4SBf z;`0gF8fzhGdpS&pj9Bfrw8dM{}w= z<;*vY&9VQ_nZXzcJ_N#@ubLgcyaR3b;2x}ysoT6=kr|`^2D;VB6W57vcWW3YoDo+B zbyRPN2GH++mT{*aQUHmWA%atPvaDMG*9asooRK|%77rcv?3OeP42!#&v%o)TMAGGf ze4M`l*vFrL?+pHCLyCaP^5`?-5QSSm0L6Q0YW^V7qoXQVDr+APWwpuE8w7*?zyyT0 z!OXfffzybEB+zFZ3IxExzjXR#ji(YL~m}8wLx#W_QOpZ{{V>KMR;J z8q)%Z`((|6zmilT=b*RS5Ja(+(Daj`6mOJ*sTf)cs{lk)f>Dw&4OLWcl}s+Hb~J5?{U1 zEXo1Y!*9EvGSXM!(RW{5M8CqRorAxKle$kleG2M9bui-b@!6oi1hP4dqe%fnC$Vaxi$;>^UQB$xAsPL%XiRxBiZaCwo324Cph~# z%KviN3n+1<*_W?C*k6bl`GNg$oh7XH|2hlWCDTspJ}&mKb_!pa0W;`jjvCn(Oy|4^ z2s~lDIrgBiL*0RXbIQg7^n$9-(oeIY35F4^5vc-pU%%ZJS&5c0zXHRjRp&>h13lU7 zj8j^o6VGm2MZc0A1!Rpk0KZ)|52C8uNfwbIR~Qm&CH4M;oSkix)24rNJzskXc9ui!>vH>ni222dF;i$o<;#Cu-wc6zG0DNFE|vb$rZ; z>nx(b)A;YRmS+ymb-HB*@4?7H|x=!{dRU;VOERWv$AmJl`25T05^4G(9# z5C>wN8)jYwMwPi3s`Zpf>Dy|Y7S!pV;Rm}_Pq^P#g8%{`Y|8t1JalyFKQxbw%f+`? zPFuD#`t{G@Fcx$13uWOio7WhY975R9cS4}Is8d~&HY{-MOLri|kzQ=EOy-dr5-=4m7g;iP*n|=eC)_^vwa^936EM{hHT$Ae<;@C!tsLl+Zc+qq7F$ zT<#9TcJQrRYj&s6{*3cIL3l=>G|eZU?liuWuPdE+L?TUW8GAlJ{HhH2`HW8|AZ|?Qx4Efrqc$Ge? z#^u-mpthuBGTGjj7B`E^->PzDSN4a$|7gxnj6Yo`I)tC<{BynqY>Rd4=iCkN!*-Cp zXiyg08=yMt92s}H55l+&-ki!|@OarxdOX#w;yx&8tTMu-Pb67PN{8u>K?HI)tY?HX+A2jyD<^CYiv+o3N`fV((0=#9N~h>gy=-% zp!nm`3fXVkl5UaNTEp_NMqRZXOtLbMcAk3#O*rV~y8-30*ePG5v4qkvrMA^INWf_> zONwDxn$D)Q$CigJaaH_1P8$Yf)b!*1-6L!N6Kf2uhsI;X7sPgR&JNer9KxL^^sPJl zg@bi4<)~DG;~;P)@!1q4X}P}-t){?E|)z2~s zd=>l;1^r2St8Q|-v18E37;o~j_YI=xFP?;;Hi400B-#UqXCBZM6fvcZ%U6Jd;~(l< zQQ3o8FG6r+|NdC6fBau*#)nx*ckF=J5XddMrI0yi01G;bKRv#>OqA5LUdIQ%+(Ji$ zN<7kvNkcf7m%?p-hrsq}qfu*>TR&@%NRJjXU;3S0()fOUg!_|4HHITy*6_lyYK^;V z{UN<}n;%x|97{{Lwsu0{j@^wOm%}s54a~e}s(#(VmQ?W9IJ^PW43QqVJlF}l0{r}) zVhAZ1!-hjL=g45f`2fh3v?WG!)2~OngI0>jZ}{|3S-Sm=>~gR}E8-7NX8OuDm#d1( zyoXMr+UJwS=Xta>&T&?YBaOHnES^Oj^1}k(hjunw4H5ier8Tg|){T07^G9QC_HG-RJ@K~c2cg}MH*n#6*>=#2$k1_L#*gN|QL0p2I=PyPoAtS$hO zoK*b~6l~Alt#@Y!N}pbv_Y6&MjoEiE(oicV%}thrEy|Sfsj-~4cV@>&Cr?Pxn#t!s zKS*Y42;*_V0Tb@ocSyGLkir-9!h)e}wD>l^wLA2_{ZS9kIO04oB!u0hM$Z~d_v7F` znwvKR^h)(ATxl6zt7cKVVr}>S_-z>??~563jzqZL3if?E!O>$GgbPJ-gBi9_zSS%o8cN6VH3GSPI5wQ*Xx`gIlgEO#%#2 z*-W>vVPpAp%}y|C)sbm_-x20X+1oc@0Q)Nt&2eM~`&--_xSMLtc99nY#d-9UtG2GL z&{w`WB{jvO#zZ7u2dESD8M+}htpzN)$PJ${h#1+W+e}CW@ON!uX2fUv#r19AIED>J6Z4 zQg3#9dr#fezdaCKR1pybs+}b#H1|1HVZ2h&+h58IT}3nY9T%OK4P4nqZWNAL)W|^Y zm?$v|JjR5(48OtIer#l{Wv18j-8n$=P&xAM8BlZ$G_0rxwiskuk3`ugNHC1E=o$u! zp_rp0gi6G>2^z5MGXzSgr-XzjLKU_xSXl@TrX%W?g_=Kb{!PUG--*r9lK}~l&yXM3 z=W$g8>N+fT4whOU&FMp1lcd$02`?MebRnyOP^kGQtvipkNlm65L}=muA(p?Ijnp+| z!U6*WE!uayl#3>cl>~`+3XBQDm(fY)TQj(LF;B#6a=R>sji=Ct-9;i)kP(`rj3pMhnSVExAlTn#27I) zA#qoe(I-kus`1&8EQ(#;vHAJD2_1%m6#nd5Img8*BJ|4v8*OjH;+J)`A1#!H>WW)y zz2=4mAhfBeGH0(Ha2-2s+7c!_`^)NsC1vTTyi{oC%Eqq8d|SDZ^|gZ>hQmu@eezV; zK)<)x_dyy&S}0|xnxKO*IOLvvhS(EjV{~JkOKphX*~51(jP?tAnu%JX@~BmspjwN6 zR4$XG%W{E@DbnCP6@g+S-_|W@TI5FJcL1Fl*ckfUGu{J~$OkCV-9KZlfQ&HjKSmTLMM zKbv+=CSBDlQ0r%D*>FvJ^hOjHX%I&ZcdDbnS~g?r=1J%UGnLx|?C3I`8$9?Z2plkx zQ`45yyvH8F)w0+>xJ=L-ZOeds&Fn=%oAF zxDrdOF>^xqI<>HzUu6tIVS#LDRg^M2pHZQPPMg+{4SjhfORlsi(>#@4XracU_?yk> zM6E*dz@+YgcM`OLuVcZc^I$4Mr4e* z`#V-zURz6cYZc1JBal*;&v|4hR)k2yKnL0v_h-XUTHnJ(ttYNYAr_Q&3 z9VM+4cdp5v`~Fm!NjBQwEVEyn*ILae`g{2Uvki+VOFF$Ha*e2fpE4svjtF68b8IZE zvGLK|(wREj?|Bgh)qbTqW%{|}^riakA}fwKr*%ce#mXOKWLV=uzfH^8lB1$-a3Wc= z&QCmpn>)r5pK$%VH|VBKtuX+mQo1P@O|;&Cs2ER)HE#Ro`JZ6Gn(+$6W9IvF=xgySMbo9># znNurWeTuqQhRBDS$cOR!UVPVW8md=QqN~AEy-^X+-)?TUhimKxwRZ73C3>01R1?rm zK|V1aTJGSV*&nMA^1g!lfB)((n8=CV3H_YPhQ4e@YkO3IqHLAR2jVD_ra;oos%vU)#(cm*qCg(FY$pcVeuqRneeM&l z#Q*;1sgy&x*VeyEI!iiy=VkyMZg2-LpZCBYQ4UCyDclx;AiHb}xKt^C zmhu2l6h=pKl|cg%<*JF3Q@My}a}6jTcRSr1+nCa`;mkC8H7UBfhs$yH?~}zdsBofM zn=^q9K2f&jLFSAi11*f1i;aw*9lIUd_3;>%tEnrMB? zIEauiy$~=|Eo3~FKI4jc!yKpf2weSwva((EU&qI%k4?MP{soK)4UCCFrh_~|7f-Cm z^6fos!Koi6rJt8unlhn||hf zYjam!BBQ*Gssg2#gAORRTi}i`ReNbp|C+^Oh=VE+AUW+pRUJV1LU$5>Q;_Udwj|jj ztn^I1Lt7Ur67ucV3qb)JDzzxh_z=cb{`F_eu5qxY*{a8Q)g9%n>%?B9iprC^jLOr?}hx}-!jqK*Wr6f!p zts=)XW$fXpCtBw)j)W!_*L&x?Y^E7d8pbP5KF;8jUrg*>)nX&9&9-j_FJU26mn~}l zAnXQ5jn*Ci_3tp4&taqn`J{p$+X|@`TcY%D4p}W0YhnW26F`jF0_NmlppQvloSk04 ztdZ~0eld5Bp_PH>@)F;NAKQO@}H?nNXPB~(`htE?T*?fdCs5Oilc2!xWiyQAAXdsOYV z_+}#D>yFRGD^AF>gDpjs4q}V)=v_gM(>57PS9OkqBZI$?PgmYn)Vi8s+_D6;LV{U_ z;ylOcua_`wT?CxJR3gbW9n;`H)Vev=pjzZq8jVQ=jijV^Zv%X?UGG}|*6c7v_Qh7! z_?-9j7Yoo)Zhf2)Sgwf2X9S2^lHphz)Vm zlWP7~@^iz-CQRm>D-Me>> zWIY@^xsw+XWxoX}PP)qz=5BWZxxQ)!+R`h-$F0(WbrI6-E8KN+n~9Ci?e1+WsSk6+ z_tX)VCXU8yN;G@CSg1TxrV{U=LNn+b z6i+SwOJ5#v{bXGeerl^2@JC}h37HuNH&+z@01_P8%Bgkx&li3MZ&9Y!zUa|#>q{-4 zI7u$Dx9ON?p+l}6-d?>c2Q2!*6&l@CRK4~;E2&yJ_oztD zf_YY`)@Q4IA_GI%A7;4JMEziJ&X8@3=ts?b0`|MiGUJ=jEs7H{+v+;w7;%E_4+q20 z(l6Q6jvABac`e6^;^}0r3>gC5)z%KXqxh3GG88!vk9S@F#&5&QVlkfl-yyMcq?F9M zfbj{aqafnr9{x6zk(gD7nDsG{+`_;+sojmOiO9WI5#g)ax(fyRu4{InajB?x9NMGH zLdwr0LP?^5rHJ4$ncs)fDYVXiYUK_EUe`vgvR6m&9(l75)Uub(uatTjNjom1ZI-9n*rf<8<4(;7`6ahS37(AQIH7Y19Kmt$&AaDM<5kLw1^40IL>@9;WMA&Ma1l^0GR zkUW{PP56C;s@lfv?Gl z56d4+3Z+9#kmMr;0yAoV? zzXjQ5}x^G~{1%&Y^&&ow}DNsP<>(Nl}2tJ*G zUms$j3=GG)U0=rOJnvr5HVPhj7%m>G_=Jl1@Wu2#OD`7`3V@a>-Bf_D%4RDX+g~J+ z7iBIs>-Ol2|+$19U>5dP8q>Tf`_zi|ji$NQ$sz}qbvZ(p@9b=%LMq6P?=b%j~ zuehv9NpSYHjDt0ps-#Hz``y>=F2v|8>WnUb5)5C3FofbrN#Lw3Q2|I2i)ggc!r6ew z{{n0Q!lik8@S_CY(FwY%VY<%TN*q=cCO=WK@cX3xKzKBt-8y&YwEfRj52}yJxJ}mnOYVMscmH<+NuMRUiLmQ!WE>a&-OH>`2mK%14coG6pcJ1~6*s!9 z-v42a1e1m$81O&8*sGPj`*KjnSfuoK^g?qr)Yn&(Uu-J>a^rKT1tfD`PXVT_bo()N z??-Mo50B0eF8UmD^ZfkgM@k`O>3a=Q&_JfvkeqK8!QbH6#9f*Y=4dSizLkOZx9?`D z@H{p%ze!=H{_@NB71~~JK)7;}fBfbf?xPNzHq1YPWZ3{nr?ay+uLq`i_ZrR5K^BCMJqjnJa$a0X)Bf8zyLVgn+>LR=3`};5>YL)zndEdG0Ge3;HZ640l z4FYVsN5HHK%6e8Ph8+~h)+}OP>b|CXhfE4i^TzmR2YuFfn|^+~`yaUJx)dKo8eje5 zBS;t7UaU59JI(uIVl-lRpy@QkyRjLUmT|uOutD!PXRC?Zz$x!?gArDiaeCr*$kd_{(j=b41H#9idD$RP9K$W;r3D~{1;R=n*z1Ycyxvqu&`pPcqZgY&PDeT)wzz}ctE9sI_T-kvwRd;?ogpZQf&k3*gxL5yt)-F z6HFCAI1-8@oRvr@)y;T|l*m1y8&c_fGiw)93z6jvW*A{rQLxHcl4z*xPx`&!SIgad zZ3-+2;S9YtcayuWmZKz1F!15YO6fJ(4*>VOke2)_vbUSVV~Fy;90S zfjMXHsG6ks32Y-X2;tg6b;^#4XnS{}dO3ah_BWddTtNW!!iVij98-r${L-7*hgp_l z?^(N;C_)W0{gq$r6()L*P4nitN?$CCL>A?l!{&#k2e!5`QI)*U7!Ew~AHaIVw|)wz zgj)&=Oorb+fZn1T{M5lK_d+9*1u6lNWzWEMrT2>A@k#e7Pa?Vc8#N#{3CewRIpW90 zYqgC<1&ZtK4Haf;eGjq3Plj$b?WZ@_n4dwm_IK4?WuCc;plV<5QSY{TR-VjQo6=p;5-BIobkhkCC==S|;o>29)& zH*;jPO6|kI=%+XmwaDG+81~Pyl1D8pDoWy3dSbedmU7I_VX6Sx?XT_4Mn$ zykuFpMCRS1D^$MKH3^L)4}1?xJnBsmJ+3kjs*aKZe#qo-`=>uz!pBtzpgnwP)NOpe z#GA#A7S>1@4DqIhmtrPj$02Gy(ivAf))pO$?SC%XyBZ&I62;a8J@w%2aDkxnk z`t6)X--D===T{&0IeX6C`Zshm6$`&bUMyTKRkHrZ55=WNR2kkP`i8R@`1Xdt+oxQW zafxkDoQd{hKhlk0hd_OE2+Z%=+L+&2gf687;4#oqAmB|nkOBBJ92TT&1lN-IYlkmi zMHlwK3j#VTH~6+RBTq^QsO~;Q??^NCd{Q=1egU@|_^L~>d@T=#8D5{@0ddj5~_q2X@DswGwjU`4;!W%P4>mX4J@0EhT z4Q?N<1auxW>%*XLdgS*kifG8Ha2LOi7$?{hyLDYGM(g9ySW*&4x%6Ex44hq}g_hY8 z3lza`q#wyEkAI{Z?$x$=I306RlOn}Z5b zrbJ3!f8X+)6G2R3nUaa$%c(N3FdBsjRFHoO1{|KSU2HZ8Qo;QVPo5f$b=dAw&O_u)5o86fRw&Tz(OXXCK;jFYh1ti7 zV8LWhTG#l?L$acRIZ!v&EMvmFf>|PsKD1s$Ld-+avfqg) zN-Rs45zPPLY4qp5u(N9#k>_lstHRX*Fpu+E>p|CR=RYW3)0OxW6G!_cky&K3CvcO9 zce@zvsYhB{ovONz)Yt0Y&A7gBFM)>SpiAi2EIC;Q@9r>NFI@jy?>gf2y~o`VfuXh# z>Hp`v>5ko@kgHN-m?7fv#_pnS7ggySbnbPL#m2DZw2V-7wutCYt?$^77%3XJKl<{4 z!1g$$*301x={ehATP@%?Pr0Qdr|D#4adF~EB7Nt2r^f7JrkT}fGugY3*QB=|mQGzy zG`4mEag|@RA~^)j!&Mi%$oMDO0n#|?*hGPP(gO6&-OIG*y_V$rBsupwC>=`xG#<$` z@7pz{!Z?C;O!y+03B{bj6a3^nQx!>neKUxM5#1=C27JZb&J?>~qR&M?uQz}k(Mi8+ zWgA+Sc7X{pWQCv3(bIV3oBrq)Q~kv)80$b? z2&L#?@O94UR!+aWym&S&@h!^vr!d@lkepodXUWc?ncvu>&^W8kNh4R)lCD6imUV*b z3JXVF>k5WG`Fc&gLcdJJz^Jj3X02cE|5;G)+Lo}mo#Gd{uD@-%2&^eB)=jPV=&EtH zPZZdnDttFwg@q<{BM3*-WP9js*S#sM_SZ^n#vT*q%FC42@P)_$#gIuUqu(&vA26T5 zv~jba4DCGqbMGy+dGaIkun3l5TA9AeqZ6uyUx6IWJ3MpFT{P;~=wU2TvRS|wm=@b5 zZb>7Gj3JcA>{cyATcG$M9~-2#7!n8#8MpMa=vK(|=x|t2wN)#_SZL6&@Kh!gGahxW z#$6c$o}vs?Z9ZfyL4;PW1Z&bYbQX)#2Q=B7|31ov zz!v-S#v2d4OWZfekEx!pP0dO)>ut2h#PK^KP0^uwNl$!tlN4M z0u%t9x=guvOZh^F4>8kTq)6Ox{=w&pn@NeKP8#}(_x5MARGKKq0|_Bsc#AR%X@Y|M za%SI?!i_u4>E9=D_UcUAKBx@jlEoPo;0g1tfzru7JeJZDuts#9{ZAwdfZzrLRH%JoiOsRnMfr&Xo;)y(oYdEB#pk@qT_pkst+JJv>z zmHFs25;jCqgT@!rth(903k)l+(D)uiLbm{^QB1-Vq5A1Nz9?H=R8Fl`ml)OB*TZ^Q z$rWBF;=#yDkcl3m&FG~2b3oT6OmbU+$_M)w+poTjjDw$L3KHuE;4Fo;i9C(k$Gs7^ zALuLTS@wmu&i{|QiD9I_4E64X74(N_=lYj*YqxyXzq;8t-TzaMc_4C`zE*3cw#q^b zLfC)*R@fI7A^CEbp@(D->5^g>qz430I(~21Ijg&Rd|IbC|6Ue zg-tt5QFh5%|+*9zi z6l{z7iE(IHtk9wHLQdXXp~Y-S8oGEFZB`4EDGb-}YC5gWZp1rJRYM>0SR?v`y7TX) zXC7Plj?x}3rT7Ww<1J8R2}+(e4tCl|buc-;&JezCk)?W`*%qV~unxXLTtDn@5)!M`*FN_Eq*x>RpGpFwcS36JsN@9b-Jw^|ZKRWPE zdBJx`iedB48J;!;{ant-g}<4tu6~5fI;IjR0LQ`e`G&98cXu|&CfKQJ-H$C$EoKoD z<;G=h_H>$)kCKXWvUtudk#`5v!d>Wn?IkfqP73CB};mnay%c#U3Gi7r3+7`>mw} zD1}g2>ZJ#`HKJEM@_X#M%dE$tdRy`IzCn~mK zfWn?kF1gN#o;FT8cpiDCF_@qo+t79TqR3zDpWOyO@cJ;tjQ0~ozsBLq!QCHy@}QU@ zaL$W#VMmugI$lY5CBpWNrQX?P0cO@OAW$StUv922I~a7)&1*?Ia!2uFUnU(xw4O^c zHQ6gQY+r=JcLxC_NB}&94-tBc=gfjQ#G7#szC&pfrBH8x+YLXDlS`+Edp=-}5YotS zIl@gF<%C*l9ZzeKIF0}zRCo)LO{iCEksWTf`q;v?%GvoI^8$;tMZFIq;yJ*czbFyo zJ@;<%bL1GS7JJxUM<<%k2@V+s;sal|^fF!y=AEQ@g@5#llX0lxOS}h-4z2ml%&_wf zh$Glt`*2*vEi^UgUR9JrPW}5G>s9F$y+E$UX!L|`ZS&aK+G*Ll#-a>T#^xuo0?$h* zrTx6^V;>BB%8jI8PZ=G45LG>+Sei1NHrk@kyL6luz2??8)JIXujS91BVZ3!;U7u}r z)z_{rj+EX~EIwTdkG@QZAyA0HQ^9CbiMHe6A>X3q5_Tgriw5bMLw)=>?9CEtE9LVo zr0X;A{Aj187BED)Frsb0Bd`0;c<#_vh1=Wr?!^xNQV<}hkPwO?n4pF3RnDGb{HMby zz+~qV_fjA-e>>580_rF^Tiv#kCOl~}iUk|Hx#j%I3XJz%YwHwOxZN3^PQSkOwF|A~ zew_OW0ObYS=!RC9s=!WQ{niRMmP8jBiP%kl>x}O~-rs>Rlp9$-wh>l( z=r_B5DBH@k%

uS;&Fcc2%+x&}QI;c$~%)%g$;iw#Fpg_Z`KuBII!_bv&?X`cYW( z?pTg0m<|QWy6aGwFL!moWt+HdkaYJLo)(Tz~ZQJ!8I|c|r>%wy1D< zVPyj%Dinxs)i2W&#LsitRz24qRLaw?D$Z6*IN($FR5xM96ry<5ww>ERrXdG>KS}NO z8ndwz3yXBT(*kR^#x;eDv{8xO*ZpD+9E|AAN%UK5wGJk@FQ_jvo{j#n{+3Y!XtYOp zf~~gFO8U)XOi{mW%H7jBAN(@85dEJ7s)gxO?|5I?DehpF3ndFBrVtjYup4;ShlYmk z@2lqP43~?JjSp8tz~19q1W?) zPNYa2s0%B!M#JqZ-0c=YBd`7z_X8Bucx1BoQ_--?!9uJrV-QhBfj#9~cr{xh{+WGX_YAG_*#6H`1p!m!&&DBI3#NqxPggM2;I1RP z`;S7`cpVcgK-ct0h%5#*DpJbg!Zm~F{5*}2y}q}e9z(@qnC~x@zkX(EYO{FoE&$TJ z9~e6{|7&C%t{O>N(5PM@LMBT#iPB#++}5Mgc}Iy$Y|i(Ci`sbLT=gO7 zs1$BL@zMpUmsR`TXKey`TQ^H~{?sjqTX`UaNn`gV=Y&pbF7ML(PpE8^$8s)YJ3R6p z`q%Xg6_*TZEo$Po9(CG~NfwfiW~`+ufyM^5#v4>mhniQJf>?CGP75HhtXs+k1cCuMyfrdG8PJ0gJ4iXqUE8F=NYRG4u zWn19qFfaHQRaNA#{=hS^i<(B9LFu>eE7s21oRtx-BHN|WX zVRm55Eb~qO4edJf#oLxGdHkJghG1x%1gz}R^RHDLrqo#iyej#qo>b^Vg^dgY78n-g)n%OQN zoqMtR+9I3My9ys!y|UB!8H|Kk6; zLzCc+T5Ak4oa?=}KTB4T2SFV>CwpB?7A;vEdbrjf_6^uAQbj$=UJj<0Mn-~Q`(zie zAYRS2z%XaXoW1nDU+L{5znEzl|Jv1qdzqdB2Fi7^WWzx}Z+iRFX+)l$(q9-D8jhnz zC?S`WR8$rWQ3~x$$_YT*!ym%HIDrU>u#rSj)6zy_kg%V&_V=T{mFscn8vuMT(g{=3FhN@)E;8b`blim3^6*#_* z0(;IM@Hk=7Ao`9zJX#vV2%V@dhp!-oUfV~4#7W+;`Ql}#6_nkTl)+O^i}#T6>) zomlbR3EJJ*E~305`{zL}eL*6z>sS-mtDcgO^#te@Cevtg*Q(SJd84bxqUK*9WepBxwYf0kTJewuYJ z>kg6RO0rNDcM@R{4N0_9$cd^(_0=g0_5=sd9b70yK)$gdjM%CW z>dk!SX`Ol8<4e3QAQ_gFqiX*Fa`iL(Ef2Gvut5%!Yc_SymU@tlBEougBWS%>90?zU zpQrIkP+53*oXYBd5S|ip+<9@B&zZ&N+Gy0mte!5>y`0A5b0X$iaOESsuD}rGtnd5R zEu*%!(;x$7hLX+p%?QxHd9yMY@mBmsjDmdI z(v)Y$L)WJBt>RjjRix)LM&NLs9hS7nqCv=|*7P6J69@gj`SXXw?+a)ZW@scg3A=^gxzqh!aWYFm2u9}bSSc?=wVJDdmUk;}5fv+4DY!*$o| z7ql*p>$R&YSTm>CZ&~y2$GAf#X~qzkDLxny0(t zz2fcIdufxQfrUSQ15n1B;!x((n2<*k20xSZ z?)M|;Pk2y6(k!R+pe5;zjsSfhj?A|HkGEzaQ81KGfs;9*!(MF{%JqEd%y}h+^Seje z;U}R-E6iLvE4lHRgN3LY^UXFRU0D|mT@7E>RA|?~5bt1kHW)e+OBZi;TP;EOhYo(| z3pOl346|PQ+nfZt{^EVvW>oV>lpc+6^EiQ#4wBiDj>xH-W~Syt_ftzcX}n5Vot6;^G+1XOIM#cVUs%pEKr3YIB{ zt(GErY9hFn(|nLA4mD@`#>m&IbVnc0_|0sj2N1n!{~uLv!4_q^hHXEJN+a#iF?0{z zNT)Q?AVYU|DIG(1cOx)>(nxoQbeD8D;&;2g_kGv1wv8XaHg#Xuc^=2Ue+Cc}lQh>K zwkNZheT#{YDN&n3Qwz}#9AdnD#{M>|#)G|SEImVJzSVZ@4bEFcdJD2fe|{y!{qp+X z59j|FCaSTY$$#0-(d;tv(e>RzWBY;ao~vQfi!Tn+a9{1dC*2_+3V-ln=>QSn zP)2*IC7(~e-GVeC{>5{)|M`7-7=)u3W;?;M1yoLBovy~U?lH~?rt!ng->)Sa`y}q@ zhu--7&r4G#`&TgkEp8xn<;6V|OPl7~|53|xO0~S$3`6IA6N*^SC@w}3Q_13F{;|K3 zE>}vbLST&2jV=&rn{dB7E(q7E)H>`pMWhA7{@;*8DyCK#IgZ+!9og-owX3M7aQF z)PtKP`#JHqLbxIce;0oga^9%#SYQEHBF|eQE;B09y>Kwt^W(_r2YWBICqavhorx4Z zr||z6Bt6MhJX!wgk@ZMN&27i`S}(^UZ^Lu{-b9W6$mmn80e!Qo1SMhZziZ3P436Jt z%i%+R`#Dcz097W4NvntU5BKr%(7Eq>G_`D>_HMUV`Of!#S%0C^Ys|NIx0e4Z{q~o2 zo!UHpg#~WhUIEJ;kiYY#z@ne^96NKIZwL13WZWcOBc^JYaN*DV;Knr{PXvqhZ^l>;>q4j_p ziam^Wh=XMt&l83-7&15zD^&29`&FV<2VBJH&7bBS$}V& z4rhHaScu{^H~!W`3s}A` z{|zVcDm{pwQBaqKzK04kt$aUM*#l&@VdIg_629@Ip4)M}BLB-97UTjt8kkB*-`bzs z>6&6i0dON0mT#Ji)Ncep(QI1BZNEyKa!Xl^X@NvbxyMV{9ClT}WJ@vp4nV+cz&Rz|4u^Ivx2@8_I9m;O)@7^;dHIn(}956#3I=LYMZ{lADT zx--xcB2YfRW9j~DhEkbO!QUUO+qe`wuAknVr|ZW4^`E4Q)XQBu^BqFVlb za6LFV0LX4KiIP_>bq&(`D_KK({WFU{^VRu_T=SN@V+1P!vUikpsfPLcd#eZpGc6T- ziX9oGfrk*lbUn;-po+p5er0WI9k~M%!1Dbt`hov0Hn3m@I*pW^g5Ar*#*6t#z8M=P zxyy&&C#|0>Iz>7eg~a}e1yCp3;QGvzoLH9RknRQ85Q(#g>{(yLnpPoID}biu0Uf*+ zY1~liPy6{?UAs07X=2a98=YFJA=Lw{jedcX*e{OH8zQgF{bgXGw{+=^1*}7CEQ?hB zt52Sg^cr{>eFSelu6ccfCbfyoNDd#d(*T8IGz`nLzgCZuEsgb{k=O^s;Z;5p? zVQ1)XEzu@LIE-|AfusGA%!AyI1xc}QoWppZ&6u)0uF3IV)cNQwh0&lRq7Nm;LnZMH z!OKt?jL)RYQP+o;x z8dQjQ4HLP}Pd?JEq|OzSFkVkYw0v_~5nue-rjpgBTi@ng*X(9DXYu8ia3SF-9%IOP z4VZcIQ(^qnzPe8e=yYD(1q%|EDj)R3W`)<)>U4cAXU=iXpHUHcDK02l)h?PurBsxE zqfJt&5iN!utA5C1H&G<=^yPWe_jKCR-lJ_y_?D~r_RFL3qj7IQAIj_~EA|FKOpxDW zHy^u=nMLIJ8CqQ$>`iDWv{08nLqmA`EgCw5re;a>LS5opV~JvXd1~AFdX=QJkw4F5 ze?&`KH`TK)hBSX2&9%~;(D~f9-CB{?tD)r(Y<@MB(*0%sJ`mh37b>i51`Ab3%OPWj z8;{wL{HevcnyvQNC%c;ny-FDNBg{swOM26#MXyVrGE0=Lo2izExfCrQ)P1)*l2Jmk z0&q!ABYnxeydS(~2UQOzq?bnAM)lbY(W9auU2J3rZ^6#pZC?et_TCLm?;WmPNwr;E zP3L3&-i!GYS=Dr+W4qi+f76`;JNHO$|5s9H>@uYUkiCjz43uJ0;ctKn`Y4quWfQeD z%{6gl(Hy3Nw37R@Ro`fe~?$64cSs zV_ZSriQ)kiSVTmGTTEdO;z-Fuc+_fxcNh}F2+31^`)lomk>Y<00rd0L%BioCVcRx# zmh;aABhqlT#h?f8M<~_CQEZq4)lfByZix#6$mu8NH?TgcYs)?HdS>sBHm~FNO`RaVS#%Ykodd$ zL|{Q_vpvKT!vS|=^_z#sZW8(g#R@GtId6jM}eMItb^5lS7I<{b5VQdyU)T_!Q-KM}_1chTh8WR_(e!1?=Ycn}QoefwY zHtW*5xc7*0gpf)ZCb=KUWPX(P23mZC-5lEG2?l=9qtm0hu_HUN%gX+NuViTVh$ zW$&H{h@53Z;8guPWRnR6-QDHGChCToB@!MJupB>6n(Zhwbj!o!Q5U@3?C@Gh#x6-f z_}Rjkvav7llIxV>7`g?F32t5S#&C@D0<~|y=4jyq3sOZNJNOHw>E!b9`Tx$5GWWnZ zwX7QSy@{2HNivDlsxxw1^h2%N03fb7HidXRgKjur!GS4BongD+A3{RV#M%0%M9ETy z{0=u4H_M%Ju7jhKs4ja8s%jruLJa?;1Ojvz{x@d_&Y^bZthU8=TL}aN`lJZH9ci`2 z1+MZ<3wBtOL9mLYMQN@&J5k@4Bl%Fs42p=nMpRtI`2LaR4}O$-AfLl{@^V;;LSii{ zD%z?)`WUk4;9;R$uFl!zG|%>~CkZ)@?pTcUHr7%fP?{i#zFHn3;80H)wK_iFI?oE+ zIG3OR6eiNaQ%_CK;v&H0 zpw|ig946i(;O#ZD&;%At=;vVu{!2|m`|5{BuWk*=Yu6@+@g7svG+gZ3(&MFuvNE^| zE8huk+is?_3jiYjm3;=Ae^sPjE+@LZ^WM-9H*}!qWK2=$ssmwY)|AWqhH2EoI&v-O zS=o$fBxet9vC&0`Z{K#EdQAcpXnlW?q*LeCmRzkxaLFh9Fn@9WU@Z=H zjyWhM1{hMn5D&FKO9druevwNxjZ{CAYm28B?%IEJ$M8~~ zd>mG6cq@4K`R`3|$*zUEPnIjB@EMk5+K^!1;^Xx$5UYJ6tfRp#HIm=3RcQ*_IBfFA@1mNfE`h0*R+$& z7X2-6IpR!6i}Ya;l-)gG=hHX#C!6-!+wH$|^1b`+prhfMdX7RDZ8^LIIk>!ZYnFO2 z9ba-k%{=@&VP`1jdqIjSY~&#!7NjAjEfQi>OZ8reoVhU};fVENvLY}LQUe>SIv)~$ z8`UW-{>~@&v%!iJr$%`xZ2k(ZVp^{WIbDR*ZJ(;zgGux}ESy%YE?IiL4`O_+W z`lkc&de;7_3TB8SK%Hi>%2QUH=o zA&G-OB`1G^hA4mYd%Ia#xF`gI!9bm6vQpdmYVyyivDbnu?cvGe$y_kmbJ&>o@5Af; zaX-a|HP;3=7x(^AVb!$y)oa?l!))bUj*V5*KbIUu(*&mMOx}vYF)TjqMx;ypEmzaS zf2950I&%)Ws)dx^{a#tKl-5iUe*Pm|CjM4L@Ja}KO=*64F<+Wgx9Y9FN8alBiTk0z zT8sMSJC>=nrtP1K zqh#1OY3MebhB1a4;+*98958M^KDD0}u^xV__HKLL{7p9D?{~Q5^=q;}k%kcBlOXuP z-s#_;6#2gr=S4WwkJl{hSij%@tWF)^=73`8#L*~b8n?OS8OeOFR7kvl(>uM{^4D)a zlr*1@%n3e@UOzp#Vzybx59{r2ct-w?qZOr_8gY`y0uuAl*+oP>cy$Ag7ZY1R4fo&X0H(mX= zc>**ZCp!^IaNXf;`@0$6T=rF&b?WI7w>-DGCPZ>uPMc^GGw`x+^B{z=)NR&tU#Gtx zFLkAcY}V@JUjeuC>if&xGU^co6`7z_09TcWatp|tN_r!2BTh*qE+B~9qw8tax-HyN zTL8_X9(oA-egNt+?^W!aGq@yby9Edtn(v+sNjy4USYeC@t^_Yc-2$d@)ErA+;&|w!F^$iK(~dm4i_%sUJh4cdjx#=&BZihbye zbmQ0?T4t?3#5~@l0VH0&hbV!K5Rq1-n09UU$kvLtzA)UrF1`$Vb#t_zq^EC8NUuUI zjD&qc1W_N%(;!5y%Gq#7VZw{emfvFchK9}`mM!lXfdt-`p_888Dp3O(vauOGOPqhq z_}5AsUW}ONKFC?-R@>PKw8=ps*gD&k29~1ly9Mgz8)rU|A|7j!7S1ufPH-v4S=DW_ zSzu>&TliNfYnj16CNudip<)s=-E3*N5lljU5;(yE z8>i|rX|Yq3R?I`;Y-T-_wZlOYJ~{ejgfGGdknr+7%7;qS72HPKsNyTN!Y19%_5CQw zUf{d8ppOR#WBMG~6Ps$5+Fvlp!DGiy>eF0k>NWAmw!zJtsa<427JQxYM#;BkoAX)> zx@gDgR2u_=7NPvT=l5|jpY!~aNiKzjdT1_Va_HkB_2CwU!bI+Dh`SQ{lq8lQ!`@7Ust-*l&b0yydw!lsT-n+$ygoImjiT&j)+4ig4$P`6(212QuLIiD7N)P@Y^2vFqe7Kzp=hY!ttIhFb1z z0*#~$G4f)I+d)e!!b2ixJ;^cpo^*aH)Le@uo%SLMsU9jkMuZj8%E3z;;~mx-{ryIn zmaJcytdF0tk8E80@dJ9)xsZ=Vl~@tsETTkvQMdGGL8^e5um*`zac{73fr=kRd#oPB zi(zJ?Ni8~zw=Yt!5_QW0!&x*PJsM52NOn!)1`AzA*wdyNJ4dCKfijJ8(zPB!_+YHZ zv9y?P0+y>TgVT?Q_eOk2BYXEkC(VO=E;g8v@Z8yw9rZGC0= zwk_m)QzfiXF1`BM+`dyj8C7Lfy-1#qg-pNQDG5XPNM_WA?48|Uq1%lskmgc1&Re-t zY=pPUQAhsDH**bqTT{v@z^Yj;Z-PEHwS%P*Yd8qQCZO!Cl`HOoBxpE`w*jnnz_JbD z9N8txV_uB34Ot9tJ_plhiskc@t=m78DA{R-UU<~&3C82q9DJMmNFy~AY^*0NkZrEz z8RlAEf^$6E)i!wTt`Xg-)8s&Sh-Jy6@tABrFLdxeQyk*cUS@Nx9eR(+$H&i}Yojlu zHeKy^j_UghKw z){$@fV#cn3C3CUqu-#<7WTY=a^?w5nf|37!kSWmp<~ZbNtKY+s^Qlq#SI^6het(r1 z$p!bIMRS)@RSeUv(YU9Yh+eV87s*2^H^I0>AB20~dWT%yeWFj6PZIx~k7A@C2}k2D zlHjLxOR^m>o6!Q;#4vRb3YVBZ4G}Fd1Fc7jL8@q`F3cO%_G1BRR=?3bO9pg?htnA^ z*pOehx@+Dc5hAey6h1$=FZ)$Q3tkK~zL515S6h~lWGEvR-g=cIkIR!6eCEkFE+W<6 z=Q8}&-QpbjKbaSsiy!g6!Dk3$LwR420jw@+O`xq0Z4QH65{e@qGsz@{%KKr>>N!y$ zE^w3fJu%s<*Wu5(d!1vZg99hw(9Cm{|EAKDmEpI<*szU(JKlmn5|kvbapP>3jT6E` za8HL(kE3Ezl84JrCUnJa;JIA5GQsDQ@t3@U=*imVhfYvuT;VzZtHMyEiu*oXf*~Rc zYUI-$b=Dh^A2UwnSL5@d0y||qduNjVk%4cVD6B`{ZP@mI;zZgZ|0(>%n3A>Bh$UJk z_&Lxr9D#I`ENs%=zar^k&SAB%u+SQwvz`5dNK%+qCt%`6YnnA%Z)#4QGvB@4N-GsK zIkOHTk}VXI!^Q={`E5`-ouHJ+8P0hMN6vzT5M|sr^Kf2-?|2(_8h`i_M5oT5P&Pa% zwK&oe0sTNoh-c_0>Y+57fvv1Bg71Zi+O>+#N<9(5SUKnWp|!!Pr8!d18cS~3WUvRj z*8dJ?E;HBD2HI(R^@UcHZ28H(P>1q_USZ~&n^fk)#pn;XzN-v`j>!baD!XrMSCaIX zC~AD*i4)5~Z-P0c6?XY!%}|}2)@N<{(R@vQ(8EpoNfv(Bb^w9Z(X34;s6l*q3pQAV zIf_)zS0asjokptd2k4U&Eje@EhVRU5IHTVOpS{?T4-;}D#yAdYT^zS_BFoMu`xkFI zbB|W7wT=}FN4>9zR34f@ajw|#be4Nit4&>pj^f|@>#M82@j>cJahfgP4m&c|V|T{o z=VT@VAmN0E=1|F0B)-ucJCDQD_Z-m|8U}Ei+`Ev?mW_i5uw&?|6GMr*Z{5K`Mm@&@7PTRTN=bkd z+%6rngY8+tl&SEb1sQeF<%2)Q*^Z@+jk>?TejuW@QIc#`0<~~P)>wLQA3$+ z^+C?Ec$xj5Nh@Bz;&u*5KhWjaO`Z*u_Nk8M^=*kP?`PDOwuA@$1F~;eM7ZDZ(!1#W zA&!sb`8C{@@YkV=>8DV_p+Kw3@iL9qc$db6^yV5voQw(9s!HAMMo$D$)P%9L>?P{j;_aQqWCNY~KWkS#98J)b91S@u7m zavu%^CO04bmphtij`bfAuh434psrH;ZwU*2971DFJwk9fMw%K!5?8=F5s69}FX6?d z5vM0LLvZXxd1Aj9p+>DL8c95g9Cf)YMGiYeJ4+e3-!rAqX{tH8Xbn2aa_A6#JDqB~ zno1$AFpd$>K_qd?H5<(W7flgTzGQ|K^~?_yI)UV`^#flf*Wx%9G)U1j9}wi)N_P56pP|{e~Rh=Wi2t?lSw9NbK!5wR^ox0dmIm&5h)XZYP8Gx4;J+ zlJ8{0{R6~n+Q>E%fDTGb}Dc$1^=$$$Brrcc2{cl@nb3E&u6m|&2n+;nU7i( z+B+X~l$XoWerF^d>Q_HJef&n%;yQh-xxewv-NN_O&6cr9!y>CFPC>_bK_OLICmbpl zt}3@+BDQZL*()Jeuur3Lu(YWs5u}qU&yO=yghRN+XTm--!&MGh%70M)t~C$@)y9jE z#T!=To-0)lFNjDwQ;$HN4fgxvfAfrRRJ+A+*}Imd_&k;(zGvt9F_~JLPud0srs+P* z`O*Y4XX$(*STXu^wmgj19SrY8cw@uHmJM0|XH4rmwRUZiv;4}}F+RWh?Yb>}`>g8b zm_=r7cIL8RpnTAFTcxA%t^CGLXxy`{e8^l|A`Zhr?`_p-xNS)C74~ZwKU(YqdrJI; zC0fZ!EI$E?pLQ)usVhm49kM|l(wh8)-RyeAnq%n8WW*oJ@y&dk-AH9H#vB~7Sm_e1 zh%Eb~!uL-sGHb$%E3whJv5b_o$tzMzVPz^>(h(q1CFW$$^mk;7e(<^K5hdxTat!qR zg_B%x;PsHkxe%=>mK;L6td#5>W?H?^kRO|3DzHPNYb2Fcc%Jy1NLFQ6&F^eOGOu zf-Cud{u1|yR)KEZndqwW9eG<(eH+<>P+E-AvMp-@5hCQR{=glRKRcpP=K8u=gfXom z1&r`lNF=YXdrXoC}f*KD%Vm5qlZYPuxmliLliRiv{jca&R2bklnAra{OJ<239U?NLGPDez0mnY`tGy{z+iEYvq?lOYOHiwJ)z*j*DgJt?H^wpVRo%?3T#DI^ciQ&Xb!N-2GBL_Pysg;eqriC-?wNGOtt za;6^bkyKHmwY_2TdPP~#Sm{nw1@(*!RE3@b#!F+$k$DyJ-*s=x`ky^xdzFP_e8^o3 zx9q4c>0HT=*@niF+?5CTN)s9BcHhJnD5nWe#{efI~-5Wx-$NObfUVyP?AXYlXh z)vLXXA{mDaPFJZ)RsEur#+qrZn+c0otRlap^$Yj!?U00PNfazVuJy~LDJqxZ_Uet3 zKRounFfkBagY1-$7f8RFT7{FUL1flXazzB&W#UIt~sKr$l4}XY(Beku$!cl+^-CE(*50C7!s5QrQJ;)cZR|fQ^Q9tV0^MQvHm8rryykA6tow%9 zOIF_$ew`o8R*cUG~98P;{C0MTi(d}N+dc%1wnt;_(d<94G@ks9j~*6E(RgXe{d(Lml3WXM zO&5Hx=8*6{9+gagJM;B+Tl$P;_#S$X_Pe}0l0iD`hOiM<@C!-i{xBiQ@wk$nOew-w zEONcXQzw*|v+E8`5NO&bg6b66CIp^6FoqKn$xwcLPOnS>l5Fb0QzKGlYdF&gR5qN| z^M%#*_nGlDHJgM|^d5b6RAHyInM0RJTiP)2r#$TxS=V0aTq5_LTg+fpavZV(obQwI zxW!ob8JijNqO0fHEtxP{QQbxR2z{P#_{1+tR3L)zwt7XhiU=Ci1xo32YkbvWyh=vKc4J!PQ)MmG#mm`*@p`{IU4$ivJr$}AyzKu%|;kf!u`0Q2XgRxPuY#k zdJfvR6tzC2$gd07I0t3PZ25Bv%w?!8b_IvOsmqyAJ#CMn_^{RZqw zjM&r>y2l^Z#iDafU_-nbHQF5monFVVAy^HRm%bUgbh9B?h#u}E>&8oHalG$g(c^DN z$OvgL+L{ob=e)rD4zuk}Qft|1CdrT{>q|aIgnJ%QC`6z3*VL;Fj1#IbgwwNpecxX5 zqIXm3)w2%OByTGf?H~v3>F&zw?PWJc|@uF;A`Ycd5q%H zXhd1~_!+K2`&-ALdC;~HIEO28w_SY zNZ%e_9_GOweCp3)iFcEJuoys`m%t1v{l>cNL4U`mEr)ZLX^tQ6-(G*(GsSaDdK6^Q zu$@H>CHfi19PS(Zepi}ddncp#*P#r_i8y_mtm^;!Wyh6lYP@HG$55^|WsVNR=w@`A z`%P=wioEr*31102iGDhrrbx4$Sc#0?AP^|-(NVnurxXolWMaYx zEIinFc`4>(6Quf+i83PQlNkc(k}E)jq{*%2+Ftv7u!zXGzC0|8#+b<&Q9?T40l+XC*NicR;|!9Ezh znW3tyl7sS5slSRyJhWdOD9C`(J`RIO2*ir?5KJ!A)d zK}kVE96-bfc{8l~BfzQbGc=Ff`6G7$Qjc9}rt~*ZE}a{AHlDArP%pk&(wg=;v)|5Q zu!_zNVFx86!B1bEQB~=IIc%i{Zv2Cqz}R55%v$1y4EA0ERwOPv;t*e&K-w`C_twVL z4(~q6MW^Yg5iaGCTik%}*e#nDy^)j6tBkqoB5vu@Gp z*PJ^MTlNKala4rbb!e`m$(= zkCxlsqFyKPdU}zOdYuV)%vC5+V(d+70l!~61pkCFk;YckE9plgg4r097um(oUPE$9 zI`G+=ZRWSBwZ$_nrm-v^!)89P5$i>2fUCRb=U)l`?$u@Egtk{6Lhy{mFv#5^X?mI{V(u|AoQe)m1$o!_&iIHP0h+E$nn=k^cWa?CBz= z0)c9pz3ItZ|Jt8O-EdndG-w zUGwkmzJvTw=JB4LwX)?T)aTU2A{+&wMv{z-aIy(3^(^bDRiQijU`58SS;=QW-X;u8 z90aMB(7O^^G%V4SkKg*>Ucw@_R44>}z!KQ*;|l-07L>YygxAriC=Vc3rt}BY->-kS zZM?h@aQGwSz7>af(mOJ^1tKc{G&^JpA|(;eQ|DIk&ESsNlAKe3t{gDz785mIC$~6W z@|<);+dfW|yF!$=jmRNyDmvyZKFd*KiZqr`+(sTsimFcJpJ|a0ca&=|*7)4>3XXSv zLExB&|7Po|Vf>8&y)353A>kDiq#I|fak{5+n)(?B=qRlpyJea`R2pda7Jt|R5fJp9 zO~kL%Ue15bT?v8{5fb(dC8-q~lA`(i>YzAUZYObUh|=h@(|aJsHx#=3m_Q|lJES+9 ziIIq^#e@Bca}DR3E;Hl;BcBd}Xz1YeoBVxVERRZfu-+%d_hYgmTY;o+B3a~uiJk5< zu{`csbU(-*NAs=s#<}y+CAjDH$!DNthaVZ6WN(Fo+5fPwTzoZg*vWDjmt8xDLzy}p zdZpJLL2UyfU}1RnNN}r=L^xm9zaIX$JNbK-VZLTY1>u_%%RDh+_@;0xG*{J{6qd63a)g%EIdA`8jwbiP0J7C)}+( z;@pV9xGh}21!Gf1R$6H^W2d#E-x)OK3Yp06G&S~wp`mX&4EL`*9&GMWl4h9oXurq^ z@lj}EMsq24-1m3^?iS}=#^|XE5x5IY7rRvQq^8@RcOsfnl&3T&EM4NLuSFzP7MV>TL&y85 z+^e!bz}39*_V43IehdpemO2c5;E8bb$_sDnFFY1^M%od;*aEFkiEa)wBr=kpJ_wq< zYWHDPB1?*Dz%>_E5ZBYf2_srZE8F%$vIEu)#M44!;BaVo2-+qlMg+r~9phGSi% zRFeBp!`5lArnDYn{WwI#j=p1=q0?KE$9YrXTKps8s3PM`XTsrNJW+x(W z>GmYjW$H1;h6`u3uQMMppNcq7gt7w|t&kawk6F)skWDIf{cgiku*RG4QIYx6B0)#J zKp0pB!}}2VW=^c*Xe}jToSk(A!^hbnXe0#eoFq%?>yCY15sX(L9N`CBSa?0c%JeC3 zp{LWhL`ApKi0%}+kQEcBJTGPit^ZdKEb6SV?9F&XITUpy-?Q3Ylj9bPIny^Uf-DL{dx5 zwjfBmGH0m9#}n_QgRR*P>zg7hz4g#XxVsrbrddVQVVc39)leQ5ULNxzB6EeN+2~!= zM;u!GcQuDIoNmMd6qjiO4V2S4&GCXg$hRX1A6Zz9M=vN~n!`xjSsw-7*|l0TPK9Qq zoAwIIrpPyYenu*I`ZYcRC3;k1g3je>*YD^aT(}sVi2m|$cp?_O4-hJtyx=tOv?oZZ zDxWX;e^))aq2BT7u%p9H>S)}@slo=4cPZCzn*I^Y?Y{J&v_EBgg(8w=SJF_=v*k1n zQX$S~^wf(O7W%2RYB&6CT8amIi3)cKRF#v@9@{_PL!SH<#1X!k{WN}bj(7LvD#r-^ zvHR)@4Ih7Z^aU2*d0?dbLJpRBsR{~A#^lq-^qZ&M4QwMG4n(;-(h*UD9o1E)x+Eiq z-jX;sLHf*0;>4C7i4E69eS#eGn|R$+!;YurfchjO7Z>7QF@mO!1nxqb7JT!jdJP&G z8|<`DiA7h4Fn7rLWagEA_sEDKifXkENnd!vG_NNVKUj}Im&d+uW(K#f@4}3=H!bb^ zJe~>(anE2v3DqchPW7kZ%&-eOGrG6cD5frQJ&LFl)x6OvHs(IH+&+{6;b@L&q z@uXS4@OUlzy!kD$(n{(JRfM-#@+r}I#3s_)?DhSxdR_8mp|p)}l3ppo zBM0nr(?*Qo0#O6VSaLcxb8Ks+Y2A=TOPp^fF9|1QB|0IwE+~lRcAe1$hJD?SB7w)D zB7rzjOY4mk7kp%%KmHk~Cyx{|2@maHO~cVaL_`IZ?qY6(O)%cxtbakl>M9MUlEdQV zyQAxnjXbNx-c*Fa^DvvP*z$w*wK%djrqCJLluQfSR9UKkVi(ji`)n- z`^9~mgEuxoZ{%^L9(Zu$*3`T7Z8ix=j5vkdu9~lnHeQRi^dj)YCtk zK6uRMb<~EkL@&&jkrGGkEGSycgE!kWuF{@`#jTXWo5ZJpIFteYl zJd~%({K=sacA$)m5cnErr$4)Zj+;`cWmXg6zfW zcB(~!PHF5@=l0MfJhD0cCOnb5rDUgRY>MSOdK;)#QFD@S|^g+>u&G+s?%u>5EZ{;GR0U)vNtnqf6x(;gC)Qtxp86F0@yt;WbU zOd|oS4-s1(6^p$A)>B6A?6h7x;luL2NLh;7kUP#rh9qbOl$Z`B(Aazo`%N5A&RXD7 zn0=RIc9wwahY|!2^%Cm!DF{=GGCC5#TmRC=q|;LW}O0p6X42vdG>UcXurFqw1k1TWYIPr&xOE3o^wi%Vq^-S@#Rc5pPIegDRK|E~BYwfMM4qy*Nj3HNNABv3`XpHe^Esj)~&Vw?^~-4x2Mlin*H=Mt={AgfZFU7LY2E6zD4Z}^J_I! z1xFEy+pH}1;%QPf9+ZPL4HkNLQW_jbx!{7K^R;Ec*N24jt$Bf5jD#SvbSY|{K=Q0W zoUwcoPoz3*y&qvd!fYzKU%J)QeD9o!a&hkX3abKpoH4z#u^wR)Nv1MWxZ3d@QP{kW z5nh=_ud?#Pcu8=38*qluCnMRMj?4$P$G(6^2l@fWGJ#qVdedV4jd03fre!E2S6}UE zOcfM29Cy>KBEMKR=o3=v?sfdSxAtmS2)*|W?m*)^aLQ<(g&RI*ksda)uY+Z^rbHfl zkPf3*+luIJG5ap&ay45ld=JQZvT^xOr83e8yAsQ>EjLkWPEP;w zawb+@ZQ_k!M_!;^L!?)m-H@62E}KY0SZ{E;p81D1y69M$BD)#Mud7}8g&%b!1+LLvzW;z_*%u-VESVF4SU3Q?F8wV@#YllHldb z=^9Py{qvF}=?1FKhzE;_{$ykF>RyjD$_qiST}~wq%e)FU&D4!9AX8{q_0JBwvh%ON zt^8rS9;&2!2Z*q6MA;xDXeIgX^iwYXXPr9BUGDfFpyQNFVukRP+Hip5!9(p%5oGl~)+dBiZ2mMlawg$GC*J&|Q1<^9AuYF6_ zF!ti(ytQqfY-X)3rfIfYBi_b0s`Id3_UsIm%^U$JaZHq_%p>^{rIr18rz%1p%xC(>tmbn_ba+gaq2 z@67nT0%IBVzSDxmWO_f1DNBQ$nv2U9y>(P}C$YY-)AzeNJ zILK^=dz0tH9tN*OKFxq?=x#WsMqN~U;BKEEV_YVyqCXu!d~5W%rwRdWxLts{aQZg} zKu!h*A&0VU{Di7M&p!`@-r<0cN5C?NbaxOZ&om;Q&U3pka2D=;9Lh(aX}-W4=HSQE zcga<$mn^Px?lAtGkXqca5CIWZ?lLtdZB1*7zxiZ3(NT&JnGo{25K+RB}|~{S`)@7P4a5$Bfm2UM--NnkbSk32q*m_ zq7%duADAMqd;?`7VlwLO!Iioc^q@+H&rZThAk?QDz@|t`=%XYpg5n&7Do-X~hz}mVD~^P>98k z_peC^i(9r?BX&}2J7ahI7(zjH;~r7LzAB612hfLEDUj<%GsZ=*pd2ihF$KcT9*~!3 zaY^$TTiqilMGduc^#}KH&OKK>;|P=Tm-1qe8OxW)fo{h>)Ov{R|XQ#6ZkL z%c7EqOFNQBK;IaV80xa^)UIuHc%KcF@>E#TnA3$;__Medzv3T672kp6vtxYVu!|0I z3=*L)ZXl>rc^Gqs6x?octlk802Ut8)XM6+4dE7~+F~$r zgGjcJbiIA)8DqNbVIdb!9&YTitxKn|BZSU@!p=6CmKpiYTDrz^%gM9EG?U-6O9I=V z^gNr_kyr&7q%jsd>?cP#>4)$hrCUr^!MZnE22^JX$Gzh7m`^h&{Ty-Add3%-@KWs2 zECUoPEZZ&r^&TfilAMMR+1-)bPf@WGq%B4M1VSNt+>ApMg*@bOP=GV_b;`Gxp zj7FO&JPa6;^?TwN=`jUPVl*W*Pe{yBW-kcQ(4(Mh8~AT3z%tG-mdlOw(q<66-nx)r zaQ)eu&+y4zpCdaUrGKIWQ>t9pOo>OOnpOt;ZId1;Tr((zHRJr!A(VG?!iK5i96<($ zZg^QaNT*IP^txgwHJvUG(*%jg`Tt|^jNNC!; z_AP}6{ITlA3a9Bu7mBejd`Nf0T`A2DzF>w&0V$5XwQz^2&2I_h<#rz_uoC$=hxR>a z5#I}%O!d|$+e`-#KZoN^47yS#-{AynI$DW%R6$1zDetFtBt^wj(3UdBU6}l+8MYUK ztsRX~q|$s9-4eo#&F=yzsFUlJWqh`l7hwvi!t@+T(aR%a?+h#JLsfK1%Uu+R-)%v_ zKbRr9NoA~7NeX3p+6wRHeVGqFcr{~f$U@z}8yo>%P(OTqyQXnm3UKV=`iX!K;?dJevn%Yl(aZ_>5_OmD#_5uZd&5O z;9zUKsqJqHdg#qb zK8IlC1Fqo1g8ebU?6Z?{e8A@sGb!06dBdSG1z*nJf@ZkdABW?{Q0fkMEM%05jmsB->AfI!* z<<*X_?jTBhwQer*YN~(ey|o#7iS;Z)DHi)({>6*$w!|Z9!S@#shZ@9R49Us0B;4|Y z=L$bWf)!E4R>|PjW4?+Hx&m&l=E>oiJA`0A7=F!YdwI4}3x7;&$1*tZCgh{Yr^cu1 z>%_}E3QGGO!fboNvB;uxF%cn#1H!WfD z{??>eqP&Yzq$0I)L)6y;2Kv=iY83B66j3?HMd?Q)WiE1M2c@pIg2B5Jl;xN67Z5Q) zE%`L&l=LEVH#g$39GPG5H9ONZO)&S#cZn6NtctOf+1+?>7zi z$wL<~!ny{;);@CWPR>FpjSZ<64|>QKB#aGOf6`%5SDFcN!u$$d4Kvu zq0DcebRaZM;Ep92;*zl|atQIvw*fLmwiUn~BpeB*DH0lqqh*65)QT;W*bB3eDs3S< z8@w}Td=mWOBK_ttN~S211c(JwLID(Cx;vc(-Y@ci5Or8z+07O~0Zfi{O|#tE0c*9% zzL(+77ARa8;hreXqo~#>dW(Ga76W#3lL-t?!srK2iK=XYLG~~X^wfpkXDb5YoKS@Z z@D;_+VVi^(SbCwymW}WrpyQtgE(!`B-J9 zv^#tV)LKCE40&LEYxwL^6w28%WZxqm<4+TixAMI#-|bfuY0ZCBaiv&AT1j_Aehk{x867oL~uwG?BjG&rbH1Ufit z(C})$v-LAFds1NGHY7}!Us5tkk=_?DVdCH9xxtyYIdMnN`|snWNzs^;by1?46WvmX zq4L3gkYo3%o07RfN?2L^Kzd}tur+0;)W!WQ{@k7~Y@|k6X&nA>wPQQL!=<$cke|B& zA?i!FA$09e(C-ZN;8^JSNcYB75`9N<4eB$9lF8Waz=Oezy<`@KjwFtNpmfP;KZ& zdb^+Q%_k_vPVHcNeed3uhO)fOTZ>}nUfP$<>mw2!B%K@c{3}RhVr(wRDVW~x*$Lx4 z+N|F6UOiE6n5$u>p`NA2?e?1l+kG3Rl?{PomgomyN1=y&OFb<`C!GcD%obnx5@iLp zy1EUMKf)emMDj$su1@#GuJ$gBW5MDtf4P3=syp+tdVV}fCZK0G29z%}YfMJ-NpA9(vAE+01kZr4Ly zME5(-7p{uzc?l(O4Duq<# z?o6XAOz&tbNOme3wNecI5BfgH2MRE_EV+K$*__qRs1dmRSaZQXK{JDIB*gaYEeE-Z z<$(}Lg|EG2&Vd0>Tq%x1Gb2T*m^p0Gd)9y}CP83WCrEZ)-tJv7mJ8QwIzm||%~FhM z9I+{QL@r0~Gr7mEnnVsRgDXDE69OJNR}axgKbe3dT@deS;=B?HcDB8ZBhPd2LBkhj zlemyqrKJaLrxG4ggqw%nxzST0TvoY964_`vAJ~c+ozww~=4YpZW-B|a58%Oqw!`B% zJg1aK3?47MCj#zU^=$H+EbQCSx?{~%FIDB9`5^9M;alA=>+l(x~h+%+<) z9%NoAKjTI)NPX~ildukStCl=-oPO5geba-_Yn~Tvb+dMzA~pFvwI$2Z?#^zR#_9G$ zss5mAq~uQO+9*xDNrIoXuApzP3V)cZwH0Ei?>LCAEY+l$P)el^R`{9C{^WXtX0=lt z#Zi9i?67(j3*Q`_ze;xQX*`|wF=HO!ySw?fhKCNjM!=tGQ3r)S6swyghJm+CCI%`yD(&FgtwvJJ(d zpX!X;+=$^078=pFvJIkNIVn3}9ikZ`TJ5`AWYg(x4kS}KW#_hgb`9e#n{zxi)whEu;=}ViDPiii8{u`dR+4ui?_qqKB zI9HzLCG~v!Y(=8u~LJl$3<0aQs=w1B0 z>Yi^;FxOgJKL@%+C2~>G8X5dd_!Vc-T$Iz`e7wgegt-W8Na*Xru$(wwoDW^|=%X4L zbB6^b>JGL$3Vl*n7{t|`q)bZK;4 z%22=#)^VWJurf*BpbOLP-36Uc$O6~dn)tGN_9PpD#_moONHf9jweeT?5{WOx)j$3w z%m!KRm|;Bo8rR!l=BYLTQ!0u~oLp-bD>BWWF+h&mE|ar|M{t0+dscg?B`j8B< zCRJXd`p|YYvAa>NKAlXfob>c&)r=AePTjZ9V7_QCZ>ka`#%(tKKhU!SX2>OCWz=+^iJ=#aN^} z2umH7f0j_c-Tha{m9P=xA2&(vcGcND-(=e<#l2zF+sBX7yusA#^NA6s-;G?HU?7j$ z6*jq@I}9bRJ?fC|SyVxkA_my;k;G-ssclLp9fXW&oxM8;qSA_HxITTc}n}XjP12h<&J$GN|L`GSW}Uf9*h{*Gn#0 zYBG9JU_=AcrCxCRuywH1iB&tiF9T=){W4yYbT;2gUIBMg^Td#2%LoZh_+2qVlkSCcA`jj&7R$@t2Ylj5#T=emAKo)JT1C=&_qW||P=`Cc;@?<) zGOs0GwWIWtazg-+`!I<2D~Uo%A_&57dPed{>ZtSC3-HEkV@Ild=rvD*)%ZSwyy1X_ zGv%>4*tcDs#-QJP_7zP9K46nb`xVs|?r1*IzDhHz{k7g_@f?Ffca3{^(cVc{ z^P`@66@^hUS#PAP_aNI@HNM{q`3WFwlhvLLCw8XB{;Bac>u}wL9ABBi#`oi1>BEv)XiH;jma{cDFt0 zNR=A9_U=hsM!sj#8fRW<${_ZEFjbDM*!@W=TkZ8cU`IiN^6!vML=l}UyH6p7#uqFe1Cc#>wd#4Odq(i=Cuu`>Vrgb0$+F7)oqX* zj5v@Rx87WNr9xY(8g+N0O7aDt3==neJ&~Dt+e;CZ5^p8YUlreGsrfM3-g(jIJ|KAa zLuICXS8p_(e zQ3t7h<#Y#Il?c92LV}%1{&^Lt4Q9I<%^z>`MUThtraIP6G0U0aI>rCFI@uSZCnv~D zKjfFs?@=cCd02sR`8t*3GPXh)!*;|%>+P8y)Y=5cr6d-->-B1$O=m_+Ks%Pc(8PHx z^yT5oI|nfmbxNKsRGQ^V9!uKfZQ`Uzy5cDk(#GA&+RaxzHV|@X{{s)ClyA^i%y8?E zGaZZ-Q@Np#PKsmeA5Dt&2QV8_;*i)^I7~1n4R;wl;{vi)IE-&GeEc1wah?k#MmVvg zR}744Ok2Dli@^Ob|Dj#g-|9$$Fbl~w^Bs7aG1=o0)e^T6Vc&9R zX&5JBLPNiYWz!K`uTPP7hBrdE!@{pSSKk;g$yR;sc5WAUiedKU zq&!BGYq^rT3MGQrnjJ5nR|6c#SG||Th0eg{U41YW{uLuC@|lgX0faOfV{_{7@)vL| zL6}1ZUhJuD`onXMadL@IK|M6`aH6F*g`31tng!*iB)cv$bBWj-P`O&hvKoD0C!E++ zir~uE*Zx@*n;*yT3DTan-)H;ou$Kw88jG?Ap@Fx+W#5W44_z=t%%dC$jo@mVj>7O`2fe_S<3^ z9OGQ7g*cNCUle8ZAzM!6LHV3cw-i3Ly?YV^8ylN*$w36HQh zCZZ4rMGfPC1hs&@J?C=as^E^gMPzy8@JB$7Dn-F6jKF}uk7TV^J||)?6<4IlJS&;8 zZ<1%GMUHt)Mw_o4>0S0))B^S*aamV#tdru;^ei`i0Z;Xl%O$x{ z`9Hc)>7mMX8#qG*a--0&u6Unmt?LHUV@AKjtXr!4(Z@U&0J@9$&F42cZz|`6H2v*5 zn#IrLp4DyWUHCMH)11RE5+&b0UEklvHL8X&!jWN@K_4Z89lcStD9 zUgRt&#C3^yd?2s3YoZxzdq(HIYvyxSnXQmFC9XMfDpyN~##)WOVSz<^pVt9qFpY^; zoDe%6YxiuHUWeSP3KrOab0;@3(T?*zB*vJ8L-V7nc1bfV{G5m7-J_t96tN-vOiND=q2O%dqK5)s;(#v1 zcrt)MpjePusFswYJf~D;>A|U3)tuIY4-J57%;lg#=@k#YTt45z+1Q_|&I_GN^q$tP z-A(;qtr$z}iHy_5`xC56gYuH2f+@Pf(?WecLIO#~#?qM;sm+ zd!n2Zle>gX8Pb^jy>T@61oP;VhihT6_}M}!%?)~)Jgqvgu-0$1Ci9&rRH#;Oee86e zAwZfjkPaym-2b$qt&XHoeelw?i%Ba>>p4iB4`N71VdYfK)tGRH@zV?e{?a=4s#lZ9Ak?_fSNR zxfE_#rjny>=-6kHBf%P{N7pCv41)87;ECx+J0z}h;IhIqHc<<~$@trlv<^)*e6Wqkc# z6aB}P_PQkE94DvqwH_QRIYf#%k-DH|=Rt|yOd5a2J?v(ybzp*!!Co~nl7dA7L!J@F# zxk~NrDYqeyY>koppAyb6mL?YO)X^`ZznIKEOz=iNTCZNjetvZE46lu9_3MMhekSMd z#EN`FTU1Mr2FN0#TBvNtOFNC+r9&s0_D}U%99SN4u>49fe99eP5F@|vCNS;O%oa1Q zi-aJfLWxi`ZG+qLc)Fuv<=FC2t0b1>LI*}ko1f$U=pO6x~tYH11T zsw4jgpk3++c{5X`UupT2yKeOv8sCzkw{iVE1}c&A9jheNwwXn2J#XRiFV(2rzr~z z%Xs;89d0tMMT89F%G~LO6ln;q_&y(s^|$?B1GvR+{4aLlsm52|Gva9R@wvJMWtn@& zQ(>PNVVe;W_Y9`HfJGn4Q-!-`V~T#A7p%p5)qGthY@{u z{nQXA?G7-5(9w1A3f9uCFfoag-x0+DWqtj~LS=uwuj>tUTFxN8sa>E?j_?Y@npjbi zWh9-?*c8y%Kd7x;I1ra?mb%*ha$XgPk2_c@^lbv)6@)%mz4aisx*OS-(7# zzC|ehL)F?%S3M?)Ct1H*ywe*Hi*%r&`H{9+->_eYztFgqKH20=e!FyVM@K@kqF3WU z8>|@YjpM6Ci|1!Jjg+LnFmr)&{}DNtI^Sns)bd!CeggWM6t2LD#aGMIs<<~z$76x-hW@CRorD27sE35x7@#?C-fuFP-*uk z&!Cr@x$oRo12*tN8rEVL<;jsvk65(|3tl!H!h6NoxyHo_?(T84Ey<(u3?TS z(+wwtkhTlD7s)si;V>x}{|dRVo)w~~q1xTL))-}p4}lz~=m~|H+(oxQ)%}j*&oL(B zD%bl5x7?-mM=xd}XvtcPBYtODnkaMr{Ern!>6@DKNrY4|t+?Cb(=vao+kdS3ThQJu zA^%HDygc2BN)r>|1)BddzdWfT769&I8$0rdf#iagc@A)Tu{<`HLJp%%t`wQHZnn#A zUr1pxKGH^m9GfpJeJ)3{h1&HC3_Z{G;Eo_#EnuF8Z4z6=pgEKIksnz?w6v3Sv)jsqYX17*&FtV3+W!vij8UL1ViG4iH>A*_D=mL` z5B{8bKcHtK^YqO*l%*>E7ecs(to)kX2xLuN;aIJa(q68QyEcBC-GTdyA`{d{GarjS z>dp!MwZnj~{qS(9;W1537=serj{*|G-97;5`dQX~`OdL(6>gy?0Fwt#~n{2eX>Ix0KXDn#OJVls^$QCF#E;8v6*zDW!LK(rFcu>=s9Y@Dj&PdrB6C`%%PJN#6VQ2;A3>-hDGpdoL$)Q#h9I z&g<^5bWNOJR!djVBBE)9Ww$FG+)rvL1?)xGGUyu=nGQrtYpK!l4J-}7NL9gX;w%1c zZ^NMM+pz2||8w-+{W8C`Vm=8$yDs&mgs|g&Hz3eiA40|RCc!BOF|&6FpX&LA1L!}*lLVFZ* zU2>unzq^$XA#`5dxE3er7e2R@(pLgLi&*r$n$CB(3wPcmZxAeHfQjwXdZfMOnJqOP zoshjtD(kWS!v0_vZhN%<^$M$OoUay9W_EUq=}PUv{8c1H4}W<1I&C*SNKXqlpKGG# zxYym7_6K;aH*tM#-v92#@=4%IHTtiH5jco(&}%-H$?5oh40{WD-XCf+4&4beaGE9H zx4LiU@?>t4BDeMn^AZseS6!JZz;?w=e(mDxT(RJOH^e)BUEXXO$r5S z%waNSM-X_GstK~s?Ag_(UXHpA!U`W;E&D+L)j*vJ!fH8GgoOk{;8p+LeA5hW)L`}8 ze3N&-0QSoPEZ@Ba_b8`Ez+MuM2{fKfec9IJDvkKs>maYjF7>$MgPHL`(4IIh z_q`DuDQ&|hjg>_@A0PLVurQ&MkV2M9shfP-o8|LO>P}_W{lV^Py)G}+{~Qr{R_>&` zl%2l+`}Us2OQUm{P-Lnin^>gR&b_pL(NAK(jX&~yn|lv$H+&lP9>=WaDPEw=kWBG! z`pU{^Q$I6ms!T=Vf~x{ekp3ybLsOGx^$G1duk}}vA6+#nu0kMC^@RHc%|VOa$STTR^M#2bVAp|T_K_Msl!zZ7%AWuGOV z^2bGb?uI;|QZKf6OVcS7u7zpD&@l7)V&l^n@vN+@kxE_8&%$jaWKREytw7zPTjNiyjTHtKt24+ToHGG1}wC}*=BxpgFIqG7L+nvi7{LYcu zrj);XZZEhW&wwe9C+Yf9Cre4iychnv=`n~Vx*8}jk?;lsi`E{{2Q}ghst|!*a*nIz z>4>vTXs4KnKxTrgh*5WB>Xk;zx zH{$xghV7msWjKk8)}i z#Ru;Du6Y;YcJt{`|J#3PCPbu4!J=!_69z!=(}yi=&!kcYfj>Q>A({r(xijm_1`8RRUdyX|y!DR9>rr|QZd`alaVIkeY_MAiI>c$D$9Wr} zACdo6jlb=)ysH$CdJgo)3SSo>mMSWk*cX?n*+X3>R0Ek?E^oG36e{xzu_-ccZtx3X zEfB8OSlA8s0sWZjZnv6o`0xRa3js9$np6-^dRP2TfS|eqgZ%UZarx8sv%L5DF36yX zu2WJ9V%3;rvrizAJfKw`GtWt5IlWY+?^%cU^bkFsUlr^iW)j`U->g1xAK`eTo%0t+ zp0I&$MH*d*-5JeB{z_u7v-K4=#*>+Xn!a_&X;^-54WBZu3Tk;JNY$^s8H~UA31bv9 zm^+%Pm?orIoY{p$YO1Q!BTj7Qc|M1D2vWUaBIB!w{z{Nn1w3@}v}zicXEzDj(Q7*h z^UWU;&?rjrJ?Cz@cCnkQACphfbfTi65nKa!03Dn!xY3_iZa?it=Mpa^=2BoYH_RS; z%ZP*EFpPF;lD=#=J~ohsFqyavquSE1S&4fO4FgkQi{Y7MII@oC-2u+^Lcl<#GwTjH zMG?RC$~Pda3W97(8AXxoDi|Zxf#+{24S0jb%C|Rn4UCN&&uPY?jHN+RFgiq=^RU%4 z%m^9)HuX37`^F;Yz(~K4TLs79ph^h5QXs@dqB)~ot%2_OQ=_fcFB_~!?!XI)-!Elr zMnT2HuNtOaQH&EQK82g%*f6`<1Y6qQkEn!LBZESzvA-F_mA>xAe&=CKXOpJX#BE5Z zSNE}wc&E0p>W+Dy&dPUN;f5Pi1ditrB0Pbd5M|8rIe5-zJgPMfF)tpIiX2&HQ>e1L}wUJ zKq0DnhD#%$?qr0^qhrS;n%oG&wfuHkYNe~r&-sVSgZS&Pyvja_lFt^|CEapnVOPDH z*Pe@CaGWrSwM+0Y5-WLb(3NwU7#+9FRpEmmwk$f;wy?rU7eX;SGl|FO!=;YzJls(Q z-DieVDEar?qXk5u*hnxr0?{o@=O+WqwO?qSlYe)_Kp5^@Wl{0q`OabLq~~w&;ukC& zr^7`VrfI$d*+Kl7O5Z~x;(gNYdcehgTwT4I=sEDLUy88cC3=#okV6*kc9IeKc6uieM?9MMQ;g zJPeJ0lBxD!a7DS81tNmzRw7-~qAabVJr^zNkfZQgCP|Lhid^9z7%LtJM=^77p6a=D z4Ut>}@1gDyju&!a1HH*O6C)#Nqml8%3~x3;^24v+pjlsoM^Q73n)MiBY!j z?*n?R?;1rjQPCp899b7$sDw#+T$tHgZcP#yY57A((ZWvA4=SKd1W#=uLMY^zv~8ytI)Uj{C}z z%wL&t`go%Qg!5m0zlZRwQwNS`Jn!_tt5_tSMWjP+_j6s6HXNojjoH364<{^j?vk8Q zd$5ZS3tquJMd)5zO$&qsq)XE|r1!Ng7H|Q^F{}FQLP?Pg%r)#~>Z4({!3oHtUCo!+ z?!vuh9Cy@Xy?FUh6vbBhjW4?+%v?b4m$pdX^NlFeZxR+~yXFaVFLjNhU?}I6f`e5e zmZGVe!8M`^fHPUI?>+USv=Ua!b<3#5lkScgqznzfouWs94}s?jC;>WXHm*o2Fzk~n z!nQFz!855dSad2!CvW)MS$rttAEsytFr0h;#(nIaZt4YimBLJ%EIb@g{12|>#8zi+RA3G8mV+NbOt2ndf_F|=lDZJQF%YG+oQ0?Ez#CY?6 zauHS9;Qt3PSVH#qQz%-g~nVvazY?O;}u zDLIy*4b5S9*Zy}VwP)`}Co5c^e*Yd!#P*cw+1j;4(i<*0nhOz6ADQie#kI2cAr6!6 z1B!*iHe`oKx`q*{bxb0j+sp(9SWsFgn4T?zbFc<5iI1Yi440Z@fHqRIbx zm`29njAP3jZWBOJG(^?M9}o~mdUQUp*YtbkYP|(#S|IxXY5e$(7Uf-kd%4r^QyF<| zxA#{i*lPO+=;1#lyUkkv38JpG^tVMr{9ruC`GIJi#Y!+e;mo0T1h_L8Y|=2=-i#%l zpVv=8U$$J@&=N#=D}CqRcs^F>G#A0svHdb0H?Mw*ydN!q*|-MLJat+fvdNHerE&C{ zu@p3j@L0^UsrrcJ(Pv=%#jVCBurg#{A3p4Rzc=xI!lQ}YvF&eKhF^Xcm$qX}&&sV1 zxEqgY58&%>4BFP3`0_5{(hE5ZsSFB%%qmMc*10!IjPdf%f$_zpKY*qh<=>P(VB@#< ztV&`(R?8yY|Mkj@Em6p>Bk8ob*lon7i(QnV}S_Xq?H#(*b{4eekhrdGR zZCcKl8;oa}*PNz0uIt;``Pz2)RGV*v4>r~q{y_0hx!@%jmiGI345B+nlcpZQTy1O- zA7)_9M;Ae#!+rm4G+dlp|2fGcZPKfOU|xNGA%-5q7V6R9go4Q2_$$Mh!iTsMsHFyM z5AL9s7bqS}jJs0e>nToGE&yhQ;q%Dn=r3*e)_z9FiY5YbM2Md2sxEENy#0wy67*6+ zg3&%%^99tj@4M|`{X`>avdI}ZGMXp(zB45+b}!Lfx$UF<_T*+PW&V7Y3Cey&6Qvht zMc18=U#xl;9!~e<4w+)JuWNobt+>z${kAFR+|e$<;iM-MTJ2Naj(xx$bNJdhR?pk` z@ea{doShfSpean@Fs~*UwnN=6!XIwR_m3e_7Q^__*Z5X$Z3-w&gGF8 zsy>$VItdkF7iV)s>%+mk(LL1#|0$l|3gaX!fKbjbBNg!DGJu>1Fl6UZx7NfYa~P8@ z43lY1Veea%74L#4bKj>W6x{qZL;Q2j{c!Jq9|>3Fw+kcO#>~- z9+%Jhh!HgQB7<;-4$;LoePNyzmA|4jbt}4+?RaK;RflQ4o?=U}&OLgb@1)UYZ`IhaHgwAe7-(3Q9-UAt8Mppr=)_m%&X{InFw8Bdtb=T z_FZ_zR~Wz=iPZOO9+zq>P-$!AQ7GiFhk#2tn^)hgW}$d?6c$-*B0t$s@>@_XxW^nz z!1lsq3@w$GLdyxT>B7AM7)?MwNjS$n4MpLDa5mC2NA9*lszb+`c=h*oq-R= zBWw1Po&!+Pve66mgLEi1d`QmyK?(}XB%eO1QM?~*&TJRaGw_908l$>E^}1Adz&-tc z-b#{+v(Avu)EtC`xHz>)ogwOdq5(JO&F2n}kI$hc<;9GDNt}R#7(=PaWQ#a04kx~} zQnqkSB3N$ss{*i^uB5LQU9_F|6ZeIid;4`)G7^{F+{z8k6Gnu~XEO(#zG$gP+8Oln z_jEeAea$=%`U>W_;N0Vni(aUCsz{LiiUyrq4qX1T^YqiS(I+t@h)FT392XCgHOcpr z^Zuz+R=sT?wKH1}@LVCTCI8{smQYoW$A>($T^21)5MB6N0Cq|11kXMy^-L9MWe{na zM`SVcZSs$_w5nZl&pEI>GxPRtTiYn(HC1 z4>u+kVbXmqQPZDKc%P5N|6 zOp-0{m#zFgJQtZ)_hY;p6ERiEm`21&>5;Ix4+YJOCD$@vuB105LJwF@Q#e(S&uN=R zJ>+Z*N|p7q*jF`FH&5HDOIqc#cP0Ba>B}OYR6a-|<+rxQ#+&BlEAu}}MR>DTck++t z;DTqE8Y>*6f2nxSJo2pG6~O^OHaYt2jV}AWS;2ji(Eh?j^C0?hvs}!jba{M^Y__8& z^B&~byUgL*K~H${i&v|g=&7}wH%L9+g zYw-4S;W^@wHzP5%x&WPrCOmD-fR|q=xNS-1$rBwu4O)2jCN74V!cgL!KW5ccRZV^D zXl!*#;YUU6oF>gQ#X4nOy<;3Om$4VW-ox-Xbbqz2p1%e}uq5CqL`FrL_NR^M)wq?| zn2&Zv{m}V|E3scWeq2DUxtQme^@Ka5HE~s?ju7%0M?s2?bwrx-WC3)c=~LeK_Hj15 zrADi%d^#yxXeK6&h3+hi=d1m*h9^njM_c40zn;QeA76QM?^>H9`X!`h(Dn#KHSSFL zDeeKKB*JI9l4qJfJ8V#uv&Qv8RRmpiV|*)_i5hWicW)DOyWl>sH>s8?qoFT5$#ktt zNR7=LSNt1!+Q9OC{m5~Tsgs{IV%18j;t$`|5am)a!=Re(ulmIiL#9I!Y>)j{gY&pw z?va0VqaiQa=<^L!8<*W?J*1_#dB|+UfWVUWsZ*pjDhwM^DIg%fvy9t#t{fh^@`#lU zvj?}sirMCvSdl#NVJmG~f4>?|Am&UPCb(Wl5blZk!WfVGJkiRz=c-D8Na((q!j-m!!x@QV$_zkkK-+MXXFVlG+4;~%71;|>A!39vL7 zU}4jy|MmZB0@kw0eQ`bovaWcy`S#bY#`5+?pPuc{MfCBs&|u7US+heva1PMQPa3}p ze+GU#v{$`U`+S5`VUcV}+%uoAv-wpiCAbDEXq}2H`N0#e{DkdjvEV-cL>rijk$iQr zK{UT-r0t&H)6xx7ACG4p0p_90MA$7g%z}AeH%|hkx&UV)y64(}HN%t71phvQbs~%Z zSo~gGiA15>!+gyqyKZXyA4fz$HosdaD^$IIy2T~G{-Ik`;h`FxCj92ui|r4{fmV5l zKg22@Mwm69?#5pWV{@h9(_qFX3X>!i{r!gFsO7Iror=gGE z<75H&*v`nq~#$` z-PwD{$FhjVB2&gI5@bpu6X#KbCZ7S7opllKW>(Zv5l480NG%u!d7 zkB8?;@95vBLnH2mYanb7=PxhoUw^- zBEPXIjH6VwT*oXZ4DJ|}C(N8q$AGGKG7g^ve)g5Y;g`0s<&k95Q&JuTCbbpHrQ7l# z*{+TXt3zAQ4&(&!40Qg=ti04fM?(<8xDfYai}U91-;Pn!KNyUeOwO&Y4Hbb7*$Zl> zXDT$z@m(Eusw~rJ_a^YK#<=nang!|{LZi{*&F!%d$2lk1-xL$=!wz4Ad$!vzUucSh^Z0bb-t6z zIfwKOzg-cswHs$Yz2@@r&+kjzEQ@OF|H91wapUy>!KJuaxG%UX)GBMx#=M^y`kudW zamsw=)52K9oFuw>?9y)-m78fC5U+;)a(=FtD<(!bU5X57*2>TD0x zT4|vJzr1R>^S*BIM;{!9sr$5tS}xq4)i@()-4|DT;z9Q+-2g4X6xcq7YC);AoT!|~ zOJ9&M)$5C=+tCA_?lqB84_a9T39-&s^4NB`q+@s}k)&w9lf3h->|2}bR>JFherHzx zWGxrzEjC?X-yG{66oytaX=_-xaqu}UHhG!*J!p?s!h;Bt)ZP+#ceZd#eYQ7a)hnv& ze~9D%8He8IFjCs?|9fh|a48(j2>azg1&?Ue1qi_F-oWo(J96pI)5E}V+j_ubY$ zKSxMA1Y+jt^_L!ZR}0a?Ef@VjvWVJ?^>oxfqz^_6ChYr^fFLb^e7wvgdeHy&*z%X% z1nA0lm5H9O8Iy5D%Axa%*VTHI9@hPr&H@-Tpbm1D}h%U%2 zdb&K-cmd$kDE$_LkJPVoNTN!A4P&_IG?r2*i4;fGDe({H3qoRhDpyX_?iHLTC>ybI z;|qbaO_Ol?n9#c`etweTrgonIm?%L;q^kPLIcmky^F<%9osU-31s?iY;?2p=VodUv zR@*;Zr6HvMCB_R?|GkdS88d*xNAs6Cox&SD2w+`ChKrQ zz|=1WrC;}ysqsEzT(5j1H_cCI^9ZKKIgC+wjVrVVyj%8;-qio3H=1??KlM0Y3UMBN zBTV@`-dB6qTvVa)YQJF+iFM<<8mov24J2Hw=;QAAJz;FS{>{G;0~1g;>i%7Oxw*3_ zVd~xOM9PH_o!w&^fQ#tr1NsZY&ss3HGtEWF2t2=W1N`c!ud@-Rn8v8WUkk+<j*Y{H>Sc=8Jwu42O70wqLiAC2)l| zlGR*kaydzn?NCxn={|Y^3SGJN{zb@+4p_vfGI+wHu`vsq{2~M+@>y$1 zXWUHdNwPOUE&Vn-J1~u|v1bAC6|jphyF3#?=Zy?~59qd5@xxG_)c)ro{?kjFZAFnX)RN=R-tAB0}xc}Y39WDLsVF=2$<;)h7 z87~!x5nWMyftp*cGVIm*)1f=BFe=EW<{0r6+8d;_O6Ogbv)I+VH)0Hpp18Y^zGK5A zM~3J&A0=AO*OoCHKb|qcaI*7I{jVWFvP5?fUD>T_no{|Tvh_d4as^*+#ub_ zSP!wuW7Bgf-8FXHGzqnZ?qO6lxA#u zAQcz-8i1fNDK2&jx_B`*Zr^S_j9HGy^V=U2_IY=i+GMO4(FkC8(Uo#PZ~a(|Z6CR829 zB&XUEb*lYoKF&Z8naAF_P+6c?A|48>U$qbty~M5pg?4XcKai8h)r%>CoW6HhS;Z9ZlDIvJm#?ZkK2qWMK^ZI#!$D8C(w|ez|1d$f z)l{i_jz%|B4JPpa=Nn2s17%VFERkc=X#UdGj2_>`v9Pu)G43-`w6e--IpVjT82lj5 zl~w;7)_Nz~;ikcOoE3D_oz?@t?2#-D#O8l)cWPD=gDhG!G#uTPik?qdXaaq2zj~-N zh&!0u{BUjZLdHhQH*sjY_WZ1$J805#R!~I^cs(~zga`-{?tvO0S43_B5=(8!9LMd# zHn|hTg)R!EVKH%8izHef#IHHtD*vN=|96~MYtjO3xHMJZv{_WxxM(zbnzGM6IEf~Ka9^<6#~sd z_vboW!q{AzFR4+?E5=8k5mU^4p%c_?07ln*&EUSX{% zi_082biCf%rL@rNW6q@lGSB3H3Qh9;8+rm_J;7QpRteW$Be9En2?F77pm!sc=<iuXLg_kvdDu0C+xrm@sw zQPwP;2zjCQ*;TF1-IGGQR}@xMWdcsxsf{ZuWWXl1UyFN4j}#Go4;q3V)HCKCp*>Q*7#*(e z$)wWxBE%q5kY7Fns;JKgi@DD6>^h{zz!6uL-Uof06}mV>nx0+Ui64UpU*$%ETtg zZwqbJobB^$Hl1y}pM7z*H*wIa>VG@)2)EI0b(*<9T_S(raJ=R3Epa-nP|qAsMh?mq zGC93#^TDI;y^}J>M%Hv)96qw&EGA>AW^sV(z)CBz;dr}86UaA~7o5o(!TRo1BZi>^ z&WC&<62KEUT0Ne)j|i0UJ*uE>Vf0cBh&}{OLyokA%|H|V=`xybtXZB=AMVMY(U({@ zm@amz$x;TdhKl>_uZwYlPfSSw+5qh>Un@NY#trYE^5k;N0cx3}64*yANS-S%I_G{^ z44J;1I@zB>8ag)6JNoUU_B95|(gc$(yHK|Y`L3vspzTw)LV(G@E;8JAU+HYsXi&Ak zz*l})_wPSpFUu{5>Y0z!+aI(0-YZ?)*mC0K$7D|WI;QX>g}dM+GLF0RDtcx7wi$0P zE;(c9L3~c~2Z%wtikNn$vR{uI zhT1zflWDiOJH7(JVC399`5LdqD_LIjB%0c}^-=(h3=ZP0%AdKd+Jo4%_*n)%h7YS_ z!?XDczG*I62Jrz)CgrIpxZj=AlqMmFv5?`|I zqbSLA?xW^NnHS$s5&ms^ObUm+@`Y z4mVyDsy&i6@)%thDfJ@*&Q;jue~Nx^?+@WCY|3fB#RaQt_%x=|-^kFHEN#7Pzvekf zL|3KAktC_21H#o`@D6D|%!yGojf{0djLcnr@qDiF{cgx?=eY`c6%w~u3W$8FA8*+K zYAxvHtWH%9q)@25la~NS7~N%2VYeT_SKM4+a`LYHSF*O3?tJ-a85`jNCsKY;0j=o| zxg(;8q>Vs`%!|CC3 z?{UNEY>-S&Gn5BXpHeWd>SBBIzji;}E8@7f1>RGxDxvl;Kst)f;m4)unM_R1(QGU`AyrVxDD6ca<9XzTe z)=$sE7>>PHj(C|fRbSO(ECmKtK_Z-RvWuuRp`iWS^5t4%`W(UgAW9|!Z2hSpZ`g<| zS_{lOWWY@9si`87XpK|!JK9f4o)VXYdc<^*GQIzyFu=f(0(R7 z>@t@rll{mQ`~GyYg~N+Yv`|N#=Ee80CTc(1pMU?hBzRVs zOPT3O&qkO9F?FWluR~1kKn8!R`*i@QXz?Kb_S?9Bfk_*-+zBs=hD7&uRfA@qVgj87 z6PJLdJ{Ud+4{(!yTyDaYYXez9o^B+nai6mjN$*|8Kz1H9FBO|KLKW?pgyuxCHa9na zx$%ubc+sn&O&qT9D69FKV(Ybo$-s*k@(6oSucuZ^gx~Y}FV8y)Lb933Z$#Wd9L*g- zI(=aNc?J`b^>UmwW`>E%V*PC+7A_I}`do}P$U@D{i zT5a^nr*mzhe3-&CI53Vrt58x!JN%MS)AK#$fse3H^q3L065V>3A*i%4%pyd-wYEP0 z5muQtD)uRe2q_d6MxKs(+hvv_a3}WV2qGf6Y^_Yto_AV!D#CiTEy7i!HwOpODB4Kl ziY_uS2mXD@%Ys$L58dZM;rm5vH3>bW0K=#e9SNwklRex`lDSk5ghJmU@4}p8-GzZpS+`&54iuk?gA%45vQr z-W>NDuG(r2+JB4ZqJF!-&YEnZ|EQ`*t!8rjRRnMj9TCD!bltq~KOfKjfKiT(;7p>Y zzJ@PTq$iJOK~2`SIe!W%EJI5DW`N3?4Vx(;bAy%`(r0a&a&;pw zoCsf_0SBU$#T2g#x?3+YDsH1?2>mq}jqA0lLpu^e18|)5q)-rMRR!tfn;hUYEVKr84STjPI zCecDgX%X*?LYxk$GVfUten=cyoS^0Jyw?BK`FQx9ZD6YcBY!Gc73;nl z6SY3o1lBWhzSn`}NtAv%T`kuVN|klw|qWUbQ3|Pp_}mRI~v1 z-Nk2?tRJo?EHV7&krD8_zEjlYju3Mx&p??rv|=k%F4Je$M2KnTf$L1oMI%+e-ZoJ^ zhQ_X)2^P3eXfuv$^X}HWcv({1b`7V?W7IzXA!00ty&265eePB|djWHW?~?o*{j>1P zA2$;IeMLNFD@#VHu4{+AH5L0yV;QN#TK=Oo9$IdHYjm?^Wn_U&gKrQW2uztuMV!*1 z!Qva@Qtm;U@2n@|WiEHdz8x(sDhtI-n6XyTs>;Pt-zD!>`jbCyV{uxk8@giujx+*z zxP?p^E%jBiirf2EIRjQIjgrxQa^zLUW!fHy)!GN)N9k)k6!3V@DY|3EiQ!0Tja%zp zRw45A$A80{pBQdy8LrN?+^E-q&*{9Tgd6=(k40u1X$2FmIWgSVuD;$D9lJ3iM4gnJ zp58J(&VT#%?dJu*X`mcW^CCu4DgWLsBenzQY&J%**76Z>&_p#NiA3CcW~ONX(Flc z_mu)FlSSZvSLML1i_u)j3T!U;McdvA}nZM4JMZ zua3;{@N37@ct486`l$8taO;Z9Y}r@R=~NC)Cmw~bD=C$fl%T(?W+&4?#A_kGr!wU| zhhC0r2OG|B-!_kkHdTdMZ9M&&pTD)7gPc9@YSV7!H~Umt`Ip2i@9MvXsC=zV@&Af` z1S~-Mzf5@J!TaH~zhrS@`$f9fRijxtfIHcTJXHlxd~}?2220;H7blS>9%4d` z>*PYm@8#nbg`dWB#vkHonQ=v5eo|RHPysB_RLl|%?V3~dOeMZ+gi@CU%1=G7?8Zw~ zR^#(N+x-;w&(K&ckB4C>WCBk1+s9&0k9h)#=&1a6XAP?vo%yZ=xde1Z*lBJ3lIris z5%~6LGGt`LYTF#hp{i^u;VDxv@mX3Nwc1u*i`l9Cz0q@cENL>SVD(?@Y=J>c{(WNl zn+`&IgZX7~OHu9lt$R;$p1l*SF`5j6YbSH6`#uqz&5e?@&k=d_(*RB?#?88snDx~h zpg-$wnx1;N9dCSzo&1P)l~FtsciVL5cWXpI4+PrRC)9H5qe@JLk8ge6(zL!;^(AFI z%=^Ch!TrTCiR?Kz^LL+12D2Pg7^S8UgIm0IBc=4V!^NSVcd71%l~c9m20_A;wlf$b z(m)cDQXUG5P%54F>4eShb{AVSo}b1B4m(~JbqJS=S+%qI{VGjsEty3_{eG*&SiiHd zqMw*gWFMy} z3Y*GrtQ{QMVc+7HWXgbv%9Qk_s1oIMGFP#7D^YY*n6@z=nq&H>6hTD$k5%H=u!qHe zLHaHz7u{8#h!KkP^083a9jy)}nS(S=#26zRubESI*!H6jI-~Ig$fshVf8Mew#VJS| zw;Z&8Kz+Y4f0vw*rP~Zv20Z9c;YvHkITr4oW-c-hc?4xcjYByNUL{z7G75_ttWp!6 zV#dquZMBZ5#Z7$^Q3{hU%*(vQN8v{ZOUZ*UbL_ zYimaSAFdH0*@6(q1r)V5LR&a(Ss_)Yg{pryegBDVGZpQ%CNFQx6dj zpU|S)zox7H>f^BcMAPj%lJJZ7%TK{9e6ia7?BC9VH+frca4{4n5+Jj={C){}!~ay% zwxi~-4=U}R!qsvMqhhl-oKXxI6$Lq-7+Xhvo7%h2T_Df2a&tCC zWn>~l!G4TW$BPo0fRnJU)MwwJ7l&h}{r2*H%8~AB)&*tZh`pi@mKez59v)SwfSgUD zzhw}5NbGJW>f;u7Qc9!zOb1$tJ{#FQMr#vw44Fs2@*hY^g~L$Rmme&ioa_0aP^bnQ zhQAff8)krt86oO-$R-X_Q`+fXCV*}IVY!o2$9+@D-XMxmh8-I%M!Ee?E@ZOQWW858 zuaybUv$W>-jXSL4o^w$L#Hyw!fcB+1H5tm!E7WQf|vM0N8g*B{AU^Zvg9- zuHT;6Vs)wEok~ilhGE;3m#>p|kv5vZ5>xQKP9p<JMiC(JAbt;22|( z_FG3qj=tA3kLP)!VydgFJuQt+vbkqSU)j6%HCndT>o&wUQcg;v%&^fZ`>F@emYe7c zmYe?Cc)NE3te~rOags{;tXV(|{qI%IKROVy+VqimF=qRU*KaED<4OXGb#&*gVGa(o zd)~}8iyRb;>AEpKdaJ2WIMbaw4|~jHCIkUXoCW7_wbV?bFNqe4s1|iG(fUaU%8nwu zrsWfBrQw=PhA5x3YaL(qJKOgMH=W>{qB4)WGTXmurXmFBLIj9??P2!>ALl)2_u*FD z>W(m}7s>S7ez(H}RL^_|7-?$o*#TBF_wyTvZsn1yBrX7yl@&(1rxb2>h%>pXDCj>+ z*7AW;6FYpwy$2}u(e^|6{%}i$-L`1t+G`ZTUSzw-gl0A0azZ+e!MIQ^|b9t$9yLOCr`MI>c;NG(-hTvwg_mzs1l;S{yk&-8h>J^Jh^wS%@Th}OTj z@#tMWX=0<}ob$iT#rIWv7X@QV;?#M`WtR!0qx}N5+HVgRU+{D4O2VcTagO0lMFf}XxU}k*wL5E9(RifWuw1vh}gjc;)#>5Nc z-deJhu5FK>L8INM2v;{C_K?>{#W@uM+fc|#q&t(r6BGYME59p1gpL$o_%8K3@vaA$ z`i`dKq_ZCtSoG*^I+8I87Ge{6vN>S_5cxnimwMFO&V~AlCHRl4GR8v-^-e27O$kZ$ z^Y@?{^CwLwRcAv7<-Q~jx(V!&?&Z7XtyX-DQ@5PYJ2T;RV~();5@ z#yfQr1;DsAQ=WOk&*nLO@q9tRAdfqd$=*g^k5;t&?=+nfnj8{yPqAkQ;4L6xrHA_n zIm?d4-77s?Iy>~AUGiIB+AILL=z}(eW2K)8Cpnp*-^*9AKZSY?F2-fEswgU=wzbZ7 zwKRkA|1zSVyWjdmj1omrg1*NRXNyzG_ptteHpy~zga#~=h?gkHK zG-TST#*+{hDZ(sbA1iTr~hpoyxpS#3b?F4-baQqrlOPs8(vOZU~Jw zB)eLfHjEP~w~!D{YdXFZ708m);(6gpQqH7Hj{x*hb{OVpyd;{`j6wv?0A(@=ijLZG zhG=`yJN9t+IK@kZdEK9X3N#)vCtVT!u?9pu+yUegCCScZLeM)R-bfOqi`HXG{Ujro z@KHpp&$zP`A4ZbWT6Z9~v{QJ1M@Eoudn={q4ezS<%Up=Kn3%;XKr?k)%Zxd*n4BFA zYl1-sy2VB|@O%N42gKp2tC^#08`nz2GhHh*Rg#Oo_0YS znR82W7^&zWv;459Gl_F=rIrE*KDKh<=aL}@D6RfFdM=ytcfaqu;yUG9#gj*bNPqL} zUX7c*VZPB1S5=*Y)Eqo7*vmO0!OuHS|&Y268%a^K_o zvUFuWq|~e>m)NU*VpGW)h&*~4$GGNk-QY84(!TJ6=N$*_j6if2;!%8xLaWZB+aVl> zJ>&VjNfl`A5+Rs(?fW~B5HO`Wg1em792A!9sRgNUPT|4SWuA;?iuL%_{Q3lA43UCP z8f=9P!a%fB`;GfEWZ;7jqh`^ZbQpf<&Bl{8M=W4QdRh)YI9~>1;fVE|4{FZgv`!#` z-*Tv5C%+@W$pM7QKI4q)+2mgF`xFY9qZaY%DHg4yykv~?0Cj)KUg%tv_B#;s=v8tN zN>9*D|5dDCFMo6z%XO813&OqRH%}sc<<=ICpDTygsr{gF{_&Wn zFiSV{af26S-_XOcbOX0qSMELmmbUs_sM($9pO{UjYw_`~G&COE1cFwaja<4Y=_*zF z@+Q`s_sO_^^!4@6h-RK34DOKnQXLc-c`R3oO`puUdaW&}8Ek;Y02rVR8i1pG3(BF$ zX@yW~Uzn96FP*4+&nz6-%FNXUxplfpeO5|a;F-N1%_@WV(g6_@yl#=p7V?mYawq7{ z@!?HDOym%>iXn6dWkSRqts-fCWWNO3KcI|h+*E!8-Eua09N_>$SMToT`~AsBoFT$; zzNn4x?6aAJSTGXk9^n_k<|eyAM3$dDCZ{fRv{*Q0>~4Y?igZCl`%zA*+QoQR#WP9A zCCaz(uhy!jQLQQTOB5%ihX~zy$-~~mxwz$j>3X!i{rB|zLllm36hyQ`$6v$_hS?2M zB0+k*lrGoy98VV=@8HiRg&qEau9h|Q8~UL;J%^_L<6N<>X81Wdz@D&_ymv%0<4jZF z{tIIHv_uBLO6ot$IkmKY36D)0+-=(vX`d8+2_@(^)C3g?txtCoxazmw(E2U&7}X!j zaTt@((v#c8f|Ez<6dQ-vzzVw=myCB_GYr2OZp}-2HaEub+i5h>3!%IJaCT{Go7crz(-ac*>vMd@yA z;H@4T-06xC3FWtw;m(gJGn;4MIp=A)x@pl`gHfC;q`ZNS+SMQ;2s=WZm5kXcb%b8i z0nH`w(3}Cn)(_!-d~iOKX2i>~-n-vuXCb#mP*ivrpj?niBSSv{+wHQD&?opUVw=dA zt6!lUo*UseFCPlp@Ptru@%k_5`C%y1Ks=*YN%JhiwK3Y!M#jerLjI$xuAN7qlg0y# z&>f%azSDJOvjY1^bM>xpc>ZpX*KVYl8EJ^8pLOj(!{qO&EQ1;jj`eL=KrXe`pV|$R z1*SzBwWu^svnS!^r9Og#oyNs~6b9TaQn%G?e{xg%rP62gG)Y)GoxHb?72CUejcfb2 zttJq?@zW2bFA^SjE6Gx4AnSvY@HhNgT8^wd+aDX~NY?@oQwtPLC6G1^ghSV!vt(DB zhw!5vL%0_W1|tD)W08}xp^tZQI$#%qu|l_Lht?QMtb4>I+17V}oCN5K0U!a}SYcLC z8_FswYWwyWT8d&F8&Dl9MV=mM)aMKB#%Is?9}1n3|I1_^i-fncG`A?XM3ETBYQAJH z9oa2?l;7;O0@5lT51GE{C+k^lxexjcNfW*wck(ksd^boB%BQnzRxzsutTlI-DZR&$ z$hGIn=A>S$-CRx3BQF|C!i)I|BIuEnWk1tWXeKw{Rjgg_02jx~Lv90kQ4faR80aY-%t6q#8^et51>WR>|3k~1zHA|tN z84+LO&8Fo`L}?QHr0esG=$9bb+Lz61LS_@_`FO2IiA8U1GFNislygm1=e}2rOT@%F zJNn@98nZg4i72)#FpB~mK4*jFx~WfZ4w!{62V7&681ChfR$)5N!9~eCu$A)M$xMp$ zMUGrsaBV@yVsc}&92*uKyjCs;=xr=!s|`Y9ku3R@R(OrVw)O?AY+iDxlT+6NJbN17 zQIA&^yY)%s$!0_8f0Ud4&-y09BMvip9(Gr3<_f4=`hybczT#(PwON`x_(cb9sdPqz z`E16Q(=xne)z^8R(U@O+>PP!~8*-@xA?ZOI!o+gG$a1H3QB!t)=3ez`nY#7^bzz?q z1ttvaqN6XtFehJ?nB$3Q_gO^sMyP>yJRNf;0EL%%HoNNMSQyKiwGwTlB0WZkP2+J^L@NVz{IpH& zZV+t%=$;J@y{co->XE#f#kw00W5ds9ambDb$^%iW4~srL$%>Rm2~E1P)Vr+OVH4?y zPqbHo@J->?Q9wLy75s9s{7wJQ9Hjia>@~>zAn@*&vI2Zz)TDe!rxkR}xffA5S3SZr zb(Mh9#EU0!x!xc;-6Z7G1Ltn12gGjs1GUc=z=j4SnDnOYC>yW4s{z|Sb9>`|xyf({ zz|^WX3uYt!{pSr!Oi_Q86Exc-s6fM43UrtqQ zL^KP*Jk4*Akp)3O)%nKH{wzg7#HCO-Y=;Zm$}D=4r2R#_pT0GQdjn!Ct6&~VV1G); z|D$s-^TRmeRn??-0qu9l(ew7~=a@DQ$ffP(8*H2{C&$%RY~+t4-ss-Qfmm?$c;aLe zwUx6+Iw(XVo{q;yUxhsgjB2Ip(GQ)ax zLqc}~Ulat`o-<)g80De-7?s!<#XBKFY$A#QaisAHpN$5~G`sjYbZqqQZ*EFk&SPqIxVKf1bbnmS@lAq;pTJ@_2)wX zyJ!pmAZ}Y|`<%?Ih($jTs7h3ijot_g-|1*PJt{dHLm+#ccws;E2+T_{uI=(Q3LGki z0PcH?w#Qq-a|EbgW?6UdV}#4uO8CpO|5gTfW1|0W$mx?iDTs{I_r+I|FH+^CRy!q< zD!1Bo*gm!_6ml#wd$Wf)9UUZxYODHe`tc|ATl1N2iz?Wfeck6%7<(!Oo9q|B`iu><1);vR$LXyb1+Oq0(W;5FkOBoMK&E=hZ78^ zL}=llIY5qL7mNZ3kZ#!*`q>W-QgHB&FhxhtHs;=z+q$cy@g;dH&d%bxb?NdZmKin| z&@cTn-wmJ4@2Xo7m+*l8COp-lwo7?Q`$F!fs!v9T`b>le2(voF0;=_-z?_{~lGZ32J{zse0|9YCFKg?jgBN;)LG4G?ofKc%HAQ(&t#Q;bPUe_L0No8=v=NjZgmAM(r3;)t;efLia7c{U!jN zpW1?YrQV}G&WB&CrsMVqPWvZS)2e>EzSV973!rW;V~$f)F^it$JtxTr-c?qxU{@9lXpF zB21~rpF`1ElS{oD4r5WE9CP_DfYq9b0y0CsRChV!y%D=}8J;g*ynr5LDS(0swKRsb zhL-B)81<8Ox}*{R%&`ayD_gH9+eqR2RjB@l zYIKzEs4T!6$jC>BpFs*8dzZ$w-DcA^Cq~6rz;v1RhC(37xX4lL&Xa)QMw3S7MlM*0 zY=1&SmB;aZtK-i`+y2akTYliC$yqw#nng zq%3BSsBe{Pjkl>bx>}syeEP>sM3P})oV)NdoLAhg{xaTUGJL;bBNSybXE}E-hcu## z+zIyug?lS%bGC(NXFPxG9O!h3x85Kt6qojJ>_H|@_Yo{xUm3!=mElfda*04QR8Hqe zFhn7U_4yxK@BLG;P`(#B5PevY^}u5PXU@%?Ynf}2RgI`LFJcP1H%AEVzqm8U-ka}O zVJ43^pB1HB1Msl1AAo0A7)fim8njj_xTBxsKEgvNO*8vaYPl4MYhSnMlpX>rGQ%+3 zX#11t`WI=lGH*#5cAP!_Vr1&cc{t6bclpXk*&9)d zG$1ZXzX%t1npCt=L{-Bct0$(%B#OSLW>*L>m{J_qk21tOFMNU%ZQl4ur#$ng&>9<# zx|G~tcbbf`4kn= z3L=?F;pcx--q#3jET_tf((JABy){(nvw=eBDb9S{ABx`JMJWg)!+~gzOELENvEAXS z{UI$G|K6p9y`l@MQ!7E5!5%}ZJ~(;np2IKf81WRWNUO+LnL>2JZ1&zybRSVa6J2OU zcXME&;~7n#mOLxLRd=9imqt2yWLd`uIN;fty6)Vca(zVmk|S%kJ%SN=dk&=P&@ z_3qAvr!KhV=C~(R)^uv|pxW)Pf+`I&GqgaWlZ7XS&p3)tuxX5cW0%ZEVNIYfw;9Vv zqC#w1seCSn^}R>t=ylx71O-5D34ab>)89h0(x|XSC+M3-p)1Vf5)A{wIB(e(>8r|c z8zL=$l@?p8TQQeV3K-|OKqZj%!Hm6{iiB8Kw8Bxdvh~kQTy4YqK_5LhTHAg%Woru0dV7O%&I?d+@JcAhGfhb+i%!~UtLGnf8}-Ln zH}1SBj*8E$)iv=Ds$Uo{aK`bI?8~NARk?{h2-dFm2cb&BzFKJ(m_jL?kS{BJe7?e6%?p+g)L2of9b2S|Bt-2URmQkX)+bmsoU)c zE<`pg_$AB!<%i5R7|6GG?i7YqMjuC0?y%u&sHtT)A#C1GSAqHu8V|dvAfF9iiuV&< z3T*JVS3TBc5p^4WjQbOHtoSPMr==j5vzT9Am$68#9J~dHb-lM9Wb_VM9Apt zr-=@Fj-MefJwpvd%F49OP26tu;Xm1pl&|;aq+R|#CP%B4_ok<2?FGCHcXD{*adM~y zZ91X*EmgG&N5j9GoTT9vjrxZZ^O{!S9!otw_jhsNL%*@-e?jj+td7qk)e$i!?g%7@ z^NBe~#k)0ZZkKb!d)8QJ4PSLVwaTvJ;2>9OWp@bQ82!*7c%#lK=!#C&N6BZg7_aXH z&OUE-$(*oIi3@(yITL?45wMq6BQ|$3Y$Vtr3pID$ zxyq+9=60!`PTMq&*9kq*X+JaZ*QJMS-cqH5ptlrsv|t=VbjK%E6nPlRn+@C-v_Y%5gP0Vtk40%g~~G^tCTnD-ypx zBP?5^06I>-EYU){B`s``c3h7H~!38|7*0$e0+3bNu~Av+V*uz=g< zChVKy>nhGKrdM~qQ{Dk6N^FTb z945f?&?3(^TLfJkrZzXjUfCwD7l?~Qgs>;NwdvlcHgt9E`rOUhG3*hk!cUf(cEQf! zU5D-iOhJBy62O4pR*G@&)3m5sFZdlA&s;Y1PAd+-blZ{wb*;$8Q~>(FpDJ5pk_ z{*YDo{ypBkcX4gH=8YK<*Z+NA{&jmS8QyL^yH4YsR+o=tb;3|@v|y#!SPcgsF#820x5%;vbKox$_CCwZ57~|n#~;U4KC}W)L6SZf z)!8=oWi}0b{`}eAyxKA+$Nfz4LD|6v>}6w{4#xkwAN8@ZTqJhv{4)eB%X;A!-hzV^ zF*H6NN1khbS~(7Lt+hyXbiI`^%LML)rl0NjU%VeO>*a5{0dT#E-4Z?J1)SRxub zYX3h^$Nzmu9trcR*AbYjsO3%|Mi?UGlTORqLNBB%N7z#TwL5|OQ^Q=o($iq zNIPq*#Z|2jsTS%mSANc%(<^^1=DLV?w_VXe!S0hM`tXuV<(B^ApTKa2XfDwIK4K2I zkJZ<|RaT-$F$^Yl0AE>CWL!S>x!ZyOUVg9R6!=TV-Gw?*u^;?P?pfzlCPh3(>Qxh6 zKs8TU4w*L^iGF49=B|t#YmE}C!-bMsf3b%hDg1rk+q;4`;c3S^T|ch; z|39+xA0^G^{PA}7iqr5*nImK6;^KO?EhZsmpspT&^Tw`$tLrFkx`6xaT_r524Qp5J z@*s~9?cxjxUquae`xO|LB=j`s@)PeGgvP6HL5cd6e!t#%Krnc-zM{|~9lTY@tW&qk z-H^R}w;*vHyoIjxr|tEsnVC;onANOMaOqN}|IpFZ81)(sP>qi9g+mwECFuN(y_K~! z_tuh<62Lcb(5ZHt>N_u|lL!qob$-rCt5)`q?1{9bq-WYxcqhk#dQA*QbcK9dDe+PA zEWtqJ+jSfn85vuDLvegSX;*h15T5P4+nca$ZTGF$Xvbbj-{1ca?#iJzv78}39&Uci z#P^H%E3mVcQGn#(agCmwm#*RTr_fol_1GwXE;TNuc3RJ`(eHx(`?l2^Ls;bDBcsl+xH-2}q3n&=@6gQt+GO$^B>jcK%xi_-f~%zDxxiadp&2p>~G~Xl8;H3D`TN zOT6x)NE=$TEO=EQE-eyankiQ0G&vPkS?PX|Ux6 zK5!)vdoY4(>l>Ot_4mtc$l{jjcZTq@_a@MA46N=?L>0H!eOETlS?;h$-7oGZbX@yq z@g>@Y=||Zg_e-p--;Gubz+3^Hw8FoZLs`tHRqq_{@{5F7ZG)6M+&w(JWw$XU_bMZ2 zd8vgj_O1Cf|WFz!^u1o!UN3+Y|F}COL|9tn*tDCkCl5$-{ zApOT)=$)%MrvlH0sv6zvn|`g2U;X=O?);D;@Wu=9(G+Vrj#m%5JM3DojCrNq9u0Ns zU!LQ-lS5BYIQYq7#CU8g?Tu55%8*=D8=K#OF=?miK-6|k+MG4LNcZVe*~?&MHT4(J zvZ&~?i7LAy0ReFxrf#+1KhJBy6^VD7bnN#C*5tSH9g6IWbOmpZ9~IX)MHdRjp6awofwVce`L10$P|8)&0Ay? z{r9PSao298JO3#$U0hVV;8oG!L2 zFV|`MU5KN^X!A+X7iDBT%?lJTC>wba?zOX+=Mq+$OUvy__a)sL$Y+Y*V~x6O5J1l` z6Bu?wYWBW-)H5z6I1P8o!O{~BwTG=w+UP4PD-XpSClZsWYXXhBKUKOXDm)V0r{U(* zbZQ^=HOuFUcS=b0=4AeM%t?yj-Hvd3lzYRO7rBi=R{T@<<6;!uBk7FT`~w=N3bOqJ zjz|&bywh#2qHd(ZerNWL3PWST{eO_%`DK*B7xQStB450T34$*OIBChoBr*TTx%uFY zq(B*Upq5p6fXj=~Cn=^p-R%C|L*t(eo&kg7)EHhyN77KoFwQ?U0b6W0Xm4YqWm+^e z(a>C5yJ$F^HeXvtI^1Evr|pgpODZmpqZk3PNP6SLJ6 z(F$=3_`<~0#lIDX#7qN*({v6w&&@A}Y+J%Kf4JiW)J12N84qVVP1n?|MwP~%GG4YdHo$_(c{5(4o8!_m5lH4Ajl*absad^wCkw)T_t`Y5&ik`U z#owPvj=fk@lb$a)_U2C8o!5H3?%d8{WdB6IVy^g2-MF(B(=WYWA#SpSr-n^{Uv`o@ z_-99Fb|EFc?tRH{kkz~cARRJ>Z>gMza)b^N?{lpwN_lKd+RMqw2}X*{;TpE{96n|G zbgNAUm;0t#bw|VJ*;5amD*4vTewbgD`g>e_^j6v_0`JN5yr?@cQgmOWbveUlqcl>m zwo{kiK=04R~*{?|$~(a@D>B_bWV(De^1j@i8}TT3rCM0G;9`hcUPC+0DAt`s;{gI;(yk{_2pw{L z5xu__b2^TM~Rwgvl$C?+O`KTm+3Iy>E;xUn6=VCdZ%e_nFzjj~7BO%xOFh`EDeu#Q#G) z#;je-tg8{wWQ|e#A1L{3ck!gB83MW|f!7tClPBx%@pY4Xf>XE0CMMRly3R(eGAqwN zrt;iMNIVzz5MLEqaBW(oCNwuVI7jo_RUD9Bw6m>^-ETTOL7R8o;<>DF zvZ93t$do@{i@?XzSJ!|bnr)9OpA3ATN_DFH2CM>Q7p&n>s@a7`-md$WZdwY!;UBHXpfiHPo(IV5%Bi*?7u9Agono znHDZU?LBkj#=k{HyL_C07Z~zdn3Q{8yj~XZ>LV4v3=ZiSJ4~e8rU#E!`sBx720YqZ zLfgikGnRa}+1)z2eJ@RZ5Wj!B#{=j{8;O#bTuwBN@u5*P;`qi+$H1@uwkJ`*RV1h* zWVzW{tMMT^jB*U=X{6#w^D(n(I2k-sHRV@Fr>`FcoJHiu+-hR8;p{CUQ58?AnR*iH zV=xWd7mPCfPrXWhY8|(0Nw!=;9gMr;MyJkBrep>^`lXht;R*CEPq!^l6rNu*Xrwx6 zvQ&(^Ic9Gv#EOMdv6h{C!9AvY_oZgYW62IZ01B=a0jT=MfUeUtM&;?lGc$JV|Z>d7Hu`LG&ksPv~{)il7XfOTY^WK=l@#(&u7zKk1tm0dX{=zM-S_} z3bv?A9!|(yD;?lVQrFdBdffV(V3%rMXY0DnXDAo0vGAb4T|e)KWo5S!-%ygFx>uA~ z63%^isJ>tN^>7~HwXN-pcd;1#@M{b65LWqzSSs*(X~sO&{zH6=k~_Z6e*{_(B>r&R zmMo}2TtwPDgfo|2*TtR3X5`ZA|HIW;hDF_W?VcW#7!c{0p#-EskseY&kW^_9Y3c3; z0jZ$`q(neOx;rFf2x&>_?v5E4*ni&r-0!}h{fXn?;L9*qtZS|F{GII03BSb|LM`Pe zD9k#LbzU3O;wj!@@I91fLzP4Gh$=pG8Ug~vT7NXk$%fZ#@jFD0Q*KFY{)o7a(9EV3 z@hqYnxJ!JpeJ?v9?gV139V%@=@LcZ|{%yed&`nz6$JLj*d*}DI zO>vn=4$(Pp`BF~zn?kPYk@qP?#5Fo|&uy)6zB)jYF?U4qDQg)fOJ05W{=Qa|fA$-4 zd(pQU^J=2b-aX-75kG>dmeI63+?<-iiqhD(Lf=D!5}z5tjL3qZHxp4OKI)$(;P4gR`rz@ zst9|C6}{JO`;%2ok+(01PLYiZJK!(`QqQF*gkXzci{K>HNK%-6BR{{^%W=#cQ2UoD z&ROfxGRCOOFX)nOvfuSlj|9V7!Wp9zImEGcel%aXh1*)J_;Bj}3w`I`i6(iuFDvZ3 zm=27uqXL%fLc_ARWm}YF{e_DM4zBewHo%d6l4C?FgKi4R+0B(|NSuC`ehyO=SchKeJwqCLcR{yG|8+B z2+z*bZ0g6E2IoTo<-(jLTm4NdR%UELXW}+#HX}vBCr@^9z^gC+tipif3JDO(`Y3b@KPQFl3BS3vZh#qAR88U(1IuOwiW#7s(Ela#H8xL1H? zRO<@37j3TYSZudPup5P|p<5(uR=W-LnhmeQ;D4r`SL@H(w(_X+mddXC%BCz~SGr(J zg8e%Q6^g=m@CP$^zv5L0h8(d!;->8OtZnA*&=Rh|Ut(X56x}40z<=ECH~$zeoOH!| zd(11*ws0AB623G1anG6h%6ruC_KfmhF%_8a>WyeLsVB8&#)2S%Ao9w z^cB~pMeKXwD!(%0YFOgx^JMXGnEc@t*~Z+VowuoxX=`c?%5^{0Y5Sw)i;x+TS;v7R@%GQKHoIqe`i zzn*BT+Z&oQlK>sm+_V~T0&O89yUL)OcaPttySdfO);KiH_;001zJsC1V99;GUS3}> z$`_Y+3T5feQWyyKVFIG>B@|-oYiRUkL_&xNgrHy5SSSJqf*e==igiF+Eb;N0PMugz`w&=_45ux9oLa;0f-gRom)hl3LRI?)CN zno(uE_Bdxo5Uh_wGqIZFM_zbWK13Kq@{Sk_?k_`e#3+ur#Yh~l*T3X64;Io`d z3Q(-h1*G5ou(Q_g3lSg1{r6iIU*(xmHyww<`I>zuSK(wr;zrjJYDG~R{+u5=FVdfGi0-`mIRt(sPO4MDXkK?j)C0$>aFA`g|HQraWr% z(C@*A!ATED@#6FMex2UiSaxb<*sA!ROZ1&!W6bFjbd||mj=_xDlKPmGP|`VwoJF9J z)2%(SJtXydQvtTE?^&VO^_aw&Od#Pp5+wa0pKd4#n^W4Z&b*)NlN*T(6RCV({D&L( zA134qJKK=(FH$*(-AjnZn?4!6)=OoBV@f53rq!#@oNHhF#m8!ef#>9!hQF|sXzzdB zek7?t-sN;kh(40lV-);0I!C?g(*(m#)}%w!r_<>KbA?2)cl|>mRvY%4HAIxe(;uQf z*rM0{_1m_-m&|&0y~&=mu^FUQGLaWM2>zN`W&b3fx~9Q}b7Fq*T(6Z$pYPWB;1Etk z1lChfD7b>`i`eoukyC&2TFoZM7G@N@$@qiaHcBOpU4dP}q8Vu!FAg`iFM3JuT-=mC^e zM4ROJz9v{%nAA*ol?*btSTiq^LI|X|wxDA1YShU40&m>mF-cl`@>y`Z zK=&ka`br$LG1WYIianOfgw_z9JcxN12NoC!_Y{_*xyD#yfz=c&Z=f;#o&Q}0;$V$M zYB660;qpK}#XsI=u{*qbcju@117#~~^H={HVf1l1ddC^T#Gzm3jmq!^tt|H@eBFCsvw4 z+EcpJ#I2_VUmC#`!7}K~EXhJ9m$i{)1~5!{ENB1jqZ74#kG&&3<0cY3p|7(y$WiV>i5<({34Z8 zj~R)OKVtumQH*B_s0&5tDoS)962WoEyryzwyF}hA_{2^x$x3h-{ZtZF-rtcHvcpG#U=I)QGW^=z%3OO$Y}ecuDHGiXO(Rn}D+L)v{wjjjV) z3mt+6;dhD>Ei2Bxh-cH)IhXz*O|g_x*NhE!Wy0!>^CVFaCj*Tt$uXES^4uhWy+X;d z`6AelnYoc9&yjFtY9;0QaT7G^?EFqO1?&4s9qMZvT6J~Fhp+Fgq~N{%qrj zbGm)W8jo*AWQ#t)V~1=BE0D^>HKGe{PqVN4D5I@m5wKBsmgThW{asr=dHHbJw~-ze&KtE`=kx26 zj3@+Ej(RBHZZ)7{LEp_x>gA~^EJ&W}cuF5jVMyCbKYq7{7axiX-B^|uC8cc8!IKeY z0>8Vztb{|3LrP&zqR$lV2)Fbl3FR(E$VeVTk#fY#a*s6dYKJw25L}Y)F&X6zIyl7i zKSs)WooWyQSanBMaKuw7gVmEi2E~htvT2ML?7VKk0*KFYhf3N>H4RI4v4QCp@AH4) z8jWWo4p==h{|i`75pR|EfS|J_$jM?fKoV|S7-GF1FOXa_?hK1~osDsJ_1u_Fp$Q&Nl*TG548N=(zyhE#VUXlP9*x8nvHg=wfCz5pbZXY*@M2^-{s7$?aXF zkV6)5Jtt?Sm$a1Fo@@+Od2Kb=sJh&DczNSe+|n}kg0=Rm`%H$2q3Y;cd0|n3giK_u zcP7xa8TY;-*3HqWddI>1?tY>7wec4Nod*dwmj3IOe;bLg@S}vd`lj&illCY)p2E7{ zfEW2oK?FKt4+K@SB{|;tcI~DsyN5okFw^N|Fs6tDd_V|+z&>NhcR?KbC6X6ith^Dd z5ps%8i*XrKpr`c@CI=7lOM^uBn>%gXDBXX7dxeX zfXM9ufE-THuV9R;ACGU;dZc@#@w@}tc^Fdsko4Y`Jk+HOC^fV7 zf!5wpx0>t-`32v6X@#1^hr|^Q_dWU)7dZSaf$DC<{`RMb z-5-(Mi9d4|b8{sVqj;FVKqMbZTB?ANZ-%yZwVq$sIyV=abgujECS13rd!ks^qFX?2 zGUD}fy<*W(>$Asss_8`FKP3h_W>M>9Re*ALrr*b7TAtF-R{0ZGm1d+}1@+JM&?tRc ztZlfRaS5$kvB=WY{DerWJUOReL{rm2>ug=h@lT>!yfXc|K_&K8lb0Y}oFH6MdGX_5 zxyJJu8$;p=vtupMh-a%ZAJDr)H^o=du8dKFtI+cZY+X=A*U6m}U$h}qMUg#%TM;%m zVLHq2RmGa@az6E#Ao96Ci8YCtPR>X+NdQSS{L|$a3~o}X-kIGjvJmTX0v&+1ojMT= z4zCoqc(h&3seU%e2O&Vi-w@%4!R4z(-6&E1y~x~H#A*nAHJOt`^U-q$W3H>aaL^0v zYNEYxu0HJEK9C|*8ncVz#`c~?5&vpBzF6&nrcoa<2<+Rw!c3XQq{m1fucW`EU{tw_ zeK`WI=A{TE`I(e`>`2VW!equQ@Y-W0&1d!S`ZsN(7gdLT1Lzzv4VSFdNGYG}*HmrVKI}!>KOW;ytm&TH& zk)Dx$VsZ~OBA+W`VcoR(THA0flza)%UU{cwa6js%j3cwtX{*>vwM6h-!@*0S9H5{7 zI0+X?;(P$;eS0FGE}Ql~_bRUpVF@`5fd|#28nUlM*vg>h@P6_lm|4Pe6B4sa4awsU zhybaLWFOHM$of$*@NUYzCNZog!y#U!@KbG&d#>rR-HE{xeV#LzM0!kENm2>MBJ(2o zw(2x(p5i4EQ6T*S-O$rWV)MLD83IlBBuN{`wvp(=iBQXAl&g3cn#U?>ZZ4y4KZAi4 z#Y<(Q&+q?2&7f3;YE{dB=7Xr_tp|V?SG58L1)+yYuc3B9{c7!1)Y9kwej4!!!Z&DC zlXp4qCa~1}x;ExG(?=}23}$drW+iFf35i_FT9{qBkX?0m6veE>#k_SJa<ax2zbZYt3VypCmQvH zGou#WG6YFRkJ;qFpM1xqi7+&R!XW;*w^aCMxS_;dM1jEamX=$MWZ!>ba65zuju8Qjg9;Y^xhiClWfrdI!5MSKHu>>=V-PCBU2KZ{_qJIRuW{9vXhE*ixN!-gVA^$J=juf5~ImXvG`Uv1#aGs-; z&}(#O5D|)-eGU@&L~4dbEsXhUu=dxxwUpzR7om^3=k%^c_3Ub6MWRY<_GN6gtm;Hp z-=37$cu?BzFITz^7U|?WcD*QRnGk9FT)NdD`95>*W$oLgcjY)>X@;?s+@b?RJf>#;sqXGhDeKD(zN>q`;&-i*FE`RzIoypm#so+du!fvO;8IQzq0bzn~5px_X8M&3Rt(fF6b%UJO*y1JDt zZgkh|aLm&C(%F7^KFSczWVnpV$jg4^Uz29`{cq#O3mPt3a&7bX zTm*Y^G#&LR1~vRgtg_~Z73tTCAfa*xua@X%Z4XL+v}{N?)GozK`>dyxoc~QMRAIK+?c-dqO)Bp+O0EiI4+>KDS6*iPhSpK0}p1y^@dk z`KGcb)bagw$WlJ&W1UR)qE-;WfBGN`r z^@14}`k(Ug*%#t%3fH2U0xHELhW_lt_tBn3c;jnnfJ=Q_()I>Z@yICCq}&)G?c^et ziC~I&k{?M?jc_(~YQ-pX$ZUm3S-cQaNyl;M+kbcFs^}f>ZBA^?(mnfiL0JxL=G;Uw zg~q3;e{punNib7MDcIMxU`4OTYCa_ERawqVF4!#+-Nd_1*T~fX4pq2nBb^ zbV7cVt*XcLb{OW)p2YjG<3Kl$p$g5MF;wgk?xbw3ExEoEL2ZCTb_`+iHD}R7hHuUX zr5~S?e?o#Nh3x;DU0p!RvpQRZL=#A%B+YV(O^!5sfIADlA zyd>(%xJkf3kWB}Q%d00)1jMH?F_XtRjvl+2O1gHY1(HN1CNuwgWGVG+HkUJ38_2BL zX$9tPjS!SDv4oNL(>p0ZyCIZatcbV4`!_M&lnol~_vZ1*NQlO4TR>T|o_BFgN$+R< z_ixyV_4D6U$P_O0aj3S;Smer@(=t$3YPz(!d3ekms*ilY8&JYR$kmDvpV#U(luwC@ zi6&;ex&87SfD3nXUcHgU$1l$KPBKf*L5U;#{JG=J)c0pEa^T3Smb!e_*;bu0OUo*2 zeO6)-1e8vplaaH?*G8h=9P^bUGo+WGJ&e}KIQ4XObG38AUaanwG_|qS{1qQWPCsjU zbrq-QaKO*`EE!lgzXfUNBtJjA{Xt&b`=UEqiakklog_5*Rl8nOfJ&}CdSYV!eewwD z0$%j;0x;0lj*zI``}8sY;lxJwdmEeDcZG|_o_3Gbe0wPben#T@spk=!1coRwK5ta^ z7Y2KzkTNn+(kS7XMTKG@9rOFe5QGTgQeli?3<++J&)TKGdAPYD-v!h462ZqL!t37>+*eBNdhu;37zT) z$iKFV6g0%u_SE{+;?&yJmd>AQbT!6OHGQ|}ji9{I3*T?4md5I9x%9TZVXK@JMM59V z4fD->b5fqXL2zY4ukN44u4-56`9YD)1qse2!qkD?YUfUvlK0`H4`1E?R{83MWcTMp z5dEHEp*Y!u@~d)t>Mac}DHXF>VtJWYMLM#8P8r`jEm10->ixr~REN$Ra0eCXt$HRl zUS+QpSGCmT;OnZ<$mC>kK8?8O(C20GOL2I9We+~;P- z;QS+9cWq<$@}`@(DEf_^Ck|8jd%NDUSKH+|KXYDvjZaehAys0}U6RScAZPmgQ@yom z5m!orr3B{*jwa^LsuCKiAh19dgqK@UuW9_0&#Mq~jd%6nME%+q)IyUqStPy0)~5v( z-*yszcxz@(_P@XJ{Wcj=iI7W`$Ns8n1~K^(Up~-pHxTkP1gS)W-*$a&F@seg|MD6*ByZ8eH4EmAik-UzEV|8JutX9K&mBtC23h@k;Wkspo|F;8{DeZyw7M zCY2a*>gIQVFv}N7mM1~vAMcI!*_*o?2H!@7zB|jfs80eaqMJf zhpm*9TuYJUM5I!S4(DM?KI8%+$8hoQ>H|k$4q|AaV0sj7#Eng}wK!Vnb?#aqdv2aq zJvBV+sJW;NXrM@aWjoO;gr-_r(=#)Aws``ZCU0yi^zu2(4QllKhI=IiiT4VmGH`{K zA!}CeOY{GJ$ZV$F(p#!;Gls=$D$&jHOe69U?Ph_q?|AO-d!b89 zj76_R2x5*YIr!5=zWcUh9haCx2;1MmJrhYEVI9devNI5+K8x#$usmR)rV)8U%UX9d+sw4~^Tw;VOo@>WXR%-gw>=B0Uo7tzSm6k@u!S zEaBZBeb!#mCqMenx6UW{(R8T4aqr7IIP$8TR2a;g-(0XfwzNpq#2%|MW;ryNIF8 z66YKEloG2Ph*XNQSUwT7S;#SO#j>$`}lZbV!|G9 zQ_rh4&yt1vxO#8l1p2tIuyVYy?;{KtChyCi*j%6v*^S8Cl`SM=5l45vJ|Vw-UI~ua zP4w@fva+{7yLuiz&@B6A@eI>XAw4%Gw`ff5vQW+Gl+mf(hUto=I4B6R9+GD{x>Vlt_exu*gfMyG#{Qaw*rrrJ*`E-Cqqx&md=kxYXkr4}NkFfN@2B^MD~v4b4~ za(d-DuE&srV&-p_gS6854^_jL@d_u1B#MVyBepqD? zoq#R0iTfSUEuxDIQFl|M`vn~pf=r>FIfqBJmphm62#w!qVgK1qb|DV#(-i$wxi z0|qCM5Aevsud0cs>)3dV}{m+N{sW7$2d@YKGt7gXT6WZy3 zx?RkKPgq!7^>A2w^kmBz1#kVWm>Ycw2ny=vTTMB*!s5)V@4m3YhKAinBlm^f#tal2 zOl`}CZm5eb`evO09qSKs-%SNl&Yt`};>!AK7QugrlG;;&jKrNzHhZ1?B09n^IKZa_ zE>LWmhg^sC*dXG|4$LM*dmlwS-wqLu#e>wU0he`Ff?x`)bv^>wo3e_KkdPbfpW=9z zNADE$S_q*b)&FdUY1|g@{mpQyz{PM$gxE>~tMauUKOZ`9z{|RaU@OzVtL|WMYu7p9 zt*5Y0PXWxUZ1PI-$SKg%7A#)_^FED8^h4;>Ls1^z0$as=FBj`u!?WtFMwl*A1{J6- zPRp*Hwtd$5+P{J;%K!vN5B3Q)n}GU}(_cY6IJL0LmxCv4Ba_HOR&+HWIn2)97IQtn z&Jee&jEhZ5h+|#>^0cO|ktZchO=4G-_B0O(Nkn0QYKo4}IIWEfU$6Ygl&1=>Jg{7G zccE?JR&#iIddumMt(C$Vtpn%z=ZpeY9V4*n2(acsBe#H3eI*aWn{la`7taE3Gb(4v zU(7kLM^8`B&i%|lo;yT5cPDb8a`A%5bxSX&d}4tN5v4uAXN+UIU+b{2FV^yj*LZ)4 z2^&ig+l{sc01D|VyV#7*Z8SlQjqU91sSgx%-($|7crAo)ZXF!BN!$W(exWDgRyz|V zRhseKZ6+g6wE{nX7_7CKv?^1Ac^{|Jiw|Wn2^k8Y1n!-PZOFfMoV}FyoP~lnx|y7U}i-|J7C&AkSN}=^hCj zYzis;I{Pz2Bc1faEGn}FLK*Kc-jspbw)#2YDu<)5tc3wT=Hr;IhB3$MyuZb5&Cbu~ z`p?xllqLF6+P4mCI#YGE6Jngi2wtT|@wQ&R+b%QGZ}Uex;nAM5xjdMQy-79-hYi64 zyTUg+Meo#qFRiP%=Y@ZpkFL>gVOug(3;HP;55sm#=jxqSJYg7ISDVHX*|l@rnHdL; zq!{?!)o^*KFnmg$zXMK4+|Rg>0?Mb{_+9axagN=+vI5xu*?Lt3F$XaiXz<~H7*if* zU*SA;=gX?1iQ>6%_|2$< zHi{(WKwH{7r=C8TTw&{&C+^oqdm5$-MeECjUJOaLv;|YYzf)xecGEo=Qus%t_j@k@ zqVvp0a?WdltiT{4TG+63lNHvm5Y|u(5mtk3#%2==TGQK2L&`@m%W)gb{Qwn3V(G-CQ{GNVv3JW>y6gl&Q~c+kZ)$IX9>Aq= zWEB*`56)#kKnme>J?=BYRO1tL*o7Q&|Bw$h@kds@I%D zM;BM#5@YvpX21${K{lLV`vL9?tM zWIS#2Ds1dzC&?oC!ewy{zCefB2RS_HbecV3rv85=xG(2V{(SkFBQ4t)0s@P2d_gam z;nZgn5f3ALNy8vdXM!TbAWPNMv5uuLm;*$R*sF)h=N9QrAMR-8@x_&w44|jbjEt|VxX`BzY=M{dxe-VynBBKE@KX^`sc> zoE47YA{0!Y5A~(zPisT_KOi&epcPk9Z@-0gHhhfkKO#wQ$#RYC>Uehd)(Q z`{Co~8jN9{F(U=pQv2x|zyHVJdF(S)gxIo;-?Jq9V4h$eRz-$Y?2p{{OI^8L zp{MubgEW7QSMsXv1t}t(DJ%la3Nl${I26`$Y{^z$5UQ&2Npr8R)bKb zp-diiJco^nMs*8=RwFNoYn7nu6Z14euYmot48tK?q$&;yNHFU$Uk&Yit9Wv)QcdqS zjz1&4S8%lzeEbGdXdtyVRz>nnL<@uFb%4_%2w!34q_Yc}%xtuRu_SVUaimyeIeHvz z`h{$eJb*%KzgNp7A>UKwqxjNuD)T>b2`gSdq;KAX_MP5y)=-+?|6eO*6&OhDG;yYR7Rc9~srsE~JmgrhPJ8+uLGy!Zu%9X|Ke5%bQYnKEe_w^% z{QRlwX#Us9+;**JW;#?*sPD;$MM&-#(&jO8`S=Nc}*LL+!YYBZQvpIHvl zrC?JHplryojieqyGkzj4%%n7lk15me8ooIKqlLms-7%O<0j#Jy+2!kff~c!szI+>M&;BSFFx_?h8?tH^Q;>j*(z z$*S2Hw?0B#^Sn4rbktx(7CZXs!B8p{71g1?HqG*jwhVhdovOZ*pJAKTxWLCZHC9XYV$BL`A!y}iA~iSs7R zutQy?Lhz}54Z|Q$I-qCJnv3vuYJ2TkDxtbD ztvvZ?FO4Zo6?b!fvo%lvBJ9|%WYrf6Gf(1y2s2d2;7fWEqiC6oaX9?An2QQ=&@#8> zrEoSILRIm;=^2Lt^i!kPuf7};zU6KY-2h$MVz`;}9e2w3@bMk`oQ^2$#ft5h8#cMe z++JFhfA<^zBu@4m#@BwAaf<>&%;@$aRnUi|DV)YCXdsboQQ$IuY$=~BIth2w0VtEs z;{ZA~4<&5Wx*}(Q1aLzgg^C>)KnmWIXz?UtcbotLxm9;Mj88m<+ldd=W3}jgf+j(y zAmg|1xQ5ke%4ik>`DJe3)BV#nvH+bLefN;t@@Mb6%)Q7M^kMRjo#*$uSfx)o-IN5} zJ=tggI~(G6HM>v?u(|)ZMY3i4j^1IPc-yWkxee-v-Jq@?n@b;BNok58X?R7cMSUhS zN-HEk4=Z5W6I(kpp$ZPP5&ZM>3i5RJVR{utEAVoJdL8K@9vtA-?Pc$*J{ z_WgA{q$8c7u$*_*+_BDEQm*zw(y!d8SDcxi$S4hLy z8JOIN8chsMdw}2-w$sjF9|`Fjc*7w%94Y3$>sT7;JpAF5+v#}4^s$tFIq}a~_%kHj zvb6;f*ZR7tWgan^awBhC5cjkc!@R#otm%6%^?e%h<>3;u~Q4qfRpZoRBnNgVDnHDMt(U;5#y^1z?)p7ss0 z5)O_cN&I}Nb?+rfUl$hoawYBx2P;$q-1ndN!CeUoQ7^{4FOIrzhV6}oy1)8B6i2l0 zR}8HkAd4tn#EQtiGj4JR46_Y$8aX&G7yl&Gt~>TqFkYbolhoS|SrA&K*D7X<_ex(k z^9-O}H&c^9Z1kzeUu|;Bw4NhfAgOfKM1J_aY?W~hvDv@EP=1epdYVKpA(Rbp)w3UL z*e>$~@1|o7BS~zy9us2irww2K3F^iVAW|oSKLb-u8Ma-UB)I$PtY|Fp-u7p$+yraugZwNFdC+6+DHAg_m z_O~>STM+CF;qu?9b^=sNv#?=@5Dp>%ktVf`L(=sn4h8`m4!YYV7Rx8x-w@0(yb--f z<3E~~nmpGi%|>Pp*fEnap5&IIY%|wEkdv12GE=&*PBAiPKE})Kxa^y^fHeNb3kmSW zXYC>;S6N>}jH}H-<^}GpXq!o0f+k1&V!w!pX{Tho_Z!`60i-Y+<}l`c4qr2~@iVMtz|hJy)<60u`64MLh0}1vWtef8EcT3{ z-!=FlW#+w1i~v{MNr!~n#o;Lhn6sOf-l@bSn`Rc1K7N^1+UzljZ=`wCqwFr7@=9AwLMy!p86zv#{Ajl~UU_^gg7Xk9%79D569;lz;2OfiosFIu6g zkFPoX;w|~Dm7>UC#Ky>N?Y-JrhdLlsV2;7}l2OcY^NXF&mVDFcM^0OyTm0q)LFe#h z1UkFyy2Njf*0@ez>u*8!^!qC?eK}>VgNKaM>jZt3s2;_S+9ls4U!Xg|k8oDT)L?c+ z6Qwm_Tj3R~FA0zBUW80jetx2!?YH0X>Q|@x&99h=rw$AOAIb}A?Q0wYHpI?<2V5&O zE&Zb1i|=7LsT3-x&ZPUAJ{oWnaO*I8-a+8b#oZoolFvK62fGaVZCKvkxIS28=yN{x z%)b6->p5TnXt`K$P!mVyNj#&tpoblixi5r(XRv#ZOzYPFx|C>^eN)n^p>=5N-IAT^X zku0-6yB+}tZTXF__aXtaU77#h!EUyF)9=#e?Y{9(=03ZTJuOB70iCeR28f!S-%J44 zn_>g&+++wGA0Q2fbNm8Q}7kGD5*)6LkY zQ^r?DMr3CUUQ9h#W?D8;$(|dks7=RF^I>LAB?i=1RtxWJJhOt`O$Q)n+N#Wh*8!$i ziPY14w(R-$fLpM(zpQyQhW^8^qyySkuiUxo5{Kyvz*VuElcz8J)lSh{m32#zqccxI z!ZaIo5Maf;)pFaka8oMmOxAXQaPcEnFZ~W=!g9Ue+gd_B;*+y=iX2co3YwO7N z_qppmhiQ-2E5ob7i&jg3idSIg!K;bpyb_OXOFDin``6U60s@yJo>CabYImC zd2NhN-kYzQZQS@>vp%jW=Dt^*p1|e$O~3UbGf&^H%Go8cM0|s;sNiD1cV@Sy@%>Y= zvX+`;u)%0)Y}3W^HAav~!Drs_M^XH1EDA$Ckg^jBK3vZsTR^_)QDiZz3L*UsN

LoU5Pfkbh}o^DzfJB64td4Ol518 z?9ZYDyMk8;<}%rj0bjX$K>^EBANT(@jCm0b-iEGr!|%5worbO~z=W+XV3&as+3N$X z=qtOOsv%*8N{7Z?^G-DS|=w!ZEc`+Qt{jzYn<^~Ufx(k z6VKGt(jEGT@)4D?xX!!5HlM>BklpW@rvUn3BPRQ)6m6$!r=093r|+8=qrsgM6L*Ge zo`%y05FNO(K10CY(#Gw_Y+oWSO41!_a}{Ez0QjeS6KcE2P8#(S@aNW0*0#KL@E%JS z&+@3IWUC3-rfi2U8nlHef0_!$CGd4tC)Tm0u<2`hie9WBIZAnu>hg#*U^*D13H93n zSkLtxS1Bck@s$n&-c!+|BhuE={@{8MWXoP7zzjO=>_P_KIRjI!WFa@@{d<6r?^fWK zj@M+<{F*tGAu$YM-0npi2Ro{_TWFnQlUSd=PL2&rEF%+KS~wc?xa?@NIHLhdpSoGp z*$#Szg=-=jmW7O(e_7Z9kl<18P)wCxKl2O@P`V$?FTo_FDUnl%52VSu|#$I>7LrO@;9a!w+<-TIP@2Xn|g?vy9$75!cj3kabY_ z;S5`>hUI3mXLFSdw^_4o*`D${LBlu6X6Q%nw@lR`zg7EAg$biP*S`_b8d%EJnGfV> zGadW{L0spOJaKnd{6nUt+cK=AYuq%kJ!B)@kEeY*u!_F zd3rt)1$ct#yUeh)kUl~zLJou|sZ)&VVlxIN3DH2zNIt!pdX4S~zHCQd zcwaL6T6dj0!^co zX+S`l=8waxmRQtzez1_zH@4!Wr>{SXHI>y@!-7sVT5~$`S;B3Nt?9`Iy1-vw6)5j^ z5IidgbA0_dBlWM|XLahrh-Q=9pfVpHzTQ;dYH|XhA(BvdG+keR;5sN1^432?q$~vD zvR!X;!EAKj2ZTv%*(t*gnFm=`JK4CP)r`k?VA||ME~{=kP3KWPxS}-(_W~#f39%FD#|W+moXHvh?x11k)g~?#;TP?ko*daPE+(Eq#6dj4 zFi{*irVSh!_MAHDbrbxcc&x66C>$J80X|#!Pf#Ns4(k~+@Al>gaG^g{_?HwYI5_@t z_ajrAZpe6m>c`kKv6Ud9;OmDnw&orry5epJuM<%!di33N%g~T2KrsC`v_*BAA zng+pF+oPNvP~(Z9U0lNftFuWA zXcP}syl5n5^#=vto-o>7{g&bK>=D6wXSrFQHWb>`w_t?B+;yei1jf|#J zCH3N;Cx=zs(?jA_e*kZL&}rwn^J78KoAZKWx)aExbHnOvIOFm9;GR46b;JWo-ASXX zi6{5&t76D7EW@LGm+NY3^1^ByqQ#3vWd~P-2F8Mw(Tu9P>PS+v9-8{b62V|L8~pD{VTv63Hy?e}-WsmrnfC zJ{GyyItDqyJ1yxK0rPWf!)3$aP>rmYjrT(5fj)pK5HIK?1E$A5CZ@NXEX^zpXs$=& z&Q`8zJd>fKrdCB^fv#Ot;!+0apJS3>tP)iOO5#ytkYZ@CjK(@9-^;{FAWn$8Orq|< zzyeDUXJRD);wu&&iWi&gn|_;)!lw_+02oLGOA`=Sl}Uo*qa}J;`0LzzZ?Jptt(mdd zYoD))Bler2NsyM}Hzf@a)RU#n0NahVKEW+(pBd_w1R;e3*#H(gkr!&WU$vYcK9plX zj`kpA@U*X;lEO_^#M<@rnyXJ~8^w>IxlXI{B8;(AobP@5!o*h>J>4^1+@1W49Hi1Q z&Ylf1p$>EHoCY9-1^5MJI@na;yd%%T11q(4`qfq_>qNUh}rEvM?kDAM;uJb>!ERd|B=73Ukc$ga2mg zGyVm?#~%(Oc;t+#Jv7&wFtHD$DVJ=Tk04YDo4#n)jz;kh7DKgcOB;5+aj6RkI^C&a}U5nW}m(UdeTwgM81pw^6vjU>{8D^~| z{^AN^LyH$zub!V8&$P9a%zaM(>?h90Gu(|~mk<*_a9<_%2n%XDOg{aw!MvZsK_LQ$ z!Wky+QdkJxmCcF}L1i>k6PyzG$-~GBYVx1P#K>o5)hA*0P)P^)0W=1~XLsZTzYN9j zJcGx%uJ76fDWU}%#Jd|C+A473cNEr z7vAod8vpv6!KE}C;H2-iU8mf>yWI&u^JZTYvT%)1u?I?bw4402BsKlA_(6AAQ;Z^E z<*`EQZFkQ9DeAnV+5F%CkF;owP_5A-wiZQc%}DG`I;|R2RF~FjiIGsVRzl3yYOPLM zdv6k(RH?m!Xp4#%iHL9dJDbBPe%IL`#se zrd^murteW(M6-zq++mvOm=H7-s6$38rO@t&k}!FdbJWjs<$onPgU%SqXC5hE^Yv3N z4L_UB``IR8mW7LucXi0$S}JobToC3Y6^7oKy5zDNg9hEhU7L(H>jo-AJO84GA-wYn z%`OL3E)}rQpEu|%`B53Qewggto5c}k8>N@UR}(K1N4M9FV8zvua}aXE2*Y2yh*97zKd%$p#H z0^BOqzuwK>XUf8AD!H5B(~Oy1V<`mVi!DCY$$mk3AY3Mea4sttSOPU)R&Wp_pHfW(z7Jnb04ztm_7| zP&0S0p5uw`K+$Eg{b=GX;vEYdVm1qMwfM~o;Uzk}Qrqt9mcqK=jSB_)yuEEf5m1VX zT5jmBJxE>fG_N(S8~i8l1jDw1ihHMmx#9?UT&+Qu=puOfgwyW+IqZH)i~ftQF{;}_ z`w@(aq3;O#DsuLvDu0lSx{qDR{cKf(3T%pgdl-u!R{m9=~wa$pG{vg$b|+ise{Dhtt7ER29t}4+!p;$El1F%-a`@{{ z-rsC)8uKH(+lx|IWjc&?#%jFzyjF3>?CG@kEM={}{h0+p>w3etJe(vKRXkKrc|Ha{#rl}}IsZV}V{5MsNF4Jwkn>aqfGFhN=u{Rru%?ixEhHd-8V0FmA zo8IZi#u-1kp`-c^p=X)FLkxc<|3wEP2sfXtJvi9G>ud&yX%XZFZa?Er52e1B4b_%I z-jV=8Ug%|yF0Qpy3NGOFQx4Bu^ou-mUCAxk-cntmFod9apC4DYu^ z0*gS(?e_ga-y++!HvYWn5Cy6D-1o+PyEB&HIM~-A;d6~D?%4b|EARauq9m&Dk;9h* zJWICBao_!q=5=8z{Xt|>lHd<}hCs)Vq}P_U*0st=e+3A((FBo4ic^U1$a8exI zG^MUR!nVMc^;YE8z8i(@LcnTH097FH+Tb2YLO5EZDK@gVrWB56s|yqNZ=j+fMSKpc_=)AU5j6SV>KNZpHltiSd|8hbfxH zR|ZE|snZb9AX>?!qO@j1G_hKix!tX~eu0}mF0?6hV|R1vskZZ^tJ0?4P~WG4d&eD* zdQJJLv0J)R#NP(b3rAwmK|Bg*eLIJBSdcKQY-dT6X8R-NGLVjt0*a>0C-8y{5{Pjq;V^&{wM9&jwDH_SJqLef(05fYUtCB1Wn#?%tacP=KKBxS) z+_(JI#N3kPx4eHsV|6wzO~2g~N35CT`+THcY7Wf(e>cICmaKA8>QlP$mDN>Lm&ei3 ztcPejb_sSVoOZJs$D02}-J@?}+CQcEK=on2BTh#Wg9*tohhHYJAY^}!tZDJ+hmF_( z2BlccTsK|~rpD^fh7IUO7Y4%wWe>!5?jKxwp|p8E@9Cix-${wSqoC-gaj?u-2x7P# zq*0Ny`#XkMs=Zmxwt;NTMf`kl&K$$6!gC4@Qbmwy7`&S=y2pL8(g&0C8W9OX5M6@_ z3nnSSKafm&&5i1ZV%))q?2cC89ottL=#Z&gO)oA0HcF8+&Bg}2^hk+#>8vNdHo{Yc@-)eqOI;fnkg zr3nSue!J=oA^UegouVhf+TlOaBJ~GtwA=T^7(Itap*5udoH0mm1NX-iFCupE%1PkV zNDOqZC^auZQki`|Xz;@=gcMKEKmiK;%D>{`?kEN(?X&`Kpq2M|dB_P!9Sb@!&Y}MF zdBlmEFRLDn1tGHJf7JU8Rc#Mx4>?aa&+HVjfdX51USDMNE;R(Ki^}HT<1kFseT$on zN8lAzW%qVHJ?9NbQ%;zsJ65hD!(um5qk`y1`?mCE(}``QZxo zvqPgf&Bb%Ii}(2^Pf4ICNzaNl*)0#>`n>UE*P0Zx4)-T>xkVt*5z`&*r#^zI)SHVV zn+cM=MW|AyT=4^3o>MkDDdEc4sb0x)6!n^oiv@n$Qg@Bf1bHN-e!2C%_^;F9=6D{J zOC{(jKWBIndi zS?aso_JCUH>7U;pU58K6II6c-mAtE8>yRvE+(%kC1Qh=$xwIiFa4JHSAjNVavgNR} z#l9iM7h3X%VUyg6LN#V`qMtIF$(~`e`ta6MH7)a=!mX14Liv~Li01Y_fwI^kmjGN0 z$Oj@w>g0p%$5@;{w>%#MYD5!!Yye)?&CU5GpvQFcR;l8d{lzn0W(OH~l;WY@6u{~Z zKhs6iisPdPxa5Ln;s}tOLq#d0T?+qQsTFKA@ti&VSpZypR9ifViB5#-ZYIhCOlW^L z5;1&#%e-gYpuu~)txSD$(k)BJ48t0wW61<}~u}5I81){c{&oyqsb+(^If?B{aBjQOSKRO}xgnshpPNFmqQwR^Q z$jhJ`&wFEM+px?%M?XK7?Rv6{nSw~g(}Z|u127r%bI@j#5znZjt9Y3HWDgcbV4peJkc#fbhm+eHN3Gk`p$JZuC(UO)XTj58D@CF>o>RY zbRwE4M?@06_uoh^F$R@H4txOsUqkF-?D|^mik?)e+qfiLqDD3fnGi~riqTxdt`k|& z;pQfuF&TQR$Mr>hP{PRuVOxR3d2ytS4yCyJO8?Vh{)x;2PU4S*TZtVZM<+>`q>LTE zHmr=w@M_bi8%(6P(Bu*lT6TyWPYnxMQ(w94@n$_Uf-Def)pQ;-HT^ZIaOP!<<%+oD z$-RE{?&E%mUJK-bk6i|89^%jyc8W6hNDRXQxvxhjx#lO_=z)wCkZwQ~A}5M)YUIbZ z%cJ2c?5RdmFE7-z-yaWY)Q=-HSu(?pzqe{^zs#*D`LH_Jm~*z9^B&;c(;H{KE29E@ z9mKrP3Q~@{zRq?L9ucVdKFUBpaRii&cT)$|VsU zuaq1%pL=}w%NUxPHK;FbegvXHjbZFAK-hnS`iTgKiwAFuW_g`pW}8Fj+xH8^wsUd8 zJMZ%c8=g~PTW0YFHqPqvB547knv40F22ACAvKF~(06XTl9Xv7qjUA^-{fS^)+TgKR z6Cs>@JZ9~DFRETHJ9v?aR%^mdAuP`icbVpmyVahGhWdmPe2i+&b+vmdJeEUO3d_P) zpAS_OS!5LX-gQ;mvW7X2>cJP;%GLV(WuF&vp*S@c?8~ggd9%WrPmsYsaxT3l_O+{VH?kf;d*Au$)#}ALKc6_Wq(!@_yUy4PV0Aoh z&3)ivh&@Z1UUv9NryUJ8Z1{7>)n5<5J#YWMn$j~yw~aTs?F}?k*4H{^vRvrII1did z`iC?q@5(>Y;!0NEZ@BavCzsLL^+3F=@7TN~EiZ_a^@q!WZwmv$qxt3gRfjuS9IvN@Dg0)JULw6BTfn)SWyvggsDy^L_XPWsd$_{GW=zEE4j>G=AUDk)j7Rn_n|M9~%J z`b%zkvCPBq9W1iJ!;CkWQIUkJ&Qy=n8)~^Feo|(F(`K*b)krIU%O1 z-=DBh4o;+a)o=Cs-s=?ciY(As4z#JO%b>q3(0O)C7*EJ!UWXkK80X|mCL#6&UM$u% z&OpP*ETW~?0MR*;DVfx3=}6m#56N5})1Dg>o5NCeJ1%g0eQbXCq-Svvn^q-u8(4lx z@9e;5OxMz@yfV9u1(>sj>D_(yPYAP&O_Q|GTW}dhTm>RGO0*^%89%f8US|xZO&iK=8 zB5`}jHFgS(6cpF)n0I!ddhLSZvOW@xNhi%^ z;U9fDulkpk5ryazcM`iEFJXZ%kdGs3-6;3qmYInwNZtM9EzsvwW9U^m)nQ6vF0mw91`7Kr*K(6L_zSV`mq#bM^ z@=7Q60;Dfo&jvmnbSn?&v-N{B{=G5Wt-;NF_tsv8CMXw)3yX+RmdeYB z_v(}7C)RqkA;-j{c{|%cGkrrEu!T|#eDp+)ey5{z4O42Ats9wVO_C)0#CeW6Qa;ID zo?GQi?zlQ>K_kklN?Vi&cB^r_|GTe3$hzJ;=Z)f@$u_%t&L#GQ!{6IkTU;3|N-2x2 z6}tU;8+i)O7{B8}FJ)EGI}sX%n$OAakZHjSwBw?=n^?ZOtgs8x2Vwjvw7WaT0S@`w z9Ea{ZY$VSK{yP$&r@GK2G+= z?7EOTlcV`Sv9&X;T>94qb%*}=7mC}bUv{cE*FNzUmuz@pEO7e0_mbEznqRxm`YHD8 z<99e$YY3!%+JOIoWyvb#<`VPgGrkVDGcnoUN-C&>{Ml*~>h{a`m9sZOeT27PUm3*f zKJ>ORjSq_WIA)A>bRhpqBj8cyL8Uh@9i}fS-DLQcEMJ~Y4>J>;^7lcEb;c&zMC(|c zeVvYgrTqTHnbqHGM}G#CPrsg5q;it;ezq><3VLT;;n2up!IFtKogk8~O*ymAOu#B> za_2`bJwJ*1^hM2b5LoOEPOz7(m!7HpSeq3p-6Ov*bdDC4wBFP4p02!>CRLW+etr7k zL(J*ukF`RF_o;dV?LN~>tF9NgA8dWZJZW^gVdsmJP@2qzT)2uGHFT^vkWv~?zeO0v zR&*8(TG<9c{(To3mxj4yzsVQ6=fy2`1mTSWS+7?wpB*_b30%&B#fwVz>0H=NeRb7e z;1lb}gJ~bsoSmQ+N2CZf@fRZ%`@=Z9v^h;fbMY;Q_;}fx<}JB8g+v1Cm;Mq}#5bim znby6OtX9_@WF7S=a`Gi2`_pGYcou879ASn58$I6n`JH}EEEc>{X9{1@oLYiO=R$hl zRK>$v>q5g+4@?_;mb%p;6XWa`1s(GBmR_iwn|9Asn2mhBf30J|_S`mRC--2v{o4^u zZq?Q8fnX}X8~|~!r3z2ULEuY4>a9K zjLBP3E)=tEH*H}+b4bYRKW-RCpJbP0O(v0^xDnE0nbFfu;wCD|fK|P{S?KV~#nRd2 z6lRjIbx0m-1r`FK8}7^ZWwo#o)Hk{dd98!2Vtd?I`Yyc5BcQ?^E?OTdw*5j6ldah%G50#U?;yq@) zfqIZjV-@iCnvCM_Mb`!F8MezuwJEGTwtJo&&U_OlMzRLY3;sD=xDQ(J(C;$74*ffy z!c${%&*Y*oPlIWMW9_o>7`7oC!GvhI;EP#iJo$WJy9(}l5b2(y3_g7QDQW-G8yOBp z`NL_(JN7bM+~7WmxV|jYzNWcz=I}}ywi2}DFNP^O%>{NJ%kER zdPcJB%+0|@h!{oo1>pkM5d74pp~`lWaxyzyQ@&D@H5Y}!_FqQq*~ClH!u+f`TMIvQKS$G2mnZMhgfbX+a zg;Q05RUZE`chAFZb!P$R=6H8LePJLAFm60ycII#fp>NW<1_9Y37*CVDI$8z57?(Ir zeoy@XjWckfyyHlWiiiY8T~w_UR79w=y;l`OR;d~w5@R!ZJfySuJbK(cNYeyIr?dFu#h|k;6QRXZ)?oGBaruL%<@DU1c?BI) ztxpUG;?$Em&&R4YnYR5pG#Eb+`SbHG%RXO5$RAnA>b{lj6rry;xK~15=wu8Kw3|sE ziNY|w5qADDJDV;y^YaYEmk=v0B+B}2Vl*{Nm*g6eDq$g*$vn&eiw~qQ9f7^wb_I+!O44k|6V0jA>vj}f=Q2Hs5emoGBVx+D?phYioyclVC| z24oJmsfG;rG*5YxN~Qj84F@cRMsFBH{fba)Pd9`xx1cbUipL#!%^+c6^X>gG5y-R~ zgP41NzzdQ+Ol~9{Q!D)8AJ(|NRJLWW@jVa|UaPwWlU=|00-TkxzP>(-I@sH+nPI>H zjQWJ`9@Dj734pYNN`(W{&i3K4v5h@TM6-8F%Uz1CTpBi`x7Ct4!dH&{Z198i52rB%epdjKxtp84@XwzaB2xwUOQ1L@T7$TYB zujnfWiV4Tu!ISs)4%|K8m?%fY>EC>WI|nif*S*6Uc03pU0p8!lf8y|m!H*1!fw38# z@t#nAfBaSkOA&M7!oLm|Wo@xg6l3gjj(IbJ7{w?;^J_u}D;#NQ-Nw$H1AOC{okEex z6t&Jezlwm6@d+%um@ZVGKURYY53Fx!aGJxuWykY;{K{wuic(ZI2!pUG0W?jHhLy;` z{5+2x=@+Jl#x6^SoGJlX9ZgM55NlHlL8_Q7J~{UiW-ajwOh98he;_N5>;r?LHoxes zX8R>QLXJ(*XgXfyN-cu(M0ZyHEFTY^uf+ludgS9Qs$2?#iLEfDyVKj_*+v{0R0}4g z-Y^Q{=d_(07{vWQ#H?TR&y!ed+=2KWnh9JGCaUF+OA^(2<=^M}i}=`LGfr=J;PmJt z9^Gv(I*rm?#h2e;Aut>aK@TeB^9|4O)br?d8WAZF zeP0b(A(j);IMkeeBPnjKo=K{!dh{4QDAT`dbpkMZ{Muz^YQ93}FhKl3MG_79rRj^C z(*%42&TiZ0Rg5v#N57?b+<8LNg?^sSHk@yAmSWzdkk}-i&2v^(p!^PsB##vfkVVo% z$h_RXHtq`6!rZR-h0(bL0OOb;j+qar9nZu?d=>AUNMzu}0k*aoWmVJPY{^^gQ5;R{ zSs9M@sBJ$IW3v@mah^#}drsQM9SbCDCG0*9fZ#V`|4}&Bn&p_uvhdGdf z7NP+=%Z|7qIxLUO%q~K8N;k0Evs%qC1C&LQRjIMfEP_v3NA8!NV&hTFNus>k15D5c zOb;77T-@}R00J5n`7>D&ETLVF#mF^*_RJyh>cJA@gX;l`cu6QYh`k^DWq4J<^CD@# z;;MHS36&8BCUGD|WgY&TCOowyd~PN~P6VO|GgqJVYbyiLJGRfA&kbKWN>Va`-W@HV zN9yJeiJ$(1eW{JkZY94!xGUme66?y{FVIDE>=DNd=lb^G^pos2YH>O}tkbY`MI!Dk)++u-~b&NS;QJ5w&NwJ>WI4e(cIv5kSpe|yW7 zS?1e0vg?#DKirs}kh%K5OQM3K@o3HI>gG#Fc#IUM2VVvfaN#-q3Bp%H)re$=IolfSs-+ zv3T;6g$_T|F?bDkmZ_?`66o23ohvyL*kiH)R;CCxC}R|^aB`YY_OAjVcD_%EMc40^ z?ZpjnU$QYB*M_S5r(DVunLmUT@@lytP70Rw@sZ2SDDL{(kn%gPQ?JY7l{xgmZq z&*nX;h#Pa-xpLKojp0HN*ivRHM*$Td9>^XVA@yhRs8@M#Cm^&GY{5O`8}@~E%a_HI wyHY3gBQ!s@ak;?M?kUMq

  • - + - 2.5.2 Create VF + 2.5.2 PF/VF mapping + +
  • @@ -1546,9 +1570,18 @@
  • - + - 3.0 Further Development + 3.0 OneAPI on OFS Running in a Virtual Machine + + + +
    • + +
  • + + + 3.1 Further Development @@ -2318,12 +2351,36 @@
  • - + + + 2.5.2 PF/VF mapping + + + + +
  • @@ -2364,9 +2421,18 @@
  • - + + + 3.0 OneAPI on OFS Running in a Virtual Machine + + + +
    • + +
  • + - 3.0 Further Development + 3.1 Further Development @@ -2561,16 +2627,16 @@

    2.1 Setup Server for 2.2 Clone and Compile FIM

    As shown in Figure 1-1, OFS components in the FPGA include the FIM and Accelerator Functional Unit(AFU). The oneAPI ASP is in the Partial Reconfiguration(PR) region of the AFU and relies on the compiled database of the static region(FIM) to interface with the host and board peripherals(e.g. on-board memory).

    -

    Once the server is setup with OPAE SDK and DFL kernel driver, the next step is to clone and compile the static region of the design, i.e. FIM. You can use the default configuration of the FIM for both target platforms. Additionaly for Intel® FPGA SmartNIC N6001-PL for ofs_n6001 and ofs_n6001_usm board variants you have the option to create a minimal FIM which removes additional VFs, HSSI and host exercisers in the design. Please follow steps in the Intel® FPGA Interface Manager Developer Guides for your target device to compile FIM supporting PR release.

    +

    Once the server is setup with OPAE SDK and DFL kernel driver, the next step is to clone and compile the static region of the design, i.e. FIM. You can use the default configuration of the FIM for both target platforms. Additionaly for Intel® FPGA SmartNIC N6001-PL for ofs_n6001 and ofs_n6001_usm board variants you have the option to create 2 different types of minimal FIM which removes the HSSI subsystem and host exercisers in the design. The difference between this two minimal FIM's, are the amount of VFs that must be created, in case of 1PF/1VF (built using n6001_1pf_1vf.ofss), only 1 VF is needed. In case of 2PF FIM (built using n6001_2pf.ofss) no VF creation is required. 2PF FIM could be used in FPGA development in virtual machines, see Section 3.0 OneAPI on OFS Running in a Virtual Machine. Please follow steps in the Intel® FPGA Interface Manager Developer Guides for your target device to compile FIM supporting PR release.

    -

    For more details on minimal FIM for Intel® Agilex® 7 FPGA for ofs_n6001 and ofs_n6001_usm board variants and how to create it, refer to Intel® FPGA Interface Manager Developer Guide: OFS for Intel® Agilex® PCIe Attach FPGAs.

    +

    For more details on minimal FIM's for Intel® Agilex® 7 FPGA for ofs_n6001 and ofs_n6001_usm board variants and how to create them, refer to Intel® FPGA Interface Manager Developer Guide: OFS for Intel® Agilex® PCIe Attach FPGAs.

    A pr_build_template directory will be generated in the work directory specified as part of the FIM compile command (using OFS/ofs-common/scripts/common/syn/build_top.sh script with the '-p' option enable to create an out-of-tree PR release). The pr_build_template directory is required for successful setup of the oneAPI ASP.

    -

    Once the FIM compile is complete, please program FIM using fpgasupdate and Remote System Update(rsu) command. Use of these commands has been demonstrated in section named Program the Intel® FPGA SmartNIC N6001-PL with the hello_fim in Intel® FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel® Agilex® PCIe Attach FPGAs refer to Test the hello_fim on a D5005 section in FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel® Stratix 10® PCIe Attach FPGAs for Intel® Stratix 10® FPGA.

    +

    Once the FIM compile is complete, please program FIM using fpgasupdate and Remote System Update(rsu) command. Use of these commands has been demonstrated in section named 5.3 Remote System Update in Intel® FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel® Agilex® PCIe Attach FPGAs refer to Test the hello_fim on a D5005 section in FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel® Stratix 10® PCIe Attach FPGAs for Intel® Stratix 10® FPGA.

    2.3 Prerequisites

    In addition to server setup and FIM compilation, a few linux packages are needed to setup the oneAPI ASP and develop HLD applications.

    @@ -2614,7 +2680,7 @@

    2.3 Prerequisitespatch-agx7-ofs-2023-3.tar.gz. For Intel® Stratix 10® FPGA ensure Quartus patch 0.23 and 0.01iofs are installed. You can find them in a tar file under assets in the following link patch-s10-ofs-2023-3.tar.gz.For quartus patches installation to work properly, you must have Git Large File Storage (LFS) installed when cloning the ofs-fim repository.

    +

    Note: For Intel® Agilex® 7 FPGA ensure Quartus patch 0.13, 0.21 and 0.02iofs are installed. You can find them in a tar file under assets in the following link patch-agx7-2023-3.tar.gz. For Intel® Stratix 10® FPGA ensure Quartus patch 0.23 and 0.01iofs are installed. You can find them in a tar file under assets in the following link patch-s10-2023-3.tar.gz.For quartus patches installation to work properly, you must have Git Large File Storage (LFS) installed when cloning the ofs-fim repository.

    Use following command to check Quartus version and installed patches.

    @@ -2701,7 +2767,7 @@

    2.3 PrerequisitesOFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-1) +Version 23.3 Pro Edition with patches (0.13, 0.21 and 0.02iofs) under assets on this link patch-agx7-2023-3.tar.gz Intel® oneAPI Base Toolkit (Base Kit) @@ -2752,7 +2818,11 @@

    2.3 Prerequisitespatch-s10-ofs-2023-3.tar.gz +Version 23.3 Pro Edition with patches( patch 0.23 and 0.01iofs) under assets on this link [patch-s10-2023-3.tar.gz + + +](https://github.com/OFS/ofs-d5005/releases/tag/ofs-2023.3-1) + Intel® oneAPI Base Toolkit (Base Kit) @@ -2778,7 +2848,7 @@

    2.4 Build and Install one 
        git clone https://github.com/OFS/oneapi-asp.git
     cd oneapi-asp
-    git checkout tags/ofs-2023.3-1
+    git checkout tags/ofs-2023.3-2

    Ensure the correct tag has ben checked out: @@ -2790,7 +2860,7 @@

    2.4 Build and Install one
  • Output:
    • -
        ofs-2023.3-1
+
        ofs-2023.3-2

    2) Ensure that OPAE_PLATFORM_ROOT and LIBOPAE_C_ROOT have been set as described in section 2.3. Generate the oneAPI ASP hardware and software using provided build-bsp.sh script. This script clones required repositories and builds the oneAPI ASP libraries required by HLD host application to run successfully.

    @@ -2937,14 +3007,23 @@

    2.5.1 Compile Initializat

    The output directory of the sample application is written to oneapi-asp/platform-name/build/bringup. The generated bitstreams are also copied to oneapi-asp/platform-name/bringup/binaries/. These are used in the initialization of the platform.

    Once the bitstreams are generated, create a VF and initialize the board as explained in following section. Ensure that the FIM has been programmed on the board as explained in section 2.2 Clone and Compile FIM

    -

    2.5.2 Create VF

    +

    2.5.2 PF/VF mapping

    The oneAPI ASP is located in the PR region of the FIM and is accessed through PF/VF Mux. Refer to the FIM Reference Manual for your target platforms for more details about PF/VF mapping.

    +

    For Base_x16 FIM (default) and 1PF/1VF minimal FIM (built using n6001_1pf_1vf.ofss), VF0 is mapped to PR region and you can create 1 VF when using this FIM. Base_x16 FIM has 5 PF's and minimal FIM just 1 PF.

    +

    For 2PF FIM (built using n6001_2pf.ofss) PF1 is mapped to PR region and no VF creation is required.

    +

    See Technical Reference Manual: Open FPGA Stack for Intel® Agilex® PCIe Attach FPGAs for diagram showing PF/VF mapping.

    +

    VF1 is mapped to PR region and you must create 2 VFs when using this FIM. This FIM has 1 PF. See FIM Technical Reference Manual: Open FPGA Stack for Intel® Stratix 10® PCIe Attach FPGAs for diagram showing PF/VF mapping.

    +
    2.5.2.1 Create VF
    +

    +
    +

    **Note:**This section only applies for Base_x16 FIM and 1PF/1VF minimal FIM for Intel® Agilex® 7 FPGA and default FIM for Intel® Stratix 10® FPGA.

    +
    • Create a VF using PCIe ID obtained from the output of fpgainfo fme (PCIe s\:b\:d.f output)
    @@ -2953,7 +3032,7 @@

    2.5.2 Create VF

      -
    • Check that the VF is created using sudo opae.io ls command and note the PCIe ID for the VF(s) (the function number in s\:b\:d.f will be different for the VF). Sample output for Intel® Agilex® 7 FPGA minimal FIM is shown below. Output for base_x16 FIM should display 5 PF's and the PCIe ID for VF0 will be s:b:d.5.
      • +
    • Check that the VF is created using sudo opae.io ls command and note the PCIe ID for the VF(s) (the function number in s\:b\:d.f will be different for the VF). Sample output for Intel® Agilex® 7 FPGA 1PF/1VF minimal FIM is shown below. Output for base_x16 FIM should display 5 PF's and the PCIe ID for VF0 will be s:b:d.5.
        $ sudo opae.io ls
@@ -2971,17 +3050,20 @@ 
    2.5.2 Create VF
 
    
 
    Note:sudo opae.io ls will list the accelerators, respective PCIe ID as well as the driver it is currently bound to.
    
+
    Note: For more information about pci_device and opae.io utilities, refer to the OPAE FPGA tools page here.
    
 
    
+
    2.5.2.2 Bind PF and VF
    
+
    For Base_x16 FIM and 1PF/1VF minimal FIM for Intel® Agilex® 7 FPGA and default FIM for Intel® Stratix 10® FPGA :
    
 
      
 
    • Bind the created VF(s) to vfio-pci driver, use the PCIe ID for the VF(s) for this step. Verify you are using the PCIe ID of the VFs you have created. For example:
      • 
-
    • From sample output for Agilex OFS target platform having minimal FIM programmed shown above, s:b:d.vf will be 0000:b1:00.1 in command below. For base_x16 FIM should be s:b:d.5.
      • 
+
    • From sample output for Agilex OFS target platform having 1PF/1VF minimal FIM programmed shown in Section 2.5.2.1 Create VF, s:b:d.vf will be 0000:b1:00.1 in command below. For base_x16 FIM should be s:b:d.5.
      • 
 
    
 
        sudo opae.io init -d s:b:d.vf $USER
      -
    • Sample output for Intel® Agilex® 7 FPGA OFS target platform minimal FIM. Output for base_x16 FIM should be similar.
      • +
    • Sample output for Intel® Agilex® 7 FPGA OFS target platform 1PF/1VF minimal FIM. Output for base_x16 FIM should be similar.
        $ sudo opae.io init -d 0000:b1:00.1 $USER
@@ -3005,35 +3087,69 @@ 
    2.5.2 Create VFSample output for Intel® Stratix 10® FPGA OFS target platform:

    • -
    $sudo opae.io init -d 0000:12:00.1 $USER
-Unbinding (0x8086,0xbccf) at 0000:12:00.1 from dfl-pci
-Binding (0x8086,0xbccf) at 0000:12:00.1 to vfio-pci
-iommu group for (0x8086,0xbccf) at 0000:12:00.1 is 149
-Assigning /dev/vfio/149 to ofsuser
-Changing permissions for /dev/vfio/149 to rw-rw----
+
        $sudo opae.io init -d 0000:12:00.1 $USER
+    Unbinding (0x8086,0xbccf) at 0000:12:00.1 from dfl-pci
+    Binding (0x8086,0xbccf) at 0000:12:00.1 to vfio-pci
+    iommu group for (0x8086,0xbccf) at 0000:12:00.1 is 149
+    Assigning /dev/vfio/149 to ofsuser
+    Changing permissions for /dev/vfio/149 to rw-rw----
 
-$sudo opae.io init -d 0000:12:00.2 $USER
-Unbinding (0x8086,0xbccf) at 0000:12:00.2 from dfl-pci
-Binding (0x8086,0xbccf) at 0000:12:00.2 to vfio-pci
-iommu group for (0x8086,0xbccf) at 0000:12:00.2 is 152
-Assigning /dev/vfio/152 to ofsuser
-Changing permissions for /dev/vfio/152 to rw-rw----
+    $sudo opae.io init -d 0000:12:00.2 $USER
+    Unbinding (0x8086,0xbccf) at 0000:12:00.2 from dfl-pci
+    Binding (0x8086,0xbccf) at 0000:12:00.2 to vfio-pci
+    iommu group for (0x8086,0xbccf) at 0000:12:00.2 is 152
+    Assigning /dev/vfio/152 to ofsuser
+    Changing permissions for /dev/vfio/152 to rw-rw----
 
-$ sudo opae.io ls
-[0000:12:00.0] (0x8086:0xbcce) Intel FPGA Programmable Acceleration Card D5005 (Driver: dfl-pci)
-[0000:12:00.1] (0x8086:0xbccf) Intel FPGA Programmable Acceleration Card D5005 (Driver: vfio-pci)
-[0000:12:00.2] (0x8086:0xbccf) Intel FPGA Programmable Acceleration Card D5005 (Driver: vfio-pci)
-
-$ls -lt /dev/vfio
-total 0
-crw-rw----. 1 ofsuser  root  235,   3 Dec 3 16:25 149
-crw-rw----. 1 ofsuser  root  235,   0 Dec 3 16:22 152
-crw-rw-rw-. 1 root     root   10, 196 Dec 1 07:28 vfio
+    $ sudo opae.io ls
+    [0000:12:00.0] (0x8086:0xbcce) Intel FPGA Programmable Acceleration Card D5005 (Driver: dfl-pci)
+    [0000:12:00.1] (0x8086:0xbccf) Intel FPGA Programmable Acceleration Card D5005 (Driver: vfio-pci)
+    [0000:12:00.2] (0x8086:0xbccf) Intel FPGA Programmable Acceleration Card D5005 (Driver: vfio-pci)
+
+    $ls -lt /dev/vfio
+    total 0
+    crw-rw----. 1 ofsuser  root  235,   3 Dec 3 16:25 149
+    crw-rw----. 1 ofsuser  root  235,   0 Dec 3 16:22 152
+    crw-rw-rw-. 1 root     root   10, 196 Dec 1 07:28 vfio
+
    
+
    +

    For 2PF FIM for Intel® Agilex® 7 FPGA :

    +

    Bind the PF1 to vfio-pci driver, use sudo opae.io ls command and note the PCIe ID (s\:b\:d.f) for the PF(s). Verify you are using the PCIe ID of the PF1. +

    +
        $ sudo opae.io ls
+    [0000:b1:00.0] (0x8086:0xbcce) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)
+    [0000:b1:00.1] (0x8086:0xbcce) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)
+
    + +

    Output below shows the command to bind PF1, in this case s:b:d.f will be 0000:b1:00.1.

    + +
        sudo opae.io init -d s:b:d.f $USER
+
    + +
      +
    • Sample output for Intel® Agilex® 7 FPGA OFS target platform 2PF minimal FIM.
      • +
    + +
        $ sudo opae.io init -d 0000:b1:00.1 $USER
+    Unbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci
+    Binding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci
+    iommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 13
+    Assigning /dev/vfio/13 to ofsuser
+    Changing permissions for /dev/vfio/13 to rw-rw----
+
+    $ sudo opae.io ls
+    [0000:b1:00.0] (0x8086:0xbcce) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)
+    [0000:b1:00.1] (0x8086:0xbcce) Intel Acceleration Development Platform N6001 (Driver: vfio-pci)
+
+    $ ls -lt /dev/vfio
+    total 0
+    crw-rw----. 1 ofsuser  root 511,   0 Feb  2 22:47 13
+    crw-rw-rw-. 1 root     root  10, 196 Feb  2 16:56 vfio

    If the driver fails to bind due to an error related to iommu_group (e.g. `No such file or directory: '/sys/bus/pci/devices/0000:b1:00.5/iommu_group'), ensure IOMMU is turned on as explained in step 2 in Section 2.3 Prerequisites.

    -

    Note: For more information about pci_device and opae.io utilities, refer to the OPAE FPGA tools page here.

    +

    Note: For more information about opae.io utilities, refer to the OPAE FPGA tools page here.

    2.5.3 Initialize Board and Run Diagnostic Test

    @@ -3269,8 +3385,11 @@

    2.7 Release VF
        $ sudo opae.io release -d s:b:d.vf

    -Sample output for Intel® Agilex® 7 FPGA OFS target platform having programmed minimal FIM is shown below.The output for Intel® Stratix 10® FPGA target platform and base_x16 FIM should be similar. For Intel® Stratix 10® FPGA you will need to release an extra VF as for this target 2 Vfs were created. -

    +Sample output for Intel® Agilex® 7 FPGA OFS target platform having programmed 1PF/1VF minimal FIM is shown below. The output for 2PF minimal FIM, base_x16 FIM for Intel® Agilex® 7 FPGA and base_x16 FIM for Intel® Stratix 10® FPGA should be similar.

    +
    +

    Note: For Intel® Stratix 10® FPGA you will need to release an extra VF as for this target 2 Vfs were created.

    +
    + 
    $ sudo opae.io release -d 0000:b1:00.1
 Releasing (0x8086,0xbccf) at 0000:b1:00.1 from vfio-pci
 Rebinding (0x8086,0xbccf) at 0000:b1:00.1 to dfl-pci
@@ -3280,7 +3399,27 @@ 
    2.7 Release VF
    -

    3.0 Further Development

    +

    3.0 OneAPI on OFS Running in a Virtual Machine

    +

    +

    Virtual machines (VM's) can be used for FPGA development, 2PF minimal FIM (built using n6001_2pf.ofss) is provided to use oneAPI on OFS Intel® FPGA SmartNIC N6001-PL reference platform in a VM. For more information about 2PF FIM configuration refer to Intel® FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel® Agilex® PCIe Attach FPGAs.

    +

    The setup flow for your virtual machine could be found in the KVM User Guide: Open FPGA Stack, there are additional things for oneAPI flow listed below which you should ensure while configuring your VM:

    +
      +
    • Assign the enough memory to the VM for running your oneAPI workloads.
      • +
    +
      +
    • In section 5.1 Passing Devices to the VM when adding the PCIe Host Devices to the VM, ensure to have PF0 and PF1 BDF adjacent (s\:b\:d.f, s\:b\:d.f+1). The following example shows the address element of the PCIe Host Device XML file of PF0 and PF1, keeping the same value for domain, bus and slot attributes and only changing the function attribute (increasing its value by one):
      • +
    +

    PFO-XML

    +

    PF1-XML

    +
      +
    • Install libnsl.so.1 library with the following command: +
        $sudo yum install libnsl.so.1
+
      + +
      • +
    +

    Once this setup is done, follow Section 2.0 Setup Flow for Using HLD Tool on OFS to finish the configuration of the VM for oneAPI .

    +

    3.1 Further Development

    Once you have completed running the oneAPI sample application, you can start developing your own applications.

    For more information about developing FPGA applications with Intel oneAPI, refer to Intel® oneAPI Programming Guide and FPGA Optimization Guide for Intel® oneAPI Toolkits.

    diff --git a/ofs-2023.3-2/search/search_index.json b/ofs-2023.3-2/search/search_index.json index 110af3b9e..541885327 100644 --- a/ofs-2023.3-2/search/search_index.json +++ b/ofs-2023.3-2/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Open FPGA Stack Overview","text":"

    Open FPGA Stack (OFS): OFS is an open-source hardware and software framework that reduces the development time for creating your custom platform. Provided within this framework are reference designs targeting different Intel\u00ae FPGA devices with upstreamed drivers and management software tools.

    The reference shells, called FPGA Interface Manager (FIMs), provide an integrated, timing closed design with the most common interfaces for host attach applications. After selecting your starting shell, you can add or subtract interfaces depending on your application requirements. Then leverage the build scripts, RTL, unit tests, Universal Verification Methodology (UVM) environment, software and drivers for this reference shell as a starting point for your own FPGA platform solution.

    OFS currently targets Intel Stratix\u00ae 10 and Intel Agilex\u00ae 7 FPGA Device Families. To find out more about Intel FPGAs, visit the Intel Stratix 10 and Intel Agilex 7 pages at Intel.com.

    "},{"location":"#how-can-i-start-using-ofs","title":"How Can I Start Using OFS?","text":"

    1. If you are interested in a production card that uses OFS for your workload application development or for full deployment, please refer to the OFS Board Catalog.

    2. If you are an FPGA developer, refer to our [FPGA Developer Journey Guide] to understand the OFS development flow as well as the reference shells, tools and development boards you can use to gest started. FPGA Developers interested in oneAPI should reference this guide as well.

    3. If you are a software developer, refer to our Software Tab for driver and user space tool development resources.

    4. If you are an application developer, preview our overview video on how OFS can help you develop FPGA-based workloads and review one of the AFU Developer Guides to find the OFS resources available for creating your own application workload.

    • Accelerator Functional Unit (AFU) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach
    • Accelerator Functional Unit (AFU) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach
    • Accelerator Functional Unit (AFU) Developer Guide: OFS for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach

    Beyond the resources we have on this site, you can navigate to the OFS GitHub repos by clicking the GitHub repository icon at the top left corner of this site page.

    "},{"location":"#what-fim-reference-designs-are-available","title":"What FIM Reference Designs Are Available?","text":"

    Below summarizes the five current reference designs (aka FIMs) you can choose from:

    OFS FIM Targeting Intel\u00ae Agilex\u00ae7 PCIe Attach (2xR-tile, F-tile)

    Key Feature Description Target OPN AGIB027R29A1E2VR3 PCIe R-tile PCIe* Gen5x8 Virtualization 5 physical functions/3 virtual functions with ability to expand Memory * Two Fabric DDR4 channels, x64 (no ECC), 2666 MHz, 8GB Ethernet 2x4x25GbE, 2x200GbE, 2x400GbE Hard Processor System Not enabled Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration)* Platform Management Controller Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported OneAPI OneAPI Acceleration Support Package (ASP) provided with compiled FIM to support OneAPI Runtime Software Support * Linux DFL drivers targeting OFS FIMs* OPAE Software Development Kit* OPAE Tools Target Board Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile)

    Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae PCIe Attach Reference FIM documentation collection.

    OFS FIM Targeting Intel\u00ae Agilex\u00ae7 PCIe Attach (2xF-tile)

    Key Feature Description Target OPN AGFB027R24C2E2VR2 PCIe F-tile PCIe* Gen4x16 Virtualization 5 physical functions/3 virtual functions with ability to expand Memory 3 DDR Channels:* One HPS DDR4 bank, x40 (x32 Data and x8 ECC), 2400 MHz, 1GB each* Two Fabric DDR4 banks, x64 (no ECC), 2400 MHz, 8GB Ethernet 2x4x25GbE Hard Processor System 64-bit quad core Arm\u00ae Cortex\u00ae-A53 MPCore with integrated peripherals. Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration)* Platform Management Controller Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported OneAPI OneAPI Acceleration Support Package (ASP) provided with compiled FIM to support OneAPI Runtime Software Support * Linux DFL drivers targeting OFS FIMs* OPAE Software Development Kit* OPAE Tools Target Board Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)

    Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae PCIe Attach Reference FIM documentation collection.

    OFS FIM Targeting Intel\u00ae Agilex\u00ae7 PCIe Attach (P-tile, E-tile)

    Key Feature Description Target OPN AGFB014R24A2E2V PCIe P-tile PCIe* Gen4x16 Virtualization 5 physical functions/3 virtual functions with ability to expand Memory 5 DDR Channels:* One HPS DDR4 bank, x40 (x32 Data and x8 ECC), 1200 MHz, 1GB each* Four Fabric DDR4 banks, x32 (no ECC), 1200 MHz, 4GB Ethernet 2x4x25GbE, 2x4x10GbE or 2x100GbE Hard Processor System 64-bit quad core Arm\u00ae Cortex\u00ae-A53 MPCore with integrated peripherals. Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration)* Platform Controller Management Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported OneAPI OneAPI Acceleration Support Package (ASP) provided with compiled FIM to support OneAPI Runtime Software Support * Linux DFL drivers targeting OFS FIMs* OPAE Software Development Kit* OPAE Tools Target Board Intel\u00ae FPGA SmartNIC N6001-PL

    Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae PCIe Attach Reference FIM documentation collection.

    OFS FIM Features Targeting Intel\u00ae Agilex\u00ae 7 SoC Attach

    Key Feature Description Device OPN AGFC023R25A2E2VR0 PCIe P-tile PCIe* Gen4x16 to the HostP-tile PCIe* Gen4x16 to the SoC (IceLake-D) Virtualization Host: 2 physical functions SoC: 1 physical function and 3 Virtual functions Memory Four Fabric DDR4 banks, x40 (optional ECC, be configured as x32 and ECC x8 ), 1200 MHz, 4GB Ethernet 2x4x25GbE Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration) * Platform Controller Management Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported Software Support * Linux DFL drivers targeting OFS FIMs * OPAE Software Development Kit * OPAE Tools Target Board Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL

    Note: Source code for BMC RTL/Firmware that works with this reference FIM can be obtained by contacting your Intel Sales Representative.

    Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae SoC Attach Reference FIM documentation collection.

    OFS FIM Targeting Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach

    Key Feature Description Device OPN 1SX280HN2F43E2VG Ethernet Configuration 1x10GbE example with 2x100GbE capability PCIe Gen3x16 EMIF Up to four DDR channels PF/VF 1 PF/3 VFs Management FPGA Management Engine (FME) with FIM management registers Interface Arm\u00ae AMBA\u00ae4 AXI Interface HLD support oneAPI Software Kernel code upstreamed to Linux.org Target Board Intel\u00ae FPGA Programmable Acceleration Card D5005

    Click here for the OFS Collateral for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach Reference FIM documentation.

    To find information on the latest releases, go to the Discussions Tab in the OFS GitHub repository.

    "},{"location":"#open-fpga-stack-repositories","title":"Open FPGA Stack Repositories","text":"

    Accessing OFS ingredients to use within the development framework is easy. The github.com/OFS site provides all the hardware and software repositories in one location.

    Development Focus Repository Folder Description Hardware ofs-agx7-pcie-attach Provides RTL, unit tests, and build scripts to create an Intel\u00ae Agilex\u00ae 7 FIM and is leveraged as a starting point for a custom PCIe Attach design. The reference FIM targets an Intel\u00ae FPGA SmartNIC N6001-PL Platform or the Intel Agilex 7 FPGA F-Series Development Kit (2x F-Tile) User Guide. Hardware ofs-f2000x-pl Provides RTL, unit tests, and build scripts to create Intel\u00aeAgilex\u00ae FIM and is leveraged as a starting point for a custom SoC Attach design. The reference FIM targets an Intel\u00ae FPGA IPU F2000X-PL Platform. Hardware ofs-d5005 Provides RTL, unit tests, and build scripts to create Intel\u00ae Stratix 10\u00ae FIM and is leveraged as a starting point for a custom PCIe Attach design. The reference FIM targets an Intel\u00ae FPGA PAC D5005 development board. Hardware oneapi-asp Contains the files to generate the support package that works with the reference shells and allows you to use OneAPI. This is an optional repository for developers interested in OneAPI Hardware ofs-fim-common Provides RTL components that are shared among all new platforms that are introduced in OFS. This folder is a subumodule in each platform repository folder. Hardware examples-afu Provides simple Accelerator Functional Unit (AFU) examples you can use as a template for starting your own workload design. Hardware ofs-platform-afu-bbb Contains the hardware code to build a standard interface between the FIM and your workload. Software linux-dfl This repository is a mirror of the linux.org Git site and contains the most up-to-date drivers that are being developed and upstreamed for OFS platforms. Software meta-ofs This repository provides the Linux\u00ae DFL kernel and the OPAE SDK for the Yocto\u00ae Project. Software opae-sdk Contains the ingredients to build the OFS Open Programmable Acceleration Engine (OPAE) Software Development Kit which provides APIs and userspace tools for OFS FPGA management. Software opae-sim This repository is used to build the AFU Hardware/Software Co-Simulation Environment workload developers can use to ensure their AFU can work with the OFS software stack. Software linux-dfl-backport A place for finding and leveraging out-of-tree backported drivers for older OS versions . Software opae-legacy Supports OFS platforms built on the legacy version of OPAE software. Not used in current OFS designs Documentation ofs.github.io Contains the hardware and software collateral that surfaces on the OFS website: https://ofs.github.io

    "},{"location":"hw/common/doc_modules/links/","title":"AFU Dev","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/","title":"Software Reference Manual: Open FPGA Stack","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#10-introduction","title":"1.0 Introduction","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#11-audience","title":"1.1 Audience","text":"

    The information presented in this document is intended to be used by software developers looking to increase their knowledge of the OPAE SDK user-space software stack and the kernel-space linux-dfl drivers. This information is intended as a starting point, with links to where users can deep dive on specific topics.

    Former OPAE and DFL software documents were combined with the Software Reference Manual to reduce clutter and more cleanly document OFS software capabilities. The following documents are no longer available as standalone as of ofs-2023.3:

    • Quick Start Guide
    • OPAE Installation Guide
    • OPAE C API Programming Guide
    • OPAE Python Bindings
    • OPAE Plugin Developers Guide
    • Open Programmable Accelerator Engine (OPAE) Linux Device Driver Architecture
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#12-terminology","title":"1.2 Terminology","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework. In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#13-opae-software-development-kit-sdk","title":"1.3 OPAE Software Development Kit (SDK)","text":"

    The OPAE C library is a lightweight user-space library that provides abstraction for FPGA resources in a compute environment. Built on top of the OPAE Intel\u00ae FPGA driver stack that supports Intel\u00ae FPGA platforms, the library abstracts away hardware specific and OS specific details and exposes the underlying FPGA resources as a set of features accessible from within software programs running on the host. The OPAE source code is available on the OPAE SDK repository.

    By providing a unified C API, the library supports different FPGA integration and deployment models, ranging from single-node systems with one or a few FPGA devices to large-scale FPGA deployments in a data center. At one end of the spectrum, the API supports a simple application using a PCIe link to reconfigure the FPGA with different accelerator functions. At the other end of the spectrum, resource management and orchestration services in a data center can use this API to discover and select FPGA resources and then allocate them for use by acceleration workloads.

    The OPAE provides consistent interfaces to crucial components of the platform. OPAE does not constrain frameworks and applications by making optimizations with limited applicability. When the OPAE does provide convenience functions or optimizations, they are optional. For example, the OPAE provides an interface to allocate physically contiguous buffers in system memory that user-space software and an accelerator can share. This interface enables the most basic feature set of allocating and sharing a large page of memory in one API call. However, it does not provide a malloc()-like interface backed by a memory pool or slab allocator. Higher layers of the software stack can make such domain-specific optimizations.

    Most of the information related to OPAE can be found on the official OFS Site and in the OPAE SDK repository. The following is a summary of the information present on this web page:

    • Configuration options present in the OPAE SDK build and installation flow
    • The steps required to build a sample OPAE application
    • An explanation of the basic application flow
    • A reference for the C, C++, and Python APIs
    • An explanation of the OPAE Linux Device Driver Architecture
    • Definitions for the various user-facing OPAE SDK tools

    The remaining sections on OPAE in this document are unique and build on basic principles explained in opae.github.io.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#table-1-additional-websites-and-links","title":"Table 1: Additional Websites and Links","text":"Document Link OPAE SDK on github OPAE SDK repository OPAE Documents OFS Site pybind11 https://pybind11.readthedocs.io/en/stable/ CLI11 https://github.com/CLIUtils/CLI11 spdlog https://github.com/gabime/spdlog"},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#table-2-ofs-hardware-terminology","title":"Table 2: OFS Hardware Terminology","text":"Term Description FPGA: Field Programmable Gate Array a discrete or integrated device connecting to a host CPU via PCIe or other type of interconnects. Accelerator Function Unit (AFU) The AFU is the supplied implementation of an accelerator, typically in HDL. AFUs implement a function such as compression, encryption, or mathematical operations.The Quartus Prime Pro software synthesizes the RTL logic into a bitstream. Accelerator Function (AF) The AF is the compiled binary for an AFU. An AF is a raw binary file (.rbf) bitstream. A tool (fpgaconf) reconfigures the FPGA using an AF bitstream. Reconfiguration The process of reprogramming the FPGA with a different AF."},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#20-opae-c-api","title":"2.0 OPAE C API","text":"

    The following OPAE data structures and functions integrate AFUs into the OPAE environment. The OPAE C API models these data structures and functions. For more information on the object models refer to the Object model section.

    • Accelerator: An accelerator is an allocable accelerator function implemented in an FPGA. An accelerator tracks the ownership of an AFU (or part of it) for a process that uses it. Multiple processes can share an accelerator.
    • Device: The OPAE enumerates and models two device types: the FPGA and the AFU.
    • Events: Events are asynchronous notifications. The FPGA driver triggers particular events to indicate error conditions. Accelerator logic can also define its own events. User applications can choose to be notified when particular events occur and respond appropriately.
    • Shared memory buffers: Software allocates shared memory buffers in user process memory on the host. Shared memory buffers facilitate data transfers between the user process and the accelerator that it owns.
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#21-libopae-c","title":"2.1 libopae-c","text":"

    Linking with this library is straightforward. Code using the OPAE library should include the header file fpga.h. Taking the GCC compiler on Linux as an example, here is the simplest compile and link command:

    gcc myprog.c -I</path/to/fpga.h> -L</path/to/libopae-c.so> -lopae-c -luuid -ljson-c -lpthread

    .. note::

    The OPAE library uses the third-party `libuuid` and `libjson-c` libraries that are not distributed with \nthe OPAE library. Make sure to install these libraries.\n
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#sample-code","title":"Sample Code","text":"

    The library source includes two code samples. Use these samples to learn how to call functions in the library. Build and run these samples to determine if your installation and environment are set up properly.

    Refer to the Running the Hello FPGA Example chapter in the Intel\u00ae Acceleration Stack Quick Start Guide for for Intel Programmable Acceleration Card with Intel Arria\u00ae 10 GX FPGA for more information about using the sample code.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#high-level-directory-structure","title":"High-Level Directory Structure","text":"

    Building and installing the OPAE library results in the following directory structure on the Linux OS. Windows and MacOS have similar directories and files.

    Directory & Files Contents include/opae Directory containing all header files include/opae/fpga.h Top-level header for user code to include include/opae/access.h Header file for accelerator acquire/release, MMIO, memory management, event handling, and so on include/opae/bitstream.h Header file for bitstream manipulation functions include/opae/common.h Header file for error reporting functions include/opae/enum.h Header file for AFU enumeration functions include/opae/manage.h Header file for FPGA management functions include/opae/types.h Various type definitions lib Directory containing shared library files lib/libopae-c.so The shared dynamic library for linking with the user application doc Directory containing API documentation doc/html Directory for documentation of HTML format doc/latex Directory for documentation of LaTex format doc/man Directory for documentation of Unix man page format"},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#object-models","title":"Object Models","text":"
    • fpga_objtype: An enum type that represents the type of an FPGA resource, either FPGA_DEVICE or FPGA_ACCELERATOR. An FPGA_DEVICE object corresponds to a physical FPGA device. Only FPGA_DEVICE objects can invoke management functions. The FPGA_ACCELERATOR represents an instance of an AFU.
    • fpga_token: An opaque type that represents a resource known to, but not necessarily owned by, the calling process. The calling process must own a resource before it can invoke functions of the resource.
    • fpga_handle: An opaque type that represents a resource owned by the calling process. The API functions fpgaOpen() and fpgaClose() acquire and release ownership of a resource that an fpga_handle represents. (Refer to the Functions section for more information.)
    • fpga_properties: An opaque type for a properties object. Your applications use these properties to query and search for appropriate resources. The FPGA Resource Properties section documents properties visible to your applications.
    • fpga_event_handle: An opaque handle the FPGA driver uses to notify your application about an event.
    • fpga_event_type: An enum type that represents the types of events. The following are valid values: FPGA_EVENT_INTERRUPT, FPGA_EVENT_ERROR, and FPGA_EVENT_POWER_THERMAL. (The Intel Programmable Acceleration Card (PAC) with Intel Arria 10 GX FPGA does not handle thermal and power events.)
    • fpga_result: An enum type to represent the result of an API function. If the function returns successfully the result is FPGA_OK. Otherwise, the result is the appropriate error codes. Function fpgaErrStr() translates an error code into human-readable strings.
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#functions","title":"Functions","text":"

    The table below groups important API calls by their functionality. For more information about each of the functions, refer to the OPAE C API reference manual.

    Functionality API Call FPGA Accelerator Description Enumeration fpgaEnumerate() Yes Yes Query FPGA resources that match certain properties Enumeration: Properties fpga[Get, Update, Clear, Clone, Destroy Properties]() Yes Yes Manage fpga_properties life cycle fpgaPropertiesGet[Prop]() Yes Yes Get the specified property Prop, from the FPGA Resource Properties table fpgaPropertiesSet[Prop]() Yes Yes Set the specified property Prop, from the FPGA Resource Properties table Access: Ownership fpga[Open, Close]() Yes Yes Acquire/release ownership Access: Reset fpgaReset() Yes Yes Reset an accelerator Access: Event handling fpga[Register, Unregister]Event() Yes Yes Register/unregister an event to be notified about fpga[Create, Destroy]EventHandle() Yes Yes Manage fpga_event_handle life cycle Access: MMIO fpgaMapMMIO(), fpgaUnMapMMIO() Yes Yes Map/unmap MMIO space fpgaGetMMIOInfo() Yes Yes Get information about the specified MMIO space fpgaReadMMIO[32, 64]() Yes Yes Read a 32-bit or 64-bit value from MMIO space fpgaWriteMMIO[32, 64]() Yes Yes Write a 32-bit or 64-bit value to MMIO space Memory management: Shared memory fpga[Prepare, Release]Buffer() Yes Yes Manage memory buffer shared between the calling process and an accelerator fpgaGetIOAddress() Yes Yes Return the device I/O address of a shared memory buffer fpgaBindSVA() Yes Yes Bind IOMMU shared virtual addressing Management: Reconfiguration fpgaReconfigureSlot() Yes No Replace an existing AFU with a new one Error report fpgaErrStr() Yes Yes Map an error code to a human readable string

    .. note::

    The UMsg APIs are not supported for the Intel(R) PAC cards. They will be deprecated in a future release.\n
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#fpga-resource-propertie","title":"FPGA Resource Propertie","text":"

    Applications query resource properties by specifying the property name for Prop in the fpgaPropertiesGet[Prop]() and fpgaPropertiesSet[Prop]() functions. The FPGA and Accelerator columns state whether or not the Property is available for the FPGA or Accelerator objects.

    Property FPGA Accelerator Description Parent No Yes fpga_token of the parent object ObjectType Yes Yes The type of the resource: either FPGA_DEVICE or FPGA_ACCELERATOR Bus Yes Yes The bus number Device Yes Yes The PCI device number Function Yes Yes The PCI function number SocketId Yes Yes The socket ID DeviceId Yes Yes The device ID NumSlots Yes No Number of AFU slots available on an FPGA_DEVICE resource BBSID Yes No The FPGA Interface Manager (FIM) ID of an FPGA_DEVICE resource BBSVersion Yes No The FIM version of an FPGA_DEVICE resource VendorId Yes No The vendor ID of an FPGA_DEVICE resource GUID Yes Yes The GUID of an FPGA_DEVICE or FPGA_ACCELERATOR resource NumMMIO No Yes The number of MMIO space of an FPGA_ACCELERATOR resource NumInterrupts No Yes The number of interrupts of an FPGA_ACCELERATOR resource AcceleratorState No Yes The state of an FPGA_ACCELERATOR resource: either FPGA_ACCELERATOR_ASSIGNED or FPGA_ACCELERATOR_UNASSIGNED"},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#opae-c-api-return-codes","title":"OPAE C API Return Codes","text":"

    The OPAE C library returns a code for every exported public API function. FPGA_OK indicates successful completion of the requested operation. Any return code other than FPGA_OK indicates an error or unexpected behavior. When using the OPAE C API, always check the API return codes.

    Error Code Description FPGA_OK Operation completed successfully FPGA_INVALID_PARAM Invalid parameter supplied FPGA_BUSY Resource is busy FPGA_EXCEPTION An exception occurred FPGA_NOT_FOUND A required resource was not found FPGA_NO_MEMORY Not enough memory to complete operation FPGA_NOT_SUPPORTED Requested operation is not supported FPGA_NO_DRIVER Driver is not loaded FPGA_NO_DAEMON FPGA Daemon (fpgad) is not running FPGA_NO_ACCESS Insufficient privileges or permissions FPGA_RECONF_ERROR Error while reconfiguring FPGA"},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#211-device-abstraction","title":"2.1.1 Device Abstraction","text":"

    The OPAE C API relies on two base abstractions concerning how the FIM and accelerator are presented to and manipulated by the user. The FIM is concerned with management functionality. Access to the FIM and its interfaces is typically restricted to privileged (root) users. The accelerator contains the user-defined logic in its reconfigurable region. Most OPAE end-user applications are concerned with querying and opening the accelerator device, then interacting with the AFU via MMIO and shared memory.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2111-device-types","title":"2.1.1.1 Device types","text":"

    The C enum fpga_objtype defines two variants. The FPGA_DEVICE variant corresponds to the FIM portion of the device, and the FPGA_ACCELERATOR refers to the accelerator, also known as the AFU.

    An FPGA_DEVICE refers loosely to the sysfs tree rooted at the dfl-fme.X directory, for example /sys/class/fpga_region/region0/dfl-fme.0, and its associated device file /dev/dfl-fme.0.

    An FPGA_ACCELERATOR refers loosely to the sysfs tree rooted at the dfl-port.X directory, for example /sys/class/fpga_region/region0/dfl-port.0, and its associated device file /dev/dfl-port.0.

    The number X in dfl-fme.X and dfl-port.X refers to a numeric ID that is assigned by the DFL device driver to uniquely identify an instance of the FIM/Accelerator. Systems with multiple FPGA acceleration devices will have multiple dfl-fme.X\u2019s and matching dfl-port.X\u2019s.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2112-tokens-and-handles","title":"2.1.1.2 Tokens and Handles","text":"

    An fpga_token is an opaque data structure that uniquely represents an FPGA_DEVICE or an FPGA_ACCELERATOR. Tokens convey existence, but not ownership. Tokens are retrieved via the OPAE enumeration process described below using the fpgaEnumerate() call.

    An fpga_handle is an opaque data structure that corresponds to an opened device instance, whether FPGA_DEVICE or FPGA_ACCELERATOR. A Handle is obtained from a token via the fpgaOpen() call. A handle conveys that the /dev/dfl-fme.X or /dev/dfl-port.X device file has been opened and is ready for interaction via its IOCTL interface.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#212-enumeration","title":"2.1.2 Enumeration","text":"

    Enumeration is the process by which an OPAE application becomes aware of the existence of FPGA_DEVICE\u2019s and FPGA_ACCELERATOR\u2019s. Refer to the signature of the fpgaEnumerate() call:

    fpga_result fpgaEnumerate(const fpga_properties *filters,\nuint32_t num_filters,\nfpaa_token *tokens,\nuint32_t max_tokens,\nuint32_t *num_matches);\n

    Figure 1 fpgaEnumerate()

    The typical enumeration flow involves an initial call to fpgaEnumerate() to discover the number of available tokens.

    uint32_t num_matches = 0;\nfpgaEnumerate(NULL, 0, NULL, 0, &num_matches);\n

    Figure 2 Discovering Number of Tokens

    Once the number of available tokens is known, the application can allocate the correct amount of space to hold the tokens:

    fpga_token *tokens;\nuint32_t num_tokens = num_matches;\ntokens = (fpga_token *)calloc(num_tokens, sizeof(fpga_token));\nfpgaEnumerate(NULL, 0, tokens, num_tokens, &num_matches);\n

    Figure 3 Enumerating All Tokens

    Note that parameters filters and num_filters were not used in the preceding example, as they were NULL and 0. When no filtering criteria are provided, fpgaEnumerate() returns all tokens that can be enumerated.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2121-fpga_properties-and-filtering","title":"2.1.2.1 fpga_properties and Filtering","text":"

    An fpga_properties is an opaque data structure used to retrieve all of the properties concerning an FPGA_DEVICE or FPGA_ACCELERATOR. These properties can be included in the filters parameter to fpgaEnumerate() to select tokens by specific criteria.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#21211-common-properties","title":"2.1.2.1.1 Common Properties","text":"

    Table 3 lists the set of properties that are common to FPGA_DEVICE and FPGA_ACCELERATOR:

    Property Description fpga_guid guid; FPGA_DEVICE: PR Interface ID FPGA_ACCELERATOR: AFU ID fpga_token parent; FPGA_DEVICE: always NULL FPGA_ACCELERATOR: the token of the corresponding FPGA_DEVICE, if any. Otherwise, NULL. fpga_objtype objtype; FPGA_DEVICE or FPGA_ACCELERATOR uint16_t segment; The segment portion of the PCIe address: ssss:bb:dd.f uint8_t bus;

    The bus portion of the PCIe address:

    ssss:bb:dd.f

    uint8_t device;

    The device portion of the PCIe address:

    ssss:bb:dd.f

    uint8_t function; The function portion of the PCIe address: ssss:bb:dd.f uint64_t object_id; A unique 64-bit value that identifies this token on the system. uint16_t vendor_id; The PCIe Vendor ID uint16_t device_id; The PCIe Device ID uint32_t num_errors; The number of error sysfs nodes available for this token. fpga_interface interface; An identifier for the underlying plugin-based access method.

    Table 3 Common Properties

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#21212-fpga_device-properties","title":"2.1.2.1.2 FPGA_DEVICE Properties","text":"

    Table 4 lists the set of properties that are specific to FPGA_DEVICE token types.

    Property Description uint64_t bbs_id; FIM-specific Blue Bitstream ID fpga_version bbs_version; BBS version

    Table 4 FPGA_DEVICE Properties

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#21213-fpga_accelerator-properties","title":"2.1.2.1.3 FPGA_ACCELERATOR Properties","text":"

    Table 5 lists the set of properties that are specific to FPGA_ACCELERATOR token types.

    Property Description fpga_accelerator_state state; Whether the Accelerator is currently open uint32_t num_mmio; The number of MMIO regions available uint32_t num_interrupts; The number of interrupts available

    Table 5 FPGA_ACCELERATOR Properties

    Following is an example of using fpga_properties to enumerate a specific AFU:

    #define NLB0_AFU \"D8424DC4-A4A3-C413-F89E-433683F9040B\"\nfpga_properties filter = NULL;\nfpga_guid afu_id;\nfpgaGetProperties(NULL, &filter); // NULL: a new empty properties\nfpgaPropertiesSetObjectType(filter, FPGA_ACCELERATOR);\nuuid_parse(NLB0_AFU, afu_id);\nfpgaPropertiesSetGuid(filter, afu_id);\nfpgaEnumerate(&filter, 1, tokens, num_tokens, &num_matches);\n
    Relevant Links: - fpga_guid - fpgaGetProperties - fpgaPropertiesSetObjectType - fpgaPropertiesSetGUID

    Figure 4 Filtering During Enumeration

    Note that fpga_properties and fpga_token\u2019s are allocated resources that must be freed by their respective API calls, ie fpgaDestroyProperties() and fpgaDestroyToken().

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#213-access","title":"2.1.3 Access","text":"

    Once a token is discovered and returned to the caller by fpgaEnumerate(), the token can be converted into a handle by fpgaOpen(). Upon a successful call to fpgaOpen(), the associated /dev/dfl-fme.X (FPGA_DEVICE) or /dev/dfl-port.X (FPGA_ACCELERATOR) is opened and ready for use. Having acquired an fpga_handle, the application can then use the handle with any of the OPAE APIs that require an fpga_handle as an input parameter.

    Like tokens and properties, handles are allocated resources. When a handle is no longer needed, it should be closed and released by calling fpgaClose().

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#214-events","title":"2.1.4 Events","text":"

    Event registration in OPAE is a two-step process. First, the type of event must be identified. The following fpga_event_type variants are defined:

    Event Description FPGA_EVENT_INTERRUPT AFU interrupt FPGA_EVENT_ERROR Infrastructure error event (FME/Port Error)

    Table 6 FPGA Event Types

    Once the desired event type is known, an fpga_event_handle is created via fpgaCreateEventHandle(). Once the event handle is available, the event notification is registered using fpgaRegisterEvent(). In the example below, note the use of the flags field for passing the desired IRQ vector when the event type is FPGA_EVENT_INTERRUPT. With the event registered, the application can then use fpgaGetOSObjectFromEventHandle() to obtain a file descriptor for use with the poll() system call. When the interrupt occurs, the file descriptor will be set to the signaled state by the DFL driver.

    fpga_event_handle event_handle = NULL;\nint fd = -1;\nfpgaCreateEventHandle(&event_handle);\nfpgaRegisterEvent(fpga_handle, FPGA_EVENT_INTERRUPT,\nevent_handle, irq_vector);\nfpgaGetOSObjectFromEventHandle(event_handle, &fd);\n

    Figure 5 Creating and Registering Events

    When an event notification is no longer needed, it should be released by calling fpgaUnregisterEvent(). Like device handles, event handles are allocated resources that must be freed when no longer used. To free an event handle, use the fpgaDestroyEventHandle() call.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#215-mmio-and-shared-memory","title":"2.1.5 MMIO and Shared Memory","text":"

    Communication with the AFU is achieved via reading and writing CSRs and by reading and writing to AFU/host shared memory buffers. An AFU\u2019s CSRs are memory-mapped into the application process address space by way of the fpgaMapMMIO() call.

    uint32_t mmio_num = 0;\nfpgaMapMMIO(fpga_handle, mmio_num, NULL);\nfpgaWriteMMIO64(fpga_handle, mmio_num, MY_CSR, 0xa);\n

    Figure 6 Mapping and Accessing CSRs

    The second parameter, mmio_num, is the zero-based index identifying the desired MMIO region. The maximum number of MMIO regions for a particular handle is found by accessing the num_mmio property. Refer to the fpgaPropertiesGetNumMMIO() call.

    Once the AFU CSRs are mapped into the process address space, the application can use the fpgaReadMMIO**XX**() and fpgaWriteMMIO**XX**() family of functions, eg fpgaReadMMIO64() and fpgaWriteMMIO64(). When an MMIO region is no longer needed, it should be unmapped from the process address space using the fpgaUnmapMMIO() call.

    Shared memory buffers are allocated by way of the fpgaPrepareBuffer() call.

    fpga_result fpgaPrepareBuffer(fpga_handle handle,\nuint64_t len,\nvoid **buf_addr,\nuint64_t *wsid,\nint flags);\n

    Figure 7 fpgaPrepareBuffer()

    Three buffer lengths are supported by this allocation method:

    Length Description 4096 (4KiB) No memory configuration needed. 2097152 (2MiB) Requires 2MiB huge pages to be allocated. 1073741824 (1GiB) Requires 1GiB huge pages to be allocated.

    Table 7 fpgaPrepareBuffer() Lengths

    echo 8 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages\necho 2 > /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages\n

    Figure 8 Configuring Huge Pages

    The buf_addr parameter to fpgaPrepareBuffer() is a pointer to a void * that accepts the user virtual base address of the newly-created buffer. The wsid parameter is a pointer to a uint64_t that receives a unique workspace ID for the buffer allocation. This workspace ID is used in subsequent calls to fpgaReleaseBuffer(), which should be called when the buffer is no longer needed and in calls to fpgaGetIOAddress() which is used to query the IO base address of the buffer. The IO base address can be programmed into the AFU by means of the AFU CSR space. For example, here is a code snippet from the hello_fpga sample that demonstrates programming a shared buffer\u2019s IO base address into an AFU CSR in MMIO region 0:

    #define LOG2_CL 6\n#define CACHELINE_ALIGNED_ADDR(p) ((p) >> LOG2_CL)\nfpgaGetIOAddress(accelerator_handle, input_wsid, &iova);\nfpgaWriteMMIO64(accelerator_handle, 0, nlb_base_addr + CSR_SRC_ADDR,\nCACHELINE_ALIGNED_ADDR(iova));\n

    Figure 9 Programming Shared Memory

    If applications need to map a shared buffer that has been allocated by some other means than fpgaPrepareBuffer(), then the flags parameter can be set to FPGA_BUF_PREALLOCATED. This causes fpgaPrepareBuffer() to skip the allocation portion of the call and to only memory map the given buf_addr into the application process address space.

    Buffers can also be allocated and mapped as read-only by specifying FPGA_BUF_READ_ONLY.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#216-management","title":"2.1.6 Management","text":"

    The management feature in OPAE concerns re-programming the programmable region of the Port. To program the Port bitstream, pass a handle to the FPGA_DEVICE associated with the desired Port. The slot parameter identifies which Port to program in the case of multi-port implementations. Most designs will only pass zero as the slot parameter. The bitstream parameter is a buffer that contains the entire bitstream contents, including the JSON bitstream header information. The bitstream_len field gives the length of bitstream in bytes.

    fpgaReconfigureSlot() first checks whether the FPGA_ACCELERATOR corresponding to the FPGA_DEVICE in fme_handle is open. If it is open, then the programming request is aborted with an error code. The application may pass FPGA_RECONF_FORCE in the flags parameter in order to avoid this open check and forcefully program the bitstream.

    fpga_result fpgaReconfigureSlot(fpga_handle fme_handle,\nuint32_t slot,\nconst uint8_t *bitstream,\nsize_t bitstream_len,\nint flags);\n

    Figure 10 fpgaReconfigureSlot()

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#217-errors","title":"2.1.7 Errors","text":"

    The OPAE errors API provides a means to query and clear both FPGA_DEVICE and FPGA_ACCELERATOR errors. Each FPGA device exports a collection of error registers via the DFL drivers\u2019 sysfs tree, for both the FME and the Port. Each register is typically an unsigned 64-bit mask of the current errors, where each bit or some collection of bits specifies an error type. An error is signaled if its bit or collection of bits is non-zero. Note that the 32-bit error index may vary from one process execution to the next. Applications should use fpgaGetErrorInfo() and examine the error name returned in the struct fpga_error_info to identify the desired 64-bit error mask.

    struct fpga_error_info {\nchar name[FPGA_ERROR_NAME_MAX];\nbool can_clear;\n};\n

    Figure 11 struct fpga_error_info

    Each 64-bit mask of errors is assigned a unique 32-bit integer index and a unique name. Given an fpga_token and an error index, fpgaGetErrorInfo() retrieves the struct fpga_error_info corresponding to the error.

    fpga_result fpgaGetErrorInfo(fpga_token token,\nuint32_t error_num,\nstruct fpga_error_info *error_info);\n

    Figure 12 fpgaGetErrorInfo()

    fpgaReadError() provides access to the raw 64-bit error mask, given the unique error index. fpgaClearError() clears the errors for a particular index. fpgaClearAllErrors() clears all the errors for the given fpga_token.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#218-metrics","title":"2.1.8 Metrics","text":"

    The OPAE metrics API refers to a group of functions and data structures that allow querying the various device metrics from the Board Management Controller component of the FPGA device. A metric is described by an instance of struct fpga_metric_info.

    typedef struct fpga_metric_info {\nuint64_t metric_num;\nfpga_guid metric_guid;\nchar qualifier_name[FPGA_METRIC_STR_SIZE];\nchar group_name[FPGA_METRIC_STR_SIZE];\nchar metric_name[FPGA_METRIC_STR_SIZE];\nchar metric_units[FPGA_METRIC_STR_SIZE];\nenum fpga_metric_datatype metric_datatype;\nenum fpga_metric_type metric_type;\n} fpga_metric_info;\n
    Relevant Links: - fpga_metric_datatype - fpga_metric_type

    Figure 13 fpga_metric_info

    The group_name field holds a string describing the broad categorization of the metric. Some sample values for group_name are \u201cthermal_mgmt\u201d and \u201cpower_mgmt\u201d. The metric_name field contains the metric\u2019s name. The number and names of metrics may vary from one FPGA platform to the next. The qualifier_name field is a concatenation of group_name and metric_name, with a colon character in between. The metric_units field contains the string name of the unit of measurement for the specific metric. Some examples for metric_units are \u201cVolts\u201d, \u201cAmps\u201d, and \u201cCelsius\u201d.

    The metric_datatype field uniquely identifies the underlying C data type for the metric\u2019s value:

    enum fpga_metric_datatype {\nFPGA_METRIC_DATATYPE_INT,\nFPGA_METRIC_DATATYPE_FLOAT,\nFPGA_METRIC_DATATYPE_DOUBLE,\nFPGA_METRIC_DATATYPE_BOOL,\nFPGA_METRIC_DATATYPE_UNKNOWN\n};\n

    Figure 14 enum fpga_metric_datatype

    The metric_type field classifies the metric into a broad category. This information is redundant with the group_name field.

    enum fpga_metric_type {\nFPGA_METRIC_TYPE_POWER,\nFPGA_METRIC_TYPE_THERMAL,\nFPGA_METRIC_TYPE_PERFORMANCE_CTR,\nFPGA_METRIC_TYPE_AFU,\nFPGA_METRIC_TYPE_UNKNOWN\n};\n

    Figure 15 enum fpga_metric_type

    In order to enumerate the information for each of the metrics available from the FPGA device, determine the number of metrics using fpgaGetNumMetrics().

    uint64_t num_metrics = 0;\nfpgaGetNumMetrics(handle, &num_metrics);\n

    Figure 16 Determining Number of Metrics

    This call retrieves the number of available metrics for the FPGA_DEVICE that is opened behind the handle parameter to the call. Refer to 2.1.3 Access for information about the fpgaOpen() call. When the number of available metrics is known, allocate a buffer large enough to hold that many fpga_metric_info data structures, and call fpgaGetMetricsInfo() to populate the entries:

    fpga_metric_info *metric_info;\nuint64_t metric_infos = num_metrics;\nmetric_info = calloc(num_metrics, sizeof(fpga_metric_info));\nfpgaGetMetricsInfo(handle, metric_info, &metric_infos);\n

    Figure 17 Querying Metrics Info

    The fpga_metric structure is the representation of a metric\u2019s value:

    typedef struct fpga_metric {\nuint64_t metric_num;\nmetric_value value;\nbool isvalid;\n} fpga_metric;\n
    Relevant Links: - metric_value

    Figure 18 struct fpga_metric

    The metric_num field matches the metric_num field of the fpga_metric_info structure. value contains the metric value, which is encoded in the C data type identified by the metric_datatype field of fpga_metric_info. Finally, the isvalid field denotes whether the metric value is valid.

    There are two methods of obtaining a metric\u2019s value, given the information in the fpga_metric_info structure:

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2181-querying-metric-values-by-index","title":"2.1.8.1 Querying Metric Values by Index","text":"

    fpgaGetMetricsByIndex() retrieves a metric value using the metric_num field of the metric info:

    uint64_t metric_num = metric_info[0]->metric_num;\nfpga_metric metric0;\nfpgaGetMetricsByIndex(handle, &metric_num, 1, &metric0);\n

    Figure 19 Retrieve Metric by Index

    This call allows retrieving one or more metric values, each identified by their unique metric_num. The second and fourth parameters allow passing arrays so that multiple values can be fetched in a single call.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2182-querying-metric-values-by-name","title":"2.1.8.2 Querying Metric Values by Name","text":"

    fpgaGetMetricsByName() retrieves a metric value using the metric_name field of the metric info:

    char *metric_name = metric_info[1]->metric_name;\nfpga_metric metric1;\nfpgaGetMetricsByName(handle, &metric_name, 1, &metric1);\n

    This call also allows retrieving one or more metric values, each identified by their unique metric_name. The second and fourth parameters allow passing arrays so that multiple values can be fetched in a single call.

    The fpgaGetMetricsThresholdInfo() call is provided for legacy implementations only. It should be considered deprecated for current and future FPGA designs.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#219-sysobject","title":"2.1.9 SysObject","text":"

    When the hardware access method in use is the DFL drivers (see 2.3.2 libxfpga Plugin), the sysfs tree rooted at the struct _fpga_token\u2019s sysfspath member is accessible via the OPAE SDK SysObject API. The SysObject API provides an abstraction to search, traverse, read, and write sysfs entities. These sysfs entities may take the form of directories, which are referred to as containers, or files, which are referred to as attributes. Figure 20 enum fpga_sysobject_type shows the API\u2019s means of distinguishing between the two types.

    enum fpga_sysobject_type {\nFPGA_OBJECT_CONTAINER = (1u << 0),\nFPGA_OBJECT_ATTRIBUTE = (1u << 1)\n};\n

    Figure 20 enum fpga_sysobject_type

    The SysObject API introduces another opaque structure type, fpga_object. An fpga_object can be queried from an fpga_token or an fpga_handle by way of the fpgaTokenGetObject() and fpgaHandleGetObject() API\u2019s.

    fpga_result fpgaTokenGetObject(fpga_token token, const char *name,\nfpga_object *object, int flags);\nfpga_result fpgaHandleGetObject(fpga_handle handle, const char *name,\nfpga_object *object, int flags);\n

    Figure 21 fpgaTokenGetObject() / fpgaHandleGetObject()

    The remainder of the SysObject API is broken into two categories of calls, depending on the fpga_object\u2019s type. The type of an fpga_object is learned via fpgaObjectGetType().

    fpga_result fpgaObjectGetType(fpga_object obj,\nenum fpga_sysobject_type *type);\n

    Figure 22 fpgaObjectGetType()

    When an fpga_object is no longer needed, it should be freed via fpgaDestroyObject().

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2191-fpga_object_container-apis","title":"2.1.9.1 FPGA_OBJECT_CONTAINER API\u2019s","text":"

    For directory sysfs entities, passing a value of FPGA_OBJECT_RECURSE_ONE or FPGA_OBJECT_RECURSE_ALL in the flags parameter to fpgaTokenGetObject() or fpgaHandleGetObject() causes these two API\u2019s to treat the target object as either a single-layer or multi-layer directory structure, making its child entities available for query via fpgaObjectGetObject() and fpgaObjectGetObjectAt().

    fpga_result fpgaObjectGetObject(fpga_object parent, const char *name,\nfpga_object *object, int flags);\nfpga_result fpgaObjectGetObjectAt(fpga_object parent, size_t idx,\nfpga_object *object);\n

    Figure 23 fpgaObjectGetObject() / fpgaObjectGetObjectAt()

    Any child object resulting from fpgaObjectGetObject() or fpgaObjectGetObjectAt() must be freed via fpgaDestroyObject() when it is no longer needed.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2192-fpga_object_attribute-apis","title":"2.1.9.2 FPGA_OBJECT_ATTRIBUTE API\u2019s","text":"

    Attribute sysfs entities may be queried for their size and read from or written to. In order to determine the size of an attribute\u2019s data, use fpgaObjectGetSize().

    fpga_result fpgaObjectGetSize(fpga_object obj,\nuint32_t *value, int flags);\n

    Figure 24 fpgaObjectGetSize()

    Attributes containing arbitrary string data can be read with fpgaObjectRead().

    fpga_result fpgaObjectRead(fpga_object obj, uint8_t *buffer,\nsize_t offset, size_t len, int flags);\n

    Figure 25 fpgaObjectRead()

    If an attribute contains an unsigned integer value, its value can be read with fpgaObjectRead64() and written with fpgaObjectWrite64().

    fpga_result fpgaObjectRead64(fpga_object obj,\nuint64_t *value, int flags);\nfpga_result fpgaObjectWrite64(fpga_object obj,\nuint64_t value, int flags);\n

    Figure 26 fpgaObjectRead64() / fpgaObjectWrite64()

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2110-utilities","title":"2.1.10 Utilities","text":"

    The fpga_result enumeration defines a set of error codes used throughout OPAE. In order to convert an fpga_result error code into a printable string, the application can use the fpgaErrStr() call.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#22-dfl-driver-ioctl-interfaces","title":"2.2 DFL Driver IOCTL Interfaces","text":"

    The DFL drivers export an IOCTL interface which the libxfpga.so plugin consumes in order to query and configure aspects of the FME and Port. These interfaces are used only internally by the SDK; they are not customer-facing. The description here is provided for completeness only.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#221-port-reset","title":"2.2.1 Port Reset","text":"

    The DFL_FPGA_PORT_RESET ioctl is used by the fpgaReset() call in order to perform a Port reset. The fpga_handle passed to fpgaReset() must be a valid open handle to an FPGA_ACCELERATOR. The ioctl requires no input/output parameters.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#222-port-information","title":"2.2.2 Port Information","text":"

    The DFL_FPGA_PORT_GET_INFO ioctl is used to query properties of the Port, notably the number of associated MMIO regions. The ioctl requires a pointer to a struct dfl_fpga_port_info.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#223-mmio-region-information","title":"2.2.3 MMIO Region Information","text":"

    The DFL_FPGA_PORT_GET_REGION_INFO ioctl is used to query the details of an MMIO region. The ioctl requires a pointer to a struct dfl_fpga_port_region_info. The index field of the struct is populated by the caller, and the padding, size, and offset values are populated by the DFL driver.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#224-shared-memory-mapping-and-unmapping","title":"2.2.4 Shared Memory Mapping and Unmapping","text":"

    The DFL_FPGA_PORT_DMA_MAP ioctl is used to map a memory buffer into the application\u2019s process address space. The ioctl requires a pointer to a struct dfl_fpga_port_dma_map.

    The DFL_FPGA_PORT_DMA_UNMAP ioctl is used to unmap a memory buffer from the application\u2019s process address space. The ioctl requires a pointer to a struct dfl_fpga_port_dma_unmap.

    These ioctls provide the underpinnings of the fpgaPrepareBuffer() and fpgaReleaseBuffer() calls.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#225-number-of-port-error-irqs","title":"2.2.5 Number of Port Error IRQs","text":"

    The DFL_FPGA_PORT_ERR_GET_IRQ_NUM ioctl is used to query the number of Port error interrupt vectors available. The ioctl requires a pointer to a uint32_t that receives the Port error interrupt count.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#226-port-error-interrupt-configuration","title":"2.2.6 Port Error Interrupt Configuration","text":"

    The DFL_FPGA_PORT_ERR_SET_IRQ ioctl is used to configure one or more file descriptors for the Port Error interrupt. The ioctl requires a pointer to a struct dfl_fpga_irq_set. The values stored in the evtfds field of this struct should be populated with the event file descriptors for the interrupt, as returned by the eventfd() C standard library API.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#227-number-of-afu-interrupts","title":"2.2.7 Number of AFU Interrupts","text":"

    The DFL_FPGA_PORT_UINT_GET_IRQ_NUM ioctl is used to query the number of AFU interrupt vectors available. The ioctl requires a pointer to a uint32_t that receives the AFU interrupt count.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#228-user-afu-interrupt-configuration","title":"2.2.8 User AFU Interrupt Configuration","text":"

    The DFL_FPGA_PORT_UINT_SET_IRQ ioctl is used to configure one or more file descriptors for the AFU interrupt. The ioctl requires a pointer to a struct dfl_fpga_irq_set. The values stored in the evtfds field of this struct should be populated with the event file descriptors for the interrupt, as returned by the eventfd() C standard library API.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#229-partial-reconfiguration","title":"2.2.9 Partial Reconfiguration","text":"

    The DFL_FPGA_FME_PORT_PR ioctl is used to update the logic stored in the Port\u2019s programmable region. This ioctl must be issued on the device file descriptor corresponding to the FPGA_DEVICE (/dev/dfl-fme.X). The ioctl requires a pointer to a struct dfl_fpga_fme_port_pr with each of the fields populated.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2210-number-of-fme-error-irqs","title":"2.2.10 Number of FME Error IRQs","text":"

    The DFL_FPGA_FME_ERR_GET_IRQ_NUM ioctl is used to query the number of FME error interrupt vectors available. The ioctl requires a pointer to a uint32_t that receives the FME error interrupt count.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2211-fme-error-interrupt-configuration","title":"2.2.11 FME Error Interrupt Configuration","text":"

    The DFL_FPGA_FME_ERR_SET_IRQ ioctl is used to configure one or more file descriptors for the FME Error interrupt. The ioctl requires a pointer to a struct dfl_fpga_irq_set. The values stored in the evtfds field of this struct should be populated with the event file descriptors for the interrupt, as returned by the eventfd() C standard library API. as returned by the eventfd() C standard library API.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#23-plugin-manager","title":"2.3 Plugin Manager","text":"

    The OPAE Plugin Manager refers to initialization code in libopae-c that examines an FPGA device\u2019s PCIe Vendor and Device ID and makes an association between a particular FPGA device and its access method. OPAE currently supports three device access methods:

    Access Method

    Plugin Module

    Device Feature List drivers libxfpga.so Virtual Function I/O libopae-v.so AFU Simulation Environment libase.so

    Table 9 Plugin Device Access Methods

    The Plugin Manager allows code that is written to a specific API signature to access FPGA hardware via different mechanisms. In other words, the end user codes to the OPAE API; and the OPAE API, based on configuration data, routes the hardware access to the device via different means.

    As an example, consider an API configuration that accesses FPGA device_A via the Device Feature List drivers and that accesses FPGA device_B via VFIO. The application is coded against the OPAE API.

    As part of its initialization process, the application enumerates and discovers an fpga_token corresponding to device_A. That fpga_token is opened and its MMIO region 0 is mapped via a call to fpgaMapMMIO().

    The API configuration for device_A is such that the fpga_handle corresponding to device_A routes its hardware access calls through libxfpga.so. The call to fpgaMapMMIO() is redirected to libxfpga.so\u2019s implementation of the MMIO mapping function, xfpga_fpgaMapMMIO(). As a result, the call to xfpga_fpgaMapMMIO() uses its AFU file descriptor to communicate with the DFL driver to map the MMIO region.

    Subsequently, the application enumerates and discovers an fpga_token corresponding to device_B. That fpga_token is opened and its MMIO region 0 is mapped via a call to fpgaMapMMIO().

    The API configuration for device_B is such that the fpga_handle corresponding to device_B routes its hardware access calls through libopae-v.so. The call to fpgaMapMMIO() is redirected to libopae-v.so\u2019s implementation of the MMIO mapping function, vfio_fpgaMapMMIO(). As a result, the call to vfio_fpgaMapMMIO() uses the MMIO mapping performed by libopaevfio.so during initialization of the VFIO session.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#231-plugin-model","title":"2.3.1 Plugin Model","text":"

    The OPAE SDK plugin model is facilitated by its use of opaque C structures for fpga_token and fpga_handle. These types are both declared as void *; and this allows the parameters to the OPAE SDK functions to take different forms, depending on the layer of the SDK being used.

    At the topmost layer, for example when calling fpgaEnumerate(), the output fpga_token parameter array is actually an array of pointers to opae_wrapped_token struct\u2019s.

    typedef struct _opae_wrapped_token {\nuint32_t magic;\nfpga_token opae_token;\nuint32_t ref_count;\nstruct _opae_wrapped_token *prev;\nstruct _opae_wrapped_token *next;\nopae_api_adapter_table *adapter_table;\n} opae_wrapped_token;\n

    Figure 27 opae_wrapped_token

    An opae_wrapped_token, as the name suggests, is a thin wrapper around the lower-layer token which is stored in struct member opae_token. The adapter_table struct member is a pointer to a plugin-specific adapter table. The adapter table provides a mapping between the top-layer opae_wrapped_token and its underlying plugin-specific API entry points, which are called using the opae_token struct member (the lower-level token).

    typedef struct _opae_api_adapter_table {\nstruct _opae_api_adapter_table *next;\nopae_plugin plugin;\n...\nfpga_result (*fpgaEnumerate)(const fpga_properties *filters,\nuint32_t num_filters,\nfpga_token *tokens,\nuint32_t max_tokens,\nuint32_t *num_matches);\n...\nint (*initialize)(void);\nint (*finalize)(void);\n} opae_api_adapter_table;\n

    Figure 28 opae_api_adapter_table

    When libopae-c loads, the plugin manager uses the plugin configuration data to open and configure a session to each of the required plugin libraries. During this configuration process, each plugin is passed an empty adapter table struct. The purpose of the plugin configuration is to populate this adapter table struct with each of the plugin-specific API entry points.

    When the top-level fpgaEnumerate() is called, each adapter table\u2019s plugin-specific fpgaEnumerate() struct member is called; and the output fpga_token\u2019s are collected. At this point, these fpga_token\u2019s are the lower-level token structure types. Before the top-level fpgaEnumerate() returns, these plugin-specific tokens are wrapped inside opae_wrapped_token structures, along with a pointer to the respective adapter table.

    After enumeration is complete, the application goes on to call other top-level OPAE SDK functions with the wrapped tokens. Each top-level entry point which accepts an fpga_token knows that it is actually being passed an opae_wrapped_token. With this knowledge, the entry point peeks inside the wrapped token and calls through to the plugin-specific API entry point using the adapter table, passing the lower-level opae_token struct member.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#232-libxfpga-plugin","title":"2.3.2 libxfpga Plugin","text":"

    2.3.1 Plugin Model introduced the concept of an opae_wrapped_token and a corresponding plugin-specific token structure. libxfpga.so is the plugin library that implements the DFL driver hardware access method. Its plugin-specific token data structure is struct _fpga_token.

    struct _fpga_token {\nfpga_token_header hdr;\nuint32_t device_instance;\nuint32_t subdev_instance;\nchar sysfspath[SYSFS_PATH_MAX];\nchar devpath[DEV_PATH_MAX];\nstruct error_list *errors;\n};\n
    Relevant Links: - fpga_token_header - error_list

    Figure 29 struct _fpga_token

    A struct _fpga_token corresponding to the Port will have sysfspath and devpath members that contain strings like the following example paths:

    sysfspath: \u201c/sys/class/fpga_region/region0/dfl-port.0\u201d\ndevpath: \u201c/dev/dfl-port.0\u201d\n

    Figure 30 libxfpga Port Token

    Likewise, a struct _fpga_token corresponding to the FME will have sysfspath and devpath members that contain strings like the following example paths:

    sysfspath: \u201c/sys/class/fpga_region/region0/dfl-fme.0\u201d\ndevpath: \u201c/dev/dfl-fme.0\u201d\n

    Figure 31 libxfpga FME Token

    When a call to the top-level fpgaOpen() is made, the lower-level token is unwrapped and passed to xfpga_fpgaOpen(). In return, xfpga_fpgaOpen() opens the character device file identified by the devpath member of the struct _fpga_token. It then allocates and initializes an instance of libxfpga.so\u2019s plugin-specific handle data structure, struct _fpga_handle.

    struct _fpga_handle {\npthread_mutex_t lock;\nuint64_t magic;\nfpga_token token;\nint fddev;\nint fdfpgad;\nuint32_t num_irqs;\nuint32_t irq_set;\nstruct wsid_tracker *wsid_root;\nstruct wsid_tracker *mmio_root;\nvoid *umsg_virt;\nuint64_t umsg_size;\nuint64_t *umsg_iova;\nbool metric_enum_status;\nfpga_metric_vector fpga_enum_metric_vector;\nvoid *bmc_handle;\nstruct _fpga_bmc_metric *_bmc_metric_cache_value;\nuint64_t num_bmc_metric;\nuint32_t flags;\n};\n
    Relevant Links: - wsid_tracker - fpga_metric_vector - _fpga_bmc_metric

    Figure 32 struct _fpga_handle

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#233-libopae-v-plugin","title":"2.3.3 libopae-v Plugin","text":"

    libopae-v.so is the plugin library that implements the VFIO hardware access method. Its plugin-specific token data structure is vfio_token.

    #define USER_MMIO_MAX 8\ntypedef struct _vfio_token {\nfpga_token_header hdr;\nfpga_guid compat_id;\npci_device_t *device;\nuint32_t region;\nuint32_t offset;\nuint32_t mmio_size;\nuint32_t pr_control;\nuint32_t user_mmio_count;\nuint32_t user_mmio[USER_MMIO_MAX];\nuint64_t bitstream_id;\nuint64_t bitstream_mdata;\nuint8_t num_ports;\nstruct _vfio_token *parent;\nstruct _vfio_token *next;\nvfio_ops ops;\n} vfio_token;\n
    Relevant Links: - fpga_token_header - pci_device_t - vfio_ops

    Figure 33 vfio_token

    When a call to the top-level fpgaOpen() is made, the lower-level token is unwrapped and passed to vfio_fpgaOpen(). In return, vfio_fpgaOpen() opens the VFIO device matching the device address found in the input vfio_token. It then allocates and initializes an instance of libopae-v.so\u2019s plugin-specific handle data structure, vfio_handle.

    typedef struct _vfio_handle {\nuint32_t magic;\nstruct _vfio_token *token;\nvfio_pair_t *vfio_pair;\nvolatile uint8_t *mmio_base;\nsize_t mmio_size;\npthread_mutex_t lock;\n#define OPAE_FLAG_HAS_AVX512 (1u << 0)\nuint32_t flags;\n} vfio_handle;\n
    Relevant Links: - vfio_token - vfio_pair_t

    Figure 34 vfio_handle

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2331-supporting-libraries","title":"2.3.3.1 Supporting Libraries","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#23311-libopaevfio","title":"2.3.3.1.1 libopaevfio","text":"

    libopaevfio.so is OPAE\u2019s implementation of the Linux kernel\u2019s Virtual Function I/O interface. This VFIO interface presents a generic means of configuring and accessing PCIe endpoints from a user-space process via a supporting Linux kernel device driver, vfio-pci.

    libopaevfio.so provides APIs for opening/closing a VFIO device instance, for mapping/unmapping MMIO spaces, for allocating/freeing DMA buffers, and for configuring interrupts for the device.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#23312-libopaemem","title":"2.3.3.1.2 libopaemem","text":"

    Each DMA buffer allocation made by libopaevfio.so\u2019s opae_vfio_buffer_allocate() and opae_vfio_buffer_allocate_ex() APIs requires a backing I/O Virtual Address range. These address ranges are discovered at VFIO device open time by way of the VFIO_IOMMU_GET_INFO ioctl.

    Each range specifies a large contiguous block of I/O Virtual Address space. The typical DMA buffer allocation size is significantly less than one of these IOVA blocks, so the division of each block into allocatable segments must be managed so that multiple DMA buffer allocations can be made from a single block. In other words, the IOVA blocks must be memory-managed in order to make efficient use of them.

    libopaemem.so provides a generic means of managing a large memory space, consisting of individual large memory blocks of contiguous address space. When a DMA buffer allocation is requested, libopaevfio.so uses this generic memory manager to carve out a small chunk of contiguous IOVA address space in order for the DMA mapping to be made. The IOVA space corresponding to the allocation is marked as allocated, and the rest of the large block remains as allocatable space within the memory manager. Subsequent de-allocation returns a chunk of IOVA space to the free state, coalescing contiguous chunks as they are freed. The allocations and de-allocations of the IOVA space can occur in any order with respect to each other. libopaemem.so tracks both the allocated and free block space, carving out small chunks from the large IOVA blocks on allocations, and coalescing small chunks back into larger ones on frees.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2332-configuring-pcie-virtual-functions","title":"2.3.3.2 Configuring PCIe Virtual Functions","text":"

    Before an AFU can be accessed with VFIO, the FPGA Physical Function must be configured to enable its Virtual Functions. Then, each VF must be bound to the vfio-pci Linux kernel driver.

    As of the Arrow Creek program, the FPGA hardware allows multiple AFU\u2019s to co-exist by placing each AFU in its own PCIe Virtual Function. Upon system startup, no PCIe VF\u2019s exist. The pci_device command can be used to enable the VF\u2019s and their AFU\u2019s. First, use the lspci command to examine the current device topology:

    # lspci | grep cel\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nb1:00.1 Processing accelerators: Intel Corporation Device bcce\nb1:00.2 Processing accelerators: Intel Corporation Device bcce\nb1:00.3 Processing accelerators: Intel Corporation Device bcce\nb1:00.4 Processing accelerators: Intel Corporation Device bcce\n

    Figure 35 lspci Device Topology

    In this example, VF\u2019s are controlled by PF 0, as highlighted in Figure 35 lspci Device Topology. In the figure, each PF is shown as having the Arrow Creek PF PCIe device ID of bcce.

    Now, use the pci_device command to enable three VF\u2019s for PF0:

    # pci_device 0000:b1:00.0 vf 3\n# lspci | grep cel\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nb1:00.1 Processing accelerators: Intel Corporation Device bcce\nb1:00.2 Processing accelerators: Intel Corporation Device bcce\nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device\nb1:00.4 Processing accelerators: Intel Corporation Device bcce\nb1:00.5 Processing accelerators: Intel Corporation Device bccf\nb1:00.6 Processing accelerators: Intel Corporation Device bccf\nb1:00.7 Processing accelerators: Intel Corporation Device bccf\n

    Figure 36 Enable Virtual Functions

    Figure 20 Enable Virtual Functions shows that three VF\u2019s were created. Each VF is shown as having the Arrow Creek VF PCIe device ID of bccf.

    Now, each Virtual Function must be bound to the vfio-pci Linux kernel driver so that it can be accessed via VFIO:

    # opaevfio -i -u myuser -g mygroup 0000:b1:00.5\nBinding (0x8086,0xbccf) at 0000:b1:00.5 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.5 is 318\n

    Figure 37 Bind VF's to vfio-pci

    Here, myuser and mygroup identify the unprivileged user/group that requires access to the device. The opaevfio command will change the ownership of the device per the values given.

    Once the VF\u2019s are bound to vfio-pci, the OPAE SDK will find and enumerate them with libopae-v.so:

    # fpgainfo port\n//****** PORT ******//\nObject Id : 0xEF00000\nPCIe s:b:d.f : 0000:B1:00.0\nDevice Id : 0xBCCE\nSocket Id : 0x00\n//****** PORT ******//\nObject Id : 0xE0B1000000000000\nPCIe s:b:d.f : 0000:B1:00.7\nDevice Id : 0xBCCF\nSocket Id : 0x01\nAccelerator GUID : 4dadea34-2c78-48cb-a3dc-5b831f5cecbb\n//****** PORT ******//\nObject Id : 0xC0B1000000000000\nPCIe s:b:d.f : 0000:B1:00.6\nDevice Id : 0xBCCF\nSocket Id : 0x01\nAccelerator GUID : 823c334c-98bf-11ea-bb37-0242ac130002\n//****** PORT ******//\nObject Id : 0xA0B1000000000000\nPCIe s:b:d.f : 0000:B1:00.5\nDevice Id : 0xBCCF\nSocket Id : 0x01\nAccelerator GUID : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n

    Figure 38 List VF's with fpgainfo

    When the VF\u2019s are no longer needed, they can be unbound from the vfio-pci driver:

    # opaevfio -r 0000:b1:00.5\nReleasing (0x8086,0xbccf) at 0000:b1:00.5 from vfio-pci\n# opaevfio -r 0000:b1:00.6\nReleasing (0x8086,0xbccf) at 0000:b1:00.6 from vfio-pci\n# opaevfio -r 0000:b1:00.7\nReleasing (0x8086,0xbccf) at 0000:b1:00.7 from vfio-pci\n

    Figure 39 Unbind VF's from vfio-pci

    Finally, the VF\u2019s can be disabled:

    # pci_device 0000:b1:00.0 vf 0\n# lspci | grep cel\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nb1:00.1 Processing accelerators: Intel Corporation Device bcce\nb1:00.2 Processing accelerators: Intel Corporation Device bcce\nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device\nb1:00.4 Processing accelerators: Intel Corporation Device bcce\n

    Figure 40 Disable Virtual Functions

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#24-application-flow","title":"2.4 Application Flow","text":"

    A typical OPAE application that interacts with an AFU via MMIO and shared memory will have a flow similar to the one described in this section.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#241-create-filter-criteria","title":"2.4.1 Create Filter Criteria","text":"

    Refer to 2.1.2 Enumeration. When enumerating AFU\u2019s, if no filtering criteria is specified, then fpgaEnumerate() returns fpga_token\u2019s for each AFU that is present in the system. In order to limit the enumeration search to a specific AFU, create an fpga_properties object and set its guid to that of the desired AFU:

    #define MY_AFU_GUID \u201c57fa0b03-ab4f-4b02-b4eb-d3fe1ec18518\u201d\nfpga_properties filter = NULL;\nfpga_guid guid;\nfpgaGetProperties(NULL, &filter);\nuuid_parse(MY_AFU_GUID, guid);\nfpgaPropertiesSetGUID(filter, guid);\n

    Figure 41 Flow: Create Filter Criteria

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#242-enumerate-the-afu","title":"2.4.2 Enumerate the AFU","text":"

    With the filtering criteria in place, enumerate to obtain an fpga_token for the AFU:

    fpga_token afu_token = NULL;\nuint32_t num_matches = 0;\nfpgaEnumerate(&filter, 1, &afu_token, 1, &num_matches);\n

    Figure 42 Flow: Enumerate the AFU

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#243-open-the-afu","title":"2.4.3 Open the AFU","text":"

    After finding an fpga_token for the AFU using fpgaEnumerate(), the token must be opened with fpgaOpen() to establish a session with the AFU. The process of opening an fpga_token creates an fpga_handle:

    fpga_handle afu_handle = NULL;\nfpgaOpen(afu_token, &afu_handle, 0);\n

    Figure 43 Flow: Open the AFU

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#244-map-mmio-region","title":"2.4.4 Map MMIO Region","text":"

    In order to access the MMIO region of the AFU to program its CSR\u2019s, the region must first be mapped into the application\u2019s process address space. This is accomplished using fpgaMapMMIO():

    uint32_t region = 0;\nfpgaMapMMIO(afu_handle, region, NULL);\n

    Figure 44 Flow: Map MMIO Region

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#245-allocate-dma-buffers","title":"2.4.5 Allocate DMA Buffers","text":"

    If the AFU is DMA-capable, shared memory buffers can be allocated and mapped into the process address space and the IOMMU with fpgaPrepareBuffer(). Refer to Figure 8 Configuring Huge Pages for instructions on configuring 2MiB and 1GiB huge pages.

    #define BUF_SIZE (2 * 1024 * 1024)\nvolatile uint64_t *src_ptr = NULL;\nuint64_t src_wsid = 0;\nvolatile uint64_t *dest_ptr = NULL;\nuint64_t dest_wsid = 0;\nfpgaPrepareBuffer(afu_handle, BUF_SIZE, (void **)&src_ptr,\n&src_wsid, 0);\nfpgaPrepareBuffer(afu_handle, BUF_SIZE, (void **)&dest_ptr,\n&dest_wsid, 0);\nmemset(src_ptr, 0xaf, BUF_SIZE);\nmemset(dest_ptr, 0xbe, BUF_SIZE);\n

    Figure 45 Flow: Allocate DMA Buffers

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#246-make-afu-aware-of-dma-buffers","title":"2.4.6 Make AFU Aware of DMA Buffers","text":"

    The process by which locations of shared memory buffers and their sizes are made known to the AFU is entirely AFU-specific. This example shows the method used by the Native Loopback AFU. Each buffer I/O virtual address is cacheline-aligned and programmed into a unique AFU CSR; then the buffer size in lines is programmed into a length CSR:

    #define CSR_SRC_ADDR 0x000A // AFU-specific\n#define CSR_DEST_ADDR 0x000B // AFU-specific\n#define CSR_NUM_LINES 0x000C // AFU-specific\nuint64_t src_iova = 0;\nuint64_t dest_iova = 0;\nfpgaGetIOAddress(afu_handle, src_wsid, &src_iova);\nfpgaGetIOAddress(afu_handle, dest_wsid, &dest_iova);\nfpgaWriteMMIO64(afu_handle, 0, CSR_SRC_ADDR, src_iova >> 6);\nfpgaWriteMMIO64(afu_handle, 0, CSR_DEST_ADDR, dest_iova >> 6);\nfpgaWriteMMIO32(afu_handle, 0, CSR_NUM_LINES, BUF_SIZE / 64);\n

    Figure 46 Flow: Make AFU Aware of DMA Buffers

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#247-initiate-an-acceleration-task","title":"2.4.7 Initiate an Acceleration Task","text":"

    With the shared buffer configuration complete, the AFU can be told to initiate the acceleration task. This process is AFU-specific. The Native Loopback AFU starts the acceleration task by writing a value to its control CSR:

    #define CSR_CTRL 0x000D // AFU-specific\nfpgaWriteMMIO32(afu_handle, 0, CSR_CTRL, 3);\n

    Figure 47 Initiate an Acceleration Task

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#248-wait-for-task-completion","title":"2.4.8 Wait for Task Completion","text":"

    Once the acceleration task is initiated, the application may poll the AFU for a completion status. This process is AFU-specific. The AFU may provide a status CSR for the application to poll; or the AFU may communicate status to the application by means of a result code written to a shared buffer.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#249-free-dma-buffers","title":"2.4.9 Free DMA Buffers","text":"

    When the acceleration task completes and the AFU is quiesced such that there are no outstanding memory transactions targeted for the shared memory, the DMA buffers can be unmapped and freed using fpgaReleaseBuffer():

    fpgaReleaseBuffer(afu_handle, src_wsid);\nfpgaReleaseBuffer(afu_handle, dest_wsid);\n

    Figure 48 Flow: Free DMA Buffers

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2410-unmap-mmio-region","title":"2.4.10 Unmap MMIO Region","text":"

    The MMIO regions should also be unmapped using fpgaUnmapMMIO():

    fpgaUnmapMMIO(afu_handle, region);\n

    Figure 49 Flow: Unmap MMIO Region

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2411-close-the-afu","title":"2.4.11 Close the AFU","text":"

    The AFU handle should be closed via fpgaClose() to release its resources:

    fpgaClose(afu_handle);\n

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2412-release-the-tokens-and-properties","title":"2.4.12 Release the Tokens and Properties","text":"

    The fpga_token\u2019s returned by fpgaEnumerate() should be destroyed using the fpgaDestroyToken() API. The fpga_properties objects should be destroyed using the fpgaDestroyProperties() API:

    fpgaDestroyToken(&afu_token);\nfpgaDestroyProperties(&filter);\n

    Figure 51 Flow: Release the Tokens and Properties

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#30-opae-c-api","title":"3.0 OPAE C++ API","text":"

    The OPAE C++ API refers to a C++ layer that sits on top of the OPAE C API, providing object-oriented implementations of the main OPAE C API abstractions: properties, tokens, handles, dma buffers, etc. Like the OPAE C API, the C++ API headers contain Doxygen markup for each of the provided classes.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#31-libopae-cxx-core","title":"3.1 libopae-cxx-core","text":"

    The implementation files for the C++ API are compiled into libopae-cxx-core.so. A convenience header, core.h, provides a quick means of including each of the C++ API headers. Each of the types comprising the C++ API is located within the opae::fpga::types C++ namespace.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#311-properties","title":"3.1.1 Properties","text":"

    Class properties provides the C++ implementation of the fpga_properties type and its associated APIs.

    properties::ptr_t filter = properties::get();\n

    Figure 52 C++ Create New Empty Properties

    Class properties provides member variables for each fpga_properties item that can be manipulated with fpgaPropertiesGet\u2026() and fpgaPropertiesSet\u2026(). For example, to set the AFU ID in a properties instance and to set that instance\u2019s type to FPGA_ACCELERATOR:

    #define MY_AFU_ID \u201c8ad74241-d13b-48eb-b428-7986dcbcab14\u201d\nfilter->guid.parse(MY_AFU_ID);\nfilter->type = FPGA_ACCELERATOR;\n

    Figure 53 C++ Properties Set GUID and Type

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#312-tokens","title":"3.1.2 Tokens","text":"

    Class token provides the C++ implementation of the fpga_token type and its associated APIs. Class token also provides the enumerate() static member function:

    std::vector<token::ptr_t> tokens = token::enumerate({filter});\nif (tokens.size() < 1) {\n// flag error and return\n}\ntoken::ptr_t tok = tokens[0];\n

    Figure 54 C++ Enumeration

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#313-handles","title":"3.1.3 Handles","text":"

    Class handle provides the C++ implementation of the fpga_handle type and its associated APIs. The handle class provides member functions for opening and closing a token, for reading and writing to MMIO space, and for reconfiguring the FPGA\u2019s Programmable Region.

    handle::ptr_t accel = handle::open(tok, 0);\n

    Figure 55 C++ Opening a Handle

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#314-shared-memory","title":"3.1.4 Shared Memory","text":"

    The shared_buffer class provides member functions for allocating and releasing DMA buffers, for querying buffer attributes, and for reading and writing buffers.

    #define BUF_SIZE (2 * 1024 * 1024)\nshared_buffer::ptr_t input = shared_buffer::allocate(accel, BUF_SIZE);\nshared_buffer::ptr_t output = shared_buffer::allocate(accel, BUF_SIZE);\nstd::fill_n(input->c_type(), BUF_SIZE, 0xaf);\nstd::fill_n(output->c_type(), BUF_SIZE, 0xbe);\n

    Figure 56 C++ Allocate and Init Buffers

    Once DMA buffers have been allocated, their IO addresses are programmed into AFU-specific CSRs to enable the DMA. Here, the IO address of each buffer is aligned to the nearest cache line before programming it into the AFU CSR space. The number of cache lines is then programmed into the appropriate AFU CSR.

    #define CSR_SRC_ADDR 0x000A // AFU-specific\n#define CSR_DEST_ADDR 0x000B // AFU-specific\n#define CSR_NUM_LINES 0x000C // AFU-specific\n#define LOG2_CL 6\naccel->write_csr64(CSR_SRC_ADDR, input->io_address() >> LOG2_CL);\naccel->write_csr64(CSR_DEST_ADDR, output->io_address() >> LOG2_CL);\naccel->write_csr32(CSR_NUM_LINES, BUF_SIZE / 64);\n

    Figure 57 C++ Make the AFU Aware of DMA Buffers

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#315-events","title":"3.1.5 Events","text":"

    The event class provides member functions for event registration. In order to register an event, provide the handle::ptr_t for the desired device, along with the event type and optional flags.

    int vect = 2;\nevent::ptr_t evt = event::register_event(accel,\nFPGA_EVENT_INTERRUPT,\nvect);\nint evt_fd = evt.os_object();\n

    Figure 58 C++ Event Registration

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#316-errors","title":"3.1.6 Errors","text":"

    Class error provides a means of querying the device errors given a token::ptr_t. The token and integer ID provided to the error::get() static member function uniquely identify one of the 64-bit error masks associated with the token.

    error::ptr_t err = error::get(tok, 0); \n

    Figure 59 C++ Query Device Errors

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#317-sysobject","title":"3.1.7 SysObject","text":"

    Class sysobject is the C++ implementation of the OPAE SysObject API. sysobject provides a means of creating class instances via its two sysobject::get() static member functions. A third non-static sysobject::get() enables creating a sysobject instance given a parent sysobject instance. The read64() and write64() member functions allow reading and writing the sysobject\u2019s value as a 64-bit unsigned integer. The bytes() member functions allow reading a sysobject\u2019s value as a raw byte stream.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#40-opae-python-api","title":"4.0 OPAE Python API","text":"

    The OPAE Python API is built on top of the OPAE C++ Core API and its object model. Because of this, developing OPAE applications in Python is very similar to developing OPAE applications in C++ which significantly reduces the learning curve required to adapt to the Python API. While the object model remains the same, some static factory functions in the OPAE C++ Core API have been moved to module level methods in the OPAE Python API with the exception of the properties class. The goal of the OPAE Python API is to enable fast prototyping, test automation, infrastructure managment, and an easy to use framework for FPGA resource interactions that don\u2019t rely on software algorithms with a high runtime complexity.

    The major benefits of using pybind11 for developing the OPAE Python API include, but are not limited to, the following features of pybind11:

    • Uses C++ 11 standard library although it can use C++ 14 or C++17.
    • Automatic conversions of shared_ptr types
    • Built-in support for numpy and Eigen numerical libraries
    • Interoperable with the Python C API

    Currently, the only Python package that is part of OPAE is opae.fpga. Because opae.fpga is built on top of the opae-cxx-core API, it does require that the runtime libraries for both opae-cxx-core and opae-c be installed on the system (as well as any other libraries they depend on). Those libraries can be installed using the opae-libs package (from either RPM or DEB format - depending on your Linux distribution). Installation for the Python OPAE bindings are included in the Getting Started Guide for your platform.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#41-opae","title":"4.1 opae","text":"

    The Python API is coded as a pybind11 project, which allows C++ code to directly interface with Python internals. Each C++ API concept is encoded into a Python equivalent. The functionality exists as a Python extension module, compiled into _opae.so.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#411-enumeration","title":"4.1.1 Enumeration","text":"

    Enumeration is somewhat simplified as compared to the OPAE C/C++ APIs. The fpga.enumerate() function accepts keyword arguments for each of the property names that are defined in the C++ API. As an example, to enumerate for an FPGA_ACCELERATOR by its GUID:

    from opae import fpga\nMY_ACCEL = \u201cd573b29e-176f-4cb7-b810-efbf7be34cc9\u201d\ntokens = fpga.enumerate(type=fpga.ACCELERATOR, guid=MY_ACCEL)\nassert tokens, \u201cNo accelerator matches {}\u201d.format(MY_ACCEL)\n

    Figure 60 Python Enumeration

    The return value from the fpga.enumerate() function is a list of all the token objects matching the search criteria.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#412-properties","title":"4.1.2 Properties","text":"

    Querying properties for a token or handle is also a bit different in the Python API. In order to query properties for one of these objects, pass the object to the fpga.properties() constructor. The return value is a properties object with each of the property names defined as instance attributes.

    prop = fpga.properties(tokens[0])\nprint(f'0x{prop.vendor_id:04x} 0x{prop.device_id:04x}')\n

    Figure 61 Python Get Token Properties

    Properties objects may also be created by invoking the fpga.properties() constructor, passing the same keyword arguments as those to fpga.enumerate(). Properties objects created in this way are also useful for enumeration purposes:

    props = fpga.properties(type=fpga.ACCELERATOR)\ntokens = fpga.enumerate([props])\n

    Figure 62 Python Properties Constructor

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#413-tokens","title":"4.1.3 Tokens","text":"

    Tokens overload both the __getitem__ and __getattr__ methods to enable the SysObject API. Both of the following are valid forms of accessing the \u2018errors/first_error\u2019 sysfs attribute, given a token object:

    tok = tokens[0]\nferr = tok['errors/first_error']\nprint(f'first error: 0x{ferr.read64():0x}')\nprint('0x{:0x}'.format(tok.errors.first_error.read64()))\n

    Figure 63 Python Tokens and SysObject API

    Tokens also implement a find() method, which accepts a glob expression in order to search sysfs. The following example finds the \u201cid\u201d sysfs entry in the given token\u2019s sysfs tree.

    my_id = tok.find(\u2018i?\u2019)\nprint(f'{my_id.read64()}')\n

    Figure 64 Python Token Find

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#414-handles","title":"4.1.4 Handles","text":"

    Tokens are converted to handles by way of the fpga.open() function. The flags (second) parameter to fpga.open() may be zero or fpga.OPEN_SHARED.

    with fpga.open(tok, fpga.OPEN_SHARED) as handle:\n    use_the_handle(handle)\n

    Figure 65 Python Open Handle

    Like token objects, handle objects overload __getitem__ and __getattr__ methods to enable the SysObject API. handle also provides a find() method similar to token\u2019s find().

    err = handle['errors/errors']\nprint(f'errors: 0x{err.read64():0x}')\nprint('first error: 0x{:0x}'.format(\nhandle.errors.first_error.read64()))\nmy_id = handle.find('i?')\nprint(f'{my_id.read64()}')\n

    Figure 66 Python Handles and SysObject API

    Partial reconfiguration is provided by class handle\u2019s reconfigure() method. The first parameter, slot, will be zero in most designs. The second parameter is an opened file descriptor to the file containing the GBS image. The third parameter, flags, defaults to zero.

    with open(\u2018my.gbs\u2019, \u2018rb\u2019) as fd:\n    handle.reconfigure(0, fd)\n

    Figure 67 Python Partial Reconfiguration

    Device reset is accomplished by means of handle\u2019s reset() method, which takes no parameters.

    Finally for handles, CSR space reads are accomplished via read_csr32() and read_csr64(). Both methods accept the register offset as the first parameter and an optional csr_space index, which defaults to zero, as the second parameter. CSR space writes are accomplished via write_csr32() and write_csr64(). Both methods accept the register offset as the first parameter, the value to write as the second, and an optional csr_space index, which defaults to zero, as the third.

    print(\u20190x{:0x}\u2019.format(handle.read_csr32(0x000a)))\nprint(\u20180x{:0x}\u2019.format(handle.read_csr64(0x000c)))\nhandle.write_csr32(0x000b, 0xdecafbad, 2)\nhandle.write_csr64(0x000e, 0xc0cac01adecafbad, 2)\n

    Figure 68 Python Read/Write CSR

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#415-shared-memory","title":"4.1.5 Shared Memory","text":"

    The fpga.allocate_shared_buffer() function provides access to the OPAE memory allocator. The allocation sizes and required huge page configurations are the same as those noted in 2.1.5 MMIO and Shared Memory.

    The fpga.allocate_shared_buffer() function returns an object instance of type shared_buffer. The shared_buffer class implements methods size(), wsid(), and io_address(), which return the buffer size in bytes, the unique workspace ID, and the IO address respectively:

    buf = fpga.allocate_shared_buffer(handle, 4096)\nprint(f\u2019size: {buf.size()}\u2019)\nprint(f\u2019wsid: 0x{buf.wsid():0x}\u2019)\nprint(f\u2019io_address: 0x{buf.io_address():0x}\u2019)\n

    Figure 69 Python Allocate Shared Memory

    The shared_buffer class implements a fill() method which takes an integer parameter which is applied to each byte of the buffer (similar to C standard library\u2019s memset()). The compare() method compares the contents of the first size bytes of one buffer to another. The value returned from compare() is the same as the C standard library\u2019s memcmp(). The copy() method copies the first size bytes of the calling buffer into the argument buffer.

    b0 = fpga.allocate_shared_buffer(handle, 4096)\nb1 = fpga.allocate_shared_buffer(handle, 4096)\nb0.fill(0xa5)\nb1.fill(0xa5)\nprint(f'compare: {b0.compare(b1, 4096)}')\nb1.fill(0xa0)\nb0.copy(b1, 4096)\nprint(f'compare: {b0.compare(b1, 4096)}')\n

    Figure 70 Python Buffer Fill, Copy, Compare

    shared_buffer\u2019s read32() and read64() methods read a 32- or 64-bit value from the given offset. The write32() and write64() methods write a 32- or 64-bit value to the given offset.

    print(f'value at 0: 0x{b0.read32(0):0x}')\nprint(f'value at 4: 0x{b0.read64(4):0x}')\nb0.write32(0xabadbeef, 0)\nb0.write64(0xdecafbadabadbeef, 4)\nprint(f'value at 0: 0x{b0.read32(0):0x}')\nprint(f'value at 4: 0x{b0.read64(4):0x}')\n

    Figure 71 Python Buffer Read and Write

    The shared_buffer class provides three polling methods: poll(), poll32(), and poll64(). Each method takes an offset as its first parameter. The second parameter is a value and the third is a mask. The value and mask parameters are 8 bits wide for poll(), 32 bits wide for poll32(), and 64 bits wide for poll64(). The fourth and last parameter is a timeout value which defaults to 1000 microseconds.

    Each polling method reads the n-bit wide item at offset and applies (logical AND) the mask to that value. The masked value created in the previous step is then compared to the second parameter, value. If the two values are equal, then the method returns true immediately. Otherwise, the method continues to loop, attempting the same comparison over and over without sleeping. Finally, if the elapsed time from the beginning of the call to the current time is greater than or equal to the timeout value, then the method times out and returns false.

    if b0.poll32(0, 0xbebebebe, 0xffffffff, 250):\n    print(\u2018Got it!\u2019)\n

    Figure 72 Python Buffer Poll

    The shared_buffer split() method allows creating two or more buffer objects from one larger buffer object. The return value is a list of shared_buffer instances whose sizes match the arguments given to split().

    b1, b2 = b1.split(2048, 2048)\nprint(f'b1 io_address: 0x{b1.io_address():0x}')\nprint(f'b2 io_address: 0x{b2.io_address():0x}')\n

    Figure 73 Python Splitting Buffer

    Finally, the shared_buffer class implements the Python buffer protocol to support memoryview objects. The Python buffer protocol allows access to an object\u2019s underlying memory without copying that memory. As a brief example:

    mv = memoryview(b1)\nassert mv\nassert mv[0] == 0xbe\nb1[15] = int(65536)\nassert struct.unpack('<L', bytearray(b1[15:19]))[0] == 65536\n

    Figure 74 Python memoryview

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#416-events","title":"4.1.6 Events","text":"

    Given a handle and an event type, the fpga.register_event() function returns an object of type event. The event class implements one method, os_object(), which returns the underlying file descriptor that can be used to poll for the event:

    import select\nevt = fpga.register_event(handle, fpga.EVENT_ERROR)\nos_object = evt.os_object()\nreceived_event = False\nepoll = select.epoll()\nepoll.register(os_object, select.EPOLLIN)\nfor fileno, ev in epoll.poll(1):\n    if fileno == os_object:\n        received_event = True\n        print(f'received: {received_event}')\n

    Figure 75 Python Events

    In addition to fpga.EVENT_ERROR, fpga.EVENT_INTERRUPT, and fpga.EVENT_POWER_THERMAL are also supported.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#417-errors","title":"4.1.7 Errors","text":"

    Given a token, the fpga.errors() function returns a list of objects of type error. Each error instance represents a 64-bit mask of error values. The error bit masks are platform-dependent. Each error instance has two attributes: name and can_clear and one method: read_value() which returns the 64-bit error mask.

    for e in fpga.errors(tok):\n    print(f\u2019name: \"{e.name}\"\u2019)\n    print(f\u2019can_clear: {e.can_clear}\u2019)\n    print(f\u2019value: {e.read_value()}\u2019)\n

    Figure 76 Python Get Errors

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#418-sysobject","title":"4.1.8 SysObject","text":"

    The Python API\u2019s SysObject implementation is introduced in 4.1.3 Tokens and 4.1.4 Handles. When the index operator (__getitem__) or attribute reference (__getattr__) is used and the referenced string or attribute name corresponds to a sysfs entry in the sysfs path of either a token or a handle, then an object of type sysobject is returned.

    The size() method returns the length of the sysfs entry in bytes. Note that a typical sysfs entry is terminated with a \u2018\\n\u2019 followed by the \u2018\\0\u2019 NULL terminator. The bytes() method returns the sysfs entry\u2019s value as a string.

    afu_id = tok['afu_id']\nassert afu_id\nprint(f'size: {afu_id.size()} bytes: {afu_id.bytes().rstrip()}')\n

    Figure 77 Python sysobject as Bytes

    The sysobject read64() and write64() methods provide a means to read and write a sysfs entry\u2019s value as an unsigned 64-bit integer. The sysobject class itself also implements the __getitem__ and __getattr__ methods so that a sysobject of type FPGA_OBJECT_CONTAINER can retrieve sysobject instances for child sysfs entries.

    errs = tok['errors']\nfirst = errs['first_error']\nassert first\nprint(f'first 0x{first.read64():0x}')\n
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#418-python-example-application","title":"4.1.8 Python Example Application","text":"

    The following example is an implementation of the sample, hello_fpga.c, which is designed to configure the NLB0 diagnostic accelerator for a simple loopback.

    import time\nfrom opae import fpga\n\nNLB0 = \"d8424dc4-a4a3-c413-f89e-433683f9040b\"\nCTL = 0x138\nCFG = 0x140\nNUM_LINES = 0x130\nSRC_ADDR = 0x0120\nDST_ADDR = 0x0128\nDSM_ADDR = 0x0110\nDSM_STATUS = 0x40\n\ndef cl_align(addr):\n    return addr >> 6\n\ntokens = fpga.enumerate(type=fpga.ACCELERATOR, guid=NLB0)\nassert tokens, \"Could not enumerate accelerator: {}\".format(NlB0)\n\nwith fpga.open(tokens[0], fpga.OPEN_SHARED) as handle:\n    src = fpga.allocate_shared_buffer(handle, 4096)\n    dst = fpga.allocate_shared_buffer(handle, 4096)\n    dsm = fpga.allocate_shared_buffer(handle, 4096)\n    handle.write_csr32(CTL, 0)\n    handle.write_csr32(CTL, 1)\n    handle.write_csr64(DSM_ADDR, dsm.io_address())\n    handle.write_csr64(SRC_ADDR, cl_align(src.io_address())) # cacheline-aligned\n    handle.write_csr64(DST_ADDR, cl_align(dst.io_address())) # cacheline-aligned\n    handle.write_csr32(CFG, 0x42000)\n    handle.write_csr32(NUM_LINES, 4096/64)\n    handle.write_csr32(CTL, 3)\n    while dsm[DSM_STATUS] & 0x1 == 0:\n        time.sleep(0.001)\n    handle.write_csr32(CTL, 7)\n

    This example shows how one might reprogram (Partial Reconfiguration) an accelerator on a given bus, 0x5e, using a bitstream file, m0.gbs.

    from opae import fpga\n\nBUS = 0x5e\nGBS = 'm0.gbs'\ntokens = fpga.enumerate(type=fpga.DEVICE, bus=BUS)\nassert tokens, \"Could not enumerate device on bus: {}\".format(BUS)\nwith open(GBS, 'rb') as fd, fpga.open(tokens[0]) as device:\n    device.reconfigure(0, fd)\n

    Figure 78 Python sysobject Container

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#50-management-interfaces-opaeadmin","title":"5.0 Management Interfaces - opae.admin","text":"

    While the OPAE SDK C, C++, and Python APIs focus on presenting the AFU and all its related functionality to the end user, there is also a need for a maintenance functionality to aid in configuring the platform and performing secure firmware updates for the FPGA device and its components. opae.admin is a Python framework which provides abstractions for performing these types of maintenance tasks on FPGA devices. opae.admin provides Python classes which model the FPGA and the sysfs interfaces provided by the DFL drivers.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#51-sysfs","title":"5.1 sysfs","text":"

    opae.admin\u2019s sysfs module provides abstractions for interacting with sysfs nodes, which comprise the base entity abstraction of opae.admin.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#511-sysfs_node","title":"5.1.1 sysfs_node","text":"

    A sysfs_node is an object that tracks a unique path within a sysfs directory tree. sysfs_node provides methods for finding and constructing other sysfs_node objects, based on the root path of the parent sysfs_node object. sysfs_node also provides a mechanism to read and write sysfs file contents. sysfs_node serves as the base class for many of the sysfs module\u2019s other classes.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#512-pci_node","title":"5.1.2 pci_node","text":"

    A pci_node is a sysfs_node that is rooted at /sys/bus/pci/devices. Each pci_node has a unique PCIe address corresponding to the PCIe device it represents. Methods for finding the pci_node\u2019s children, for determining the PCIe device tree rooted at the node, for manipulating the node\u2019s PCIe address, for determining the vendor and device ID\u2019s, and for removing, unbinding, and rescanning the device are provided.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#513-sysfs_driver","title":"5.1.3 sysfs_driver","text":"

    A sysfs_driver is a sysfs_node that provides a method for unbinding a sysfs_device object.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#514-sysfs_device","title":"5.1.4 sysfs_device","text":"

    A sysfs_device is a sysfs_node that is located under /sys/class or /sys/bus. sysfs_device provides the basis for opae.admin\u2019s FPGA enumeration capability.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#515-pcie_device","title":"5.1.5 pcie_device","text":"

    A pcie_device is a sysfs_device that is rooted at /sys/bus/pci/devices.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#52-fpga","title":"5.2 fpga","text":"

    opae.admin\u2019s fpga module provides classes which abstract an FPGA and its components.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#521-region","title":"5.2.1 region","text":"

    A region is a sysfs_node that has an associated Linux character device, rooted at /dev. Methods for opening the region\u2019s character device file and for interacting with the character device via its IOCTL interface are provided.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#522-fme","title":"5.2.2 fme","text":"

    An fme is a region that represents an FPGA device\u2019s FME component. An fme provides accessors for the PR interface ID, the various bus paths that may exist under an FME, and the BMC firmware revision information.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#523-port","title":"5.2.3 port","text":"

    A port is a region that represents an FPGA device\u2019s Port component. A port provides an accessor for the Port AFU ID.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#524-fpga_base","title":"5.2.4 fpga_base","text":"

    An fpga_base is a sysfs_device that provides accessors for the FPGA device\u2019s FME, for the FPGA device\u2019s Port, and for the secure update sysfs controls. fpga_base provides routines for enabling and disabling AER and for performing device RSU.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#525-fpga","title":"5.2.5 fpga","text":"

    An fpga (derived from fpga_base) is the basis for representing the FPGA device in opae.admin. Utilities such as fpgasupdate rely on fpga\u2019s enum classmethod to enumerate all of the FPGA devices in the system. In order for a device to enumerate via this mechanism, it must be bound to the dfl-pci driver at the time of enumeration.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#53-opaeadmin-utilities","title":"5.3 opae.admin Utilities","text":"

    Several utilities are written on top of opae.admin\u2019s class abstractions. The following sections highlight some of the most commonly-used utilities.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#531-fpgasupdate","title":"5.3.1 fpgasupdate","text":"

    fpgasupdate, or FPGA Secure Update, is used to apply firmware updates to the components of the FPGA. As the name implies, these updates target a secure FPGA device, one that has the ability to implement a secure root of trust. The command-line interface to fpgasupdate was designed to be as simple as possible for the end user. The command simply takes a path to the firmware update file to be applied and the PCIe address of the targeted FPGA device.

    # fpgasupdate update-file.bin 0000:b2:00.0\n

    Figure 79 fpgasupdate Interface

    fpgasupdate can apply a variety of firmware image updates. | Image| Description| | -----| -----| |Programmable Region Image| .gbs or Green BitStream| |SR Root Key Hash| Static Region RKH| |PR Root Key Hash| Programmable Region RKH| |FPGA Firmware Image| Static Region Device Firmware| |PR Authentication Certificate| Programmable Region Auth Cert| |BMC Firmware Image| Board Management Controller Firmware| |SR Thermal Image| Static Region Thermal Sensor Thresholds| |PR Thermal Image| Programmable Region Thermal Sensor Thresholds| |CSK Cancelation| Code Signing Key Cancelation Request| |SDM Image| Secure Device Manager Firmware|

    Table 10 fpgasupdate Image Types

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#532-pci_device","title":"5.3.2 pci_device","text":"

    pci_device is a utility that provides a convenient interface to some of the Linux Kernel\u2019s standard PCIe device capabilities.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5321-pci_device-aer-subcommand","title":"5.3.2.1 pci_device aer subcommand","text":"

    The aer dump subcommand displays the Correctable, Fatal, and NonFatal device errors.

    # pci_device 0000:b2:00.0 aer dump\n

    Figure 80 pci_device aer dump

    The aer mask subcommand displays, masks, or unmasks errors using the syntax of the setpci command.

    # pci_device 0000:b2:00.0 aer mask show\n0x00010000 0x000031c1\n# pci_device 0000:b2:00.0 aer mask all\n# pci_device 0000:b2:00.0 aer mask off\n# pci_device 0000:b2:00.0 aer mask 0x01010101 0x10101010\n

    Figure 81 pci_device aer mask

    The aer clear subcommand clears the current errors.

    # pci_device 0000:b2:00.0 aer clear\naer clear errors: 00000000\n

    Figure 82 pci_device aer clear

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5322-pci_device-unbind-subcommand","title":"5.3.2.2 pci_device unbind subcommand","text":"

    The unbind subcommand unbinds the target device from the currently-bound device driver.

    # pci_device 0000:b2:00.0 unbind\n

    Figure 83 pci_device unbind

    In order to re-bind the device to a driver, eg dfl-pci, use the following commands:

    # cd /sys/bus/pci/drivers/dfl-pci\n# echo 0000:b2:00.0 > bind\n

    Figure 84 Re-binding a Driver

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5323-pci_device-rescan-subcommand","title":"5.3.2.3 pci_device rescan subcommand","text":"

    The rescan subcommand triggers a PCIe bus rescan of all PCIe devices.

    # pci_device 0000:b2:00.0 rescan\n

    Figure 85 pci_device rescan

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5324-pci_device-remove-subcommand","title":"5.3.2.4 pci_device remove subcommand","text":"

    The remove subcommand removes the target device from Linux kernel management.

    # pci_device 0000:b2:00.0 remove\n

    Figure 86 pci_device remove

    Note: a reboot may be required in order to re-establish the Linux kernel management for the device.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5325-pci_device-topology-subcommand","title":"5.3.2.5 pci_device topology subcommand","text":"

    The topology subcommand shows a tab-delimited depiction of the target device as it exists in the PCIe device tree in the Linux kernel.

    # pci_device 0000:b2:00.0 topology\n[pci_address(0000:3a:00.0), pci_id(0x8086, 0x2030)] (pcieport)\n    [pci_address(0000:3b:00.0), pci_id(0x10b5, 0x8747)] (pcieport)\n        [pci_address(0000:3c:09.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:b2:00.0), pci_id(0x8086, 0x0b30)] (dfl-pci)\n        [pci_address(0000:3c:11.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:43:00.0), pci_id(0x8086, 0x0b32)] (no driver)\n        [pci_address(0000:3c:08.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:3d:00.1), pci_id(0x8086, 0x0d58)] (i40e)\n            [pci_address(0000:3d:00.0), pci_id(0x8086, 0x0d58)] (i40e)\n        [pci_address(0000:3c:10.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:41:00.0), pci_id(0x8086, 0x0d58)] (i40e)\n            [pci_address(0000:41:00.1), pci_id(0x8086, 0x0d58)] (i40e)\n

    Figure 87 pci_device topology

    The green output indicates the target device. The other endpoint devices are shown in blue.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5326-pci_device-vf-subcommand","title":"5.3.2.6 pci_device vf subcommand","text":"

    The vf subcommand allows setting the value of the sriov_numvfs sysfs node of the target device. This is useful in scenarios where device functionality is presented in the form of one or more PCIe Virtual Functions.

    # pci_device 0000:b2:00.0 vf 3\n# pci_device 0000:b2:00.0 vf 0\n

    Figure 88 pci_device vf

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#533-rsu","title":"5.3.3 rsu","text":"

    rsu is a utility that performs Remote System Update. rsu is used subsequent to programming a firmware update or other supported file type with fpgasupdate, in order to reset the targeted FPGA entity so that a newly-loaded firmware image becomes active.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5331-rsu-bmc-subcommand","title":"5.3.3.1 rsu bmc subcommand","text":"

    The bmc subcommand causes a Board Management Controller reset. This command is used to apply a previous fpgasupdate of a BMC firmware image. The --page argument selects the desired boot image. Valid values for --page are \u2018user\u2019 and \u2018factory\u2019.

    # rsu bmc --page user 0000:b2:00.0\n# rsu bmc --page factory 0000:b2:00.0\n

    Figure 89 rsu bmc

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5332-rsu-retimer-subcommand","title":"5.3.3.2 rsu retimer subcommand","text":"

    The retimer subcommand causes a Parkvale reset (specific to Vista Creek). This command is used to apply a previous fpgasupdate of a BMC firmware image (the Parkvale firmware is contained within the BMC firmware image). The retimer subcommand causes only the Parkvale to reset.

    # rsu retimer 0000:b2:00.0\n

    Figure 90 rsu retimer

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5333-rsu-fpga-subcommand","title":"5.3.3.3 rsu fpga subcommand","text":"

    The fpga subcommand causes a reconfiguration of the FPGA Static Region. This command is used to apply a previous fpgasupdate of the Static Region image. The --page argument selects the desired boot image. Valid values for --page are \u2018user1\u2019, \u2018user2\u2019, and \u2018factory\u2019.

    # rsu fpga --page user1 0000:b2:00.0\n# rsu fpga --page user2 0000:b2:00.0\n# rsu fpga --page factory 0000:b2:00.0\n

    Figure 91 rsu fpga

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5334-rsu-sdm-subcommand","title":"5.3.3.4 rsu sdm subcommand","text":"

    The sdm subcommand causes a reset of the Secure Device Manager. This command is used to apply a previous fpgasupdate of the SDM image.

    # rsu sdm 0000:b2:00.0\n

    Figure 92 rsu sdm

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5335-rsu-fpgadefault-subcommand","title":"5.3.3.5 rsu fpgadefault subcommand","text":"

    The fpgadefault subcommand can be used to display the default FPGA boot sequence; and it can be used to select the image to boot on the next reset of the FPGA. When given without additional parameters, the fpgadefault subcommand displays the default FPGA boot sequence:

    # rsu fpgadefault 0000:b2:00.0\n

    Figure 93 rsu Displaying FPGA Boot Sequence

    The parameters to the fpgadefault subcommand are --page and --fallback. The --page parameter accepts \u2018user1\u2019, \u2018user2\u2019, or \u2018factory\u2019, specifying the desired page to boot the FPGA from on the next reset. Note that this subcommand does not actually cause the reset to occur. Please refer to rsu fpga subcommand for an example of resetting the FPGA using the rsu command.

    # rsu fpgadefault --page user1 0000:b2:00.0\n# rsu fpgadefault --page user2 0000:b2:00.0\n# rsu fpgadefault --page factory 0000:b2:00.0\n

    Figure 94 rsu Select FPGA Boot Image

    The --fallback parameter accepts a comma-separated list of the keywords \u2018user1\u2019, \u2018user2\u2019, and \u2018factory\u2019. These keywords, in conjunction with the --page value are used to determine a fallback boot sequence for the FPGA. The fallback boot sequence is used to determine which FPGA image to load in the case of a boot failure. For example, given the following command, the FPGA would attempt to boot in the order \u2018factory\u2019, \u2018user1\u2019, \u2018user2\u2019. That is to say, if the \u2018factory\u2019 image failed to boot, then the \u2018user1\u2019 image would be tried. Failing to boot \u2018user1\u2019, the \u2018user2\u2019 image would be tried.

    # rsu fpgadefault --page factory --fallback user1,user2 0000:b2:00.0\n

    Figure 95 rsu Select FPGA Boot Image and Fallback

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#60-sample-applications","title":"6.0 Sample Applications","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#61-afu-test-framework","title":"6.1 afu-test Framework","text":"

    afu-test refers to a test-writing framework that exists as a set of C++ classes written on top of the OPAE C++ bindings. The first class, afu, serves as the base class for the test application abstraction. Class afu provides integration with CLI11, a C++ \u201911 command-line parsing framework, and with spdlog, a C++ logging library. The second class, command represents a unique test sequence that is called by the afu object. Instances of the command class implement the test-specific workload.

    class afu {\npublic:\nafu(const char *name,\nconst char *afu_id = nullptr,\nconst char *log_level = nullptr);\nint open_handle(const char *afu_id);\nint main(int argc, char *argv[]);\nvirtual int run(CLI::App *app, command::ptr_t test);\ntemplate<class T>\nCLI::App *register_command();\n};\n

    Figure 96 C++ class afu

    The afu class constructor initializes the CLI11 command parser with some general, application-wide parameters.

    Subcommand Description -g,--guid Accelerator AFU ID. -p,--pci-address Address of the accelerator device. -l,--log-level Requested spdlog output level. -s,--shared Open the AFU in shared mode? -t,--timeout Application timeout in milliseconds.

    Figure 97 class afu Application Parameters

    The register_command() member function adds a test command instance to the afu object. Each test command that an afu object is capable of executing is registered during the test\u2019s startup code. For instance, here is the hssi application\u2019s use of register_command():

    hssi_afu app;\nint main(int argc, char *argv[])\n{\napp.register_command<hssi_10g_cmd>();\napp.register_command<hssi_100g_cmd>();\napp.register_command<hssi_pkt_filt_10g_cmd>();\napp.register_command<hssi_pkt_filt_100g_cmd>();\n\u2026\napp.main(argc, argv);\n}\n

    Figure 98 hssi's app.register_command()

    Next, the afu instance\u2019s main() member function is called. main() initializes the spdlog instance, searches its database of registered commands to find the command matching the test requested from the command prompt, uses the open_handle() member function to enumerate for the requested AFU ID, and calls its run() member function, passing the CLI::App and the test command variables. The run() member function initializes a test timeout mechanism, then calls the command parameter\u2019s run() to invoke the test-specific logic.

    With all the boiler-plate of application startup, configuration, and running handled by the afu class, the test-specific command class is left to implement only a minimum number of member functions:

    class command {\npublic:\nvirtual const char *name() const = 0;\nvirtual const char *description() const = 0;\nvirtual int run(afu *afu, CLI::App *app) = 0;\nvirtual void add_options(CLI::App *app) { }\nvirtual const char *afu_id() const { return nullptr; }\n};\n

    Figure 99 class command

    The name() member function gives the unique command name. Some examples of names from the hssi app are hssi_10g, hssi_100g, pkt_filt_10g, and pkt_filt_100g. The description() member function gives a brief description that is included in the command-specific help output. add_options() adds command-specific command-line options. afu_id() gives the AFU ID for the command, in string form. Finally, run() implements the command-specific test functionality.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#62-afu-test-based-samples","title":"6.2 afu-test Based Samples","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#621-dummy_afu","title":"6.2.1 dummy_afu","text":"

    The dummy_afu application is a afu-test based application that implements three commands: mmio, ddr, and lpbk.

    Target Description # dummy_afu mmio Targets special scratchpad area implemented by the AFU. # dummy_afu ddr Execute dummy_afu-specific DDR test. # dummy_afu lpbk Execute a simple loopback test."},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#622-host_exerciser","title":"6.2.2 host_exerciser","text":"

    host_exerciser markdown document.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#623-hssi","title":"6.2.3 hssi","text":"

    hssi markdown document.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#70-other-utilities","title":"7.0 Other Utilities","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#71-opaeio","title":"7.1 opae.io","text":"

    opae.io markdown document.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#72-bitstreaminfo","title":"7.2 bitstreaminfo","text":"

    The bitstreaminfo command prints diagnostic information about firmware image files that have been passed through the PACSign utility. PACSign prepends secure block 0 and secure block 1 data headers to the images that it processes. These headers contain signature hashes and other metadata that are consumed by the BMC firmware during a secure update.

    To run bitstreaminfo, pass the path to the desired firmware image file:

    # bitstreaminfo my_file.bin \n

    Figure 100 Running bitstreaminfo

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#73-fpgareg","title":"7.3 fpgareg","text":"

    The fpgareg command prints the register spaces for the following fpga device components:

    Command Description # fpgareg 0000:b1:00.0 pcie Walks and prints the DFL for the device. # fpgareg 0000:b1:00.0 bmc Prints the BMC registers for the device. # fpgareg 0000:b1:00.0 hssi Prints the HSSI registers for the device. # fpgareg 0000:b1:00.0 acc Prints the AFU register spaces.

    Figure 101 fpgareg Commands

    Note that fpgareg is only available as of Arrow Creek ADP and forward. It will not work with prior platforms, eg N3000.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#74-opaevfio","title":"7.4 opaevfio","text":"

    opaevfio markdown document.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#80-building-opae","title":"8.0 Building OPAE","text":"

    The OPAE SDK uses the cmake build and configuration system, version >= 3.10. The basic steps required to build the SDK from source are:

    Install prerequisite packages.

    $ git clone <https://github.com/OFS/opae-sdk.git>\n$ cd opae-sdk\n$ mkdir build\n$ cd build\n$ cmake .. <cmake options>\n$ make\n
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#81-installing-prerequisite-packages","title":"8.1 Installing Prerequisite Packages","text":"

    The OPAE SDK is intended to build and run on modern Linux distributions. The SDK contains a set of system configuration scripts to aid the system configuration process.

    Script Target Operating System centos.sh CentOS 8.x fedora.sh Fedora 33/34 ubuntu.sh Ubuntu 20.04 LTS

    Table 11 System Configuration Scripts

    For more information on the environment setup instructions for your exact platform, please visit https://ofs.github.io/ and refer to your platform's User Guide.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#82-cloning-the-sdk-repository","title":"8.2 Cloning the SDK repository","text":"
    $ git clone https://github.com/OFS/opae-sdk.git\n
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#83-cmake-options","title":"8.3 CMake Options","text":"Option Description Values Default -DCMAKE_BUILD_TYPE Configure debugging info

    Debug

    Release

    Coverage

    RelWithDebInfo

    RelWithDebInfo -DCMAKE_INSTALL_PREFIX Root install path /usr/local -DOPAE_BUILD_SPHINX_DOC Enable/Disable docs ON/OFF OFF -DOPAE_BUILD_TESTS Enable/Disable unit tests ON/OFF OFF -DOPAE_ENABLE_MOCK Enable/Disable mock driver for unit tests ON/OFF OFF -DOPAE_INSTALL_RPATH Enable/Disable rpath for install ON/OFF OFF -DOPAE_VERSION_LOCAL Local version string -DOPAE_PRESERVE_REPOS Preserve local changes to external repos? ON/OFF OFF -D OPAE_BUILD_LIBOPAE_CXX Enable C++ bindings ON/OFF ON -DOPAE_WITH_PYBIND11 Enable pybind11 ON/OFF ON -D OPAE_BUILD_PYTHON_DIST Enable Python bindings ON/OFF OFF -DOPAE_BUILD_LIBOPAEVFIO Build libopaevfio.so ON/OFF ON -D OPAE_BUILD_PLUGIN_VFIO Build libopae-v.so ON/OFF ON -DOPAE_BUILD_LIBOPAEUIO Build libopaeuio.so ON/OFF ON -DOPAE_BUILD_LIBOFS Build OFS Copy Engine ON/OFF ON -DOPAE_BUILD_SAMPLES Build Samples ON/OFF ON -DOPAE_BUILD_LEGACY Build legacy repo ON/OFF OFF -DOPAE_LEGACY_TAG Specify legacy build tag master -DOPAE_WITH_CLI11 Enable apps which use CLI11 ON/OFF ON -DOPAE_WITH_SPDLOG Enable apps which use spdlog ON/OFF ON -DOPAE_WITH_LIBEDIT Enable apps which use libedit ON/OFF ON -DOPAE_WITH_HWLOC Enable apps which use hwloc ON/OFF ON -DOPAE_WITH_TBB Enable apps which use Thread Building Blocks ON/OFF ON -DOPAE_MINIMAL_BUILD Enable/Disable minimal build. When set to ON, disable CLI11, spdlog, libedit, hwloc, tbb ON/OFF OFF

    Table 12 CMake Options

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#84-building-opae-for-debug","title":"8.4 Building OPAE for Debug","text":"
    $ cmake .. -DCMAKE_BUILD_TYPE=Debug\n
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#85-creating-rpms","title":"8.5 Creating RPMs","text":"

    To ease the RPM creation process, the OPAE SDK provides a simple RPM creation script. The parameters to the RPM create script are fedora or rhel, depending on which distribution is targeted. For rhel, the build flag -DOPAE_MINIMAL_BUILD is set to ON, omitting the binaries which have dependencies on external components that RHEL does not include in its base repositories.

    In order to create RPMs for Fedora, run the create script on a system loaded with all the Fedora build prerequisites. If prerequisites are missing, the create script will complain until they are resolved.

    In order to create RPMs for RHEL, run the create script on a system loaded with all the RHEL build prerequisites. If prerequisites are missing, the create script will complain until they are resolved.

    $ cd opae-sdk\n$ ./packaging/opae/rpm/create fedora\n-OR-\n$ ./packaging/opae/rpm/create rhel\n

    Figure 102 RPM Creation

    After running the create script, the RPM files will be located in the packaging/opae/rpm directory.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#851-updating-the-rpm-version-information","title":"8.5.1 Updating the RPM Version Information","text":"

    The RPMs will be versioned according to the information found in the file packaging/opae/version. Edit this file to update the version information, then re-run the create script to create the RPMs.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#90-debugging-opae","title":"9.0 Debugging OPAE","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#91-enabling-debug-logging","title":"9.1 Enabling Debug Logging","text":"

    The OPAE SDK has a built-in debug logging facility. To enable it, set the cmake flag -DCMAKE_BUILD_TYPE=Debug and then use the following environment variables: | Variable| Description| | ----- | ----- | |LIBOPAE_LOG=1| Enable debug logging output. When not set, only critical error messages are displayed.| |LIBOPAE_LOGFILE=file.log| Capture debug log output to file.log. When not set, the logging appears on stdout and stderr. The file must appear in a relative path or it can be rooted at /tmp.|

    Table 13 Logging Environment Variables

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#92-gdb","title":"9.2 GDB","text":"

    To enable gdb-based debugging, the cmake configuration step must specify a value for -DCMAKE_BUILD_TYPE of either Debug or RelWithDebInfo so that debug symbols are included in the output binaries. The OPAE SDK makes use of dynamically-loaded library modules. When debugging with gdb, the best practice is to remove all OPAE SDK libraries from the system installation paths to ensure that library modules are only loaded from the local build tree:

    $ cd opae-sdk/build\n$ LD_LIBRARY_PATH=$PWD/lib gdb --args <some_opae_executable> <args>\n

    Figure 103 Debugging with GDB

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#100-adding-new-device-support","title":"10.0 Adding New Device Support","text":"

    As of OPAE 2.2.0 the SDK has transitioned to a single configuration file model. The libraries, plugins, and applications obtain their runtime configuration during startup by examining a single JSON configuration file. In doing so, the original configuration file formats for libopae-c and fpgad have been deprecated in favor of the respective sections in the new configuration file.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#101-configuration-file-search-order","title":"10.1 Configuration File Search Order","text":"

    By default the OPAE SDK will install its configuration file to /etc/opae/opae.cfg. 

    /etc/opae/opae.cfg \n

    Figure 104 Default Configuration File

    The SDK searches for the configuration file during startup by employing the following search algorithm:

    First, the environment variable LIBOPAE_CFGFILE is examined. If it is set to a path that represents a valid path to a configuration file, then that configuration file path is used, and the search is complete.

    Next, the HOME environment variable is examined. If its value is valid, then it is prepended to the following set of relative paths. If HOME is not set, then the search continues with the value of the current user\u2019s home path as determined by getpwuid(). The home path, if any, determined by getpwuid() is prepended to the following set of relative paths. Searching completes successfully if any of these home-relative search paths is valid. 

    /.local/opae.cfg \n\n/.local/opae/opae.cfg \n\n/.config/opae/opae.cfg \n

    Figure 105 HOME Relative Search Paths

    Finally, the configuration file search continues with the following system-wide paths. If any of these paths is found to contain a configuration file, then searching completes successfully. 

    usr/local/etc/opae/opae.cfg \n\n/etc/opae/opae.cfg \n

    Figure 106 System Search Paths

    If the search exhausts all of the possible configuration file locations without finding a configuration file, then an internal default configuration is used. This internal default configuration matches that of the opae.cfg file shipped with the OPAE SDK.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#102-configuration-file-format","title":"10.2 Configuration File Format","text":"

    The OPAE SDK configuration file is stored in JSON formatted text. The file has two main sections: \u201cconfigs\u201d and \u201cconfigurations\u201d. The \u201cconfigs\u201d section is an array of strings. Each value in the \u201cconfigs\u201d array is a key into the data stored in the \u201cconfigurations\u201d section. If a key is present in \u201cconfigs\u201d, then that key is searched for and processed in \u201cconfigurations\u201d. If the key is not found in \u201cconfigs\u201d, then that section of \u201cconfigurations\u201d will not be processed, irrespective of whether it exists in \u201cconfigurations\u201d. 

    { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      ... \n\n    } \n\n  }, \n\n  \u201cconfigs\u201d: [ \n\n    \u201cc6100\u201d \n\n  ] \n\n} \n

    Figure 107 Keyed Configurations

    Each keyed section in \u201cconfigurations\u201d has four top-level entries: \u201cenabled\u201d, \u201cplatform\u201d, \u201cdevices\u201d, \u201copae\u201d. 

    { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      \u201cenabled\u201d: true, \n\n      \u201cplatform\u201d: \u201cIntel Acceleration Development Platform C6100\u201d, \n\n      \u201cdevices\u201d: [ \n\n        { \u201cname\u201d: \u201cc6100_pf\u201d, \u201cid\u201d: [ ... ] }, \n\n        { \u201cname\u201d: \u201cc6100_vf\u201d, \u201cid\u201d: [ ... ] } \n\n      ], \n\n      \u201copae\u201d: { \n\n        ... \n\n      } \n\n    } \n\n  }, \n\n} \n

    Figure 108 Configurations Format

    The \u201cenabled\u201d key holds a Boolean value. If the value is false or if the \u201cenabled\u201d key is omitted, then that configuration is skipped when parsing the file. The \u201cplatform\u201d key holds a string that identifies the current configuration item as a product family. The \u201cdevices\u201d key contains the device descriptions.

    \u201cdevices\u201d is an array of objects that contain a \u201cname\u201d and an \u201cid\u201d key. The \u201cname\u201d is a shorthand descriptor for a device PF or VF. The value of \u201cname\u201d appears elsewhere in the current \u201cconfigurations\u201d section in order to uniquely identify the device. \u201cid\u201d is an array of four strings, corresponding to the PCIe Vendor ID, Device ID, Subsystem Vendor ID, and Subsystem Device ID of the device. The entries corresponding to Vendor ID and Device ID must contain valid 16-bit hex integers. The entries corresponding to Subsystem Vendor ID and Subsystem Device ID may be 16-bit hex integers or the special wildcard string \u201c*\u201d, which indicates a don\u2019t care condition.

    The remaining sections in this chapter outline the format of the \u201copae\u201d configurations key.

    \u201cplugin\u201d: libopae-c and libopae-v

    The \u201cplugin\u201d key in the \u201copae\u201d section of a configuration is an array of OPAE SDK plugin configuration data. Each item in the array matches one or more PF or VF devices to a plugin library module. 

    { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      ... \n\n      \u201copae\u201d: { \n\n        \u201cplugin\u201d: [ \n\n          { \n\n            \u201cenabled\u201d: true, \n\n            \u201cmodule\u201d: \u201clibxfpga.so\u201d, \n\n            \u201cdevices\u201d: [ \u201cc6100_pf\u201d ], \n\n            \u201cconfiguration\u201d: {} \n\n          }, \n\n          { \n\n            \u201cenabled\u201d: true, \n\n            \u201cmodule\u201d: \u201clibopae-v.so\u201d, \n\n            \u201cdevices\u201d: [ \u201cc6100_pf\u201d, \u201cc6100_vf\u201d ], \n\n            \u201cconfiguration\u201d: {} \n\n          } \n\n        ], \n\n      } \n\n    } \n\n  }, \n\n} \n

    Figure 109 \"opae\" / \"plugin\" key/

    If the \u201cenabled\u201d key is false or if it is omitted, then that \u201cplugin\u201d array entry is skipped, and parsing continues. The \u201cmodule\u201d key is a string that identifies the desired plugin module library for the entry. The \u201cdevices\u201d array lists one or more PF/VF identifiers. Each array value must be a string, and it must match a device that is described in the \u201cconfigurations\u201d \u201cdevices\u201d section. The \u201cconfiguration\u201d key of the \u201cplugin\u201d section specifies a unique plugin-specific configuration. Currently, libopae-c and libopae-v use no plugin-specific config, so these keys are left empty.

    \u201cfpgainfo\u201d: fpgainfo application

    The \u201cfpgainfo\u201d key in the \u201copae\u201d section of a configuration is an array of fpgainfo plugin configuration data. Each item in the array matches one or more PF or VF devices to an fpgainfo plugin library module. 

    { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      ... \n\n      \u201copae\u201d: { \n\n        \u201cfpgainfo\u201d: [ \n\n          { \n\n            \u201cenabled\u201d: true, \n\n            \u201cmodule\u201d: \u201clibboard_c6100.so\u201d, \n\n            \u201cdevices\u201d: [ \n\n              { \u201cdevice\u201d: \u201cc6100_pf\u201d, \u201cfeature_id\u201d: \u201c0x12\u201d }, \n\n              { \u201cdevice\u201d: \u201cc6100_vf\u201d, \u201cfeature_id\u201d: \u201c0x12\u201d } \n\n            ] \n\n          } \n\n        ], \n\n      } \n\n    } \n\n  }, \n\n} \n

    Figure 110 \"opae\" / \"fpgainfo\" key

    If the \u201cenabled\u201d key is false or if it is omitted, then that \u201cfpgainfo\u201d array entry is skipped, and parsing continues. The \u201cmodule\u201d key is a string that identifies the desired fpgainfo module library for the entry. Each \u201cdevices\u201d array entry gives a PF/VF identifier in its \u201cdevice\u201d key and a DFL feature ID in its \u201cfeature_id\u201d key.

    \u201cfpgad\u201d: fpgad daemon process

    The \u201cfpgad\u201d key in the \u201copae\u201d section of a configuration is an array of fpgad plugin configuration data. Each item in the array matches one or more PF or VF devices to an fpgad plugin library module. 

    { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      ... \n\n      \u201copae\u201d: { \n\n        \u201cfpgad\u201d: [ \n\n          { \n\n            \u201cenabled\u201d: true, \n\n            \u201cmodule\u201d: \u201clibfpgad-vc.so\u201d, \n\n            \u201cdevices\u201d: [ \u201cc6100_pf\u201d ], \n\n            \u201cconfiguration\u201d: { \n\n              ... \n\n            } \n\n          } \n\n        ], \n\n      } \n\n    } \n\n  }, \n\n} \n

    Figure 111 \"opae\" / \"fpgad\" key

    If the \u201cenabled\u201d key is false or if it is omitted, then that \u201cfpgad\u201d array entry is skipped, and parsing continues. The \u201cmodule\u201d key is a string that identifies the desired fpgad plugin module library for the entry. The \u201cdevices\u201d array lists one or more PF/VF identifiers. Each array value must be a string, and it must match a device that is described in the \u201cconfigurations\u201d \u201cdevices\u201d section. The \u201cconfiguration\u201d key of the \u201cfpgad\u201d section specifies a unique plugin-specific configuration.

    \u201crsu\u201d: rsu script

    The \u201crsu\u201d key in the \u201copae\u201d section of a configuration is an array of rsu script configuration data. Each item in the array matches one or more PF devices to an rsu configuration. 

    { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      ... \n\n      \u201copae\u201d: { \n\n        \u201crsu\u201d: [ \n\n          { \n\n            \u201cenabled\u201d: true, \n\n            \u201cdevices\u201d: [ \u201cc6100_pf\u201d ], \n\n            \u201cfpga_default_sequences\u201d: \u201ccommon_rsu_sequences\u201d \n\n          } \n\n        ], \n\n      } \n\n    } \n\n  }, \n\n  \u201ccommon_rsu_sequences\u201d: [ \n\n    ... \n\n  ] \n\n} \n

    Figure 112 \"opae\" / \"rsu\" key

    If the \u201cenabled\u201d key is false or if it is omitted, then that \u201crsu\u201d array entry is skipped, and parsing continues. When disabled, the device(s) mentioned in that array entry will not be available for the rsu command. The \u201cdevices\u201d array lists one or more PF identifiers. Each array value must be a string, and it must match a device that is described in the \u201cconfigurations\u201d \u201cdevices\u201d section. The \u201cfpga_default_sequences\u201d key of the \u201crsu\u201d section specifies a JSON key. The configuration searches for that JSON key at the global level of the configuration file, and when found applies its value as the valid set of fpga boot sequences that can be used with the rsu fpgadefault subcommand.

    C \u201cfpgareg\u201d: fpgareg script

    The \u201cfpgareg\u201d key in the \u201copae\u201d section of a configuration is an array of fpgareg script configuration data. Each item in the array matches one or more PF/VF devices to an fpgareg configuration.

    ```C {

    \u201cconfigurations\u201d: { 

    \u201cc6100\u201d: {\n\n  ...\n\n  \u201copae\u201d: {\n\n    \u201cfpgareg\u201d: [\n\n      {\n\n        \u201cenabled\u201d: true,\n\n        \u201cdevices\u201d: [ \u201cc6100_pf\u201d, \u201cc6100_vf\u201d ]\n\n      }\n\n    ],\n\n  }\n\n}\n

    },

    } ```

    Figure 113 \"opae\" / \"fpgareg\" key

    If the \u201cenabled\u201d key is false or if it is omitted, then that \u201cfpgareg\u201d array entry is skipped, and parsing continues. When disabled, the device(s) mentioned in that array entry will not be available for the fpgareg command. The \u201cdevices\u201d array lists one or more PF/VF identifiers. Each array value must be a string, and it must match a device that is described in the \u201cconfigurations\u201d \u201cdevices\u201d section. 

    \u201copae.io\u201d: opae.io application \n

    The \u201copae.io\u201d key in the \u201copae\u201d section of a configuration is an array of opae.io configuration data. Each item in the array matches one or more PF/VF devices to an opae.io platform string. 

    { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      ... \n\n      \u201copae\u201d: { \n\n        \u201copae.io\u201d: [ \n\n          { \n\n            \u201cenabled\u201d: true, \n\n            \u201cdevices\u201d: [ \u201cc6100_pf\u201d, \u201cc6100_vf\u201d ] \n\n          } \n\n        ], \n\n      } \n\n    } \n\n  }, \n\n} \n

    Figure 114 \"opae\" / \"opae.io\" key

    If the \u201cenabled\u201d key is false or if it is omitted, then that \u201copae.io\u201d array entry is skipped, and parsing continues. When disabled, the device(s) mentioned in that array entry will continue to be available for the opae.io command. The device(s) platform string will not be shown in the opae.io ls command. The \u201cdevices\u201d array lists one or more PF/VF identifiers. Each array value must be a string, and it must match a device that is described in the \u201cconfigurations\u201d \u201cdevices\u201d section.

    Libxfpga \u2013 Updating the Metrics API

    Edit libraries/plugins/xfpga/sysfs.c. Find the definition of the opae_id_to_hw_type() function. Update the function to add the new vendor/device ID to hw_type mapping.

    This mapping is used by the SDK\u2019s metrics API to determine the method of accessing the board sensor information and is very specific to the underlying BMC implementation. It may be necessary to add a new hw_type value and to update the logic in libraries/plugins/xfpga/metrics.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#110-dfl-linux-kernel-drivers","title":"11.0 DFL Linux Kernel Drivers","text":"

    OFS DFL driver software provides the bottom-most API to FPGA platforms. Libraries such as OPAE and frameworks such as DPDK are consumers of the APIs provided by OFS. Applications may be built on top of these frameworks and libraries. The OFS software does not cover any out-of-band management interfaces. OFS driver software is designed to be extendable, flexible, and provide for bare-metal and virtualized functionality.

    The OFS driver software can be found in the OFS repository - linux-dfl, under the linux-dfl specific category. This repository has an associated OFS repository - linux-dfl that includes the following information:

    • An description of the three available branch archetypes
    • Configuration tweaks required while building the kernel
    • A functional description of the available DFL framework
    • Descriptions for all currently available driver modules that support FPGA DFL board solutions
    • Steps to create a new DFL driver
    • Steps to port a DFL driver patch
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#111-hardware-architecture","title":"11.1 Hardware Architecture","text":"

    The Linux Operating System treats the FPGA hardware as a PCIe* device. A predefined data structure, Device Feature List (DFL), allows for dynamic feature discovery in an Intel FPGA solution.

    The Linux Device Driver implements PCIe Single Root I/O Virtualization (SR-IOV) for the creation of Virtual Functions (VFs). The device driver can release individual accelerators for assignment to virtual machines (VMs).

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#112-fpga-management-engine-fme","title":"11.2 FPGA Management Engine (FME)","text":"

    The FPGA Management Engine provides error reporting, reconfiguration, performance reporting, and other infrastructure functions. Each FPGA has one FME which is always accessed through the Physical Function (PF). The Intel Xeon\u00ae Processor with Integrated FPGA also performs power and thermal management. These functions are not available on the Intel Programmable Acceleration Card (PAC).

    User-space applications can acquire exclusive access to the FME using open(), and release it using close(). Device access may be managed by standard Linux interfaces and tools.

    If an application terminates without freeing the FME or Port resources, Linux closes all file descriptors owned by the terminating process, freeing those resources.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#113-port","title":"11.3 Port","text":"

    A Port represents the interface between two components: * The FPGA Interface Manager (FIM) which is part of the static FPGA fabric * The Accelerator Function Unit (AFU) which is the partially reconfigurable region

    The Port controls the communication from software to the AFU and makes features such as reset and debug available.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#114-accelerator-function-unit-afu","title":"11.4 Accelerator Function Unit (AFU)","text":"

    An AFU attaches to a Port. The AFU provides a 256 KB memory mapped I/O (MMIO) region for accelerator-specific control registers.

    • Use open() on the Port device to acquire access to an AFU associated with the Port device.
    • Use close()on the Port device to release the AFU associated with the Port device.
    • Use mmap() on the Port device to map accelerator MMIO regions.
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#115-partial-reconfiguration-pr","title":"11.5 Partial Reconfiguration (PR)","text":"

    Use PR to reconfigure an AFU from a bitstream file. Successful reconfiguration has the following requirement:

    • You must generate the reconfiguration AFU for the exact FIM. The AFU and FIM are compatible if their interface IDs match. You can verify this match by comparing the interface ID in the bitstream header against the interface ID that is exported by the driver in sysfs.

    In all other cases PR fails and may cause system instability.

    Platforms that support 512-bit Partial Reconfiguration require binutils >= version 2.25.

    Close any software programs accessing the FPGA, including those running in a virtualized host before initiating PR. For virtualized environments, the recommended sequence is as follows:

    1. Unload the driver from the guest
    2. Release the VF from the guest

    Releasing the VF from the guest while an application on the guest is still accessing its resources may lead to VM instabilities. We recommend closing all applications accessing the VF in the guest before releasing the VF.

    1. Disable SR-IOV
    2. Perform PR
    3. Enable SR-IOV
    4. Assign the VF to the guest
    5. Load the driver in the guest
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#116-fpga-virtualization","title":"11.6 FPGA Virtualization","text":"

    To enable accelerator access from applications running on a VM, create a VF for the port using the following process:

    1. Release the Port from the PF using the associated ioctl on the FME device.

    2. Use the following command to enable SR-IOV and VFs. Each VF can own a single Port with an AFU. In the following command, N is the number of Port released from the PF.

        echo N > $PCI_DEVICE_PATH/sriov_numvfs\n

    The number, 'N', cannot be greater than the number of supported VFs. This can be read from $PCI_DEVICE_PATH/sriov_totalvfs.

    1. Pass the VFs through to VMs using hypervisor interfaces.

    2. Access the AFU on a VF from applications running on the VM using the same driver inside the VM.

    Creating VFs is only supported for port devices. Consequently, PR and other management functions are only available through the PF.

    If assigning multiple devices to the same VM on a guest IOMMU, you may need to increase the hard_limit option in order to avoid hitting a limit of pinned memory. The hard limit should be more than (VM memory size x Number of PCIe devices).

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#117-driver-organization","title":"11.7 Driver Organization","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1171-pcie-module-device-driver","title":"11.7.1 PCIe Module Device Driver","text":"

    FPGA devices appear as a PCIe devices. Once enumeration detects a PCIe PF or VF, the Linux OS loads the FPGA PCIe device driver. The device driver performs the following functions:

    1. Walks through the Device Feature List in PCIe device base address register (BAR) memory to discover features and their sub-features and creates necessary platform devices.
    2. Enables SR-IOV.
    3. Introduces the feature device infrastructure, which abstracts operations for sub-features and provides common functions to feature device drivers.
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1172-pcie-module-device-driver-functions","title":"11.7.2 PCIe Module Device Driver Functions","text":"

    The PCIe Module Device Driver performs the following functions:

    1. PCIe discovery, device enumeration, and feature discovery.
    2. Creates sysfs directories for the device, FME, and Port.
    3. Creates the platform driver instances, causing the Linux kernel to load their respective drivers.
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1173-fme-platform-module-device-driver","title":"11.7.3 FME Platform Module Device Driver","text":"

    The FME Platform Module Device Driver loads automatically after the PCIe driver creates the FME Platform Module. It provides the following features for FPGA management:

    1. Power and thermal management, error reporting, performance reporting, and other infrastructure functions. You can access these functions via sysfs interfaces the FME driver provides.

    2. Partial Reconfiguration. During PR sub-feature initialization, the FME driver registers the FPGA Manager framework to support PR. When the FME receives the relevant ioctl request from user-space, it invokes the common interface function from the FPGA Manager to reconfigure the AFU using PR.

    3. Port management for virtualization (releasing/assigning port device).

    After a port device is released, you can use the PCIe driver SR-IOV interfaces to create/destroy VFs.

    For more information, refer to \"FPGA Virtualization\".

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1174-fme-platform-module-device-driver-functions","title":"11.7.4 FME Platform Module Device Driver Functions","text":"

    The FME Platform Module Device Driver performs the the following functions:

    • Creates the FME character device node.
    • Creates the FME sysfs files and implements the FME sysfs file accessors.
    • Implements the FME private feature sub-drivers.
    • FME private feature sub-drivers: * FME Header * Partial Reconfiguration * Global Error * Global Performance
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1175-port-platform-module-device-driver","title":"11.7.5 Port Platform Module Device Driver","text":"

    After the PCIe Module Device Driver creates the Port Platform Module device, the FPGA Port and AFU driver are loaded. This module provides an interface for user-space applications to access the individual accelerators, including basic reset control on the Port, AFU MMIO region export, DMA buffer mapping service, and remote debug functions.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1176-port-platform-module-device-driver-functions","title":"11.7.6 Port Platform Module Device Driver Functions","text":"

    The Port Platform Module Device Driver performs the the following functions:

    • Creates the Port character device node.
    • Creates the Port sysfs files and implements the Port sysfs file accessors.
    • Implements the following Port private feature sub-drivers. * Port Header * AFU * Port Error * Signal Tap
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1177-opae-fpga-driver-interface","title":"11.7.7 OPAE FPGA Driver Interface","text":"

    The user-space interface consists of a sysfs hierarchy and ioctl requests. Most kernel attributes can be accessed/modified via sysfs nodes in this hierarchy. More complex I/O operations are controlled via ioctl requests. The OPAE API implementation, libopae-c, has been designed to use this interface to interact with the OPAE FPGA kernel drivers.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#120-plugin-development","title":"12.0 Plugin Development","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#121-overview","title":"12.1 Overview","text":"

    Beginning with OPAE C library version 1.2.0, OPAE implements a plugin-centric model. This guide serves as a reference to define the makeup of an OPAE C API plugin and to describe a sequence of steps that one may follow when constructing an OPAE C API plugin.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#122-plugin-required-functions","title":"12.2 Plugin Required Functions","text":"

    An OPAE C API plugin is a runtime-loadable shared object library, also known as a module. On Linux systems, the dl family of APIs from libdl are used to interact with shared objects. Refer to \"man dlopen\" and \"man dlsym\" for examples of using the libdl API.

    An OPAE C API plugin implements one required function. This function is required to have C linkage, so that its name is not mangled.

        int opae_plugin_configure(opae_api_adapter_table *table, const char *config);\n

    During initialization, the OPAE plugin manager component loads each plugin, searching for its opae_plugin_configure function. If none is found, then the plugin manager rejects that plugin. When it is found, opae_plugin_configure is called passing a pointer to a freshly-created opae_api_adapter_table and a buffer consisting of configuration data for the plugin.

    The job of the opae_plugin_configure function is to populate the given adapter table with each of the plugin's API entry points and to consume and comprehend the given configuration data in preparation for initialization.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#123-opae-api-adapter-table","title":"12.3 OPAE API Adapter Table","text":"

    The adapter table is a data structure that contains function pointer entry points for each of the OPAE APIs implemented by a plugin. In this way, it adapts the plugin-specific behavior to the more general case of a flat C API. Note that OPAE applications are only required to link with opae-c. In other words, the name of the plugin library should not appear on the linker command line. In this way, plugins are truly decoupled from the OPAE C API, and they are required to adapt to the strict API specification by populating the adapter table only. No other linkage is required nor recommended.

    adapter.h contains the definition of the opae_api_adapter_table. An abbreviated version is depicted below, along with supporting type opae_plugin:

        typedef struct _opae_plugin {\n        char *path;\n        void *dl_handle;\n    } opae_plugin;\n\n    typedef struct _opae_api_adapter_table {\n\n        struct _opae_api_adapater_table *next;\n        opae_plugin plugin;\n\n        fpga_result (*fpgaOpen)(fpga_token token, fpga_handle *handle,\n                                int flags);\n\n        fpga_result (*fpgaClose)(fpga_handle handle);\n\n        ...\n\n        fpga_result (*fpgaEnumerate)(const fpga_properties *filters,\n                                     uint32_t num_filters, fpga_token *tokens,\n                                     uint32_t max_tokens,\n                                     uint32_t *num_matches);\n\n        ...\n\n        // configuration functions\n        int (*initialize)(void);\n        int (*finalize)(void);\n\n        // first-level query\n        bool (*supports_device)(const char *device_type);\n        bool (*supports_host)(const char *hostname);\n\n    } opae_api_adapter_table;\n

    Some points worth noting are that the adapter tables are organized in memory by adding them to a linked list data structure. This is the use of the next structure member. (The list management is handled by the plugin manager.) The plugin structure member contains the handle to the shared object instance, as created by dlopen. This handle is used in the plugin's opae_plugin_configure to load plugin entry points. A plugin need only implement the portion of the OPAE C API that a target application needs. Any API entry points that are not supported should be left as NULL pointers (the default) in the adapter table. When an OPAE API that has no associated entry point in the adapter table is called, the result for objects associated with that plugin will be FPGA_NOT_SUPPORTED.

    The following code illustrates a portion of the opae_plugin_configure for a theoretical OPAE C API plugin libfoo.so:

        /* foo_plugin.c */\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        adapter->fpgaOpen = dlsym(adapter->plugin.dl_handle, \"foo_fpgaOpen\");\n        adapter->fpgaClose =\n                dlsym(adapter->plugin.dl_handle, \"foo_fpgaClose\");\n\n        ...\n\n        adapter->fpgaEnumerate =\n                dlsym(adapter->plugin.dl_handle, \"foo_fpgaEnumerate\");\n\n        ...\n\n        return 0;\n    }\n

    Notice that the implementations of the API entry points for plugin libfoo.so are prefixed with foo_. This is the recommended practice to avoid name collisions and to enhance the debugability of the application. Upon successful configuration, opae_plugin_configure returns 0 to indicate success. A non-zero return value indicates failure and causes the plugin manager to reject the plugin from futher consideration.

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#124-plugin-optional-functions","title":"12.4 Plugin Optional Functions","text":"

    Once the plugin manager loads and configures each plugin, it uses the adapter table to call back into the plugin so that it can be made ready for runtime. This is the job of the opae_plugin_initialize entry point, whose signature is defined as:

        int opae_plugin_initialize(void);\n

    The function takes no parameters, as the configuration data was already given to the plugin by opae_plugin_configure. opae_plugin_initialize returns 0 if no errors were encountered during initialization. A non-zero return code indicates that plugin initialization failed. A plugin makes its opae_plugin_initialize available to the plugin manager by populating the adapter table's initialize entry point as shown:

        /* foo_plugin.c */\n\n    int foo_plugin_initialize(void)\n    {\n        ...\n\n        return 0; /* success */\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->initialize =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_initialize\");\n\n        ...\n\n        return 0;\n    }\n

    If a plugin does not implement an opae_plugin_initialize entry point, then the initialize member of the adapter table should be left uninitialized. During plugin initialization, if a plugin has no opae_plugin_initialize entry in its adapter table, the plugin initialization step will be skipped, and the plugin will be considered to have initialized successfully.

    Once plugin initialization is complete for all loaded plugins, the system is considered to be running and fully functional.

    During teardown, the plugin manager uses the adapter table to call into each plugin's opae_plugin_finalize entry point, whose signature is defined as:

        int opae_plugin_finalize(void);\n

    opae_plugin_finalize returns 0 if no errors were encountered during teardown. A non-zero return code indicates that plugin teardown failed. A plugin makes its opae_plugin_finalize available to the plugin manager by populating the adapter table's finalize entry point as shown:

        /* foo_plugin.c */\n\n    int foo_plugin_finalize(void)\n    {\n        ...\n\n        return 0; /* success */\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->finalize =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_finalize\");\n\n        ...\n\n        return 0;\n    }\n

    If a plugin does not implement an opae_plugin_finalize entry point, then the finalize member of the adapter table should be left uninitialized. During plugin cleanup, if a plugin has no opae_plugin_finalize entry point in its adapter table, the plugin finalize step will be skipped, and the plugin will be considered to have finalized successfully.

    In addition to initialize and finalize, an OPAE C API plugin has two further optional entry points that relate to device enumeration. During enumeration, when a plugin is being considered for a type of device, the plugin may provide input on that decision by exporting an opae_plugin_supports_device entry point in the adapter table:

        bool opae_plugin_supports_device(const char *device_type);\n

    opae_plugin_supports_device returns true if the given device type is supported and false if it is not. A false return value from opae_plugin_supports_device causes device enumeration to skip the plugin.

    Populating the opae_plugin_supports_device is done as:

        /* foo_plugin.c */\n\n    bool foo_plugin_supports_device(const char *device_type)\n    {\n        if (/* device_type is supported */)\n            return true;\n\n        ...\n\n        return false;\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->supports_device =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_supports_device\");\n\n        ...\n\n        return 0;\n    }\n
    .. note::\n    The `opae_plugin_supports_device` mechanism serves as a placeholder only.\n    It is not implemented in the current version of the OPAE C API.\n

    Similarly to determining whether a plugin supports a type of device, a plugin may also answer questions about network host support by populating an opae_plugin_supports_host entry point in the adapter table:

        bool opae_plugin_supports_host(const char *hostname);\n

    opae_plugin_supports_host returns true if the given hostname is supported and false if it is not. A false return value from opae_plugin_supports_host causes device enumeration to skip the plugin.

    Populating the opae_plugin_supports_host is done as:

        /* foo_plugin.c */\n\n    bool foo_plugin_supports_host(const char *hostname)\n    {\n        if (/* hostname is supported */)\n            return true;\n\n        ...\n\n        return false;\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->supports_host =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_supports_host\");\n\n        ...\n\n        return 0;\n    }\n
    .. note::\n    The `opae_plugin_supports_host` mechanism serves as a placeholder only.\n    It is not implemented in the current version of the OPAE C API.\n
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#125-plugin-construction","title":"12.5 Plugin Construction","text":"

    The steps required to implement an OPAE C API plugin, libfoo.so, are:

    • Create foo_plugin.c: implements opae_plugin_configure, opae_plugin_initialize, opae_plugin_finalize, opae_plugin_supports_device, and opae_plugin_supports_host as described in the previous sections.
    • Create foo_plugin.h: implements function prototypes for each of the plugin-specific OPAE C APIs.
        /* foo_plugin.h */\n\n    fpga_result foo_fpgaOpen(fpga_token token, fpga_handle *handle,\n                             int flags);\n\n    fpga_result foo_fpgaClose(fpga_handle handle);\n\n    ...\n\n    fpga_result foo_fpgaEnumerate(const fpga_properties *filters,\n                                  uint32_t num_filters, fpga_token *tokens,\n                                  uint32_t max_tokens,\n                                  uint32_t *num_matches);\n    ...\n
    • Create foo_types.h: implements plugin-specific types for opaque data structures.
        /* foo_types.h */\n\n    struct _foo_token {\n        ...\n    };\n\n    struct _foo_handle {\n        ...\n    };\n\n    struct _foo_event_handle {\n        ...\n    };\n\n    struct _foo_object {\n        ...\n    };\n
    • Create foo_enum.c: implements foo_fpgaEnumerate, foo_fpgaCloneToken, and foo_fpgaDestroyToken.
    • Create foo_open.c: implements foo_fpgaOpen.
    • Create foo_close.c: implements foo_fpgaClose.
    • Create foo_props.c: implements foo_fpgaGetProperties, foo_fpgaGetPropertiesFromHandle, foo_fpgaUpdateProperties
    • Create foo_mmio.c: implements foo_fpgaMapMMIO, foo_fpgaUnmapMMIO foo_fpgaWriteMMIO64, foo_fpgaReadMMIO64, foo_fpgaWriteMMIO32, foo_fpgaReadMMIO32.
    • Create foo_buff.c: implements foo_fpgaPrepareBuffer, foo_fpgaReleaseBuffer, foo_fpgaGetIOAddress.
    • Create foo_error.c: implements foo_fpgaReadError, foo_fpgaClearError, foo_fpgaClearAllErrors, foo_fpgaGetErrorInfo.
    • Create foo_event.c: implements foo_fpgaCreateEventHandle, foo_fpgaDestroyEventHandle, foo_fpgaGetOSObjectFromEventHandle, foo_fpgaRegisterEvent, foo_fpgaUnregisterEvent.
    • Create foo_reconf.c: implements foo_fpgaReconfigureSlot.
    • Create foo_obj.c: implements foo_fpgaTokenGetObject, foo_fpgaHandleGetObject, foo_fpgaObjectGetObject, foo_fpgaDestroyObject, foo_fpgaObjectGetSize, foo_fpgaObjectRead, foo_fpgaObjectRead64, foo_fpgaObjectWrite64.
    • Create foo_clk.c: implements foo_fpgaSetUserClock, foo_fpgaGetUserClock.
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#130-building-a-sample-application","title":"13.0 Building a Sample Application","text":"

    Building a sample application The library source includes code samples. Use these samples to learn how to call functions in the library. Build and run these samples as quick sanity checks to determine if your installation and environment are set up properly.

    In this guide, we will build hello_fpga.c. This is the \"Hello World!\" example of using the library. This code searches for a predefined and known AFU called \"Native Loopback Adapter\" on the FPGA. If found, it acquires ownership and then interacts with the AFU by sending it a 2MB message and waiting for the message to be echoed back. This code exercises all major components of the API except for AFU reconfiguration: AFU search, enumeration, access, MMIO, and memory management.

    You can also find the source for hello_fpga in the samples directory of the OPAE SDK repository on GitHub.

        int main(int argc, char *argv[])\n    {\n        fpga_properties    filter = NULL;\n        fpga_token         afu_token;\n        fpga_handle        afu_handle;\n        fpga_guid          guid;\n        uint32_t           num_matches;\n\n        volatile uint64_t *dsm_ptr    = NULL;\n        volatile uint64_t *status_ptr = NULL;\n        volatile uint64_t *input_ptr  = NULL;\n        volatile uint64_t *output_ptr = NULL;\n\n        uint64_t        dsm_wsid;\n        uint64_t        input_wsid;\n        uint64_t        output_wsid;\n        fpga_result     res = FPGA_OK;\n\n        if (uuid_parse(NLB0_AFUID, guid) < 0) {\n            fprintf(stderr, \"Error parsing guid '%s'\\n\", NLB0_AFUID);\n            goto out_exit;\n        }\n\n        /* Look for accelerator by its \"afu_id\" */\n        res = fpgaGetProperties(NULL, &filter);\n        ON_ERR_GOTO(res, out_exit, \"creating properties object\");\n\n        res = fpgaPropertiesSetObjectType(filter, FPGA_ACCELERATOR);\n        ON_ERR_GOTO(res, out_destroy_prop, \"setting object type\");\n\n        res = fpgaPropertiesSetGuid(filter, guid);\n        ON_ERR_GOTO(res, out_destroy_prop, \"setting GUID\");\n\n        /* TODO: Add selection via BDF / device ID */\n\n        res = fpgaEnumerate(&filter, 1, &afu_token, 1, &num_matches);\n        ON_ERR_GOTO(res, out_destroy_prop, \"enumerating accelerators\");\n\n        if (num_matches < 1) {\n            fprintf(stderr, \"accelerator not found.\\n\");\n            res = fpgaDestroyProperties(&filter);\n            return FPGA_INVALID_PARAM;\n        }\n\n        /* Open accelerator and map MMIO */\n        res = fpgaOpen(afu_token, &afu_handle, 0);\n        ON_ERR_GOTO(res, out_destroy_tok, \"opening accelerator\");\n\n        res = fpgaMapMMIO(afu_handle, 0, NULL);\n        ON_ERR_GOTO(res, out_close, \"mapping MMIO space\");\n\n        /* Allocate buffers */\n        res = fpgaPrepareBuffer(afu_handle, LPBK1_DSM_SIZE,\n                    (void **)&dsm_ptr, &dsm_wsid, 0);\n        ON_ERR_GOTO(res, out_close, \"allocating DSM buffer\");\n\n        res = fpgaPrepareBuffer(afu_handle, LPBK1_BUFFER_ALLOCATION_SIZE,\n                   (void **)&input_ptr, &input_wsid, 0);\n        ON_ERR_GOTO(res, out_free_dsm, \"allocating input buffer\");\n\n        res = fpgaPrepareBuffer(afu_handle, LPBK1_BUFFER_ALLOCATION_SIZE,\n                   (void **)&output_ptr, &output_wsid, 0);\n        ON_ERR_GOTO(res, out_free_input, \"allocating output buffer\");\n\n        printf(\"Running Test\\n\");\n\n        /* Initialize buffers */\n        memset((void *)dsm_ptr,    0,    LPBK1_DSM_SIZE);\n        memset((void *)input_ptr,  0xAF, LPBK1_BUFFER_SIZE);\n        memset((void *)output_ptr, 0xBE, LPBK1_BUFFER_SIZE);\n\n        cache_line *cl_ptr = (cache_line *)input_ptr;\n        for (uint32_t i = 0; i < LPBK1_BUFFER_SIZE / CL(1); ++i) {\n            cl_ptr[i].uint[15] = i+1; /* set the last uint in every cacheline */\n        }\n\n        /* Reset accelerator */\n        res = fpgaReset(afu_handle);\n        ON_ERR_GOTO(res, out_free_output, \"resetting accelerator\");\n\n        /* Program DMA addresses */\n        uint64_t iova;\n        res = fpgaGetIOAddress(afu_handle, dsm_wsid, &iova);\n        ON_ERR_GOTO(res, out_free_output, \"getting DSM IOVA\");\n\n        res = fpgaWriteMMIO64(afu_handle, 0, CSR_AFU_DSM_BASEL, iova);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_AFU_DSM_BASEL\");\n\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 0);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 1);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        res = fpgaGetIOAddress(afu_handle, input_wsid, &iova);\n        ON_ERR_GOTO(res, out_free_output, \"getting input IOVA\");\n        res = fpgaWriteMMIO64(afu_handle, 0, CSR_SRC_ADDR, CACHELINE_ALIGNED_ADDR(iova));\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_SRC_ADDR\");\n\n        res = fpgaGetIOAddress(afu_handle, output_wsid, &iova);\n        ON_ERR_GOTO(res, out_free_output, \"getting output IOVA\");\n        res = fpgaWriteMMIO64(afu_handle, 0, CSR_DST_ADDR, CACHELINE_ALIGNED_ADDR(iova));\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_DST_ADDR\");\n\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_NUM_LINES, LPBK1_BUFFER_SIZE / CL(1));\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_NUM_LINES\");\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CFG, 0x42000);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        status_ptr = dsm_ptr + DSM_STATUS_TEST_COMPLETE/8;\n\n        /* Start the test */\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 3);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        /* Wait for test completion */\n        while (0 == ((*status_ptr) & 0x1)) {\n            usleep(100);\n        }\n\n        /* Stop the device */\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 7);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        /* Check output buffer contents */\n        for (uint32_t i = 0; i < LPBK1_BUFFER_SIZE; i++) {\n            if (((uint8_t*)output_ptr)[i] != ((uint8_t*)input_ptr)[i]) {\n                fprintf(stderr, \"Output does NOT match input \"\n                    \"at offset %i!\\n\", i);\n                break;\n            }\n        }\n\n        printf(\"Done Running Test\\n\");\n\n        /* Release buffers */\n    out_free_output:\n        res = fpgaReleaseBuffer(afu_handle, output_wsid);\n        ON_ERR_GOTO(res, out_free_input, \"releasing output buffer\");\n    out_free_input:\n        res = fpgaReleaseBuffer(afu_handle, input_wsid);\n        ON_ERR_GOTO(res, out_free_dsm, \"releasing input buffer\");\n    out_free_dsm:\n        res = fpgaReleaseBuffer(afu_handle, dsm_wsid);\n        ON_ERR_GOTO(res, out_unmap, \"releasing DSM buffer\");\n\n        /* Unmap MMIO space */\n    out_unmap:\n        res = fpgaUnmapMMIO(afu_handle, 0);\n        ON_ERR_GOTO(res, out_close, \"unmapping MMIO space\");\n\n        /* Release accelerator */\n    out_close:\n        res = fpgaClose(afu_handle);\n        ON_ERR_GOTO(res, out_destroy_tok, \"closing accelerator\");\n\n        /* Destroy token */\n    out_destroy_tok:\n        res = fpgaDestroyToken(&afu_token);\n        ON_ERR_GOTO(res, out_destroy_prop, \"destroying token\");\n\n        /* Destroy properties object */\n    out_destroy_prop:\n        res = fpgaDestroyProperties(&filter);\n        ON_ERR_GOTO(res, out_exit, \"destroying properties object\");\n\n    out_exit:\n        return res;\n\n    }\n

    Linking with the OPAE library is straightforward. Code using this library should include the header file fpga.h. Taking the GCC compiler on Linux as an example, the minimalist compile and link line should look like:

    gcc -std=c99 hello_fpga.c -I/usr/local/include -L/usr/local/lib -lopae-c -luuid -ljson-c -lpthread -o hello_fpga\n

    The API uses some features from the C99 language standard. The -std=c99 switch is required if the compiler does not support C99 by default.

    Third-party library dependency: The library internally uses libuuid and libjson-c. But they are not distributed as part of the library. Make sure you have these libraries properly installed. The -c flag may not be necessary depending on the platform being used. 

    sudo ./hello_fpga -c\nRunning Test\nRunning on bus 0x08.\nAFU NLB0 found @ 28000\nDone Running Test\n
    To run the hello_fpga application:

    sudo ./hello_fpga\nRunning Test\nDone\n
    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#appendix-a-integrating-an-n6001-based-custom-platform-with-dfl-and-opae","title":"Appendix A - Integrating an N6001 Based Custom Platform with DFL and OPAE","text":"

    The process of adding a custom device to the DFL framework and OPAE SDK requires changes to files in several locations. In this section we will walk through the additions necessary to instruct the kernel's probe and match function to associate your new device with OPAE, choose the correct OPAE plugin to associate with your board, and change basic descriptors to properly display the name of your new custom platform when using OPAE's management interfaces.

    This section does not require useage of an entirely new platform - we will use the Intel N6001's FIM design as our base, and alter only those settings in the PCIe IP necessary to change the following PCIe IDs for both the PF and VF:

    • Vendor ID
    • Device ID
    • Subsystem Vendor ID
    • Subsystem Device ID

    This document will not cover the process by which a FIM can be modified to support these new IDs. Refer to section 5.7 How to change the PCIe device ID and Vendor ID in F2000x FIM Developer guide or section 10.0 How to change the PCIe device ID and Vendor ID in the N6001 FIM Developer guide for an overview of this process. The following sections assume you have a FIM that has been configured with new IDs. This FIM can be loaded onto your N6001 board using either the SOF via a USB-BlasterII cable and the Quartus Programmer, or by using the OPAE command fpgasupdate consuming a compatible binary programming file. The following values will be used as our new device IDs.

    ID Type PF0 PF0-VF Vendor ID 0xff00 0xff00 Device ID 0x1234 0x5555 Subsystem Vendor ID 0xff00 0xff00 Subsystem Device ID 0x0x5678 0x5678

    The only value that differs between PF0 and PF0-VF in this example is the Device ID, and all other values do not currently exist in either the OPAE SDK or linux-dfl drivers. You will need to download the OPAE SDK and linux-dfl sources from GitHub and modify their contents. We will be using a validated Agilex OFS release to pair our OPAE SDK and linux-dfl versions together. Refer to the below table for a list of the exact version we will use for each software component and where you can learn how to build and install. Your versions may not match those in the document.

    Software Version Build Instructions OPAE SDK 2.3.0-1 4.0 OPAE Software Development Kit linux-dfl ofs-2022.3-2 3.0 Intel OFS DFL Kernel Drivers

    The following steps will enable your device to use the OPAE SDK. We will call our new device the \"Intel FPGA Programmable Acceleration Card N6002\". This device is identical to the Intel FPGA Programmable Acceleration Card N6001, and will use the pre-existing plugins and feature ID associated with that device. We will also use the enum value FPGA_HW_DCP_N6002 to describe our new board in the code. These plugins can be customized as you become more familiar with the OPAE SDK software.

    1. Download the OPAE SDK from GitHub. File paths assume the user is in the directory opae-sdk.
    2. Open the file binaries/opae.io/opae/io/config.py. Add a new configuration entry under DEFAULT_OPAE_IO_CONFIG. Save and exit.

    Example:

    (0xff00, 0x1234, 0xff00, 0x5678) : {\n        'platform': 'Intel FPGA Programmable Acceleration Card N6002'\n    },\n
    1. Open the file libraries/libopae-c/cfg-file.c. Add two new entries (one for PF0 and PF0-VF) under STATIC libopae_config_data default_libopae_config_table[]. Add two new entries under STATIC fpgainfo_config_data default_fpgainfo_config_table[]. Save and exit.

    Example:

    STATIC fpgainfo_config_data default_fpgainfo_config_table[] = {\n...\n{ 0xff00, 0x1234, 0xff00, 0x5678, 0x12, \"libboard_n6000.so\", NULL,\n\"Intel FPGA Programmable Acceleration Card N6002\" },                  // N6002 PF0\n{ 0xff00, 0x5555, 0xff00, 0x5678, 0x12, \"libboard_n6000.so\", NULL,\n\"Intel FPGA Programmable Acceleration Card N6002\" },                  // N6002 PF0-VF\n

    Example:

    STATIC libopae_config_data default_libopae_config_table[] = {\n...\n{ 0xff00, 0x1234, 0xff00,          0x5678,          \"libxfpga.so\",  \"{}\", 0 }    , // N6002 PF0\n{ 0xff00, 0x5555, 0xff00,          0x5678,          \"libxfpga.so\",  \"{}\", 0 }    , // N6002 PF0-VF\n
    4. Open the file libraries/plugins/xfpga/metrics/metric_utils.c. Add one entry to the switch case under enum_fpga_metrics(fpga_handle handle). The enum value used should match the enum set in step 6. Add a new condition to the if statement beginning if (((_fpga_enum_metric->hw_type == FPGA_HW_DCP_N3000). Save and exit.

    Example:

                     // DCP VC DC\n                 case FPGA_HW_DCP_N3000:\n                 case FPGA_HW_DCP_D5005:\n                 case FPGA_HW_DCP_N6002:\n                 case FPGA_HW_DCP_N5010: {\n                 ...\n

    Example:

                                if (((_fpga_enum_metric->hw_type == FPGA_HW_DCP_N3000) ||\n                                 (_fpga_enum_metric->hw_type == FPGA_HW_DCP_D5005) ||\n                                 (_fpga_enum_metric->hw_type == FPGA_HW_DCP_N6002) ||\n                                 (_fpga_enum_metric->hw_type == FPGA_HW_DCP_N5010)) &&\n
    5. Open the file libraries/plugins/xfpga/sysfs.c. Add a new set of switch cases under enum fpga_hw_type opae_id_to_hw_type(uint16_t vendor_id, uint16_t device_id). The enum value used should match the enum value set in step 6. Save and exit.

    Example:

                if (vendor_id == 0xff00) {        \n                 switch (device_id) {\n                         case 0x1234:\n                         case 0x5555:\n                                 hw_type = FPGA_HW_DCP_N6002;\n                         break;\n                 default:\n                         OPAE_ERR(\"unknown device id: 0x%04x\", device_id);\n
    6. Open the file libraries/plugins/xfpga/types_int.h. Add your new enum value under enum fpga_hw_type. Save and exit.

    Example:

    enum fpga_hw_type {\n         FPGA_HW_MCP,\n         FPGA_HW_DCP_RC,\n         FPGA_HW_DCP_D5005,\n         FPGA_HW_DCP_N3000,\n         FPGA_HW_DCP_N5010,\n         FPGA_HW_DCP_N6002,\n         FPGA_HW_UNKNOWN\n};\n
    7. Open the file opae.cfg. Create a new entry for device \"n6002\" by copying the entry for \"n6001,\"\" substituting our new values. Add one new entry under \"configs\" for the name \"n6002.\" Save and exit.

    Example:

        \"n6002\": {\n      \"enabled\": true,\n      \"platform\": \"Intel Acceleration Development Platform N6002\",\n      \"devices\": [\n        { \"name\": \"n6002_pf\", \"id\": [ \"0xff00\", \"0x1234\", \"0xff00\", \"0x5678\" ] },\n        { \"name\": \"n6002_vf\", \"id\": [ \"0xff00\", \"0x5555\", \"0xff00\", \"0x5678\" ] }\n      ],\n      \"opae\": {\n        \"plugin\": [\n          {\n            \"enabled\": true,\n            \"module\": \"libxfpga.so\",\n            \"devices\": [ \"n6002_pf\" ],\n            \"configuration\": {}\n          },\n          {\n            \"enabled\": true,\n            \"module\": \"libopae-v.so\",\n            \"devices\": [ \"n6002_pf\", \"n6002_vf\" ],\n            \"configuration\": {}\n          }\n        ],\n        \"fpgainfo\": [\n          {\n            \"enabled\": true,\n            \"module\": \"libboard_n6000.so\",\n            \"devices\": [\n              { \"device\": \"n6002_pf\", \"feature_id\": \"0x12\" },\n              { \"device\": \"n6002_vf\", \"feature_id\": \"0x12\" }\n            ]\n          }\n        ],\n        \"fpgad\": [\n         {\n            \"enabled\": true,\n            \"module\": \"libfpgad-vc.so\",\n            \"devices\": [ \"n6002_pf\" ],\n            \"configuration\": {\n              \"cool-down\": 30,\n              \"get-aer\":     [ \"setpci -s %s ECAP_AER+0x08.L\",\n                               \"setpci -s %s ECAP_AER+0x14.L\" ],\n              \"disable-aer\": [ \"setpci -s %s ECAP_AER+0x08.L=0xffffffff\",\n                               \"setpci -s %s ECAP_AER+0x14.L=0xffffffff\" ],\n              \"set-aer\":     [ \"setpci -s %s ECAP_AER+0x08.L=0x%08x\",\n                               \"setpci -s %s ECAP_AER+0x14.L=0x%08x\" ],\n              \"sensor-overrides\": [],\n              \"monitor-seu\": false\n            }\n          }\n        ],\n        \"rsu\": [\n          {\n            \"enabled\": true,\n            \"devices\": [ \"n6002_pf\" ],\n            \"fpga_default_sequences\": \"common_rsu_sequences\"\n          }\n        ],\n        \"fpgareg\": [\n          {\n            \"enabled\": true,\n            \"devices\": [ \"n6002_pf\", \"n6002_vf\" ]\n          }\n        ],\n        \"opae.io\": [\n          {\n            \"enabled\": true,\n            \"devices\": [ \"n6002_pf\", \"n6002_vf\" ]\n          }\n        ]\n      }\n    },\n

    Example:

    \"configs\": [\n     \"mcp\",\n     \"a10gx\",\n     \"d5005\",\n     \"n3000\",\n     \"n5010\",\n     \"n5013\",\n     \"n5014\",\n     \"n6000\",\n     \"n6001\",\n     \"n6002\",\n     ...\n
    These conclude our integration of our new platform with the OPAE SDK. The next step is to download the source for linux-dfl (as shown above) and configure our new kernel's match and probe function to associate the DFL drivers with our new custom platform. The following file path assumes the user is in the directory linux-dfl 1. Open the file drivers/fpga/dfl-pci.c. Define a list of necessary ID values at the top of the file. Use these values to add two new entries under pci_device_id cci_pcie_id_tbl[], one for PF0 and the other for PF0-VF. Save and exit.

    Example:

    /* N6002 IDS */\n#define PCIE_DEVICE_ID_PF_N6002                 0x1234\n#define PCIE_VENDOR_ID_PF_N6002                 0xff00\n#define PCIE_SUBDEVICE_ID_PF_N6002              0x5678\n#define PCIE_DEVICE_ID_VF_N6002                 0x5555\n

    Example:

    static struct pci_device_id cci_pcie_id_tbl[] = {\n        ...\n        {PCI_DEVICE_SUB(PCIE_VENDOR_ID_PF_N6002, PCIE_DEVICE_ID_PF_N6002,\n                    PCIE_VENDOR_ID_PF_N6002, PCIE_SUBDEVICE_ID_PF_N6002),}, //N6002 PF0\n        {PCI_DEVICE_SUB(PCIE_VENDOR_ID_PF_N6002, PCIE_DEVICE_ID_VF_N6002,\n                    PCIE_VENDOR_ID_PF_N6002, PCIE_SUBDEVICE_ID_PF_N6002),}, //N6002 PF0-VF\n                    ...\n

    This concludes our integration our new platform with the linux-dfl driver set. Build and install the linux-dfl enabled kernel and the OPAE SDK userspace libraries as discussed in their relevant sections in the user guide linked above. If the above conditions have been met, and your N6001 board has been configured with this new \"N6002\" FIM, you should see the following output when running the command \"fpgainfo fme\" (your Bitstream ID, PR Interface ID, and Image Info entries may differ). Check that the board's display name at the top and values for Vendor/Device/SubVendor/Subdevice IDs are correct.

    Intel Acceleration Development Platform N6002\nBoard Management Controller NIOS FW version: 3.11.0\nBoard Management Controller Build version: 3.11.0\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0xFF00\nDevice Id                        : 0x1234\nSubVendor Id                     : 0xFF00\nSubDevice Id                     : 0x5678\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010202C8AD72D7\nBitstream Version                : 5.0.1\nPr Interface Id                  : 8df219e3-cf25-5e77-8689-f57102d54435\nBoot Page                        : user1\nFactory Image Info               : a2b5fd0e7afca4ee6d7048f926e75ac2\nUser1 Image Info                 : 9804075d2e8a71a192ec89602f2f5544\nUser2 Image Info                 : 9804075d2e8a71a192ec89602f2f5544\n

    "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#notices-disclaimers","title":"Notices & Disclaimers","text":"

    Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Performance varies by use, configuration and other factors. Your costs and results may vary. You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications. Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards. \u00a9 Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.

    OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/","title":"oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack","text":""},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#10-overview","title":"1.0 Overview","text":""},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#11-about-this-document","title":"1.1 About this Document","text":"

    This document serves as a reference manual for platform designers wanting to enable oneAPI support on their Open FPGA Stack(OFS) platforms. The document describes essential hardware and software components required for enabling this design flow using OFS. Implementation details for oneapi-asp for Open FPGA Stack(OFS) reference platforms is covered towards the end of the document.

    Note: Table 1-1 in oneAPI Accelerator Support Package (ASP): Getting Started User Guide lists OFS reference platforms.

    For more information about developing application kernels for FPGA using Intel\u00ae oneAPI Base Toolkit (Base Kit) refer to Intel\u00ae FPGA Add-on for oneAPI Base Toolkit webpage.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#12-terminology","title":"1.2 Terminology","text":"

    This table defines some of the common terms used when discussing OFS.

    Table 1-1: Terminology

    Term Abbreviation Description Open FPGA Stack OFS A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. Accelerator Functional Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. FPGA Interface Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. High Level Design HLD For the purpose of this guide, this term refers to designing with Intel High Level Design tools like Intel\u00ae oneAPI Base Toolkit (Base Kit). oneAPI Accelerator Support Package oneAPI ASP A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and other OFS hardware, software components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Intel\u00ae FPGA Basic Building Blocks BBB Basic Building Blocks (BBB) for Intel\u00ae FPGAs is a suite of application building blocks and shims like Memory Properties Factory (MPF). BBB Memory Properties Factory BBB MPF Intel\u00ae FPGA BBB MPF block provides features like virtual to physical address (VTP), ordering read responses, read/write hazard detection, and masked (partial) writes. oneapi-asp uses MPF VTP feature. Open Programmable Acceleration Engine Software Development Kit OPAE SDK A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. Platform Interface Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Device Feature List DFL A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. Best Known Configuration BKC The exact hardware configuration Intel has optimized and validated the solution against. SYCL - SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL\u2122) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Installable Client Driver ICD Intel\u00ae FPGA Runtime for OpenCL\u2122 Software Technology supports the OpenCL ICD extension from the Khronos Group\u2122. The OpenCL ICD extension allows you to have multiple OpenCL implementations on your system. With the OpenCL ICD Loader Library, you may choose from a list of installed platforms and execute OpenCL API calls that are specific to your OpenCL implementation of choice. FPGA Client Driver FCD Intel\u00ae FPGA Runtime for OpenCL\u2122 Software Technology supports FPGA Client Driver(FCD) extension. FCD allows the runtime to automatically find and load the oneAPI ASP libraries at host run time

    Note: oneapi-asp was referred to as ofs-hld-shim in OFS (Agilex & Stratix 10 OFS) and OpenCL AFU Shim (ofs-opencl-afu-shim) in OFS early access(EA) release (for Intel\u00ae Stratix 10\u00ae FPGA with Intel\u00ae FPGA PAC D5005 as reference platform).

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#13-prerequisites","title":"1.3 Prerequisites","text":"

    The content in this manual requires readers to be familiar with:

    • Hardware and software components of Open FPGA Stack, especially the following:
      • FPGA Interface Manager(FIM)
        • Intel\u00ae Stratix 10\u00ae FPGA:
          • FIM Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs
          • FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs
        • Intel\u00ae Agilex\u00ae 7 FPGA:
          • Technical Reference Manual: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs
          • FPGA Interface Manager Developer Guide for Intel\u00ae Agilex\u00ae 7 PCIe Attach
      • Accelerator Functional Unit(AFU)
        • Intel\u00ae Stratix 10\u00ae FPGA: OFS AFU Development Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs
        • Intel\u00ae Agilex\u00ae 7 FPGA: AFU Development Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs
      • Software Reference Manual: Open FPGA Stack (OPAE SDK and Linux-DFL section)
      • ofs-platform-afu-bbb

    In addition to above, developers must be familiar with the following tools & concepts:

    • Intel\u00ae Quartus\u00ae Prime Design Software (Intel\u00ae Quartus\u00ae software revisions, Platform Designer, compilation Flows, timing analysis, compilation reports, understanding FPGA resource utilization, programming Intel\u00ae FPGAs)
    • Partial Reconfiguration (PR)
    • FPGA Peripheral IPs (PCIe, External Memory IP, Ethernet IP)
    • Avalon\u00ae Interface
    • Scripting (TCL, Python, Shell scripts)
    • Verilog, SystemVerilog
    • C++
    • Familiarity with SYCL
    • Familiarity with oneAPI compilation process for FPGAs & oneAPI code samples
    • Familiarity with oneAPI Accelerator Support Package (ASP): Getting Started User Guide
    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#14-introduction-to-oneapi-on-open-fpga-stackofs","title":"1.4 Introduction to oneAPI on Open FPGA Stack(OFS)","text":"

    The Intel\u00ae oneAPI Base Toolkit (Base Kit) is a core set of tools and libraries for developing high-performance, data-centric applications across diverse architectures (CPUs, GPUs and FPGAs). It features an industry-leading C++ compiler that implements SYCL, an evolution of C++ for heterogeneous computing.

    Figure 1-1 shows the high-level representation of oneAPI application developers using FPGAs for acceleration. The runtime flow consists of a host code running on a processor and an application kernel code running on an FPGA. Open FPGA Stack enables vendors to enable support for this flow on their platforms.

    Figure 1-1: oneAPI Application on OFS Platforms

    Intel\u00ae oneAPI Base Toolkit (Base Kit) consists of a compiler and runtime environment. The compiler converts a SYCL kernel (FPGA application code) into a hardware circuit. This hardware circuit requires additional logic to communicate with the runtime and FPGA board peripherals. This additional logic is provided by oneAPI Accelerator Support Package(oneAPI ASP). oneAPI ASP consists of hardware components that enable this generated hardware circuit to communicate with the host processor as well as software components that enable the runtime to identify and communicate with the kernel.

    Figure 1-2 shows the workload design steps and steps in which the Intel\u00ae oneAPI Base Toolkit (Base Kit) requires oneAPI ASP as input. For more information about workload development and how workload developers target a specific platform during compilation, refer to Intel oneAPI Programming Guide. The next section introduces oneAPI ASP.

    Figure 1-2: High Level Design Flow for FPGAs with Intel\u00ae oneAPI Base Toolkit (Base Kit)

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#15-introduction-to-oneapi-accelerator-support-packageasp","title":"1.5 Introduction to oneAPI Accelerator Support Package(ASP)","text":"

    As mentioned in previous section, oneAPI ASP is a collection of hardware and software components that interface with the hardware circuit generated by the oneAPI compiler. The hardware circuit generated by the oneAPI compiler from a oneAPI kernel is referred to as the kernel system. While the kernel system consists of logic controlled by the workload developer's specifications, the kernel system interfaces are generated by the oneAPI compiler based on specifications provided by the oneAPI ASP designer. These specifications are input to the compiler using XML files (discussed in section 2.0).

    Note: All the interfaces generated by the oneAPI compiler are Avalon\u00ae Interfaces.

    Figure 1-3: Kernel System Interfaces

    Figure 1-3 shows a high-level representation of an OFS hardware design and interfaces to/from kernel_system. The numbered arrows depict the following:

    • Path 1 represents host-to-External Memory Interface (EMIF)
    • Path 2 represents the host to kernel interface
    • Path 3 represents kernel to EMIF
    • Path 4 represents kernel to Unified Shared Memory (USM) Interface
    • Path 5 represents kernel to HSSI interface

    oneAPI ASP hardware components can be divided into 3 categories:

    1. RTL components: constituting various interface logic, for example, host to External Memory Interface (EMIF), kernel to EMIF interface, host to kernel interface, kernel to host memory interface as well as additional components to handle kernel control signals and perform Direct Memory Access (DMA)
    2. XML files: for describing hardware interfaces and compilation environment to Intel\u00ae oneAPI Base Toolkit (Base Kit)
    3. Scripts: to control compile flow

    In addition to the hardware components, a software layer is required for handling I/O operations between oneAPI runtime and the board. The oneAPI ASP software layer can be divided into 2 categories:

    1. Memory Mapped Device (MMD) Layer: required by the host & runtime to communicate with the oneAPI kernel & other oneAPI ASP hardware registers
    2. oneAPI ASP utilities: required to setup and diagnose the board

    The MMD uses API provided by OPAE SDK to communicate with the device. The FPGA driver is provided by the linux-DFL kernel driver.

    Figure 1-4 shows how the above oneAPI ASP components tie into Open FPGA Stack.

    Figure 1-4: Open FPGA Stack (OFS) components

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#20-xml-files-in-oneapi-asp","title":"2.0 XML Files in oneAPI ASP","text":"

    The kernel system interfaces generated by the oneAPI compiler are based on specifications provided by oneAPI ASP developer. An XML file, called board_spec.xml is used to pass the specifications to the oneAPI compiler. oneAPI ASP developers must create this XML file for their boards.

    In addition to board_spec.xml, the Intel\u00ae oneAPI Base Toolkit (Base Kit) relies on another XML file called board_env.xml to get information about the board environment. The board_env.xml file helps the runtime setup board installation.

    The next section explains the contents of board_spec.xml. Section 2.2 covers contents of board_env.xml file.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#21-board_specxml-file","title":"2.1 board_spec.xml File","text":"

    A typical board_spec.xml structure is shown in Fig 2-1. In addition to kernel system interfaces, the board_spec.xml file is also used to specify other compilation details like Intel\u00ae Quartus\u00ae Prime Pro Edition Software version used in platform design, compilation scripts to help control Intel\u00ae Quartus\u00ae software compile flows, FPGA resource utilization details.

    Elements of board_spec.xml file are summarized in table 2-1. Each element has additional attributes and parameters. Details are covered in respective sections for each element.

    Figure 2-1: board_spec.xml File Structure

    Table 2-1: Elements of board_spec.xml

    Element Use of Element Attributes board Used to specify a name for the board and the version of Intel\u00ae Quartus\u00ae Prime Pro Edition Software used to develop the platform design. This board name is used to identify the board design to be compiled and must match the name of the directory in which board_spec.xml resides. version, name compile Used to describe different compile flows project, revision, qsys_file, generic_kernel, generate_cmd, synthesize_cmd, auto_migrate device Used to specify the FPGA device model file for the FPGA part on the board. device_model, used_resources global_mem Different attributes in this element are used to provide details about the external memory used as the global memory for the FPGA oneAPI kernel/application. name, max_bandwidth, interleaved_bytes, config_addr, default, interface host Used to specify the offset at which the kernel resides. kernel_config interfaces Used to specify control signals to oneAPI kernels interface, kernel_clk_reset channels Used to describe interface to stream data directly between kernels and I/O interface

    The compiler expects a separate board_spec.xml file for every board variant a platform supports. Board variants are different hardware design implementations for the same platform, a oneAPI ASP can have multiple board variants. A oneAPI kernel developer can select the board variant suitable for their application at compile time.

    A board_spec.xml file must be located at the top most level of each board variant's hardware directory (the hardware directory is specified by board_env.xml, please refer to section 2.2 for details on hardware element). For example, a separate board_spec.xml file for each board variant for OFS reference platforms is located in oneapi-asp/Platform-Name/hardware/Board-Variant/ directory, where Platform-Name is n6001 for Agilex OFS and d5005 for Stratix 10 OFS.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#211-board-element","title":"2.1.1 board Element","text":"

    The board element of the board_spec.xml file provides the Intel\u00ae Quartus\u00ae Prime Pro Edition Software version and the name of the board.

    Table 2-2: Attributes for the board Element

    Attribute Description version The version of the board. The board version should match the version of the Intel\u00ae Quartus\u00ae Prime Pro Edition Software you use to develop the platform design. The oneAPI compiler uses this value to perform environment checks for supported version during application compile name The name of the accelerator board, which must match the name of the directory in which the board_spec.xml file resides. The name must contain a combination of only letters, numbers, underscores (_), hyphens (-), or periods (.) (for example, ofs_n6000).

    Example below shows the board element populated for a board designed with Intel\u00ae Quartus\u00ae Prime Pro Edition Software version 22.3 and board variant named \"Agilex_brd1\".

    Note: A board variant name is different from a platform directory name. Please see Note in section 2.2 for more information on board variants.

    Figure 2-2: board Element

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#212-compile-element","title":"2.1.2 compile Element","text":"

    Depending on the application requirements, the design may have different compilation flows and different design settings for each flow (for example, there can be a flat flow without partial reconfiguration support or a flow with partitions in the design to enable partial reconfiguration). Designers can control the flow and its settings using scripts. To allow selection of compile flow during application compile & to describe control of Intel\u00ae Quartus\u00ae software compilation as well as registration, automigration, the compile element of the board_spec.xml file and its associated attributes and parameters are used.

    Table 2-3: Attributes for compile Element

    Attribute Description name Name of the compilation flow. This name can be used to differentiate between flows at oneAPI kernel compilation time. oneAPI compiler allows selecting a compile flow using -Xsbsp-flow option. project Name of the Intel\u00ae Quartus\u00ae software project file (.qpf) that the Intel\u00ae Quartus\u00ae Prime Pro Edition Software intends to compile. revision Name of the revision within the Intel\u00ae Quartus\u00ae software project that the Intel\u00ae Quartus\u00ae Prime Pro Edition Software compiles to generate the final bitstream. qsys_file Name of the Platform Designer file into which the oneAPI kernel is embedded. You have the option to assign a value of \"none\" to qsys_file if you do not require the Intel\u00ae Quartus\u00ae Prime Pro Edition Software to create a top-level .qsys file for your design. In this scenario, oneAPI compiler adds a .qip file into the Intel\u00ae Quartus\u00ae software project. In this case, the custom oneAPI ASP must manually instantiate the generated HDL entity (generated entity is in the kernel_system.v file). generic_kernel Set this value to 1 if you want the offline compiler to generate a common Verilog interface for all compilations. This setting is necessary in situations where you must set up design partitions around the kernel, such as in the Configuration via Protocol (CvP) flow. generate_cmd Command required to prepare for full compilation, such as to generate the Verilog files for the Platform Designer system into which the oneAPI kernel is embedded. synthesize_cmd Command required to generate the fpga.bin file from the Custom Platform. Usually, this command instructs the Intel\u00ae Quartus\u00ae Prime Pro Edition Software to perform a full compilation. auto_migrate *platform_type\u2014Choose this value based on the value referenced in the Intel\u00ae FPGA Reference Platform from which you derive your Custom Platform. Valid values are a10_ref, s10_ref, and none. *include fixes\u2014Comma-separated list of named fixes that you want to apply to the Custom Platform. *exclude fixes\u2014Comma-separated list of named fixes that you do not want to apply to the Custom Platform.

    Example below shows a populated compile element for a sample Intel\u00ae Quartus\u00ae software Project called ofs.qpf, the Intel\u00ae Quartus\u00ae software revision to be compiled is called asp (asp.qsf). In this example, the compiler generates the kernel system (entity is called kernel_system) and this entity is instantiated manually in the Intel\u00ae Quartus\u00ae software project (e.g. in a file called kernel_wrapper.v), hence qsys_file is set to \"none\". The synthesize_cmd points to a script \"compile.tcl\" located in the same directory as the board_spec.xml, compile script performs all necessary system generation and compile steps for generation of final bitstream. The project directory snippet below is for demonstration only. The compile flow is named \"demo_flow\".

    There can be multiple compile elements for the different compilation flows that a platform designer wishes to enable in their platform (e.g. different revisions with different Intel\u00ae Quartus\u00ae software settings or a PR revision).

    Figure 2-3: compile Element

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#213-device-element","title":"2.1.3 device Element","text":"

    A device model(DM) file is an XML file that has the total resources on the device (i.e. ALMs, FFs, DSPs, RAMs). This is required for any FPGA part used in a oneAPI design. Most device model files are provided as part of the Intel\u00ae oneAPI Base Toolkit (Base Kit) installation ($INTELFPGAOCLSDKROOT/share/models/dm, where INTELFPGAOCLSDKROOT is set by the setvars.sh environment setup script provided by oneAPI toolkit). If the device model file for your part number is not included in $INTELFPGAOCLSDKROOT/share/models/dm, it must be created and placed in the same folder as board_spec.xml. A new device model file can be created using existing files as reference.

    The device model file name must be specified in the device_model attribute of device element. The used_resources attribute is used to specify the resources being utilized by the oneAPI ASP and peripheral IPs. The utilization by non-kernel logic is calculated during platform design. The compiler utilizes the total resources from device model file and utilized resources in used_resources section to estimate the available resources for application kernel.

    Table 2-4: Attributes for device Element

    Attribute Description device_model The file name of the device model file that describes the available FPGA resources on the accelerator board. used_resources Reports the number of adaptive logic modules (ALMs), flip-flops, digital signal processor (DSP) blocks and RAM blocks that the board design consumes in the absence of any kernel. If you create a defined partition around all the board logic, you can obtain the used resources data from the Partition Statistics section of the Fitter report. Extract the information from the following parameters: * alms num \u2014 The number of logic ALMs used, excluding the number of ALMs with only their registers used. The value should correspond to [a]+[b]+[d] from part [A] of the Fitter Partition Statistics. * ffs num \u2014 The number of flip flops. * dsps num \u2014 The number of DSP blocks. * rams num \u2014 The number of RAM blocks.

    Example below shows the device element added for a Intel\u00ae Agilex\u00ae 7 FPGA based platform with device model file named \"agfb014r24a2e2vr0_dm.xml\". The number of used_resources are for demonstration purposes and are not to be used by oneAPI ASP developers.

    Figure 2-4: device Element

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#214-interface-attribute","title":"2.1.4 interface Attribute","text":"

    Note: This is different from the interfaces element discussed in upcoming sections. In the board_spec.xml file, each global memory, channel or kernel connection is comprised of individual interfaces. For the global_mem, channels, and interfaces XML elements, an interface attribute must be included to specify the corresponding parameters for each connection.

    Table 2-5: Parameters for interface attribute

    Parameter Description Applicable Interface name * For global_mem: instance name of the Platform Designer component.* For channels: instance name of the Platform Designer component that has the channel interface.* For interfaces: name of the entity in which the kernel interface resides (for example, board). All port port parameter can be defined either inline in interface attribute or as a separate element in interface attribute. See section 2.1.4.1 for more information. All type * For global_mem: set to agent. * For channels: - Set to streamsource for a stream source that provides data to the kernel.- Set to streamsink for a stream sink interface that consumes data from the kernel.* For interfaces: set to either host, irq, or streamsource. All width * For global_mem: width of the memory interface in bits.* For channels: number of bits in the channel interface.* For interfaces: width of the kernel interface in bits. All waitrequest_allowance * For global_mem: [Optional] Amount of Avalon\u00ae-MM waitrequest allowance supported on the agent interface (that is, kernel-facing interface) of the clock-crossing bridge that spans between the memory and the kernel clock domains.* For kernel_cra: [Optional] Amount of Avalon\u00ae-MM waitrequest allowance that the kernel_cra agent interface must support.This parameter defaults to 0 if you do not specify it in the board_spec.xml file. A value of 0 indicates that this waitrequest allowance feature is disabled. All maxburst Maximum burst size for the agent interface.Attention: The value of width \u00f7 8 x maxburst must be less than the value of interleaved_bytes. global_mem address Starting address of the memory interface that corresponds to the host interface-side address.For example, address 0 should correspond to the bank1 memory host from the Memory Bank Divider. In addition, any non-zero starting address must abut the end address of the previous memory. global_mem size Size of the memory interface in bytes.* For global_mem: The sizes of all memory interfaces should be equal.* For interfaces: interface can have variable sizes. global_meminterfaces > Note: Support for size parameter for interface attribute is available in Intel\u00ae oneAPI Base Toolkit (Base Kit) version 2024.0 and beyond. latency_type If the memory interface has variable latency, set this parameter to average to signify that the specified latency is considered the average case. If the complete kernel-to-memory path has a guaranteed fixed latency, set this parameter to fixed. global_mem chan_id A string used to identify the channel interface. The string may have up to 128 characters. channels clock For the streamsource kernel interface type, the parameter specifies the name of the clock that the snoop stream uses. Usually, this clock is the kernel clock. interfaces

    Example for how the interface attribute is used in global_mem and interfaces elements is covered in section for these elements respectively.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#2141-port-parameter","title":"2.1.4.1 port Parameter","text":"

    As mentioned in Table 2-5, port parameter can be defined either inline in interface attribute or as a separate element in interface attribute. The definition method to use depends on the direction of the port.

    • If the direction of the port is either read or write, it must be a separate element in interface attribute.
    • If the direction is readwrite, port must be inline with the port name in interface attribute. No direction specification is required.

    Table below shows the port element attributes.

    Table 2-6: port parameter

    | Parameter | Description | |---------|---------|---------| | name | * For global_mem: name of the Avalon\u00ae-MM interface in the Platform Designer component that corresponds to the interface attribute.* For channels: name of the streaming interface in the Platform Designer component.* For interfaces: name of the interface to the Kernel Interface Platform Designer component. For example, kernel_cra is the Avalon\u00ae-MM interface, and kernel_irq is an interrupt. | | direction | Direction of the port. Valid values are: * \"r\" : Indicates read * \"w\" : Indicates write |

    Snippet below shows the inline and separate element definitions of port parameter.

    Note: Direction specification for port is available in Intel\u00ae oneAPI Base Toolkit (Base Kit) versions 2024.0 and beyond. For versions prior to Intel\u00ae oneAPI Base Toolkit (Base Kit) version 2024.0, only the default inline definition of port parameter is supported.

    Examples for global_mem and interfaces elements in sections below use the inline definition of port.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#215-global_mem-element","title":"2.1.5 global_mem Element","text":"

    The global_mem element of the board_spec.xml file is used to provide information on the memory interfaces that connect to the kernel.

    Note: For each global memory that the kernel accesses, you must include one interface element that describes its characteristics. The different attributes for global_mem element are discussed in table below.

    Table 2-7: Attributes for global_mem Element

    Attribute Description name The name FPGA application/kernel developer should use to identify the memory type. Each name must be unique and must comprise of less than 32 characters. max_bandwidth The maximum bandwidth, in megabytes per second (MB/s), of all global memory interfaces combined in their current configuration. The oneAPI compiler uses max_bandwidth to choose an architecture suitable for the application and the board. Compute this bandwidth value from datasheets of memories on your board. Example max_bandwidth calculation for a 64-bit DDR4 interface running at 1200 MHz: max_bandwidth = 1200 MHz x 2 x 64 bits \u00f7 8-bits = 19200 MB/s The max_bandwidth value will change based on global memory configuration, for example, if the memory configuration comprises of 4 banks of DDR4 configured as a single homogenous memory, the max_bandwidth will be 19200 x 4 (i.e. number of memory interfaces from kernel). Please see section 2.1.5.1 for more information on global memory configurations. Designers have the option to use block RAM instead of or in conjunction with external memory as global memory. The formula for calculating max_bandwidth for block RAM is max_bandwidth = block RAM speed x (block RAM interface size \u00f7 8 bits). Example max_bandwidth calculation for a 512-bit block RAM running at 100 MHz: max_bandwidth = 100 MHz x 512 bits \u00f7 8 bits = 6400 MB/s interleaved_bytes Include the interleaved_bytes attribute in the board_spec.xml file when you instantiate multiple interfaces(i.e. memory banks) for a given global memory system. This attribute controls the size of data that the offline compiler distributes across the interfaces. The offline compiler currently can interleave data across banks no finer than the size of one full burst. This attribute specifies this size in bytes and following are the recommended values: For two or fewer global memory banks: maxburst x width_bytes For four or more global memory banks: maxburst x width_bytes x 4 The interleaved_bytes value must be the same for the host interface and the kernels. Therefore, the configuration of the Memory Bank Divider must match the exported kernel agent interfaces in this respect (refer to section 3.1.1 for information about Memory Bank Divider) For block RAM, interleaved_bytes equals the width of the interface in bytes. config_addr The address of the ACL Mem Organization Control Platform Designer component (mem_org_mode) that the host software uses to configure memory. You may omit this attribute if your board has homogeneous memory; the software uses the default address (0x18) for this component. If your board has heterogeneous memory, there is a mem_org_mode component in the board system for each memory type. Enter the config_addr attribute and set it to the value of the base address of the mem_org_mode component(s). default Include this optional attribute and assign a value of 1 to set the global memory as the default memory interface. The default memory must start at address 0x0. If you do not implement this attribute, the first memory type defined in the board_spec.xml file becomes the default memory interface. interface See the interface section above for the parameters you must specify for each interface. allocation_type A list that specifies which USM allocator is used to allocate from the global_mem element. Values allowed in this list are host, shared, and device. The following conditions apply: If there are multiple global_mem elements with the same allocation_type attribute, the first allocation_type attribute in the board_spec.xml is assumed to be the one used by the specified allocator. If there is a single global_mem element with multiple allocation_type attributes, this indicates that allocations of the specified types use this global_mem interface. [Legacy support] If you have not specified the allocation_type attribute, it is assumed that all global memory interfaces have the device allocation_type.

    Example below shows a global_mem element configuration for a kernel system connected to four 4GB DDR4 memory banks. The DDR4 interface is 64 bit operating at 1200MHz. Note that the name of the platform designer system name is board.qsys. As mentioned in description for interleaved_bytes in table above, the Memory Bank Divider configuration ensures that the host interface matches the interleaved_bytes setting (i.e. 512 bits x 64 burst size = 4096 bytes). For information on waitrequest_allowance, refer to section 2.1.4 on interface attribute.

    Note: More details on the Memory Bank Divider and the Clock Crossing Bridges is covered in section 3.0

    Figure 2-6: Memory Connection Example Block Diagram and Corresponding global_mem Element in board_spec.xml

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#2151-global-memory-configurations","title":"2.1.5.1 Global Memory Configurations","text":"

    A board can have a single memory bank, multiple memory banks of the same type (e.g. 4 banks of DDR4) or different banks of different types.

    The partitioning of memory for oneAPI kernel developers is explained in the FPGA Optimization Guide for Intel\u00ae oneAPI Toolkits. The global memory configuration required by an application kernel must match the configuration in board_spec.xml as the compiler uses this information to generate a suitable architecture for the application. The different memory configurations are

    • A single global memory region (possible with same type of memory banks)
    • Different global memories (heterogeneous memory)
    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#21511-contiguous-global-memory","title":"2.1.5.1.1 Contiguous Global Memory","text":"

    For boards with multiple memory banks of the same type, designers can configure these as a single contiguous global memory region. This is done by specifying each memory interface within a single global_mem element. Figure 2-6 showed 4 DDR4 memory banks configured as a single global memory region.

    With this configuration, FPGA application developers have the option to use contiguous memory region in an interleaved or a non-interleaved fashion. Even with contiguous memory regions, kernel developers can partition data buffers across the banks/memory channels. Please refer to Global Memory Access Optimization section in FPGA Optimization Guide for Intel\u00ae oneAPI Toolkits for more details on these partitioning techniques.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#21512-heterogeneous-memory","title":"2.1.5.1.2 Heterogeneous Memory","text":"

    For boards with different memory technologies, designers must specify each type of memory that the kernel needs to access as a separate global memory.

    Figure 2-7 shows heterogeneous configurations and the global_mem element structure for two different types of memories (QDR, DDR4). The global_mem element in example below also demonstrates use of the default attribute. It is set to \"1\" for the DDR4 memory banks, indicating to the oneAPI compiler that the default global memory for the kernel is DDR4.

    Figure 2-7: Heterogeneous Memory Example Block Diagram and Corresponding global_mem Element in board_spec.xml

    Unified Shared Memory

    For applications that require USM support, the board_spec.xml must specify host and device memories in a heterogeneous manner. The allocation_type must be host for global memory region on the host processor. The allocation_type must be set to device for global memory on the FPGA board. Example below extends the board_spec.xml snippet in figure 2-6 to add a global_mem element for the kernel system to host processor memory interface.

    Figure 2-8: global_mem Element Example for Unified Shared Memory(USM)

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#216-host-element","title":"2.1.6 host Element","text":"

    The host element of the board_spec.xml file provides information on the interface from the host to the kernel. Figure below shows an example of host element.

    Figure 2-9: host Element Example

    Table 2-8: Attributes for the host Element

    Attribute Description kernel_config This attribute informs the oneAPI compiler at what offset the kernel resides, from the perspective of the kernel_cra host on the kernel_interface module.* start: the starting address of the kernel. Normally, this attribute has a value of 0 because the kernel_cra host should not host anything except kernels.* size: keep this parameter at the default value of 0x0100000."},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#217-interfaces-element","title":"2.1.7 interfaces Element","text":"

    The interfaces element of the board_spec.xml file describes the kernel interfaces that connect to application kernels and control kernel behavior. For this element, include one of each interface of types host, irq and streamsource. Refer to the interface section for the parameters you must specify for each interface. In addition to the host, irq, and streamsource interfaces, if your design includes a separate Platform Designer subsystem containing the board logic, the kernel clock and reset interfaces exported from it are also part of the interfaces element. Specify these interfaces with the kernel_clk_reset attribute and its corresponding parameters.

    Figure below shows example of interfaces element.

    Figure 2-10: interfaces Element Example

    Table 2-9: Parameters for the kernel_clk_reset Attribute

    Attribute Description clk The Platform Designer name for the kernel clock interface clk2x The Platform Designer name for the 2xkernel clock interface reset The Platform Designer connection for the kernel reset

    Note: Name the kernel clock and reset interfaces in the Platform Designer connection format (that is, .). For example: board.kernel_clk"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#218-channels-element","title":"2.1.8 channels Element","text":"

    The channels element provides channels for streaming data directly between kernel and I/O. Each channel (implemented using Avalon-ST specification) must be connected to the kernel via the interface attribute. The channel interface only supports data, and valid and ready Avalon-ST signals. The I/O channel defaults to 8-bit symbols and big-endian ordering at the interface level.

    Figure below shows an example of channels element for a single channel with a width of 64 bits. The chan_id attribute identified helps identify the port in the generated kernel_system. Refer to section 2.1.4 for more information about the interface attribute parameters. Additional interface attributes can be added for additional channels.

    Figure 2-11: channels Element Example

    For more information about kernel development using channels, refer to I/O Pipes section in FPGA Optimization Guide for Intel\u00ae oneAPI Toolkits.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#22-board_envxml-file","title":"2.2 board_env.xml File","text":"

    The board_env.xml file is used by the oneAPI toolkit to set up the board installation that enables the compiler to target a specific accelerator platform. The board_env.xml file must be located in the top most level of the oneAPI ASP for each platform. For example, the board_env.xml for oneAPI ASP for OFS reference platforms is located in the oneapi-asp/Platform-Name folder, where Platform-Name is n6001 for Agilex OFS and d5005 for Stratix 10 OFS.

    A sample board_env.xml file is shown below. Table 2-10 explains the elements of this file.

    Figure 2-12: board_env.xml File Structure

    Table 2-10: Specifications of XML Elements and Attributes in the board_env.xml File

    Element Attribute Description board_env * version: The oneAPI compiler version used to create oneAPI ASP* name: The runtime uses this as the name of the FPGA Client Driver(FCD) file name hardware * dir: Name of the subdirectory, within the oneAPI ASP directory, that contains the board variant directories for a platform * default: The default board variant that the compiler targets when a platform has multiple board variants and user does not specify an explicit argument using -Xstarget option platform name: Name of the operating system. A separate platform element must be specified for each supported OS for the oneAPI ASP platform mmdlib A string that specifies the path to the MMD library of your oneAPI ASP. To load multiple libraries, specify them in an ordered, comma-separated list. The host application will load the libraries in the order that they appear in the list> Note: You can use %b to reference your oneAPI ASP directory and provide path relative to oneAPI ASP directory, for example, if MMD library is located inside linux64/lib folder in oneAPI ASP, the path would be %b/linux64/lib/libintel_opae_mmd.so linkflags A string that specifies the linker flags necessary for linking with the MMD layer available with the board> Note: You can use %b to reference your oneAPI ASP directory and provide path relative to oneAPI ASP directory, for example, if MMD library is located inside linux64/lib folder in oneAPI ASP, the path would be %b/linux64/lib. linklibs A string that specifies the libraries the oneAPI runtime must link against to use the MMD layer available with the board utilbindir Directory in which the runtime expects to locate board utility executables (i.e. install, uninstall, program, diagnose, flash) > Note: You can use %b to reference your oneAPI ASP directory and provide path relative to oneAPI ASP directory, for example, if the utilities are located in linux64/libexec folder in oneAPI ASP, the path would be %b/linux64/libexec"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#30-oneapi-asp-hardware","title":"3.0 oneAPI ASP Hardware","text":"

    The oneAPI compiler generates the kernel system interfaces based on specifications provided by the oneAPI ASP developer in the board_spec.xml file. The kernel system interfaces with the rest of the oneAPI ASP RTL as shown in figure 1-3.

    Figure 1-3 shows 5 different paths, summarized below:

    • Host to EMIF: Consisting of RTL to handle data transfer between host and on-board memory (e.g. DDR4)
    • Host to Kernel: Consisting of RTL to handle control signals & interrupts between host and kernel
    • Kernel to EMIF: Consisting of RTL to handle data transfer between kernel and on-board memory
    • Kernel to Host memory: Required to support Unified Shared Memory. This requires some additional RTL to handle data transfer between kernel and host memory
    • Kernel to HSSI: Consisting of RTL to handle data streaming between kernel and I/O

    Please note that the kernel system generated by oneAPI compiler has Avalon\u00ae interfaces. OFS FIM has AXI interfaces. Additional logic blocks from Platform Interface Manager are used to handle protocol conversions. Please refer to section 5.3.1 for more details on PIM. The next few sections cover some of the important IP components required to enable kernel communications with host and board peripherals. More design implementation details are covered in section 5.0.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#31-host-to-external-memory-interfaceemif","title":"3.1 Host to External Memory Interface(EMIF)","text":"

    The host to EMIF datapath consists of a PCIe Subsytem(SS), EMIF Subsystem located in the FIM and a Direct Memory Access(DMA) engine in the oneAPI ASP.

    PCIe Subsystem(SS) has the PCIe IP and additional logic to handle PCIe packet format and routing. FIM handles routing signals received from host to the user design located in a region referred to as Accelerator Functional Unit(AFU) (the Kernel system resides in the AFU).

    Note: For more information about the PCIe SS, please refer to Intel FPGA IP Subsystem for PCI Express IP User Guide

    The External Memory Interface Subsystem (EMIF SS) consists of EMIF IP and additional logic for handling transfers between AFU and on-board memories.

    Note: For more information about the EMIF SS, please refer to Memory Subsystem Intel FPGA IP User Guide

    Large buffers of data are usually transferred between host and on-board memory in oneAPI applications. This necessitates a Direct Memory Access(DMA) Engine between host and on-board memory. In oneAPI ASP designs for OFS reference platform, this DMA engine is placed in the AFU region.

    As described in section 2.1.5.1, there are different configurations for memories on board. In addition to above, figure 1-3 also shows an additional IP in the host to memory datapath, called Memory Bank Divider. This IP is used for handling one of the most commonly used configurations, i.e. configuring multiple memory banks of same type as a contiguous memory region. In this case, the kernel has a contiguous view of the memory and data can be interleaved across the different memory channels. The host must also have the same view of the memory in order to ensure read and write transactions from correct addresses.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#311-memory-bank-divider","title":"3.1.1 Memory Bank Divider","text":"

    The OpenCL\u2122 Memory Bank Divider is a Platform Designer component that takes an incoming request from the host interface on the Avalon\u00ae-MM agent port and routes it to the appropriate bank host port. This component must reside on the path between the host and the global memory interfaces. In addition, it must reside outside of the path between the kernel and the global memory interfaces.

    Figure 3-1: Memory Bank Divider IP

    Table 3-1: Parameter Settings for the Memory Bank Divider Component

    Parameter Description Number of banks Number of memory banks for each of the global memory types included in your board system. Separate read/write ports Enable this parameter so that each bank has one port for read operation and one for write operation. Add pipeline stage to output Enable this parameter to allow for potential timing improvements. Data Width Width of the data bus to the memory in bits. Address Width (total addressable) Total number of address bits necessary to address all global memory. Burst size (maximum) Set to a value equal to interleaved_bytes/(width/8), where interleaved_bytes and width are defined in the interface attribute of the global_mem element in the board_spec.xml file. Maximum Pending Reads Maximum number of pending read transfers the component can process without asserting a waitrequest signal. Intel\u00ae recommended value is 64 if BSP has two global memory banks or fewer and 128 if BSP has four or more global memory banks. CAUTION: A high Maximum Pending Reads value causes Platform Designer to insert a deep response FIFO buffer, between the component's host and agent, that consumes a lot of device resources. It also increases the achievable bandwidth between host and memory interfaces. Split read/write bursts on burst word boundary Enable splitting of read and write bursts on burst word boundary. Enable this parameter if the Number of banks parameter value is greater than 1, and the burst reads and writes that the host controller sends to the agent port crosses burst word boundary.

    Table 3-2: Signals and Ports for the Memory Bank Divider Component

    Signal or Port Description clk The bank divider logic uses this clock input. If the IP of your host and memory interfaces have different clocks, ensure that clk clock rate is not slower than the slowest of the two IP clocks. reset The reset input that connects to the board power-on reset. s The agent port that connects to the host interface controller. kernel_clk The kernel_clk drives this clock input kernel_reset The kernel_reset output from the Kernel Interface IP drives this reset input. acl_bsp_snoop Export this Avalon\u00ae Streaming (Avalon\u00ae-ST) source. In the board_spec.xml file, under interfaces, describe only the snoop interface for the default memory (acl_internal_snoop). If you have a heterogeneous memory design, perform these tasks only for the Memory Bank Divider component associated with the default memory. Important: The memory system you build in Platform Designer alters the width of acl_bsp_snoop. You must update the width of the streamsource interface within the channels element in the board_spec.xml file to match the width of acl_bsp_snoop. In the board_spec.xml file, update the width of the snoop interface (acl_internal_snoop) specified with the streamsource kernel interface within the interfaces element. Updating the width ensures that the global_mem interface entries in board_spec.xml match the characteristics of the bankN Avalon\u00ae-MM hosts from corresponding Memory Bank Divider component for the default memory. acl_bsp_memorg_host This conduit connects to the acl_bsp_memorg_host interface of the OpenCL Kernel Interface`.> Note: Signal present if Number of banks > 1. bank1, bank2, ..., bank8 The number of memory hosts available in the Memory Bank Divider depends on the number of memory banks that were included when the unit was instantiated. Connect each bank with each memory interface in the same order as the starting address for the corresponding kernel memory interface specified in the board_spec.xml file. For example, global_mem interface that begins at address 0 must correspond to the memory host in bank1 from the Memory Bank Divider."},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#32-host-to-kernel-interface","title":"3.2 Host to Kernel Interface","text":"

    The host exchanges control signals with kernel with the help of an additional IP . The control signals coming from the host are on a different clock domain (PCIe clock) while the kernel runs on different clock frequency . The Kernel Interface IP handles the clock domain crossing for these control signals as well as handles communication with kernel CSR, interrupts, generates the reset for kernel. All oneAPI ASP designs must instantiate Kernel Interface IPs to ensure the kernel functions correctly.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#321-kernel-interface","title":"3.2.1 Kernel Interface","text":"

    The Kernel Interface is a Platform Designer component that allows the host interface to access and control the oneAPI kernel.

    Figure 3-2: Kernel Interface IP

    Table 3-3: Parameter Settings for the Kernel Interface Component

    Parameter Description Number of global memory systems Number of global memory types in your board design.

    Table 3-4: Signals and Ports for the Kernel Interface Component

    Signal or Port Description clk The clock input used for the host control interface. The clock rate of clk can be slow. reset This reset input resets the control interface. It also triggers the kernel_reset signal, which resets all kernel logic. ctrl Use this agent port to connect to the host interface. This interface is a low-speed interface with which you set kernel arguments and start the kernel's execution. kernel_clk kernel clock drives this clock input. kernel_cra This Avalon\u00ae-MM host interface communicates directly with the kernels generated by the oneAPI compiler. Export the Avalon\u00ae-MM interface to the Kernel Interface and name it in the board_spec.xml file. sw_reset_in When necessary, the host interface resets the kernel via the ctrl interface. If the board design requires a kernel reset, it can do so via this reset input. Otherwise, connect the interface to a global power-on reset. kernel_reset Use this reset output to reset the kernel and any other hardware that communicates with the kernel. Warning: This reset occurs between the MMD open and close calls. Therefore, it must not reset anything necessary for the operation of your MMD. sw_reset_export This reset output is the same as kernel_reset, but it is synchronized to the clk interface. Use this output to reset logic that is not in the kernel_clk clock domain but should be reset whenever the kernel resets. acl_bsp_memorg_host The memory interfaces use these signals. Based on the number of global memory systems you specify in the Kernel Interface component parameter editor, the Intel\u00ae Quartus\u00ae Prime Pro Edition Software creates the corresponding number of copies of this signal, each with a different hexadecimal suffix. Connect each signal to the Memory Bank Divider component associated with each global memory system (for example, DDR). Then, list the hexadecimal suffix in the config_addr attribute of the global_mem element in the board_spec.xml file. kernel_irq_from_kernel An interrupt input from the kernel. This signal is exported and named in the board_spec.xml file. kernel_irq_to_host An interrupt output from the kernel. This signal connects to the host interface."},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#33-kernel-to-external-memory-interface","title":"3.3 Kernel to External Memory Interface","text":"

    The kernel system masters the interface from kernel to external memory. oneAPI compiler generates kernel system memory interface logic (e.g. Load-Store Unit) according to the global memory configuration and interface specifications in board_spec.xml file. The kernel system operates at kernel clock(see next section for more information), hence, oneAPI ASP developers must handle clock domain crossing from kernel to EMIF clock domain.

    For implementation details for all datapaths discussed above, please refer to section 5.3.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#40-oneapi-asp-software","title":"4.0 oneAPI ASP Software","text":"

    The software components of oneAPI ASP consist of the Memory Mapped Device(MMD) layer and the board utility routine required by runtime.

    Section 4.1 introduces MMD layer and section 4.2 explains board utilities.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#41-memory-mapped-devicemmd-layer","title":"4.1 Memory Mapped Device(MMD) Layer","text":"

    The oneAPI ASP Memory Mapped Device (MMD) layer sits in between the oneAPI runtime and OPAE SDK and provides a set of API for device communication and control. The runtime calls into the MMD API for various operations like opening a handle to the device, allocating memory etc.

    Note: For more information about the FPGA runtime, please refer to FPGA Runtime documentation here.

    A header file, called aocl_mmd.h, has the list of MMD API calls that must be implemented by oneAPI ASPs. From the perspective of the caller, below is typical MMD API lifecycle:

    1. Open device to provide handle for further operations
    2. Set interrupt and status handlers
    3. Program device with kernel bitstream
    4. Allocate memory if required
    5. Perform Read, Write operations (DMA or MMIO)
    6. Free memory if allocation done in step 4
    7. Close device. No further operations permitted until subsequent open device call

    Table below summarizes all APIs listed in aocl_mmd.h.

    Table 4-1: Summary of MMD API from aocl_mmd.h

    API Purpose aocl_mmd_get_offline_info Obtain offline information about the board. This function is offline because it is device-independent and does not require a handle from the aocl_mmd_open() call aocl_mmd_get_info Obtain information about the board specified in the requested_info_id argument (refer to section 4.1.2 for more information) aocl_mmd_open Open and initialize the specified device aocl_mmd_close Close an opened device via its handle aocl_mmd_set_interrupt_handler Set the interrupt handler for the opened device aocl_mmd_set_device_interrupt_handler Sets the device interrupt handler for opened device, interrupt handler is called to notify runtime of any exceptions aocl_mmd_set_status_handler Set the operation status handler for the opened device aocl_mmd_yield The aocl_mmd_yield function is called when the host interface is idle. The host interface might be idle because it is waiting for the device to process certain events aocl_mmd_read Read operation on a single interface aocl_mmd_write Write operation on a single interface aocl_mmd_copy Copy operation on a single interface aocl_mmd_hostchannel_create Creates a channel interface aocl_mmd_hostchannel_destroy Destroys channel interface aocl_mmd_hostchannel_get_buffer Provides host with pointer used to read/write from channel interface aocl_mmd_hostchannel_ack_buffer Acknowledges read/write from channel aocl_mmd_program Reprogram operation for the specified device aocl_mmd_host_alloc Provide memory that is allocated on the host. Host allocations are accessible by the host and one or more devices aocl_mmd_free Free memory that has been allocated by MMD aocl_mmd_device_alloc Allocate memory that is owned by the device aocl_mmd_shared_alloc Allocate shared memory between the host and the FPGA aocl_mmd_shared_migrate Handle migration of non-concurrent shared allocations any time the accessor of the allocation changes

    Sections below cover more details for each API (expected arguments, return values). Section 5.4 discusses more about the implementation of the MMD layer APIs in oneAPI ASPs for OFS reference platforms.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#411-aocl_mmd_get_offline_info","title":"4.1.1 aocl_mmd_get_offline_info","text":"

    The aocl_mmd_get_offline_info function obtains offline information about the board specified in the requested_info_id argument. This function is offline because it is device-independent and does not require a handle from the aocl_mmd_open() call.

    Syntax

    \n\n    int aocl_mmd_get_offline_info (\n        aocl_mmd_offline_info_t requested_info_id,\n        size_t param_value_size,\n        void* param_value,\n        size_t* param_size_ret )\n\n

    Function Arguments

    1. requested_info_id: An enum value of type aocl_mmd_offline_info_t that indicates the offline device information returning to the caller.

    Table 4-2: Possible Enum Values for the requested_info_id Argument

    Name Description Type AOCL_MMD_VERSION Version of MMD layer char* AOCL_MMD_NUM_BOARDS Number of candidate boards int AOCL_MMD_BOARD_NAMES Names of available boards > Note: Separate each board name by a semicolon (;) delimiter. char* AOCL_MMD_VENDOR_NAME Name of board vendor char* AOCL_MMD_VENDOR_ID An integer board vendor ID int AOCL_MMD_USES_YIELD A value of 0 instructs the runtime to suspend user's processes. The runtime resumes these processes after it receives an event update (for example, an interrupt) from the MMD layer.A value of 1 instructs the runtime to continuously call the aocl_mmd_yield function while it waits for events to complete.CAUTION: Setting AOCL_MMD_USES_YIELD to 1 might cause high CPU utilization if the aocl_mmd_yield function does not suspend the current thread. int

    1. param_value_size: Size of the param_value field in bytes. This size_t value should match the size of the expected return type that the enum definition indicates. For example, if AOCL_MMD_NUM_BOARDS returns a value of type int, set the param_value_size to sizeof (int). You should see the same number of bytes returned in the param_size_ret argument.

    2. param_value: A void* pointer to the variable that receives the returned information.

    3. param_size_ret: A pointer argument of type size_t* that receives the number of bytes of returned data.

    Return Value

    A negative return value indicates an error.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#412-aocl_mmd_get_info","title":"4.1.2 aocl_mmd_get_info","text":"

    The aocl_mmd_get_info function obtains information about the board specified in the requested_info_id argument.

    Syntax

    \n\n    int aocl_mmd_get_info (\n        int handle,\n        aocl_mmd_info_t requested_info_id,\n        size_t param_value_size,\n        void* param_value,\n        size_t* param_size_ret )\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    2. requested_info_id: An enum value of type aocl_mmd_info_t that indicates the device information returning to the caller.

    Table 4-3: Possible Enum Values for the requested_info_id Argument

    Name Description Type AOCL_MMD_NUM_KERNEL_INTERFACES Number of kernel interfaces int AOCL_MMD_KERNEL_INTERFACES Kernel interfaces int* AOCL_MMD_PLL_INTERFACES Kernel clock handles int* AOCL_MMD_MEMORY_INTERFACE Global memory handle int AOCL_MMD_TERMPERATURE Temperature measurement float AOCL_MMD_PCIE_INFO PCIe\u00ae information char* AOCL_MMD_BOARD_NAME Board name char* AOCL_MMD_BOARD_UNIQUE_ID Unique board ID char* AOCL_MMD_CONCURRENT_READS Number of parallel readsA value of 1 indicates serial reads. int AOCL_MMD_CONCURRENT_WRITES Number of parallel writesA value of 1 indicates serial writes. int AOCL_MMD_CONCURRENT_READS_OR_WRITES Total number of concurrent read and write operations int AOCL_MMD_MIN_HOST_MEMORY_ALIGNMENT Minimum alignment that the oneAPI ASP supports for host allocations size_t AOCL_MMD_HOST_MEM_CAPABILITIES Capabilities of aocl_mmd_host_alloc() function unsigned int AOCL_MMD_SHARED_MEM_CAPABILITIES Capabilities of aocl_mmd_shared_alloc() function unsigned int AOCL_MMD_DEVICE_MEM_CAPABILITIES Capabilities of aocl_mmd_device_alloc() function unsigned int AOCL_MMD_HOST_MEM_CONCURRENT_GRANULARITY Granularity of concurrent host accesses size_t AOCL_MMD_SHARED_MEM_CONCURRENT_GRANULARITY Granularity of concurrent shared accesses size_t AOCL_MMD_DEVICE_MEM_CONCURRENT_GRANULARITY Granularity of concurrent device accesses size_t

    1. param_value_size: Size of the param_value field in bytes. This size_t value should match the size of the expected return type that the enum definition indicates. For example, if AOCL_MMD_TEMPERATURE returns a value of type float, set the param_value_size to sizeof (float). You should see the same number of bytes returned in the param_size_ret argument.

    2. param_value: A void* pointer to the variable that receives the returned information.

    3. param_size_ret: A pointer argument of type size_t* that receives the number of bytes of returned data.

    Capability Values

    Table 4-4: Capability Values for aocl_mmd_get_info Function

    Value Description AOCL_MMD_MEM_CAPABILITY_SUPPORTED If you do not set this value, allocation function is not supported even if other capabilities are set. AOCL_MMD_MEM_CAPABILITY_ATOMIC Supports atomic access to the memory by either the host or the device. AOCL_MMD_MEM_CAPABILITY_CONCURRENT Supports concurrent access to the memory either by the host or the device if the accesses are not on the same block. Block granularity is defined by AOCL_MMD_*_MEM_CONCURRENT_GRANULARITY. Blocks are aligned to this granularity. AOCL_MMD_MEM_CAPABILITY_P2P Memory can be accessed by multiple devices at the same time.

    Return Value

    A negative return value indicates an error.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#413-aocl_mmd_open","title":"4.1.3 aocl_mmd_open","text":"

    The aocl_mmd_open function opens and initializes the specified device.

    Syntax

    \n\n    int aocl_mmd_open (const char *name)\n\n

    Function Arguments

    1. name: The function opens the board with a name that matches this const char* string. The name typically matches the one specified by the AOCL_MMD_BOARD_NAMES offline information.

    The runtime first queries the AOCL_MMD_BOARD_NAMES offline information to identify the boards that it might be able to open. Then it attempts to open all possible devices by calling aocl_mmd_open and using each of the board names as argument.

    Note: The name must be a C-style NULL-terminated ASCII string.

    Return Value

    If aocl_mmd_open() executes successfully, the return value is a positive integer that acts as a handle to the board.

    If aocl_mmd_open() fails to execute, a negative return value indicates an error. In the event of an error, the runtime proceeds to open other known devices. Therefore, it is imperative that the MMD layer does not exit the application if an open call fails.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#414-aocl_mmd_close","title":"4.1.4 aocl_mmd_close","text":"

    The aocl_mmd_close function closes an opened device via its handle.

    Syntax

    \n\n    int aocl_mmd_close (int handle)\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    Return Value

    If the aocl_mmd_close() executes successfully, the return value is 0.

    If aocl_mmd_close() fails to execute, a negative return value indicates an error.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#415-aocl_mmd_set_interrupt_handler","title":"4.1.5 aocl_mmd_set_interrupt_handler","text":"

    The aocl_mmd_set_interrupt_handler function sets the interrupt handler for the opened device. When the device internals identify an asynchronous kernel event (for example, a kernel completion), the interrupt handler is called to notify the runtime of the event.

    Note: Ignore the interrupts from the kernel until this handler is set.

    Syntax

    \n\n    int aocl_mmd_set_interrupt_handler (\n        int handle, \n        aocl_mmd_interrupt_handler_fn fn,\n        void* user_data )\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    2. fn: The callback function to invoke when a kernel interrupt occurs. The fn argument is of type aocl_mmd_interrupt_handler_fn, which is defined as follows:

    \n    typedef void (*aocl_mmd_interrupt_handler_fn)( int handle, void* user_data );\n
    1. user_data: The void* type user-provided data that passes to fn when it is called.

    Return Value

    If the function executes successfully, the return value is 0.

    If the function fails to execute, a negative return value indicates an error.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#416-aocl_mmd_set_device_interrupt_handler","title":"4.1.6 aocl_mmd_set_device_interrupt_handler","text":"

    The aocl_mmd_set_device_interrupt_handler function sets the device interrupt handler for the opened device. When the device internals identify an asynchronous exception event (for example, a bit correction event), the device interrupt handler is called to notify the runtime of the event.

    Note: Ignore the interrupts from the device until this handler is set.

    Syntax

    \n\n    int aocl_mmd_set_device_interrupt_handler (\n        int handle, \n        aocl_mmd_device_interrupt_handler_fn fn,\n        void* user_data )\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    2. fn: The callback function to invoke when a kernel interrupt occurs. The fn argument is of type aocl_mmd_device_interrupt_handler_fn, which is defined as follows:

     \n    typedef void (*aocl_mmd_device_interrupt_handler_fn)( int handle, aocl_mmd_interrupt_info* data_in, void* user_data );\n

    aocl_mmd_interrupt_info is defined as:

    \n\n    typedef struct {\n        unsigned long long int exception_type;\n        void *user_private_info;\n        size_t user_cb;\n    } aocl_mmd_interrupt_info;\n\n

    Where:

    • exception_type acts as a bitfield that contains exactly one bit, corresponding to an exception number.
    • user_private_info and user_cb represent pointers to binary data that the OpenCL implementation return. These pointers log additional information that is helpful for debugging the error.
    1. user_data: The void* type user-provided data that passes to fn when it is called.

    Return Value

    If the function executes successfully, the return value is 0.

    If the function fails to execute, a negative return value indicates an error.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#417-aocl_mmd_set_status_handler","title":"4.1.7 aocl_mmd_set_status_handler","text":"

    The aocl_mmd_set_status_handler function sets the operation status handler for the opened device. The operation status handler is called under the following circumstances:

    • When the operation completes successfully and status is 0.
    • When the operation completes with errors and status is a negative value.

    Syntax

    \n\n    int aocl_mmd_set_status_handler (\n        int handle,\n        aocl_mmd_status_handler_fn fn,\n        void* user_data )\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    2. fn: The callback function to invoke when a status update occurs. The fn argument is of type aocl_mmd_status_handler_fn, which is defined as follows:

    \n    typedef void (*aocl_mmd_status_handler_fn)( int handle, void* user_data, aocl_mmd_op_t op, int status );\n
    1. user_data: The void* type user-provided data that passes to fn when it is called.

    Return Value

    If the function executes successfully, the return value is 0.

    If the function fails to execute, a negative return value indicates an error.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#418-aocl_mmd_yield","title":"4.1.8 aocl_mmd_yield","text":"

    The aocl_mmd_yield function is called when the host interface is idle. The host interface might be idle because it is waiting for the device to process certain events.

    Syntax

    \n\n    int aocl_mmd_yield (int handle)\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    Return Value

    A nonzero return value indicates that the yield function performed work necessary for proper device functioning such as processing direct memory access (DMA) transactions.

    A return value of 0 indicates that the yield function did not perform work necessary for proper device functioning.

    Note: The yield function might be called continuously if it reports that it has necessary work to perform.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#419-aocl_mmd_read","title":"4.1.9 aocl_mmd_read","text":"

    The aocl_mmd_read function is the read operation on a single interface.

    Syntax

    \n\n    int aocl_mmd_read (\n        int handle,\n        aocl_mmd_op_t op,\n        size_t len,\n        void* dst,\n        int mmd_interface, size_t offset )\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    2. op: The operation object of type aocl_mmd_op_t used to track the progress of the operation. If op is NULL, the call must block, and return only after the operation completes.

    Note: aocl_mmd_op_t is defined as follows:

    \n    typedef void* aocl_mmd_op_t;\n

    1. len: The size of the data, in bytes, that the function transfers. Declare len with type size_t.

    2. dst: The host buffer, of type void*, to which data is written.

    3. mmd_interface: the handle to the interface being accessed. For example, to access global memory this handle will be value obtained from aocl_mmd_get_info call with AOCL_MMD_MEMORY_INTERFACE as requested_info_id argument.

    4. offset: The size_t byte offset within the interface at which the data transfer begins.

    Return Value

    If the read operation is successful, the return value is 0.

    If the read operation fails, a negative return value indicates an error.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4110-aocl_mmd_write","title":"4.1.10 aocl_mmd_write","text":"

    The aocl_mmd_write function is the write operation on a single interface.

    Syntax

    \n\n    int aocl_mmd_write (\n        int handle,\n        aocl_mmd_op_t op,\n        size_t len,\n        const void* src,\n        int mmd_interface, size_t offset )\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    2. op: The operation object of type aocl_mmd_op_t used to track the progress of the operation. If op is NULL, the call must block, and return only after the operation completes.

    Note: aocl_mmd_op_t is defined as follows:

    \n    typedef void* aocl_mmd_op_t;\n

    1. len: The size of the data, in bytes, that the function transfers. Declare len with type size_t.

    2. src: The host buffer, of type const void*, from which data is read.

    3. mmd_interface: the handle to the interface being accessed. For example, to access global memory this handle will be value obtained from aocl_mmd_get_info call with AOCL_MMD_MEMORY_INTERFACE as requested_info_id argument.

    4. offset: The size_t byte offset within the interface at which the data transfer begins.

    Return Value

    If the write operation is successful, the return value is 0.

    If the write operation fails, a negative return value indicates an error.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4111-aocl_mmd_copy","title":"4.1.11 aocl_mmd_copy","text":"

    The aocl_mmd_copy function is the copy operation on a single interface.

    Syntax

    \n\n    int aocl_mmd_copy (\n        int handle,\n        aocl_mmd_op_t op,\n        size_t len,\n        int mmd_interface, size_t src_offset, size_t dst_offset )\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    2. op: The operation object of type aocl_mmd_op_t used to track the progress of the operation. If op is NULL, the call must block, and return only after the operation completes.

    Note: aocl_mmd_op_t is defined as follows:

    \n    typedef void* aocl_mmd_op_t;\n

    1. len: The size of the data, in bytes, that the function transfers. Declare len with type size_t.

    2. mmd_interface: the handle to the interface being accessed. For example, to access global memory this handle will be value obtained from aocl_mmd_get_info call with AOCL_MMD_MEMORY_INTERFACE as requested_info_id argument.

    3. src_offset: The size_t byte offset within the source interface at which the data transfer begins.

    4. dst_offset: The size_t byte offset within the destination interface at which the data transfer begins

    Return Value

    If the copy operation is successful, the return value is 0.

    If the copy operation fails, a negative return value indicates an error.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4112-aocl_mmd_hostchannel_create","title":"4.1.12 aocl_mmd_hostchannel_create","text":"

    The aocl_mmd_hostchannel_create function creates a channel interface.

    Syntax

    \n\n    int aocl_mmd_hostchannel_create (\n        int handle,\n        char *channel_name,\n        size_t queue_depth,\n        int direction )\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    2. channel_name: Name of the channel to be initialized. The channel name is same as that used in the board_spec.xml file.

    3. queue_depth: The size of pinned internal buffer in bytes. Pointer to the internal buffer is provided when the user calls the aocl_mmd_hostchannel_get_buffer() function.

    4. direction: The direction of the channel.

    Return Value

    If the function executes successfully, the return value is positive and is handle to the channel.

    If the function fails to execute, a negative return value indicates an error.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4113-aocl_mmd_hostchannel_destroy","title":"4.1.13 aocl_mmd_hostchannel_destroy","text":"

    The aocl_mmd_hostchannel_destroy function destroys the channel interface.

    Syntax

    \n\n    int aocl_mmd_hostchannel_destroy (\n        int handle,\n        int channel )\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    2. channel: A positive int value representing handle to the channel to close obtained from the aocl_mmd_hostchannel_create() call.

    Return Value

    If the function executes successfully, the return value is 0.

    If the function fails to execute, a negative return value indicates an error.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4114-aocl_mmd_hostchannel_get_buffer","title":"4.1.14 aocl_mmd_hostchannel_get_buffer","text":"

    The aocl_mmd_hostchannel_get_buffer function provides a host with a pointer to the buffer they can access to write or read from the channel interface, along with the space or data available in the buffer, in bytes.

    Syntax

    \n\n    void *aocl_mmd_hostchannel_get_buffer (\n        int handle,\n        int channel,\n        size_t *buffer_size,\n        int *status )\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    2. channel: A positive int value representing handle to the channel to close obtained from the aocl_mmd_hostchannel_create() call.

    3. buffer_size: A pointer to size_t that the function writes available buffer space or size to.

    4. status: A pointer to int that the function writes result of the call to.

    Return Value

    If the function executes successfully, int pointed to by the status pointer is 0. Returned void* may still be NULL, in which case size_t pointed by the buffer_size is 0.

    If the function fails to execute, int pointed by the status pointer is a negative value.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4115-aocl_mmd_hostchannel_ack_buffer","title":"4.1.15 aocl_mmd_hostchannel_ack_buffer","text":"

    You can acknowledge write or read from the channel by calling aocl_mmd_hostchannel_ack_buffer.

    Syntax

    \n\n    size_t aocl_mmd_hostchannel_ack_buffer (\n        int handle,\n        int channel,\n        size_t send_size,\n        int *status )\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    2. channel: A positive int value representing handle to the channel to close obtained from the aocl_mmd_hostchannel_create() call.

    3. send_size: The size in bytes that the user is acknowledging.

    4. status: A pointer to int that the function writes result of the call to.

    Return Value

    If the function executes successfully, int pointed to by status pointer is 0. Also, there is no guarantee that the user's send_size is the actual size that gets acknowledged. The returned size_t is the amount of bytes that was actually acknowledged.

    If the function fails to execute, int pointed by status pointer is a negative value.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4116-aocl_mmd_program","title":"4.1.16 aocl_mmd_program","text":"

    The aocl_mmd_program function is the program operation for the specified device. The host must guarantee that no other operations are executing on the device during the program operation.

    During aocl_mmd_program execution, the kernels are idle and no read, write, or copy operation can occur.

    Disable interrupts and program the FPGA with the data from user_data, which has a size specified by the size argument. The host then calls aocl_mmd_set_status_handler and aocl_mmd_set_interrupt_handler again, which enable the interrupts. If events such as interrupts occur during aocl_mmd_program execution, race conditions or data corruption might occur.

    Syntax

    \n\n int aocl_mmd_program (\n    int handle,\n    void * user_data,\n    size_t size,\n    aocl_mmd_program_mode_t program_mode )\n\n

    Function Arguments

    1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.

    2. user_data: The void* type binary contents of the fpga.bin file that is created during kernel compilation.

    3. size: The size of user_data in bytes. The size argument is of size_t.

    4. program_mode: The bit-field that specifies the mode of device programming.

    Table 4-5: Possible Values for the program_mode Argument

    program_mode Argument Value Description AOCL_MMD_PROGRAM_PRESERVE_GLOBAL_MEMORY This flag specifies that during programming the global memory on the devices are preserved.

    Return Value

    If aocl_mmd_program executes successfully, the return value is the pointer value that the host uses to access shared memory.

    "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4117-aocl_mmd_host_alloc","title":"4.1.17 aocl_mmd_host_alloc","text":"

    Host allocations provide memory that is allocated on the host. This memory must be deallocated with the aocl_mmd_free function. Host allocations are accessible by the host and one or more devices. The same pointer to a host allocation may be used on the host and all supported devices. They have address equivalence.

    Syntax

    Once the device has signaled completion through the aocl_mmd_interrupt_handler_fn function, the host can assume it has access to the latest contents of the memory, allocated by the aocl_mmd_host_alloc function call.

    \n\n    void* aocl_mmd_host_alloc (\n        int* handles,\n        size_t num_devices,\n        size_t size,\n        size_t alignment,\n        aocl_mmd_mem_properties_t *properties,\n        int* error )\n\n

    Function Arguments

    1. handles: Handles for devices that needs access to this memory.

    2. num_devices: Number of devices in the handles.

    3. size: The size of the memory region.

    4. alignment: The alignment (in bytes) of the allocation.

    5. properties: Specifies additional information about the allocated memory, described by a property type name and its corresponding value. Each property type name is immediately followed by the corresponding desired value. The list is terminated with a zero. For example, [, , , , 0]

    6. error: The error code defined by AOCL_MMD_ERROR*:

      • AOCL_MMD_ERROR_SUCCESS: No error occurred.
      • AOCL_MMD_ERROR_INVALID_HANDLE: The device handle provided is invalid.
      • AOCL_MMD_ERROR_OUT_OF_MEMORY: Ran out of memory.
      • AOCL_MMD_ERROR_UNSUPPORTED_ALIGNMENT: The device does not support the provided alignment.
      • AOCL_MMD_ERROR_UNSUPPORTED_PROPERTY: The device does not support the provided property.

      Return Value

      If the aocl_mmd_host_alloc function executes successfully, the return value is a valid pointer value. Otherwise, the return value is NULL.

      "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4118-aocl_mmd_free","title":"4.1.18 aocl_mmd_free","text":"

      Releases memory that was allocated by MMD.

      Syntax

      \n\n    int aocl_mmd_free (void* mem)\n\n

      Function Arguments

      1. mem: The pointer to the memory region. Must be a pointer that is allocated by the MMD.

      Return Value

      Returns one of the following error code:

      • AOCL_MMD_ERROR_SUCCESS: No error occurred
      • AOCL_MMD_ERROR_INVALID_POINTER: Invalid pointer provided
      "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4119-aocl_mmd_device_alloc","title":"4.1.19 aocl_mmd_device_alloc","text":"

      Allocate memory that is owned by the device. This pointer can only be accessed by the kernel. It cannot be accessed by the host. The host is able to manipulate the pointer (for example, increment it) and not just access the underlying data. This memory must be deallocated by the aocl_mmd_free() function.

      Syntax

      \n\n    void * aocl_mmd_device_alloc (\n        int handle, \n        size_t size, \n        size_t alignment, \n        aocl_mmd_mem_properties_t *properties, \n        int* error )\n\n

      Function Arguments

      1. handle: Device that has access to this memory.

      2. size: The size of the memory region.

      3. alignment: The alignment (in bytes) of the memory region.

      4. properties: Specifies additional information about the allocated memory, described by a property type name and its corresponding value. Each property type name is immediately followed by the corresponding desired value. The list is terminated with a zero. For example, [, , , , 0]

        Return Value

        Returns one of the following error code:

        • AOCL_MMD_ERROR_SUCCESS: No error occurred
        • AOCL_MMD_ERROR_INVALID_HANDLE: The device handle provided is invalid.
        • AOCL_MMD_ERROR_OUT_OF_MEMORY: Ran out of memory.
        • AOCL_MMD_ERROR_UNSUPPORTED_ALIGNMENT: The device does not support the provided alignment.
        • AOCL_MMD_ERROR_UNSUPPORTED_PROPERTY: The device does not support the provided property.
        "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4120-aocl_mmd_shared_alloc","title":"4.1.20 aocl_mmd_shared_alloc","text":"

        Shared allocations can migrate between the host and one or more associated device. The same pointer to a shared allocation can be used on the host and the supported device. They have address equivalence.

        Syntax

        \n\n    void * aocl_mmd_shared_alloc (\n        int handle, \n        size_t size, \n        size_t alignment, \n        aocl_mmd_mem_properties_t* properties, \n        int* error )\n\n

        Function Arguments

        1. handle: Device that needs access to this memory.

        2. size: The size of the memory region.

        3. alignment: The alignment (in bytes) of the allocation.

        4. properties: Specifies additional information about the allocated memory described by a property type name and its corresponding value. Each property type name is immediately followed by the corresponding desired value. The list is terminated with a zero. For example, [, , , , 0]

        5. error: The error code defined by AOCL_MMD_ERROR*.

          • AOCL_MMD_ERROR_SUCCESS: No error occurred.
          • AOCL_MMD_ERROR_INVALID_HANDLE: The device handle provided is invalid.
          • AOCL_MMD_ERROR_OUT_OF_MEMORY: Ran out of memory.
          • AOCL_MMD_ERROR_UNSUPPORTED_ALIGNMENT: The device does not support the provided alignment.
          • AOCL_MMD_ERROR_UNSUPPORTED_PROPERTY: The device does not support the provided property.

          Return Value

          If the aocl_mmd_shared_alloc function executes successfully, the return value is a valid pointer value. Otherwise, the return value is NULL.

          "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4121-aocl_mmd_shared_migrate","title":"4.1.21 aocl_mmd_shared_migrate","text":"

          A call to the aocl_mmd_shared_migrate() function must be made for non-concurrent shared allocations any time the accessor of the allocation changes. For example, the aocl_mmd_shared_migrate() function should be called indicating that the allocation should be migrated to the device before a kernel accessing the allocation is launched on the device. Similarly, the aocl_mmd_shared_migrate() function should be called indicating that the allocation is migrated to the host before the host accesses the memory after kernel completion. For concurrent allocations, this call may be used as a performance hint, but it is not strictly required for functionality.

          Syntax

          \n\n    int aocl_mmd_shared_migrate (\n        int handle, \n        void* shared_ptr, \n        size_t size, \n        aocl_mmd_migrate_t destination )\n\n

          Function Arguments

          1. handle: Device that has access to this memory.

          2. shared_ptr: Pointer allocated by the aocl_mmd_shared_alloc() function.

          3. size: Size (in bytes) of the migration. Must be a multiple of a page boundary that the oneAPI ASP supports.

          4. destination: The destination of the migration.

          Return Value

          Returns one of the following error code:

          • AOCL_MMD_ERROR_SUCCESS: No error occurred.
          • AOCL_MMD_ERROR_INVALID_HANDLE: The device handle provided is invalid.
          • AOCL_MMD_ERROR_INVALID_MIGRATION_SIZE: The migration size is not supported by the device.
          "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#42-board-utilities","title":"4.2 Board Utilities","text":"

          oneAPI runtime provides a set of options for the aocl utility.

          Note: aocl is an utility available in the oneAPI runtime environment, please use aocl help command for more information on this.

          Table 4-6 shows the subcommands that aocl utility provides for FPGA platforms.

          Table 4-6: aocl Board Utilities

          Subcommand Description Executable Call install Install board into the host system. This installs the FPGA Client Driver (FCD) for your FPGA platform. FCD allows runtime to find and load the FPGA platform libraries at runtime aocl install path-to-FPGA-platform-oneapi-asp uninstall Uninstall board from the host system. Removes FCD. aocl uninstall path-to-FPGA-platform-oneapi-asp initialize Configure a default FPGA image onto the board. For more information about initialization refer to Intel oneAPI FPGA Handbook Two methods are available to initalize the board: 1. aocl initialize device-name board-variant 2. aocl initialize device-name oneAPI fat binary > Note: The second option requires Intel\u00ae oneAPI Base Toolkit (Base Kit) version 2024.0 & above as well as 2023.3 OFS Release for oneAPI Accelerator Support Package and above program Configure a new FPGA image onto the board aocl program device-name aocx file diagnose Runs ICD and FCD diagnostics followed by querying devices in installed platforms. If a device-name is specified in the call, it run board vendor's test program for the FPGA platform * aocl diagnose : This queries the devices in FPGA platform and supplies a list of valid strings assigned to the list of devices * aocl diagnose device-name : This runs full diagnostic test on the FPGA platform

          The runtime expects the routine for each of this utilities to be defined in the oneAPI ASP. It looks for the routine executables in the location specified by the utilbinder element in the board_env.xml file.

          install

          oneAPI runtime uses the information in board_env.xml file to create a FCD file. The FCD file name matches name attribute of board_env element and the FCD contents are the platform libraries specified in mmdlib element. Refer to section 2.2 for more information about board_env.xml file. The runtime adds the installed platform to the list of installed packages(file used by runtime to track installed platforms) and then invokes the install routine from oneAPI ASP.

          uninstall

          oneAPI runtime removes the FCD file and removes the platform from list of installed packages. It then invokes the uninstall routine for oneAPI ASP.

          initialize

          oneAPI runtime invokes initialize routine provided by oneAPI ASPs for installed platforms.

          program

          oneAPI runtime loads the programming file on the FPGA by invoking program routine provided by the oneAPI ASPs for the installed platform.

          diagnose

          oneAPI runtime runs ICD and FCD diagnostics to check the ICD and FCD files installed on the system. It then queries for available boards in the installed platform and lists boards matching every installed platform. If a device-name is specified in the call, runtime invokes the diagnostic routine provided in oneAPI ASP.

          For more information about the implementation of these routines in oneAPI ASPs for OFS reference platforms, please refer to section 5.5.

          "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#50-oneapi-asp-implementation-details","title":"5.0 oneapi-asp Implementation Details","text":"

          oneapi-asp in the OFS has two reference platform releases, one is based on Intel\u00ae Stratix 10\u00ae FPGA and the other is based on an Intel\u00ae Agilex\u00ae 7 FPGA. This chapter aims to explain the architecture and current implementation details of oneapi-asp for these platforms. The oneapi-asp repository is located here.

          The next section explains the oneapi-asp directory structure, followed by sections on hardware and MMD layers.

          "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#51-oneapi-asp-directory-structure","title":"5.1 oneapi-asp Directory Structure","text":"

          As described in section 2.0, oneAPI compiler & runtime use the board_env.xml and board_spec.xml files to get information about the FPGA platform. The compiler expects the board_env.xml file to be at the topmost level in the platform directory. The board_env.xml file describes the location of the hardware files & platform libraries.

          Figure 5-1: Sample board_env.xml File

          Figure 5-1 shows a sample board_env.xml file, the corresponding oneAPI ASP directory structure must match the following format. Table 5-1 provides details on each folder.

          In addition to below folders, oneAPI ASP for OFS reference platforms have another folder for the software layer (MMD & board utilities) source code located in common directory (oneapi-asp/common). This is because a common source code is utilized for both d5005 and n6001 reference platform ASPs. 

          \n    oneapi-asp_Platform-Name/\n    |--hardware/\n    |--|--Board-Variant-1/\n    |--|--Board-Variant-2/\n    |--linux64/\n    |--board_env.xml\n

          Note: In addition to above folders, oneAPI ASPs for OFS reference platforms have additional directories called scripts and bringup which contain helper scripts for platform generation & a sample for board bring up respectively. Please refer to the README for each reference platform in the oneASP-asp repository for more information on these additional folders. * README for oneapi-asp targeting Intel\u00ae FPGA PAC D5005 reference platform: README * README for oneapi-asp targeting Intel\u00ae FPGA SmartNIC N6001-PL Platform: README

          The Platform-Name is used for identifying the platform and can be alphanumeric value decided by the platform developer. For example, Intel uses the Platform-Name d5005 for oneapi-asp for Intel\u00ae Stratix 10\u00ae FPGA as the reference platform is Intel\u00ae FPGA PAC D5005.

          Table 5-1: Directory Contents

          Files/Folder Descriptions hardware Contains hardware files (RTL, platform designer files, SDCs, compilation scripts, floorplan settings) and the board_spec.xml files for all board variants. See table 5-2 for more details common/source Source code for MMD layer as well as oneapi-asp board utilities linux64 Location for FPGA platform libraries and executables for oneapi-asp board utilities board_env.xml Contains platform installation information. Please refer to section 2.2 for more details on board_env.xml elements

          Tables 5-2 to 5-4 give more details on each of these folders for oneAPI ASPs for OFS reference platforms.

          "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#511-hardware-folder","title":"5.1.1 hardware Folder","text":"
          \nhardware/\n|--board-variant-1/\n|--|--build\n|--|--part-number_dm.xml (Please see note for this file in table 5-2)\n|--|--board_spec.xml\n|--|--quartus.ini\n

          Table 5-2: hardware Folder Contents

          Files/Folder Description build Contains all hardware design files including Intel\u00ae Quartus\u00ae software project file (*.qpf), Intel\u00ae Quartus\u00ae software Settings Files (*.qsf), IP files, *.sv, *.qsys, *.sdc as well as scripts to control compile flow board_spec.xml Defines compile flow, global memory, kernel interfaces. Please see section 2.1 for more information about board_spec.xml file part-number_dm.xml Device Model file for Intel\u00ae FPGA part on the target platform. The name must be of the format part-number_dm.xml. This file has the total resources available on the device. > Note: The device model files provided as part of the Intel\u00ae oneAPI Base Toolkit (Base Kit) installation are located in $INTELFPGAOCLSDKROOT/share/models/dm, where INTELFPGAOCLSDKROOT is set by the setvars.sh environment setup script provided by oneAPI toolkit. If the device model file for your part number is not included in $INTELFPGAOCLSDKROOT/share/models/dm, it must be created and placed in the same folder as board_spec.xml. quartus.ini Intel\u00ae Quartus\u00ae software ini settings file"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#512-commonsource-folder","title":"5.1.2 common/source Folder","text":"
          \nsource/\n|--cmake/\n|--|--modules\n|--extra/\n|--|--intel-fpga-bbb\n|--host\n|--include\n|--util\n|--CMakeLists.txt\n

          Table 5-3: common/source Folder Contents

          Files/Folder Description cmake/modules Contains Find***.cmake files to find packages required for building MMD library extra/intel-fpga-bbb oneAPI ASP for OFS reference platforms uses a library provided as part of Intel\u00ae FPGA Basic Building Blocks (BBB) repository. The oneapi-asp build scripts clone this repository in extra directory by default host Source code for MMD API include Contains MMD header files util Contains source code for oneapi-asp reference platform routines for diagnose and program utilities CMakeLists.txt Top-level CMakeLists.txt file for building MMD and other libraries required by oneapi-asp as well as the executables for diagnose and program utilities"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#513-linux64-folder","title":"5.1.3 linux64 Folder","text":"
          \nlinux64/\n|--include (see note below)\n|--lib\n|--libexec\n

          Note: The include and lib folders do not exist in the oneapi-asp repository. These are added after the oneapi-asp build flow is complete.

          Table 5-4: linux64 Folder Contents

          Files/Folder Description include Contains header files from Intel\u00ae FPGA BBB required by MMD (see section 5.4 for more information on use of MPF from Intel\u00ae FPGA BBB in MMD) lib Contains MMD and Intel\u00ae FPGA BBB libraries libexec Contains executables/scripts for oneapi-asp board utilities"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#52-oneapi-asp-build-flow","title":"5.2 oneapi-asp Build Flow","text":"

          Figure 1-3 shows a high level overview of the hardware design and Figure 1-4 shows oneAPI ASP as part of the Open FPGA Stack. As shown in these images, all oneapi-asp hardware components reside in the AFU region in the PR slot.

          Note: The architecture of the FIM with PR slot is explained in the FIM technical reference manuals, please refer to section 1.3 for link to the manuals.

          The oneapi-asp repository contains source files for components that reside in the AFU region for each reference platform. oneapi-asp expects a compiled FIM netlist and a corresponding PR tree. The FIM database is copied to the oneapi-asp during ASP build flow (oneapi-asp/Platform-Name/scripts/build-bsp.sh). ASP compile scripts import the FIM database during oneAPI compilation.

          Notes: 1. FIM developer guide outlines steps to compile a FIM and generate PR tree, please refer to section 1.3 for links to FIM developer guides 2. The steps to build oneapi-asp using PR tree and build-bsp.sh script are covered in the oneAPI Accelerator Support Package (ASP): Getting Started User Guide

          The following figure shows the oneapi-asp build process.

          Figure 5-2: oneapi-asp Build Flow

          Table 5-5 Environment Variables used in Build Flow

          Variable Number Environment Variable Description 1 OFS_PLATFORM_AFU_BBB Should point to location where ofs-platform-afu-bbb repository is cloned, if this variable is not set, build-bsp.sh script clones the repository 2 OPAE_PLATFORM_ROOT Must point to the PR tree generated during FIM build, this is a required variable and build flow fails without this 3 LIBOPAE_C_ROOT Should point to the installation location for OPAE libraries (please see oneAPI Accelerator Support Package (ASP): Getting Started User Guide for more information on this variable setting), build-opae.sh script is used to clone & build OPAE library if this variable is not set 4 OPAE_SDK_REPO_BRANCH If LIBOPAE_C_ROOT is not set, it is recommended to set this variable to indicate the OPAE SDK branch to be used for building OPAE libraries (please see oneAPI Accelerator Support Package (ASP): Getting Started User Guide)

          All scripts required for oneapi-asp build are located in oneapi-asp/Platform-Name/scripts folder, where Platform-Name is:

          • n6001 for Agilex OFS
          • d5005 for Stratix 10 OFS

          The build flow generates the complete hardware directories for all board variants in oneapi-asp.

          Note: For more information about the build scripts, please refer to README in oneapi-asp/Platform-Name/scripts directory.

          "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#53-oneapi-asp-hardware-implementation","title":"5.3 oneapi-asp Hardware Implementation","text":"

          This section goes deeper into the current hardware architecture of the oneapi-asp.

          Figure 1-3 shows a high level overview of the hardware design. OFS reference platforms have different board variants enabling the different paths shown in Figure 1-3. Table below summarizes the board variants and paths supported in each. The Path numbers in the table match the ones in Figure 1-3. Figure 5-3 to 5-6 show the detailed diagram for oneapi-asp components in each of these board variants.

          Table 5-6: OFS Reference Platform Board Variants

          # Device Board Variant Host to EMIF with DMA Engine (Path 1) Host to Kernel Interface (Path 2) Kernel to EMIF (Path 3) Kernel to Unified Shared Memory (Path 4) Kernel to HSSI (Path 5) Figure # 1 Agilex OFS Stratix 10 OFS ofs_n6001 ofs_d5005 Yes Yes Yes No No 5-3 2 Agilex OFS Stratix 10 OFS ofs_n6001_usm ofs_d5005_usm Yes Yes Yes Yes No 5-4 3 Agilex OFS ofs_n6001_iopies Yes Yes Yes No Yes 5-5 4 Agilex OFS ofs_n6001_usm_iopipes Yes Yes Yes Yes Yes 5-6

          Note: Please see oneapi-asp/Platform-Name/hardware folder for the board variants for each OFS reference platform.

          Figure 5-3: oneapi-asp Reference Platform Hardware Design - Board Variant #1

          Figure 5-4: oneapi-asp Reference Platform with USM Hardware Design - Board Variant #2

          Figure 5-5: oneapi-asp Reference Platform with IO Pipes Hardware Design - Board Variant #3

          Figure 5-6: oneapi-asp Reference Platform with IO Pipes and USM Hardware Design - Board Variant #4

          The ofs_plat_afu.sv is the top level entity for the oneapi-asp.

          All hardware design files for the components inside ofs_plat_afu module are inside the oneapi-asp/Platform-Name/hardware/Board-Variant/build directory. The FIM database files imported during oneapi-asp build (section 5.2) are located in oneapi-asp/Platform-Name/hardware/Board-Variant/fim_platform directory.

          Table 5-7 gives a brief description of the important design files in all board variants.

          Table 5-7: Hardware Design Files in oneapi-asp/Platform-Name/hardware/Board-Variant

          Files/Folder Description build/ip Contains all the *.ip files for IP components used in the hardware design build/rtl Contains the SystemVerilog and Verilog design files shown in figures 5-3 to 5-6 above build/scripts Contains scripts to control oneAPI kernel and asp Intel\u00ae Quartus\u00ae software compile flow (see section 5.3.4 for more information on compile flow) build/afu_ip.qsf Adds platform specific settings & design files, this file is sourced in afu_flat.qsf build/mpf_vtp.qsf Adds IP components from Intel\u00ae FPGA BBB (see section 5.3.2 below for information about use of MPF blocks from Intel\u00ae FPGA BBB) repository used in the design build/bsp_design_files.tcl Adds design files to the project, this file is source in afu_ip.qsf build/board.qsys Please refer to figures above for components in board.qsys system build/ddr_board.qsys Instantiated in the board.qsys system, contains IP components in the host to EMIF and kernel to EMIF datapath. Memory Bank Divider is instantiated in this platform designer system. Please refer to section 3.1.1 for more details on Memory Bank Divider build/ddr_channel.qsys Instantiated in ddr_board.qsys, contains bridges required for clock domain crossings between PCIe and EMIF clocks as well as kernel and EMIF clock fim_platform Contains Platform Interface Modules (PIM) modules and FIM database used in the design. See PIM subtopic below ofs_asp.sdc Contains clock group constraints for all clocks in oneAPI ASP

          The hardware implementation diagrams show a PF/VF Mux/De-mux module in the AFU region. The PF/VF mux routes packets to AFU component based on pf_num and vf_num information in PCIe TLP header. For more information about the PF/VF mux, please refer to the PF/VF Mapping details in FPGA Interface Manager Technical Reference Manual for your target device (links are available in section 1.3).

          The oneapi-asp resides inside the AFU region and ,depending on the FIM configuration, connects to the lowest PF or lowest VF number that routes into the PR slot. Table below summazires the PF/VF mapping in oneapi-asp for some of the different FIM configurations.

          Table 5-8: PF/VF Mapping in oneapi-asp

          Device FIM Configuration* PF/VF Mapping in oneapi-asp Stratix 10 OFS Default PF0-VF1 Agilex OFS Default PF0-VF0 Agilex OFS Compiled using n6001_1pf_1vf.ofss, i.e. PCIe Subsystem configuration is set to PF0 with 1 VF PF0-VF0 Agilex OFS Compiled using n6001_2pf.ofss, i.e. PCIe Subsystem configuration is set to two physical functions PF0 and PF1 PF1

          *Note: For more information on different FIM configurations & how to compile these, please refer to FPGA Interface Manager Developer Guide for Open FPGA Stack for your target device (links are available in section 1.3).

          Sections below provide some more information on some blocks in the hardware design block diagram shown above. Refer to section 3.1 for more information about Memory Bank Divider and to section 3.2 for information about Kernel Interface.

          "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#531-platform-interface-modulespim","title":"5.3.1 Platform Interface Modules(PIM)","text":"

          In addition to above files/folders, an important part of the design are the ofs_plat_* modules provided by Platform Interface Manager (PIM). oneAPI kernel system and oneapi-asp have Avalon\u00ae interfaces. FIM components have AXI interfaces. PIM modules are used for protocol translation.

          Note: For more information about PIM, please refer to PIM README here.

          "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#532-direct-memory-accessdma-module","title":"5.3.2 Direct Memory Access(DMA) Module","text":"

          The Direct Memory Access module is located in the host to EMIF datapath in the oneapi-asp and provides a controller to execute transfers from host to DDR on the board and vice versa. The source files for the DMA module used in oneapi-asp for OFS reference platforms are located in oneapi-asp/Platform-Name/hardware/Board-Variant/build/rtl/dma directory. Figure below shows the DMA module interface.

          Figure 5-7: DMA Controller Block Diagram

          Figure 5-3 shows the instantiation of this dma_top module as dma_controller_inst. Table 5-9 shows the signal names and descriptions. Section 5-5 covers the complete compilation flow.

          Table 5-9: DMA Module Signal Descriptions

          Signal or Port Description mmio64_if Control signal from host, connects to DMA Control Status Registers(CSR) host_mem_rd_avmm_if Avalon\u00ae memory-mapped interface to read from host memory host_mem_wr_avmm_if Avalon\u00ae memory-mapped interface to write to host memory local_mem_rd_avmm_if Avalon\u00ae memory-mapped interface to read from on-board DDR4 memory local_mem_wr_avmm_if Avalon\u00ae memory-mapped interface to write to on-board DDR4 memory dma_irq_host2fpga Interrupt to indicate completion of host to FPGA DMA transaction (read from host, write to DDR4) dma_irq_fpga2host Interrupt to indicate completion of FPGA DDR4 to host DMA transaction (read from DDR4, write to host)> Note: The oneapi-asp for OFS reference platforms uses a host memory write to indicate completion of FPGA to host transaction

          The data transfer module manages data movement from source to destination addresses.

          To start a data transfer, the data transfer module requires following information, this is set by the dma_dispatcher:

          • Source address
          • Destination address
          • Number of bytes to transfer

          oneapi-asp for OFS reference platforms uses virtual addresses in the DMA controller for data transfer. The Memory Properties Factory(MPF) Virtual to Physical(VTP) blocks (i.e.mpf_vtp_* modules) shown in figure 5-3 translate virtual addresses to appropriate host addresses.

          Note: Memory Properties Factory(MPF) is a part of Intel\u00ae FPGA Basic Building Blocks. Please refer to Intel\u00ae FPGA BBB repository for more information.

          "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#533-user-datagram-protocoludp-engine","title":"5.3.3 User Datagram Protocol(UDP) Engine","text":"

          I/O pipes allow kernel to stream data directly using HSSI. To demonstrate this functionality, reference design in oneapi-asp repository (refer to Figure 5-5 and 5-6) has a UDP protocol engine to allow transmitting UDP/IP packets over HSSI..

          Figure below shows a simple block diagram of the UDP engine.

          Figure 5-8: UDP Offload Engine

          The UDP engine consists of a separate receive (rx) and trasmit (tx) path. The following functionalilty is performed by this reference design engine:

          • Implements an Address Resolution Protocol (ARP) functionality to respond to be able to send & respond to ARP requests. This is needed for routing between different subnets using a gateway.
          • Packetizes data from kernel to add the required header information (for MAC, IP & UDP layers)
          • Extracts data from packets received by removing header information
          • Handles clock crossing between kernel clock and Ethernet MAC clock domains

          The source files for UDP engine used in oneapi-asp for OFS reference platform are located in oneapi-asp/n6001/hardware/ofs_n6001_iopipes/build/rtl/udp_offload_engine directory.

          Note: The same engine is used in the board variant with USM shown in Figure 5-6 (source files are in oneapi-asp/n6001/hardware/ofs_n6001_usm_iopipes/build/rtl/udp_offload_engine).

          "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#534-hardware-compile-flow","title":"5.3.4 Hardware Compile Flow","text":"

          Figure below shows the compile flow overview; the oneAPI compiler generated hardware circuit is compiled by Intel\u00ae Quartus\u00ae software along with design files for oneapi-asp.

          Figure 5-9: oneAPI Compile Flow Overview

          The oneAPI compiler uses the board_spec.xml to get more information about the oneapi-asp configuration. board_spec.xml file has a compile element to allow control of the Intel\u00ae Quartus\u00ae software compilation flow. The attributes of this element are discussed in section 2.1.2. The oneapi-asp uses tcl scripts to control the Intel\u00ae Quartus\u00ae software compilation flow. Figure 5-10 shows the flow and scripts used. All compilation scripts are located in oneapi-asp/Platform-Name/hardware/Board-Variant/build/scripts folder.

          Figure 5-10: Compilation Scripts in oneapi-asp

          Table 5-10 summarizes notes for reference numbers 1-5 marked in figure above.

          Table 5-10: Notes for Reference Numbers in Figure 5-10

          Reference Number Note 1 revision_name is afu_flat for oneapi-asps for OFS reference platforms 2 gen-asp-quartus-report.tcl script generates a report (acl_quartus_report.txt) containing resource utilization and kernel clock frequency summary"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#54-oneapi-asp-memory-mapped-devicemmd-layer-implementation","title":"5.4 oneapi-asp Memory Mapped Device(MMD) Layer Implementation","text":"

          As discussed in section 4.1, the MMD provides a set of API that allow the runtime to control the device and communicate with it.

          The source code for MMD layer is located in oneapi-asp/common/source/host folder. aocl_mmd.h is the header file for the implemented API and is located in oneapi-asp/common/source/include folder. Table below summarizes the APIs that have been implemented in oneapi-asp for OFS reference platforms.

          Note: For more details about the API, its arguments and enums please refer to the aocl_mmd.h file and to section 4.1.

          Table 5-11: MMD API Implemented in oneapi-asp for OFS Reference Platforms

          API aocl_mmd_get_offline_info aocl_mmd_get_info aocl_mmd_open aocl_mmd_close aocl_mmd_set_interrupt_handler aocl_mmd_set_status_handler aocl_mmd_yield aocl_mmd_read aocl_mmd_write aocl_mmd_copy aocl_mmd_program aocl_mmd_host_alloc aocl_mmd_free aocl_mmd_shared_alloc aocl_mmd_shared_migrate

          The implementation of these APIs is in oneapi-asp/common/source/host/mmd.cpp. The functions used in the implementation are distributed across various source files. Table below provides details on source code files.

          Table 5-12: MMD Source Code Files

          Files/Folder Description mmd.cpp This file has the implementation for all MMD API calls listed in table 5-11 fpgaconf.hfpgaconf.c Contains bitstream reconfiguration function declaration(.h) & definition(.c) kernel_interrupt.h Contains KernelInterrupt class declaration; the class consists of functions to handle kernel interrupts kernel_interrupt.cpp Contains KernelInterrupt class constructor and function definitions mmd_device.h Contains Device class declaration, which stores device data and has functions to interact with the device mmd_device.cpp Contains Device class constructor and function definitions mmd_dma.hmmd_dma.cpp Contain DMA functions declaration(.h) & definition(.cpp) mmd_iopipes.h Contains the iopipes class declaration(.h) mmd_iopipes.cpp Contains function definitions, these functions include iopipes class constructor as well as fucntions to detect and setup CSR space for IO pipes feature in board variants that support IO pipes zlib_inflate.hzlib_inflate.c Function declaration(.h) and definition(.c) for decompressing bitstream data CMakeLists.txt CMakeLists.txt file for building MMD source code

          The build flow scripts build the MMD library, i.e. libintel_opae_mmd, and place them in oneapi-asp/Platform-Name/linux64/lib folder. The MMD library is specified as part of mmdlib, linklibs element in board_env.xml and used at runtime (refer to figure 5-1 for sample board_env.xml file and section 2.2 for more information about board_env.xml elements).

          Use of OPAE library in MMD

          The MMD layer uses API from OPAE SDK for various device operations. Hence, the MMD layers requires OPAE library to be loaded to execute successfully. The mmdlib element specifies the libopae-c library to be loaded before the MMD library (demonstrated in sample board_env.xml in figure 5-1).

          Note: Please refer to Software Reference Manual: Open FPGA Stack for more information about OPAE SDK API. The document also has information about linux-dfl driver.

          Use of Memory Properties Factory(MPF) library in MMD

          In addition to OPAE, the MMD also uses API from Memory Properties Factory(MPF) software for memory operations. This The libMPF library is compiled as part of oneapi-asp build flow and placed in oneapi-asp/Platform-Name/linux64/lib. This library is also loaded before the MMD library by specifying before libintel_opae_mmd in mmdlib element.

          Note: Memory Properties Factory(MPF) is a part of Intel\u00ae FPGA BBB. Please refer to Intel\u00ae FPGA BBB wiki for more information about MPF.

          "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#55-oneapi-asp-utilities-implementation","title":"5.5 oneapi-asp Utilities Implementation","text":"

          This section covers the implementation of board utilities (refer to section 4.2 for more information on board utilities) in oneapi-asp for OFS reference platforms.

          Table below shows the source code/script locations for the utilities.

          Table 5-13: oneapi-asp Utilities Source Code Locations

          Utility Source Location diagnose oneapi-asp/common/source/util/diagnostic program oneapi-asp/common/source/util/reprogram installuninstallinitialize> Note: These are executable scripts oneapi-asp/Platform-Name/linux64/libexec/

          diagnose and program are compiled as part of the oneapi-asp build flow and executables are placed in oneapi-asp/Platform-Name/linux64/libexec/. A CMakeLists.txt file is provided for building the utilities, located in oneapi-asp/common/source/util directory.

          The path to all of the above utility executables is used in utilbinder element in board_env.xml (demonstrated in sample board_env.xml in figure 5-1). The runtime uses this when the corresponding aocl utility is invoked.

          Brief descriptions for the source code files are given in table below.

          Table 5-14: Brief Descriptions of oneapi-asp Utility Routines for OFS Reference Platforms

          File Description setup_permissions.sh Helper script to configure correct device port permissions, make changes to allow users to lock pages in memory and set the hugepages required for the software stack to function correctly. The helper script is used by install, initialize routines install install routine invokes the setup_permissions.sh script after the FPGA Client Driver (FCD) is setup by the runtime uninstall uninstall routine reverts the port permission, memory locking and hugepage setting changes performed by install routine and is invoked by runtime after the FCD is removed by runtime initialize initialize routine performs the following steps: * looks for the initialization binary for the board variant to be initialized * extracts the FPGA hardware configuration file from the oneAPI fat binary using clang-offload-extract command provided by Intel\u00ae oneAPI Base Toolkit (Base Kit) version 2024.0 and beyond * invokes the setup_permissions.sh script to set correct device permissions * performs partial reconfiguration of the FPGA device by invoking program routine with the initialization bitstream as an argument >Note: For more information about how initialize utility extracts FPGA hardware configuration file from oneAPI fat binary, refer to Intel oneAPI\u00ae FPGA Handbook program program routine allocates memory and loads the supplied initialization bitstream in memory followed by a call to reprogramming function provided by oneapi-asp's MMD library. The MMD library uses fpgaReconfigureSlot API provided by OPAE library to perform device reconfiguration> Note: Please refer to Software Reference Manual: Open FPGA Stack for more information about OPAE SDK API diagnose diagnose routine scans for the available devices for the installed platform and performs DMA transactions between host & device. It also reports the PCIe bandwidth. diagnose routine uses functions provided by the MMD library for scanning & opening connection to available devices"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#document-revision-history","title":"Document Revision History","text":"Date Release Changes May 26, 2023 2023.1 First release of oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack on https://ofs.github.io/ September 15, 2023 2023.2 1. Section 1: * Updated Figure 1-3 to add HSSI path 2. Section 2: * Added channels element in figure 2-1 and table 2-1, added information about board variants below this table * Added Section 2.1.8 on channels element 3. Section 4: * Added information about MMD API (table 4-1) and new sections 4.1.1 to 4.1.21 4. Section 5: * Moved oneapi-asp build flow information into new section 5.2 * Added new table 5-6 with oneAPI ASP board variants information * Added hardware design diagrams and information about new board variants with I/O pipes support (Hardware Design with IO Pipes and Hardware Design with IO Pipes and USM) * Updated hardware design diagrams to show PF/VF Mux/De-mux and added information about PF/VF mapping in section 5.3 * Added new section on UDP engine (section 5.3.3) * Updated figure 5-10 to remove import_opencl_kernel.tcl and add_bbb_to_pr_project.tcl * Updated table 5-12 to add mmd_iopipes.h and mmd_iopipes.cpp files Dec 13, 2023 2023.3-1 1. Section 2: * Update table 2-5 to add information about port parameter * Added new section 2.1.4.1 on port parameter * Update table & figure numbers * Changed \"master\" and \"slave\" ports to \"host\" & \"agent\" ports in figures 2. Section 4: * Added new executable call supported by initialize utility 3. Section 5: * Updated initialize utility information to add clang-offload-extractcommand usage * Updated table with hardware design files information, added information about ofs_asp.sdc * Updated compile flow diagram, added information about new gen-asp-quartus-report.tcl script * Updated hardware implementation diagrams for signal name and IP name changes, replaced mem_if_vtp block with host_mem_if_vtp * Updated OpenCL Memory Bank Divider to Memory Bank Divider * Updated OpenCL Kernel Interface to Kernel Interface 2023.3-2 1. Section 5: * Updated Table 5-8 to add PF/VF mapping for different FIM configurations"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#notices-disclaimers","title":"Notices & Disclaimers","text":"

          Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Performance varies by use, configuration and other factors. Your costs and results may vary. You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications. Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards. \u00a9 Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.

          OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122.

          "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/","title":"AFU Development Guide: OFS for Intel\u00ae Intel\u00ae Agilex\u00ae 7 FPGA PCIe Attach FPGAs","text":"

          Last updated: February 03, 2024

          "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#1-introduction","title":"1. Introduction","text":"

          This document is a design guide for the creation of an Accelerator Functional Unit (AFU) using Open FPGA Stack (OFS) for Intel\u00ae Agilex\u00ae 7 FPGAs PCIe Attach. The AFU concept consists of separating out the FPGA design development process into two parts, the construction of the foundational FPGA Interface Manager (FIM), and the development of the Acceleration Function Unit (AFU), as shown in the diagram below.

          This diagram shows the separation of FPGA board interface development from the internal FPGA workload creation. This separation starts with the FPGA Interface Manager (FIM) which consists of the external interfaces and board management functions. The FIM is the base system layer and is typically provided by board vendors. The FIM interface is specific to a particular physical platform. The AFU makes use of the external interfaces with user defined logic to perform a specific application. By separating out the lengthy and complicated process of developing and integrating external interfaces for an FPGA into a board allows the AFU developer to focus on the needs of their workload. OFS for Intel\u00ae Agilex\u00ae 7 FPGAs PCIe Attach provides the following tools for rapid AFU development:

          • Scripts for both compilation and simulation setup
          • Optional Platform Interface Manager (PIM) which is a set of SystemVerilog shims and scripts for flexible FIM to AFU interfacing
          • Acceleration Simulation Environment (ASE) which is a hardware/software co-simulation environment scripts for compilation and Acceleration
          • Integration with Open Programmable Acceleration Engine (OPAE) SDK for rapid software development for your AFU application

          Please notice in the above block diagram that the AFU region consists of static and partial reconfiguration (PR) regions where the PR region can be dynamically reconfigured while the remaining FPGA design continues to function. Creating AFU logic for the static region is described in FPGA Interface Manager Developer Guide for Intel\u00ae Agilex\u00ae 7 PCIe Attach. This guide covers logic in the AFU Main region.

          "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#11-document-organization","title":"1.1. Document Organization","text":"

          This document is organized as follows:

          • Description of design flow
          • Interfaces and functionality provided in the Intel\u00ae Agilex\u00ae 7 FPGAs PCIe Attach FIM
          • Setup of the AFU Development environment
          • Synthesize the AFU example
          • Testing the AFU example on the Intel\u00ae FPGA SmartNIC N6001-PL card
          • Hardware/Software co-simulation using ASE
          • Debugging an AFU with Remote Signal Tap

          This guide provides theory followed by tutorial steps to solidify your AFU development knowledge.

          NOTE:

          This guide uses the Intel\u00ae FPGA SmartNIC N6001-PL as the platform for all tutorial steps. Additionally, this guide and the tutorial steps can be used with other platforms, such as the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile).

          Some of the document links in this guide are specific to the Intel\u00ae FPGA SmartNIC N6001-PL. If using a different platform, please use the associated documentation for your platform as there could be differences in building the FIM and downloading FIM images.

          If you have worked with previous Intel Programmable Acceleration products, you will find out that OFS for Intel\u00ae Agilex\u00ae 7 FPGAs PCIe Attach is similar. However, there are differences and you are advised to carefully read and follow the tutorial steps to fully understand the design tools and flow.

          "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#12-prerequisite","title":"1.2. Prerequisite","text":"

          This guide assumes you have the following FPGA logic design-related knowledge and skills:

          • FPGA compilation flows including the Intel\u00ae Quartus\u00ae Prime Pro Edition design flow
          • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition software, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
          • RTL and coding practices to create synthesizable logic.
          • Understanding of AXI and Avalon memory mapped and streaming interfaces.
          • Simulation of complex RTL using industry standard simulators (Synopsys\u00ae VCS\u00ae or Siemens\u00ae QuestaSim\u00ae).
          • Signal Tap Logic Analyzer tool in the Intel\u00ae Quartus\u00ae Prime Pro Edition software.

          You are strongly encouraged to review the FPGA Interface Manager Developer Guide for Intel\u00ae Agilex\u00ae 7 PCIe Attach.

          "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#13-acceleration-functional-unit-afu-development-flow","title":"1.3. Acceleration Functional Unit (AFU) Development Flow","text":"

          The AFU development flow is shown below:

          "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#131-understanding-platform-capabilities","title":"1.3.1. Understanding Platform Capabilities","text":"

          The block diagram of the N6001 Board is shown below:

          The N6001 FIM provided with this release is shown below:

          This release FIM provides the following features:

          • Host interface
            • PCIe Gen4 x 16
            • 5 - PF, 4 - VF, AXI-S TLP packets
            • MSI-X interrupts
            • Logic to demonstrate simple PCIe loopback
          • Network interface
            • 2 - QSFP28/56 cages
            • 2 x 4 x 25 GbE with exerciser logic demonstrating traffic generation/monitoring
          • External Memory - DDR4 - 2400
            • HPS - 1GB organized as 256 Mb x 32 with 256 Mb x 8 ECC
            • Channel 0, 1 - 4 GB organized as 1 Gb x 32
            • Channel 2, 3 - 4 GB organized as 1 Gb x 32 with 1 Gb x 8 ECC (ECC is not implemented in this release)
            • Memory exerciser logic demonstrating external memory operation
          • Board Management
            • SPI interface
            • FPGA configuration
            • Example logic showing DFH operation
          • Remote Signal Tap logic
          • Partial reconfiguration control logic
          • ARM HPS subsystem with embedded Linux
            • HPS Copy engine
          "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#132-high-level-data-flow","title":"1.3.2. High Level Data Flow","text":"

          The OFS high level data flow is shown below:

          "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#133-considerations-for-pim-usage","title":"1.3.3. Considerations for PIM Usage","text":"

          When creating an AFU, a designer needs to decide what type of interfaces the platform (FIM) should provide to the AFU. The FIM can provide the native interfaces (i.e. PCIe TLP commands) or standard memory mapped interfaces (i.e. AXI-MM or AVMM) by using the PIM. The PIM is an abstraction layer consisting of a collection of SystemVerilog interfaces and shims to enable partial AFU portability across hardware despite variations in hardware topology and native interfaces. The PIM adds a level of logic between the AFU and the FIM converting the native interfaces from the FIM to match the interfaces provided by the AFU.

          The following resources are available to assist in creating an AFU:

          PIM Core Concepts provides details on using the PIM and its capabilities.

          Connecting an AFU to a Platform using PIM guides you through the steps needed to connect a PIM Based AFU to the FIM.

          The AFU Tutorial provides several AFU examples. These examples can be run with the current OFS FIM package. There are three AFU types of examples provided (PIM based, hybrid and native). Each example provides the following:

          • RTL, which includes the following interfaces:
            • Host Channel:
              • Host memory, providing a DMA interface.
              • MMIO, providing a CSR interface.
            • Local Memory
          • Host software example interfacing to the CSR interface and host memory interface, using the OPAE C API.
          • Accelerator Description File .json file
          • Source file list
          "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#134-afu-interfaces-included-with-intel-fpga-smartnic-n6001-pl","title":"1.3.4. AFU Interfaces Included with Intel\u00ae FPGA SmartNIC N6001-PL","text":"

          The figure below shows the interfaces available to the AFU in this architecture. It also shows the design hierarchy with module names from the fim (top.sv) to the PR region AFU (afu_main.sv). One of the main differences from the Stratix 10 PAC OFS architecture to this one is the presence of the static port gasket region (port_gasket.sv) that has components to facilitate the AFU and also consists of the PR region (afu_main.sv) via the PR slot. The Port Gasket contains all the PR specific modules and logic, e.g., PR slot reset/freeze control, user clock, remote STP etc. Architecturally, a Port Gasket can have multiple PR slots where user workload can be programmed into. However, only one PR slot is supported for OFS Release for Intel Agilex. Everything in the Port Gasket until the PR slot should be provided by the FIM developer. The task of the AFU developer is to add their desired application in the afu_main.sv module by stripping out unwanted logic and instantiating the target accelerator. As shown in the figure below, here are the interfaces connected to the AFU (highlighted in green) via the PCIe Attach FIM:

          1. AXI Streaming (AXI-S) interface to the Host via PCIe Gen4x16
          2. AXI Memory Mapped Channels (4) to the DDR4 EMIF interface
          3. AXI Streaming (AXI-S) interface to the HSSI 25 Gb Ethernet

          "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#2-set-up-afu-development-environment","title":"2. Set Up AFU Development Environment","text":"

          This section covers the setup of the AFU development environment.

          "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#21-afu-development-environment-overview","title":"2.1. AFU development environment overview","text":"

          A typical development and hardware test environment consists of a development server or workstation with FPGA development tools installed and a separate server with the target OFS compatible FPGA PCIe card installed. The typical usage and flow of data between these two servers is shown below:

          Note: both development and hardware testing can be performed on the same server if desired.

          This guide uses Intel\u00ae FPGA SmartNIC N6001-PL as the target OFS compatible FPGA PCIe card for demonstration steps. The Intel\u00ae FPGA SmartNIC N6001-PL must be fully installed following the Getting Started User Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs. If using a different OFS FPGA PCIe card, contact your supplier for instructions on how to install and operate user developed AFUs.

          The following is a summary of the steps to set up for AFU development:

          1. Install Quartus Prime Pro Version 23.3 for Linux with Agilex device support and required Quartus patches.
          2. Make sure support tools are installed and meet version requirements.
          3. Install OPAE SDK.
          4. Download the Basic Building Blocks repository.
          5. Build or download the relocatable AFU PR-able build tree based on your Intel\u00ae Agilex FPGA PCIe Attach FIM.
          6. Download FIM to the Intel\u00ae Agilex FPGA PCIe Attach platform.

          Building AFUs with OFS for Agilex requires the build machine to have at least 64 GB of RAM.

          "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#22-installation-of-quartus-and-required-patches","title":"2.2. Installation of Quartus and required patches","text":"

          Intel Quartus Prime Pro Version 23.3 is verified to work with the latest OFS release ofs-2023.3-2. However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.

          Use RedHatEnterprise Linux\u00ae (RHEL) 8.6 for compatibility with your development flow and also testing your FIM design in your platform.

          Prior to installing Quartus:

          1. Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.

            • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
            • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.

          2. Perform the following steps to satisfy the required dependencies.

            $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n

            Apply the following configurations.

            $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n

          3. Create the default installation path: /intelFPGA_pro/, where is the default path of the Linux workstation, or as set by the system administrator and is your Quartus version number.

            The installation path must satisfy the following requirements:

            • Contain only alphanumeric characters
            • No special characters or symbols, such as !$%@^&*<>,
            • Only English characters
            • No spaces

          4. Download your required Quartus Prime Pro Linux version here.

          5. Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe).

          6. After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:

            export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n

            For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:

            export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n

          7. Verify, Quartus is discoverable by opening a new shell:

            $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
            8. "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#23-installation-of-support-tools","title":"2.3. Installation of Support Tools","text":"

            Make sure support tools are installed and meet version requirements.

            The OFS provided Quartus build scripts require the following tools. Verify these are installed in your development environment.

            Item Version Python 3.6.8 GCC 7.4.0 cmake 3.15 git 1.8.3.1 perl 5.8.8"},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#24-installation-of-opae-sdk","title":"2.4. Installation of OPAE SDK","text":"

            Follow the instructions in the Getting Started Guide: Open FPGA Stack for Intel\u00ae FPGA SmartNIC N6001-PL, section 4.0 OPAE Software Development Kit to build and install the required OPAE SDK for the Intel\u00ae FPGA SmartNIC N6001-PL.

            Working with the Intel\u00ae Intel\u00ae FPGA SmartNIC N6001-PL card requires opae-2.10.0-1. Follow the instructions in the Getting Started Guide: Intel\u00ae Open FPGA Stack for Intel\u00ae FPGA SmartNIC N6001-PL section 4.0 OPAE Software Development Kit. Make sure to check out the cloned repository to tag 2.10.0-1 and branch release/2.10.0.

            $ git checkout tags/2.10.0-1 -b release/2.10.0\n

            Note: The tutorial steps provided in the next sections assume the OPAE SDK is installed in default system locations, under the directory, /usr. In most system configurations, this will allow the OS and tools to automatically locate the OPAE binaries, scripts, libraries and include files required for the compilation and simulation of the FIM and AFUs.

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#25-download-the-basic-building-blocks-repositories","title":"2.5. Download the Basic Building Blocks repositories","text":"

            The ofs-platform-afu-bbb repository contains the PIM files as well as example PIM-based AFUs that can be used for testing and demonstration purposes. This guide will use the host_chan_mmio AFU example in the ofs-platform-afu-bbb repository and the hello_world sample accompanying the examples AFU repository to demonstrate how to synthesize, load, simulate, and test a PIM-based AFU using the Intel\u00ae FPGA SmartNIC N6001-PL card with the PCIe Attach FIM.

            Execute the next commands to clone the BBB repository.

              # Create top level directory for AFU development\n$ mkdir OFS_BUILD_ROOT\n$ cd OFS_BUILD_ROOT\n$ export OFS_BUILD_ROOT=$PWD\n\n  # Clone the ofs-platform-afu-bbb repository.\n$ cd $OFS_BUILD_ROOT\n$ git clone https://github.com/OFS/ofs-platform-afu-bbb.git\n\n  # Verify retrieval\n$ cd $OFS_BUILD_ROOT/ofs-platform-afu-bbb\n$ ls\nLICENSE  plat_if_develop  plat_if_release  plat_if_tests  README.md\n

            The documentation in the ofs-platform-afu-bbb repository further addresses - The PIM concept. - The structure of the PIM-based AFU examples. - How to generate a release and configure the PIM. - How to connect an AFU to an FIM.

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#26-build-or-download-the-relocatable-pr-build-tree","title":"2.6. Build or download the relocatable PR build tree","text":"

            A relocatable PR build tree is needed to build the AFU partial reconfiguration area for the intended FIM. The tree is relocatable and may be copied to a new location. It does not depend on files in the original FIM build.

            You can use the Intel\u00ae FPGA SmartNIC N6001-PL release package and download the PR build tree and FIM images, to develop your AFU. These are located at OFS-N6001 release

            Or you can build your own FIM and generate the PR build tree during the process.

            To download and untar the pr_build_template:

            $ cd $OFS_BUILD_ROOT\n$ wget https://github.com/OFS/ofs-agx7-pcie-attach/releases/download/ofs-2023.3-2/n6001-images_ofs-2023-3-2.tar.gz\n$ tar -zxvf n6001-images_ofs-2023-3-2.tar.gz\n$ cd n6001-images_ofs-2023-3-2/\n$ mkdir pr_build_template\n$ tar -zxvf pr_build_template-n6001.tar.gz -C ./pr_build_template\n$ cd pr_build_template\n$ export OPAE_PLATFORM_ROOT=$PWD\n

            To build your own FIM and generate the PR build tree for the Intel\u00ae FPGA SmartNIC N6001-PL platform, refer the FPGA Interface Manager Developer Guide for Intel\u00ae Agilex\u00ae 7 PCIe Attach and follow the Out-of-Tree PR FIM build flow. If you are using a different platform, refer to the FPGA Interface Manager Developer Guide for your platform and follow the Out-of-Tree PR FIM build flow.

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#27-download-fim-to-fpga","title":"2.7. Download FIM to FPGA","text":"

            The AFU requires that the FIM from which the AFU is derived be loaded onto the FPGA.

            If you are using the Intel\u00ae FPGA SmartNIC N6001-PL release package downloaded in the previous section: 

            $ cd $OFS_BUILD_ROOT/n6001-images_ofs-2023-3-2/\n

            If you are generating your own FIM, use the unsigned FPGA binary images from your FIM build.

            Downlaod the FIM to the Intel\u00ae FPGA SmartNIC N6001-PL platform. If you are running on a Virtual Machine, refer to the [KVM User Guide] for passing the devices to the VM. 

            $ sudo fpgasupdate ofs_top_page1_unsigned_user1.bin <N6001 SKU2 PCIe b:d.f>\n$ sudo fpgasupdate ofs_top_page2_unsigned_user2.bin <N6001 SKU2 PCIe b:d.f>\n$ sudo rsu fpga --page=user1 <N6001 SKU2 PCIe b:d.f>\n

            If you are using a different platform, refer to the documentation for your platform to download the FIM images onto your Intel\u00ae Agilex\u00ae PCIe Attach Platform.

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#28-set-up-required-environment-variables","title":"2.8. Set up required Environment Variables","text":"

            Set the required environment variables as shown below. These environment variables must be set prior to simulation or compilation tasks. You can create a simple script to set these variables and save time going forward.

            # If not already done, export OFS_BUILD_ROOT to the top level directory for AFU development\n$ export OFS_BUILD_ROOT=<path to ofs build directory>\n\n# If not already done, export OPAE_PLATFORM_ROOT to the PR build tree directory\n$ export OPAE_PLATFORM_ROOT=<path to pr build tree>\n\n# Quartus Tools\n# Note, QUARTUS_HOME is your Quartus installation directory, e.g. $QUARTUS_HOME/bin contains Quartus executable.\n$ export QUARTUS_HOME=<user_path>/intelFPGA_pro/23.3/quartus\n$ export QUARTUS_ROOTDIR=$QUARTUS_HOME\n$ export QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\n$ export QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\n$ export IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\n$ export IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\n$ export QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys\n$ export PATH=$QUARTUS_HOME/bin:$QSYS_ROOTDIR/bin:$QUARTUS_HOME/../sopc_builder/bin/:$PATH\n\n# OPAE SDK release\n$ export OPAE_SDK_REPO_BRANCH=release/2.10.0\n\n# The following environment variables are required for compiling the AFU examples. \n\n# Location to clone the ofs-platform-afu-bbb repository which contains PIM files and AFU examples.\n$ export OFS_PLATFORM_AFU_BBB=$OFS_BUILD_ROOT/ofs-platform-afu-bbb \n\n# OPAE and MPF libraries must either be on the default linker search paths or on both LIBRARY_PATH and LD_LIBRARY_PATH.  \n$ export OPAE_LOC=/usr\n$ export LIBRARY_PATH=$OPAE_LOC/lib:$LIBRARY_PATH\n$ export LD_LIBRARY_PATH=$OPAE_LOC/lib64:$LD_LIBRARY_PATH\n
            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#3-compiling-an-afu","title":"3. Compiling an AFU","text":"

            In this section, you will use the relocatable PR build tree created in the previous steps from the FIM to compile an example PIM-based AFU. This section will be developed around the host_chan_mmio and hello_world AFU examples to showcase the synthesis of a PIM-based AFU.

            The build steps presented below demonstrate the ease in building and running an actual AFU on the board. To successfully execute the instructions in this section, you must have set up your development environment and have a relocateable PR Build tree as instructed in section 2 of this document.

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#31-creating-the-afu-synthesis-environment","title":"3.1. Creating the AFU Synthesis Environment","text":"

            The PIM flow provides the script afu_synth_setup to create the synthesis environment to build the AFU examples. See how to use it below.

            usage: afu_synth_setup [-h] -s SOURCES [-p PLATFORM] [-l LIB] [-f] dst\n\nGenerate a Quartus build environment for an AFU. A build environment is\ninstantiated from a release and then configured for the specified AFU. AFU\nsource files are specified in a text file that is parsed by rtl_src_config,\nwhich is part of the OPAE base environment.\n\npositional arguments:\n  dst                   Target directory path (directory must not exist).\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -s SOURCES, --sources SOURCES\n                        AFU source specification file that will be passed to\n                        rtl_src_config. See \"rtl_src_config --help\" for the\n                        file's syntax. rtl_src_config translates the source\n                        list into either Quartus or RTL simulator syntax.\n  -p PLATFORM, --platform PLATFORM\n                        FPGA platform name.\n  -l LIB, --lib LIB     FPGA platform release hw/lib directory. If not\n                        specified, the environment variables OPAE_FPGA_HW_LIB\n                        and then BBS_LIB_PATH are checked.\n  -f, --force           Overwrite target directory if it exists.\n
            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#32-building-and-running-host_chan_mmio-example-afu","title":"3.2. Building and Running host_chan_mmio example AFU","text":"

            The $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio is a simple example demonstrating both hardware and software access to an AFU. The host_chan_mmio example AFU consists of the following files. The hw directory contains the RTL to implement the hardware functionality using Avalon and AXI interfaces. However, this guide will use the AXI version of the host_chan_mmio AFU to go through the compilation steps. The sw directory of the AFU contains the source code of the host application that communicates with the actual AFU hardware.

            host_chan_mmio\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_avalon.sv\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_axi.sv\n\u2502       \u251c\u2500\u2500 host_chan_mmio.json\n\u2502       \u251c\u2500\u2500 test_mmio_avalon0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon1.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon2_512rw.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi1.txt\n\u2502       \u2514\u2500\u2500 test_mmio_axi2_512rw.txt\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 main.c\n    \u251c\u2500\u2500 Makefile\n
            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#321-build-the-host_chan_mmio-example-afu","title":"3.2.1. Build the host_chan_mmio example AFU","text":"

            Execute afu_synth_setup as follows to create the synthesis environment for a host_chan_mmio AFU that fits the PCIe Attach FIM previously constructed.

            $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio\n$ afu_synth_setup -s ./hw/rtl/test_mmio_axi1.txt afu_dev\n
            Now, move into the synthesis environment afu_dev directory just created. From there, execute the afu_synth command. The successful completion of the command will produce the host_chan_mmio.gbs file under the synthesis environment directory, $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_dev.

            $ cd afu_dev\n$ $OPAE_PLATFORM_ROOT/bin/afu_synth\nCompiling ofs_top ofs_pr_afu\nGenerating host_chan_mmio.gbs\n==================================\n...\n...\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n Design meets timing\n===========================================================================\n

            The previous output indicates the successful compilation of the AFU and the compliance with the timing requirements. Analyze the reports generated in case the design does not meet timing. The timing reports are stored in the directory, $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_dev/build/syn/board/n6001/syn_top/output_files/timing_report.

            Once the compilation finishes successfully, load the new host_chan_mmio.gbs bitstream file into the partial reconfiguration region of the target Intel\u00ae FPGA SmartNIC N6001-PL board. Keep in mind, that the loaded image is dynamic - this image is not stored in flash and if the card is power cycled, then the PR region is re-loaded with the default AFU.

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#322-download-the-host_chan_mmio-example-afu","title":"3.2.2. Download the host_chan_mmio example AFU","text":"

            To test the AFU in actual hardware, load the host_chan_mmio.gbs to the Intel\u00ae FPGA SmartNIC N6001-PL card. For this step to be successful, the PCIe Attach FIM must have already been loaded to the Intel\u00ae FPGA SmartNIC N6001-PL card following the steps described in Section 2 of this document. If you are running on a Virtual Machine, refer to the [KVM User Guide] for passing the devices to the VM.

            Verify Board and PCIe b.d.f. For the following example, the N6001 SKU2 PCIe b:d.f is B1:00.0, however this may be different in your system. 

            $ fpgainfo fme\nIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\n...\n

            Download AFU. 

            $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_dev\n$ sudo fpgasupdate host_chan_mmio.gbs B1:00.0\n[sudo] password for <<Your username>>: \n[2022-04-15 20:22:18.85] [WARNING ] Update starting. Please do not interrupt.\n[2022-04-15 20:22:19.75] [INFO    ] \nPartial Reconfiguration OK\n[2022-04-15 20:22:19.75] [INFO    ] Total time: 0:00:00.90\n

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#323-set-up-host-to-interface-with-example-afu","title":"3.2.3. Set up host to interface with example AFU","text":"

            Set up host to interface with the newly loaded AFU.

            List the PFs available, the default N6001 FIM has 5 PFs. 

            $ lspci -s B1:00\nB1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.2 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.3 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.4 Processing accelerators: Intel Corporation Device bcce (rev 01)\n

            Create the Virtual Functions (VFs) provided by the FIM, the default N6001 FIM has 3 VFs. If your FIM uses only PFs, skip this step. 

            $ sudo pci_device B1:00.0 vf 3\n\n# Verify the VFs have been added (device id: bccf)\n$ lspci -s B1:00\nB1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.2 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.3 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.4 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.5 Processing accelerators: Intel Corporation Device bccf\nB1:00.6 Processing accelerators: Intel Corporation Device bccf\nB1:00.7 Processing accelerators: Intel Corporation Device bccf\n

            Bind PFs and VFs to VFIO driver (except PF0/B1:00.0, which is the FME PF). 

            # Enter your username.\n$ sudo opae.io init -d 0000:b1:00.1 <<Your username>>\n[sudo] password for <<Your username>>:\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 183\nAssigning /dev/vfio/183 to ceg\nChanging permissions for /dev/vfio/183 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.2 <<Your username>>\n[sudo] password for <<Your username>>:\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 184\nAssigning /dev/vfio/184 to ceg\nChanging permissions for /dev/vfio/184 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.3 <<Your username>>\n[sudo] password for <<Your username>>:\nUnbinding (0x1af4,0x1000) at 0000:b1:00.3 from virtio-pci\nBinding (0x1af4,0x1000) at 0000:b1:00.3 to vfio-pci\niommu group for (0x1af4,0x1000) at 0000:b1:00.3 is 185\nAssigning /dev/vfio/185 to ceg\nChanging permissions for /dev/vfio/185 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.4 <<Your username>>\n[sudo] password for <<Your username>>: \nUnbinding (0x8086,0xbcce) at 0000:b1:00.4 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.4 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.4 is 186\nAssigning /dev/vfio/186 to ceg\nChanging permissions for /dev/vfio/186 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.5 <<Your username>>\n[sudo] password for <<Your username>>:\nBinding (0x8086,0xbccf) at 0000:b1:00.5 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.5 is 315\nAssigning /dev/vfio/315 to ceg\nChanging permissions for /dev/vfio/315 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.6 <<Your username>>\n[sudo] password for <<Your username>>:\nBinding (0x8086,0xbccf) at 0000:b1:00.6 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.6 is 316\nAssigning /dev/vfio/316 to ceg\nChanging permissions for /dev/vfio/316 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.7 <<Your username>>\n[sudo] password for <<Your username>>:\nBinding (0x8086,0xbccf) at 0000:b1:00.7 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.7 is 317\nAssigning /dev/vfio/317 to ceg\nChanging permissions for /dev/vfio/317 to rw-rw----\n

            Verify the new AFU is loaded. The host_chan_mmio AFU GUID is \"76d7ae9c-f66b-461f-816a-5428bcebdbc5\". 

            $ fpgainfo port\n//****** PORT ******//\nObject Id                        : 0xEC00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0xE0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.7\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : d15ab1ed-0000-0000-0210-000000000000\n//****** PORT ******//\nObject Id                        : 0xC0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.6\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : d15ab1ed-0000-0000-0110-000000000000\n//****** PORT ******//\nObject Id                        : 0xA0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.5\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 76d7ae9c-f66b-461f-816a-5428bcebdbc5\n//****** PORT ******//\nObject Id                        : 0x80B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.4\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 44bfc10d-b42a-44e5-bd42-57dc93ea7f91\n//****** PORT ******//\nObject Id                        : 0x6098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 1aae155c-acc5-4210-b9ab-efbd90b970c4\n//****** PORT ******//\nObject Id                        : 0x40B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n//****** PORT ******//\nObject Id                        : 0x20B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 3e7b60a0-df2d-4850-aa31-f54a3e403501\n

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#324-run-the-host_chan_mmio-example-afu","title":"3.2.4. Run the host_chan_mmio example AFU","text":"

            Now, navigate to the directory of the host_chan_mmio AFU containing the host application's source code, $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw. Once there, compile the host_chan_mmio host application and execute it on the host server to excercise the functionality of the AFU.

            Note: If OPAE SDK libraries were not installed in the default systems directories under /usr, you need to set the OPAE_LOC, LIBRARY_PATH, and LD_LIBRARY_PATH environment variables to the custom locations where the OPAE SDK libraries were installed.

            # Move to the sw directory of the the host_chan_mmio AFU. This directory holds the source for the host application.\n$ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw\n$ export OPAE_LOC=/usr\n$ export LIBRARY_PATH=$OPAE_LOC/lib:$LIBRARY_PATH\n$ export LD_LIBRARY_PATH=$OPAE_LOC/lib64:$LD_LIBRARY_PATH\n$ make\n\n# Run the application\n$  ./host_chan_mmio\nAFU ID:  76d7ae9cf66b461f 816a5428bcebdbc5\nAFU MMIO interface: AXI Lite\nAFU MMIO read bus width: 64 bits\n512 bit MMIO write supported: yes\nAFU pClk frequency: 470 MHz\n\nTesting 32 bit MMIO reads:\n  PASS - 4 tests\n\nTesting 32 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 64 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 512 bit MMIO writes:\n  PASS\n
            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#33-building-and-running-the-hello_world-example-afu","title":"3.3. Building and running the hello_world example AFU","text":"

            The platform-independent examples AFU repository also provides some interesting example AFUs. In this section, you will compile and execute the PIM based hello_world AFU. The RTL of the hello_world AFU receives from the host application an address via memory mapped I/O (MMIO) write and generates a DMA write to the memory line at that address. The content written to memory is the string \"Hello world!\". The host application spins, waiting for the memory line to be updated. Once available, the software prints out the string.

            The hello_world example AFU consists of the following files. The hw directory contains the RTL to implement the hardware functionality using CCIP, Avalon, and AXI interfaces. However, this guide will use the AXI version of the AFU to go through the compilation steps. The sw directory of the AFU contains the source code of the host application that communicates with the AFU hardware.

            hello_world\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u251c\u2500\u2500 ccip\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_ccip.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u2514\u2500\u2500 hello_world.json\n\u251c\u2500\u2500 README.md\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 hello_world\n    \u251c\u2500\u2500 hello_world.c\n    \u251c\u2500\u2500 Makefile\n    \u2514\u2500\u2500 obj\n        \u251c\u2500\u2500 afu_json_info.h\n        \u2514\u2500\u2500 hello_world.o\n

            The following instructions can be used to compile other AFU samples accompanying this repository.

            If not done already, download and clone the examples AFU repository. 

            $ cd $OFS_BUILD_ROOT \n$ git clone https://github.com/OFS/examples-afu.git\n

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#331-build-the-hello_world-example-afu","title":"3.3.1. Build the hello_world example AFU","text":"

            Compile the hello_word sample AFU. 

            $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world\n$ afu_synth_setup --source hw/rtl/axi/sources.txt afu_dev\n$ cd afu_dev\n$ $OPAE_PLATFORM_ROOT/bin/afu_synth\nCompiling ofs_top ofs_pr_afu\nGenerating hello_world.gbs\n==================================\n.\n.\n.\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'hello_world.gbs'\n Design meets timing\n===========================================================================\n

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#332-download-the-hello_world-example-afu","title":"3.3.2. Download the hello_world example AFU","text":"

            To test the AFU in actual hardware, load the hello_world.gbs to the Intel\u00ae FPGA SmartNIC N6001-PL card. For this step to be successful, the PCIe Attach FIM must have already been loaded to the Intel\u00ae FPGA SmartNIC N6001-PL card following the steps described in Section 2 of this document. If you are running on a Virtual Machine, refer to the [KVM User Guide] for passing the devices to the VM.

            Verify Board and PCIe b.d.f. For the following example, the N6001 SKU2 PCIe b:d.f is B1:00.0, however this may be different in your system. 

            $ fpgainfo fme\nIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\n...\n

            Download AFU. 

            $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/afu_dev\n$ sudo fpgasupdate hello_world.gbs B1:00.0\n  [sudo] password for <<Your username>>: \n[2022-04-15 20:22:18.85] [WARNING ] Update starting. Please do not interrupt.\n[2022-04-15 20:22:19.75] [INFO    ] \nPartial Reconfiguration OK\n[2022-04-15 20:22:19.75] [INFO    ] Total time: 0:00:00.90\n

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#333-set-up-host-to-interface-with-example-afu","title":"3.3.3. Set up host to interface with example AFU","text":"

            Set up your Intel\u00ae FPGA SmartNIC N6001-PL board to work with the newly loaded hello_world.gbs file. 

            # List the PF's available, the default N6001 FIM has 5 PF's\n$ lspci -s B1:00\nB1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.2 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.3 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.4 Processing accelerators: Intel Corporation Device bcce (rev 01)\n

            Download AFU. 

            # Create the Virtual Functions (VFs) provided by the FIM, the default N6001 FIM has 3 VFs.  \n# If your FIM uses only PFs, skip this step.\n$ sudo pci_device B1:00.0 vf 3\n\n# Verify the VFs have been added (device id: bccf)\n$ lspci -s B1:00\nB1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.2 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.3 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.4 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.5 Processing accelerators: Intel Corporation Device bccf\nB1:00.6 Processing accelerators: Intel Corporation Device bccf\nB1:00.7 Processing accelerators: Intel Corporation Device bccf\n

            Bind PFs and VFs to VFIO driver (except PF0/B1:00.0, which is the FME PF). 

            #Enter your username\n$ sudo opae.io init -d 0000:b1:00.1 <<Your username>>\n[sudo] password for <<Your username>>:\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 183\nAssigning /dev/vfio/183 to ceg\nChanging permissions for /dev/vfio/183 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.2 <<Your username>>\n[sudo] password for <<Your username>>:\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 184\nAssigning /dev/vfio/184 to ceg\nChanging permissions for /dev/vfio/184 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.3 <<Your username>>\n[sudo] password for <<Your username>>:\nUnbinding (0x8086,0xbcce)  at 0000:b1:00.3 from virtio-pci\nBinding (0x8086,0xbcce)  at 0000:b1:00.3 to vfio-pci\niommu group for (0x8086,0xbcce)  at 0000:b1:00.3 is 185\nAssigning /dev/vfio/185 to ceg\nChanging permissions for /dev/vfio/185 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.4 <<Your username>>\n[sudo] password for <<Your username>>: \nUnbinding (0x8086,0xbcce) at 0000:v1:00.4 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.4 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.4 is 186\nAssigning /dev/vfio/186 to ceg\nChanging permissions for /dev/vfio/186 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.5 <<Your username>>\n[sudo] password for <<Your username>>:\nBinding (0x8086,0xbccf) at 0000:b1:00.5 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.5 is 315\nAssigning /dev/vfio/315 to ceg\nChanging permissions for /dev/vfio/315 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.6 <<Your username>>\n[sudo] password for <<Your username>>:\nBinding (0x8086,0xbccf) at 0000:b1:00.6 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.6 is 316\nAssigning /dev/vfio/316 to ceg\nChanging permissions for /dev/vfio/316 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.7 <<Your username>>\n[sudo] password for <<Your username>>:\nBinding (0x8086,0xbccf) at 0000:b1:00.7 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.7 is 317\nAssigning /dev/vfio/317 to ceg\nChanging permissions for /dev/vfio/317 to rw-rw----\n

            Verify the new AFU is loaded. The hello_world AFU GUID is \"c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\". 

            $ fpgainfo port\n\n//****** PORT ******//\nObject Id                        : 0xEE00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0xE098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.7\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : d15ab1ed-0000-0000-0210-000000000000\n//****** PORT ******//\nObject Id                        : 0xC098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.6\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : d15ab1ed-0000-0000-0110-000000000000\n//****** PORT ******//\nObject Id                        : 0xA098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.5\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\n//****** PORT ******//\nObject Id                        : 0x8098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.4\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 44bfc10d-b42a-44e5-bd42-57dc93ea7f91\n//****** PORT ******//\nObject Id                        : 0x6098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 1aae155c-acc5-4210-b9ab-efbd90b970c4\n//****** PORT ******//\nObject Id                        : 0x4098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n//****** PORT ******//\nObject Id                        : 0x2098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 3e7b60a0-df2d-4850-aa31-f54a3e403501\n

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#334-run-the-hello_world-example-afu","title":"3.3.4. Run the hello_world example AFU","text":"

            Compile and execute the host application of the hello_world AFU. You should see the application outputs the \"Hello world!\" message in the terminal.

            # Move to the sw directory of the hello_world AFU\n$ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/sw\n\n$ make\n\n# Launch the host application\n$ ./hello_world\n  Hello world!\n
            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#34-modify-the-afu-user-clocks-frequency","title":"3.4. Modify the AFU user clocks frequency","text":"

            An OPAE compliant AFU specifies the frequency of the uclk_usr and uclk_usr_div2 clocks through the JSON file for AFU configuration located under the <afu_example>/hw/rtl directory of an AFU design. For instance, the AFU configuration file of the host_chan_mmio example is $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/host_chan_mmio.json.

            The AFU specifies the frequency for uClk_usr in its platform configuration file using the following key:value pairs:

              \"clock-frequency-high\": [<float-value>|\u201dauto\u201d|\u201dauto-<float-value>\u201d]\n  \"clock-frequency-low\": [<float-value>|\u201dauto\u201d|\u201dauto-<float-value>\u201d]\n

            These key:value tuples are used to configure the PLL of the target platform that provides the user clocks through the AFU clocks interface. In addition, the specified frequency affects the timing closure process on the user clocks during AFU compilation.

            Setting the value field to a float number (e.g., 315.0 to specify 315 MHz) drives the AFU generation process to close timing within the bounds set by the low and high values and sets the AFU's JSON metadata to specify the user clock PLL frequency values.

            The following example shows the JSON file of the host_chan_mmio to set the AFU uClk to 500 MHz and uClk_div2 to 250 MHz.

            {\n   \"version\": 1,\n   \"afu-image\": {\n      \"power\": 0,\n      \"clock-frequency-high\": 500,\n      \"clock-frequency-low\": 250,\n      \"afu-top-interface\":\n         {\n            \"class\": \"ofs_plat_afu\"\n         },\n      \"accelerator-clusters\":\n         [\n            {\n               \"name\": \"host_chan_mmio\",\n               \"total-contexts\": 1,\n               \"accelerator-type-uuid\": \"76d7ae9c-f66b-461f-816a-5428bcebdbc5\"\n            }\n         ]\n   }\n}\n

            Save the changes to host_chan_mmio.json file, then execute the afu_synth_setup script to create a new copy of the AFU files with the modified user clock settigns.

            $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio\n$ afu_synth_setup -s ./hw/rtl/test_mmio_axi1.txt afu_clks\n\nCopying build from /home/<user_area>/ofs-agx7-pcie-attach/work_pr/pr_build_template/hw/lib/build...\nConfiguring Quartus build directory: build_n6001_afu_clks/build\nLoading platform database: /home/<user_area>/ofs-agx7-pcie-attach/work_pr/pr_build_template/hw/lib/platform/platform_db/ofs_agilex_adp.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting platform/platform_afu_top_config.vh\nWriting platform/platform_if_addenda.qsf\nWriting ../hw/afu_json_info.vh\n
            Compile the host_chan_mmio AFU with the new frequency values.

            $ cd afu_clks\n$ $OPAE_PLATFORM_ROOT/bin/afu_synth\n

            During the compilation phase, you will observe the Timing Analyzer uses the specified user clock frequency values as the target to close timing.

            AFU developers must ensure the AFU hardware design meets timing. The compilation of an AFU that fails timing shows a message similar to the following.

            .\n.\n.\n\nWrote host_chan_mmio.gbs\n\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n\n  *** Design does not meet timing\n  *** See build/syn/board/n6001/syn_top/output_files/timing_report\n\n===========================================================================\n

            The previous output indicates the location of the timing reports for the AFU designer to identify the failing paths and perform the necessary design changes. Next, is a listing of the timing report files from a host_chan_mmio AFU that fails to meet timing after modifying the user clock frequency values.

            $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/n6001/afu_clks\n$ ls build/syn/board/n6001/syn_top/output_files/timing_report\n\nclocks.rpt  clocks.sta.fail.summary  clocks.sta.pass.summary\n

            Warning: AFU developers must inform software developers of the maximum operating frequency (Fmax) of the user clocks to avoid any unexpected behavior of the accelerator and potentially of the overall system.

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#4-simulating-an-afu-using-ase","title":"4. Simulating an AFU using ASE","text":"

            The Application Simulation Environment (ASE) is a hardware/software co-simulation environment for your AFU. See diagram below illustrating ASE operation:

            ASE uses the simulator Direct Programming Interface (DPI) to provide HW/SW connectivity. The PCIe connection to the AFU under testing is emulated with a transactional model.

            The following list describes ASE operation:

            • Attempts to replicate the transactions that will be seen in real system.
            • Provides a memory model to AFU, so illegal memory accesses can be identified early.
            • Not a cache simulator.
            • Does not guarantee synthesizability or timing closure.
            • Does not model system latency.
            • No administrator privileges are needed to run ASE. All code is user level.

            The remainder of this section is a tutorial providing the steps on how to run ASE with either Synopsys\u00ae VCS\u00ae or Siemens\u00ae QuestaSim\u00ae using an example AFU and the AFU build tree previously created in this guide.

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#41-set-up-steps-to-run-ase","title":"4.1. Set Up Steps to Run ASE","text":"

            In this section you will set up your server to support ASE by independently downloading and installing OPAE SDK and ASE. Then, set up the required environment variables.

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#411-install-opae-sdk","title":"4.1.1. Install OPAE SDK","text":"

            Follow the instructions documented in the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 FPGAs Targeting the Intel\u00ae FPGA SmartNIC N6001-PL, section 4.0 OPAE Software Development Kit to build and install the required OPAE SDK for the Intel\u00ae FPGA SmartNIC N6001-PL card.

            The N6001 SKU2 card requires 2.10.0-1. Follow the instructions provided in the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 FPGAs Targeting the Intel\u00ae FPGA SmartNIC N6001-PL, section 4.0 OPAE Software Development Kit. However, just make sure to check out the cloned repository to tag 2.10.0-1 and branch release/2.10.0.

            $ git checkout tags/2.10.0-1 -b release/2.10.0\n
            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#412-install-ase-tools","title":"4.1.2 Install ASE Tools","text":"

            ASE is an RTL simulator for OPAE-based AFUs. The simulator emulates both the OPAE SDK software user space API and the AFU RTL interface. The majority of the FIM as well as devices such as PCIe and local memory are emulated with simple functional models.

            ASE must be installed separatedly from the OPAE SDK. However, the recommendation is to install it in the same target directory as OPAE SDK.

            1. If not done already, set the environment variables as described in section, Set Up AFU Development Environment.

            2. Clone the opae-sim repository. 

              $ cd $OFS_BUILD_ROOT\n$ git clone https://github.com/OFS/opae-sim.git\n$ cd opae-sim  \n$ git checkout tags/2.10.0-1 -b release/2.10.0\n

            3. Create a build directory and build ASE to be installed under the default system directories along with OPAE SDK. 

              $ mkdir build\n$ cd build\n$ cmake  -DCMAKE_INSTALL_PREFIX=/usr ..\n$ make\n

            Optionally, if the desire is to install ASE binaries in a different location to the system's default, provide the path to CMAKE through the CMAKE_INSTALL_PREFIX switch, as follows. 

            $ cmake -DCMAKE_INSTALL_PREFIX=<</some/arbitrary/path>> ..  \n

            1. Install ASE binaries and libraries under the system directory /usr. 
              $ sudo make install  \n
            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#413-setup-required-ase-environment-variables","title":"4.1.3. Setup Required ASE Environment Variables","text":"

            The values set to the following environment variables assume the OPAE SDK and ASE were installed in the default system directories below /usr. Setup these variables in the shell where ASE will be executed. You may wish to add these variables to the script you created to facilitate configuring your environment.

            $ export OPAE_PLATFORM_ROOT=<path to PR build tree>\n$ cd /usr/bin\n$ export PATH=$PWD:$PATH\n$ cd /usr/lib/python*/site-packages\n$ export PYTHONPATH=$PWD\n$ cd /usr/lib\n$ export LIBRARY_PATH=$PWD\n$ cd /usr/lib64\n$ export LD_LIBRARY_PATH=$PWD\n$ cd $OFS_BUILD_ROOT/ofs-platform-afu-bbb\n$ export OFS_PLATFORM_AFU_BBB=$PWD\n\n  ## For VCS, set the following:\n\n$ export VCS_HOME=<Set the path to VCS installation directory>\n$ export PATH=$VCS_HOME/bin:$PATH\n\n  ## For QuestaSIM, set the following:\n$ export MTI_HOME=<path to Modelsim installation directory>\n$ export PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\n
            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#42-simulating-the-host_chan_mmio-afu","title":"4.2. Simulating the host_chan_mmio AFU","text":"

            The $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio is a simple example demonstrating both hardware and software access to an AFU. The host_chan_mmio example AFU consists of the following files:

            host_chan_mmio\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_avalon.sv\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_axi.sv\n\u2502       \u251c\u2500\u2500 host_chan_mmio.json\n\u2502       \u251c\u2500\u2500 test_mmio_avalon0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon1.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon2_512rw.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi1.txt\n\u2502       \u2514\u2500\u2500 test_mmio_axi2_512rw.txt\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 main.c\n    \u251c\u2500\u2500 Makefile\n

            This example AFU contains examples using both Avalon and AXI interface buses. This guide will use the AXI version of the host_chan_mmio AFU.

            ASE uses client-server application architecture to deliver hardware/software co-simulation. You require one shell for the hardware based simulation and another shell where the software application is running. The hardware is started first with a simulation compilation and simulator startup script, once the simulator has loaded the design, it will wait until the software process starts. Once the software process starts, the simulator proceeds. Transaction logging and waveform capture is performed.

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#421-set-up-and-run-the-hw-simulation-process","title":"4.2.1 Set Up and Run the HW Simulation Process","text":"

            You will run the afu_sim_setup script to create the scripts for running the ASE environment. The afu_sim_setup script has the following usage:

            usage: afu_sim_setup [-h] -s SOURCES [-p PLATFORM] [-t {VCS,QUESTA,MODELSIM}]\n                     [-f] [--ase-mode ASE_MODE] [--ase-verbose]\n                     dst\n\nGenerate an ASE simulation environment for an AFU. An ASE environment is\ninstantiated from the OPAE installation and then configured for the specified\nAFU. AFU source files are specified in a text file that is parsed by\nrtl_src_config, which is also part of the OPAE base environment.\n\npositional arguments:\n  dst                   Target directory path (directory must not exist).\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -s SOURCES, --sources SOURCES\n                        AFU source specification file that will be passed to\n                        rtl_src_config. See \"rtl_src_config --help\" for the\n                        file's syntax. rtl_src_config translates the source\n                        list into either Quartus or RTL simulator syntax.\n  -p PLATFORM, --platform PLATFORM\n                        FPGA Platform to simulate.\n  -t {VCS,QUESTA,MODELSIM}, --tool {VCS,QUESTA,MODELSIM}\n                        Default simulator.\n  -f, --force           Overwrite target directory if it exists.\n  --ase-mode ASE_MODE   ASE execution mode (default, mode 3, exits on\n                        completion). See ase.cfg in the target directory.\n  --ase-verbose         When set, ASE prints each CCI-P transaction to the\n                        command line. Transactions are always logged to\n                        work/ccip_transactions.tsv, even when not set. This\n                        switch sets ENABLE_CL_VIEW in ase.cfg.\n

            Run afu_sim_setup to create the ASE simulation environment for the host_chan_mmio example AFU. The '-t VCS' option indicates to prepare the ASE simulation environment for Synopsys\u00ae VCS\u00ae.

            $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio\n\n$ afu_sim_setup -s $./hw/rtl/test_mmio_axi1.txt -t VCS afu_sim\n\nCopying ASE from /opae-sdk/install-opae-sdk/share/opae/ase...\n#################################################################\n#                                                               #\n#             OPAE Intel(R) Xeon(R) + FPGA Library              #\n#               AFU Simulation Environment (ASE)                #\n#                                                               #\n#################################################################\n\nTool Brand: VCS\nLoading platform database: /ofs-agx7-pcie-attach/work_pr/build_tree/hw/lib/platform/platform_db/ofs_agilex_adp.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting rtl/platform_afu_top_config.vh\nWriting rtl/platform_if_addenda.txt\nWriting rtl/platform_if_includes.txt\nWriting rtl/ase_platform_name.txt\nWriting rtl/ase_platform_config.mk and rtl/ase_platform_config.cmake\nASE Platform: discrete (FPGA_PLATFORM_DISCRETE)\n

            The afu_sim_setup creates the ASE scripts in the directory host_chan_mmio_sim where the afu_sim_setup script was run. Start the simulator as shown below:

            $ cd afu_sim\n$ make\n$ make sim\n

            This process launches the AFU hardware simulator. Before moving to the next section, pay attention to the simulator output highlighted in the image below.

            The simulation artifacts are stored in host_chan_mmio/work and consist of:

            log_ase_events.tsv\nlog_ofs_plat_host_chan.tsv \nlog_ofs_plat_local_mem.tsv \nlog_pf_vf_mux_A.tsv \nlog_pf_vf_mux_B.tsv \n
            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#422-set-up-and-run-the-sw-process","title":"4.2.2 Set Up and Run the SW Process","text":"

            Open an additional shell to build and run the host application that communicates with the actual AFU hardware. Set up the same environment variable you have set up in the shell you have been working on until this point.

            Additionally, as indicated by the hardware simulator output that is currently executing in the \"simulator shell\", copy and paste the line \"export ASE_WORKDIR=...\", into the new \"software shell\". See the last image of the previous section.

            $ export ASE_WORKDIR=$OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_sim/work\n
            Then, go to the sw directory of the host_chan_mmio AFU example to compile the host application.

            $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw  \n$ make\n\nafu_json_mgr json-info --afu-json=../hw/rtl/host_chan_mmio.json --c-hdr=obj/afu_json_info.h\nWriting obj/afu_json_info.h\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c main.c -o obj/main.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c test_host_chan_mmio.c -o obj/test_host_chan_mmio.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/connect.c -o obj/connect.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/csr_mgr.c -o obj/csr_mgr.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/hash32.c -o obj/hash32.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/test_data.c -o obj/test_data.o\ncc -o host_chan_mmio obj/main.o obj/test_host_chan_mmio.o obj/connect.o obj/csr_mgr.o obj/hash32.o obj/test_data.o  -z noexecstack -z relro -z now -pie -luuid -lopae-c-ase\n

            Now, launch the host application to exercise the AFU hardware running on the simulator shell. The next image shows the AFU hardware simulation process on the left side shell. The right hand shell shows the host application's output of a successful simulation.

            $ with_ase ./host_chan_mmio\n  [APP]  Initializing simulation session ...\nRunning in ASE mode\nAFU ID:  76d7ae9cf66b461f 816a5428bcebdbc5\nAFU MMIO interface: AXI Lite\nAFU MMIO read bus width: 64 bits\n512 bit MMIO write supported: yes\nAFU pClk frequency: 470 MHz\n\nTesting 32 bit MMIO reads:\n  PASS - 4 tests\n\nTesting 32 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 64 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 512 bit MMIO writes:\n  PASS\n  [APP]  Deinitializing simulation session\n  [APP]         Took 1,003,771,568 nsec\n  [APP]  Session ended\n

            Finally, on the hardware simulation shell, you can view the wave forms by invoking the following command.

            $ make wave\n

            This brings up the VCS\u00ae simulator GUI and loads the simulation waveform files. Use the Hierarchy window to navigate to the afu instance located under, ase_top | ase_top_plat | ase_afu_main_pcie_ss | ase_afu_main_emul | afu_main | port_afu_instances | ofs_plat_afu | afu , as shown below.

            Right click on the afu (afu) entry to display the drop-down menu. Then, click on Add to Waves | New Wave View to display the following waveforms window.

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#43-simulating-the-hello_world-afu","title":"4.3 Simulating the hello_world AFU","text":"

            In this section you will quickly simulate the PIM-based hello_world sample AFU accompanying the examples-afu repository.

            1. Set the environment variables as described in section 4.1. Set Up Steps to Run ASE.

            2. Prepare an RTL simulation environment for the AXI version of the hello_world AFU.

            Simulation with ASE requires two software processes, one to simulate the AFU RTL and the other to run the host software that excercises the AFU. To construct an RTL simulation environment under the directory simulation, execute the following.

            $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world\n$ afu_sim_setup -s ./hw/rtl/axi/sources.txt -t VCS afu_sim\n\n\nCopying ASE from /usr/local/share/opae/ase...\n#################################################################\n#                                                               #\n#             OPAE Intel(R) Xeon(R) + FPGA Library              #\n#               AFU Simulation Environment (ASE)                #\n#                                                               #\n#################################################################\n\nTool Brand: VCS\nLoading platform database: /home/<user_area>/ofs-agx7-pcie-attach/work_pr/pr_build_template/hw/lib/platform/platform_db/ofs_agilex_adp.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting rtl/platform_afu_top_config.vh\nWriting rtl/platform_if_addenda.txt\nWriting rtl/platform_if_includes.txt\nWriting rtl/ase_platform_name.txt\nWriting rtl/ase_platform_config.mk and rtl/ase_platform_config.cmake\nASE Platform: discrete (FPGA_PLATFORM_DISCRETE)\n

            The afu_sim_setup script constructs an ASE environment in the hello_world_sim subdirectory. If the command fails, confirm that the path to the afu_sim_setup is on your PATH environment variable (in the OPAE SDK bin directory) and that your Python version is at least 2.7.

            1. Build and execute the AFU RTL simulator.
            $ cd afu_sim\n$ make\n$ make sim  \n

            The previous commands will build and run the Synopsys\u00ae VCS\u00ae RTL simulator, which prints a message saying it is ready for simulation. The simulation process also prints a message instructing you to set the ASE_WORKDIR environment variable in a second shell.

            1. Open a second shell where you will build and execute the host software. In this new \"software shell\", set up the environment variables you have set up so far in the \"hardware simulation\" shell.

            2. Also, set the ASE_WORKDIR environment variable following the instructions given in the \"hardware simulation\" shell.

            $ export ASE_WORKDIR=$OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/afu_sim/work\n
            6. Then, move to the sw directory of the hello_world AFU sample to build the host software.

            $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/sw\n$ make      \n
            1. Run the hello_world host application to resume the work of the RTL simulation. The host software process and the RTL simulation execute in lockstep. If successful, you should see the Hello world! output.
            $ with_ase ./hello_world\n\n[APP]  Initializing simulation session ...\nHello world!\n[APP]  Deinitializing simulation session\n[APP]         Took 43,978,424 nsec\n[APP]  Session ended\n

            The image below shows the simulation of the AFU hardware and the execution of the host application side-by-side.

            1. Finally, on the hardware simulation shell, you can view the wave forms by invoking the following command.
            make wave\n

            This brings up the DVE GUI and loads the simulation waveform files. Use the Hierarchy window to navigate to the afu instance located under, ase_top | ase_top_plat | ase_afu_main_pcie_ss | ase_afu_main_emul | afu_main | port_afu_instances | ofs_plat_afu | hello_afu, as shown below.

            Right click on the hello_afu entry to display the drop-down menu. Then, click on Add to Waves | New Wave View to display the following waveforms window.

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#5-adding-remote-signal-tap-logic-analyzer-to-debug-the-afu","title":"5. Adding Remote Signal Tap Logic Analyzer to debug the AFU","text":"

            The OPAE SDK provides a remote Signal Tap facility. It also supports the following in system debug tools included with the Intel Quartus Prime Pro Edition:

            • In-system Sources and Probes
            • In-system Memory Content Editor
            • Signal Probe
            • System Console

            This section is a short guide on adding remote Signal Tap instances to an AFU for in system debugging. You can follow the steps in the following sections, in order of execution to create an instrumented AFU. The host_chan_mmio AFU is used in this guide as the target AFU to be instrumented.

            You need a basic understanding of Signal Tap. Please see the Signal Tap Logic Analyzer: Introduction & Getting Started Web Based Training for more information.

            You will run with a Signal Tap GUI running locally on the server with the Intel\u00ae FPGA SmartNIC N6001-PL as shown below:

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#51-adding-rstp-to-the-host_chan_mmio-afu","title":"5.1. Adding RSTP to the host_chan_mmio AFU","text":"

            RSTP is added to an AFU by:

            1. Defining signals to be instrumented in Signal Tap. This creates a new *.stp file.
            2. Modify ofs_top.qpf to include the new *.stp file
            3. Modify ofs_top.qsf
            4. Modify ofs_pr_afu.qsf
            5. Run $OPAE_PLATFORM_ROOT/bin/afu_synth to build the PR-able image containing the RSTP instance

            You can use these detailed steps to add Signal Tap to your AFU.

            1. Set path to platform root directory and create the host_chan_mmio AFU Quartus project for adding Signal Tap.: 

              $ export OPAE_PLATFORM_ROOT=<path to PR build tree>\n\n # we will now build a new host_chahnel_mmio example based on Signal Tap\n\n$ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio\n$ afu_synth_setup -s ./hw/rtl/test_mmio_axi1.txt afu_stp\n

            2. Navigate to host_chan_mmio AFU Quartus project and open the project using Quartus GUI. 

              $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp/build/syn/board/n6001/syn_top\n$ quartus ofs_top.qpf &\n

            3. Once the project is loaded in Quartus, run Analysis & Synthesis Processing | Start | Start Analysis & Synthesis. When complete, review the project hierarchy as shown in the Project Navigator. This example will add Signal Tap probe points to the AFU region. Reviewing the code will give insight into the function of this block. You can bring up the code in the Project Navigator by expanding afu_top - port_gasket - pr_slot - afu_main - port_afu_instances - ofs_plat_afu, then select instance afu, right click, select Locacte Node - Locate in Design File as shown below.

            4. Bring up Signal Tap to create the *.stp file. In the Quartus GUI, go to Tools - Signal Tap Logic Analyzer. In the New File from Template pop up, click Create to accept the default template. The Signal Tap Logic Analyzer window comes up.

            5. Set up the clock for the Signal Tap logic instance by clicking ... button as shown below:

            6. The Node Finder comes up and you will click ... as shown below to bring up the hierarchy navigator:

            7. In the Select Hierarchy Level, navigate to top - afu_top - pg_afu.port_gasket - pr_slot - afu_main - port_afu_instances - ofs_plat_afu, then select instance afu and click Ok.

            8. Enter *clk* in the Named: box and click Search. This brings up matching terms. Click clk and >. Verify your Node Finder is as shown below and then click Ok:

            9. Double click the Double-click to add nodes and once again, click ... and navigate to top - afu_top - port_gasket - pr_slot - afu_main - port_afu_instances - ofs_plat_afu, then select instance afu and click Ok. Enter mmio64_reg* and click Search. Then click >> to add these signals to the STP instance as shown below:

              Then click Insert and Close.

            10. Save the newly created STP by clicking File - Save As and in the save as navigate to $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp/build/syn/board/n6001/syn_top and save the STP file as host_chan_mmio.stp as shown below:

            Select Yes when asked to add host_chan_mmio.stp to current project. Close Signal Tap window.

            1. Edit ofs_top.qsf to add host_chan_mmio.stp file and enable STP. Open $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp/build/syn/board/n6001/syn_top/ofs_top.qsf in an editor and add the lines shown below:
            set_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE host_chan_mmio.stp\nset_global_assignment -name SIGNALTAP_FILE host_chan_mmio.stp\n

            And also ensure \"INCLUDE_REMOTE_STP\" is enabled in ofs_top.qsf.

            # At most one of INCLUDE_REMOTE_STP and INCLUDE_JTAG_PR_STP should be\n# set. If both are defined, JTAG-based SignalTap takes precedence.\n# Remote STP uses mmlink. JTAG_PR_STP is on node 0 of the FPGA chain.\nset_global_assignment -name VERILOG_MACRO \"INCLUDE_REMOTE_STP\"       # Includes Remote SignalTap support in PR Region\n#set_global_assignment -name VERILOG_MACRO \"INCLUDE_JTAG_PR_STP\"      # Includes JTAG-based SignalTap via programming cable in the PR region\n

            Save the ofs_top.qsf.

            1. Edit ofs_pr_afu.qsf to add host_chan_mmio.stp file and enable STP. Open $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp/build/syn/board/n6001/syn_top/ofs_pr_afu.qsf in an editor and add the lines shown below:

            set_global_assignment -name VERILOG_MACRO \"INCLUDE_REMOTE_STP\"\nset_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE host_chan_mmio.stp\nset_global_assignment -name SIGNALTAP_FILE host_chan_mmio.stp\n
            Save the ofs_pr_afu.qsf and close Quartus.

            1. The host_chan_mmio AFU Quartus project is ready to be built. In your original build shell enter the following commands:
            $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp\n$ $OPAE_PLATFORM_ROOT/bin/afu_synth\n\n...\n...\nWrote host_chan_mmio.gbs\n\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n  Design meets timing\n===========================================================================\n
            1. Once compilation completes, the new host_chan_mmio.gbs file that contains the Signal Tap instance can be loaded.
             # For the following example, the N6001 SKU2 PCIe b:d.f is assumed to be b1:00.0,\n # however this may be different in your system\n\n# Enusre FIM is loading prior to loading AFU\n# Load AFU\n$ sudo fpgasupdate host_chan_mmio.gbs b1:00.0\n[2021-12-04 07:16:59,101] [WARNING ] Update starting. Please do not interrupt.\n[2021-12-04 07:16:59,740] [INFO    ] \nPartial Reconfiguration OK\n
            1. Use the OPAE SDK mmlink tool to create a TCP/IP connection to your Intel Agilex card under test. The mmlink command has the following format:
            Usage:\nmmlink\n<Segment>             --segment=<SEGMENT NUMBER>\n<Bus>                 --bus=<BUS NUMBER>           OR  -B <BUS NUMBER>\n<Device>              --device=<DEVICE NUMBER>     OR  -D <DEVICE NUMBER>\n<Function>            --function=<FUNCTION NUMBER> OR  -F <FUNCTION NUMBER>\n<Socket-id>           --socket-id=<SOCKET NUMBER>  OR  -S <SOCKET NUMBER>\n<TCP PORT>            --port=<PORT>                OR  -P <PORT>\n<IP ADDRESS>          --ip=<IP ADDRESS>            OR  -I <IP ADDRESS>\n<Version>             -v,--version Print version and exit\n

            Enter the command below to create a connection using port 3333:

            $ sudo mmlink -P 3333 -B 0xb1\n\n ------- Command line Input START ----\n\n Socket-id             : -1\n Port                  : 3333\n IP address            : 0.0.0.0\n ------- Command line Input END   ----\n\nPORT Resource found.\nServer socket is listening on port: 3333\n

            Leave this shell open with the mmlink connection.

            1. In this step you will open a new shell and enable JTAG over protocol. You must have Quartus 23.3 Programmer loaded on the N6001 server for local debugging.
            $ jtagconfig --add JTAG-over-protocol sti://localhost:0/intel/remote-debug/127.0.0.1:3333/0\n\n# Verify connectivity with jtagconfig --debug\n\n$ jtagconfig --debug\n1) JTAG-over-protocol [sti://localhost:0/intel/remote-debug/127.0.0.1:3333/0]\n   (JTAG Server Version 21.4.0 Build 67 12/06/2021 SC Pro Edition)\n    020D10DD   VTAP10 (IR=10)\n    Design hash    86099113E08364C07CC4\n    + Node 00406E00  Virtual JTAG #0\n\n  Captured DR after reset = (020D10DD) [32]\n  Captured IR after reset = (155) [10]\n  Captured Bypass after reset = (0) [1]\n  Captured Bypass chain = (0) [1]\n
            1. Start Quartus Signal Tap GUI, connect to target, load stp file by navigating to $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp/build/syn/board/n6001/syn_top. The Quartus Signal Tap must be the same version of Quartus used to compile the host_chan_mmio.gbs. Quartus Prime Pro Version 23.3 is used in the steps below:
            $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp/build/syn/board/n6001/syn_top\n$ quartus_stpw host_chan_mmio.stp &\n

            This command brings up Signal Tap GUI. Connect to the Signal Tap over protocol by selecting the Hardware button on the right side of the GUI and click the \"Please Select\" pull down as shown below:

            JTAG over protocol selected:

            This connection process will take approximately 2-3 minutes for the Signal Tap instance to indicate \"Ready to acquire\".

            1. Set the trigger condition for a rising edge on signal awvalid signal.

            2. In the Signal Tap window, enable acquisition by pressing key F5, the Signal Tap GUI will indicate \"Acquisition in progress\". Create and bind the VFs, then run the host_chan_mmio application following 3.2. Loading and Running host_chan_mmio example AFU, and observe that the Signal Tap instance has triggered. You should see signals being captured in the Signaltap GUI.

            See captured image below:

            To end your Signal Tap session, close the Signal Tap GUI, then in the mmlink shell, enter ctrl c to kill the mmlink process. To remove the JTAG over protocol connection:

            # This is assuming the JTAG over protocol is instance '1', as shown during jtagconfig --debug\n$ jtagconfig --remove 1\n
            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#6-how-to-modify-the-pfvf-mux-configuration","title":"6. How to modify the PF/VF MUX configuration","text":"

            For information on how to modify the PF/VF mapping for your own design, refer to the FPGA Interface Manager Developer Guide for Intel\u00ae Agilex\u00ae 7 PCIe Attach.

            "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#notices-disclaimers","title":"Notices & Disclaimers","text":"

            Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Performance varies by use, configuration and other factors. Your costs and results may vary. You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications. Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards. \u00a9 Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.

            OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122.

            "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/","title":"oneAPI Accelerator Support Package (ASP): Getting Started User Guide","text":""},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#10-introduction","title":"1.0 Introduction","text":""},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#11-about-this-document","title":"1.1 About This Document","text":"

            This document serves as a quick start guide for setting up Intel\u00ae oneAPI Base Toolkit (Base Kit) on Open FPGA Stack (OFS) using oneapi-asp repository. Please see Table 1-1 for OFS reference platforms targeted in this guide.

            Table 1-1 HLD Tools

            Target Device for the oneAPI ASP Target Platform for the oneAPI ASP Intel\u00ae Agilex\u00ae 7 FPGA Intel\u00ae FPGA SmartNIC N6001-PL Intel\u00ae Stratix 10\u00ae FPGA Intel\u00ae FPGA PAC D5005

            Attention: Intel is discontinuing the Intel FPGA SDK for OpenCL software product. Refer to the Product Discontinuation Notice PDN2219. Alternatively, Intel recommends using the Intel\u00ae oneAPI Base Toolkit (Base Kit), which provides core tools and libraries for developing high-performance data-centric applications across diverse architectures. It features an industry-leading C++ compiler that implements SYCL*, an evolution of C++ for heterogeneous computing. For more information, refer to the Intel\u00ae oneAPI Base Toolkit (Base Kit) web page. To migrate your OpenCL FPGA designs to SYCL, review Migrating OpenCL FPGA Designs to SYCL* guide that demonstrates important differences between OpenCL and SYCL for FPGA and provides steps to migrate your OpenCL designs.

            After reviewing the document you will be able to:

            • Setup your host machine to develop HLD AFUs
            • Compile and run sample HLD applications on OFS
            "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#12-terminology","title":"1.2 Terminology","text":"

            This table defines some of the common terms used when discussing OFS.

            Table 1-2: Terminology

            Term Abbreviation Description Open FPGA Stack OFS A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. Accelerator Functional Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. FPGA Interface Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. High Level Design HLD For the purpose of this guide, this term refers to designing with Intel\u00ae oneAPI Base Toolkit (Base Kit). oneAPI Accelerator Support Package oneAPI ASP OR oneapi-asp A collection of hardware and software components that enable oneAPI kernels to communicate with oneAPI runtime as well as with OFS components. The hardware components of the oneAPI ASP along with the kernels lie in the AFU region. Open Programmable Acceleration Engine Software Development Kit OPAE SDK A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. Platform Interface Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Device Feature List DFL A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. Best Known Configuration BKC The exact hardware configuration Intel has optimized and validated the solution against. SYCL - SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL\u2122) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Installable Client Driver ICD Intel\u00ae FPGA Runtime for OpenCL\u2122 Software Technology supports the OpenCL ICD extension from the Khronos Group\u2122. The OpenCL ICD extension allows you to have multiple OpenCL implementations on your system. With the OpenCL ICD Loader Library, you may choose from a list of installed platforms and execute OpenCL API calls that are specific to your OpenCL implementation of choice. FPGA Client Driver FCD Intel\u00ae FPGA Runtime for OpenCL\u2122 Software Technology supports FPGA Client Driver(FCD) extension. FCD allows the runtime to automatically find and load the oneAPI ASP libraries at host run time"},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#13-introduction-to-high-level-design-on-ofs","title":"1.3 Introduction to High Level Design on OFS","text":"

            Intel currently provides Intel\u00ae oneAPI Base Toolkit (Base Kit) for FPGA application development using high level languages like Data Parallel C++(DPC++).

            Figure 1-1 shows how OFS components can be used with Intel HLD tool.

            Figure 1-1 HLD Tool on OFS Platforms

            For high level description and setup details for OFS components shown in figure above, please refer to the Getting Started guide for your target device.

            • OFS Getting Started User Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
            • OFS Getting Started User Guide: For Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.

            For a more detailed diagram and more information about the FPGA Interface Manager(FIM) shown in figure above, please refer to the FIM developer guides for your target device.

            • Technical Reference Manual: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
            • FIM Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.
            • Intel\u00ae FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
            • FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.

            The oneAPI ASP is required for compiling and running HLD application kernel on OFS platforms using Intel oneAPI. It is a collection of hardware and software components that enable oneAPI kernels to communicate with oneAPI runtime as well as with other OFS components. The hardware components of the oneAPI ASP along with the kernel lie in the AFU region shown in the figure above. For more details about the components of the oneAPI ASP, please refer to oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack.

            Figure 1-2 shows the setup steps to use oneAPI base toolkit on OFS platforms.

            Figure 1-2 Setup Steps for oneAPI base toolkit on OFS Platforms

            The next section covers the setup steps in detail.

            Note: Administrative privileges are needed for multiple setup steps, ensure you have administrative/sudo privileges before proceeding.

            "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#20-setup-flow-for-using-hld-tool-on-ofs","title":"2.0 Setup Flow for Using HLD Tool on OFS","text":""},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#21-setup-server-for-ofs","title":"2.1 Setup Server for OFS","text":"

            As a first step, the server or host machine being used for developing HLD application needs to be setup for OFS. This involves setting up the FPGA card as well as installing OFS software stack including OPAE SDK and OFS DFL kernel driver.

            Please follow steps in Getting started guides for your target devices to setup Linux DFL kernel driver and install OPAE SDK.

            • OFS Getting Started User Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs
            • OFS Getting Started User Guide: For Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.
            "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#22-clone-and-compile-fim","title":"2.2 Clone and Compile FIM","text":"

            As shown in Figure 1-1, OFS components in the FPGA include the FIM and Accelerator Functional Unit(AFU). The oneAPI ASP is in the Partial Reconfiguration(PR) region of the AFU and relies on the compiled database of the static region(FIM) to interface with the host and board peripherals(e.g. on-board memory).

            Once the server is setup with OPAE SDK and DFL kernel driver, the next step is to clone and compile the static region of the design, i.e. FIM. You can use the default configuration of the FIM for both target platforms. Additionaly for Intel\u00ae FPGA SmartNIC N6001-PL for ofs_n6001 and ofs_n6001_usm board variants you have the option to create a minimal FIM which removes additional VFs, HSSI and host exercisers in the design. Please follow steps in the Intel\u00ae FPGA Interface Manager Developer Guides for your target device to compile FIM supporting PR release.

            • Intel\u00ae FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
            • FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.

            For more details on minimal FIM for Intel\u00ae Agilex\u00ae 7 FPGA for ofs_n6001 and ofs_n6001_usm board variants and how to create it, refer to Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.

            A pr_build_template directory will be generated in the work directory specified as part of the FIM compile command (using OFS/ofs-common/scripts/common/syn/build_top.sh script with the '-p' option enable to create an out-of-tree PR release). The pr_build_template directory is required for successful setup of the oneAPI ASP.

            Once the FIM compile is complete, please program FIM using fpgasupdate and Remote System Update(rsu) command. Use of these commands has been demonstrated in section named Program the Intel\u00ae FPGA SmartNIC N6001-PL with the hello_fim in Intel\u00ae FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs refer to Test the hello_fim on a D5005 section in FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs for Intel\u00ae Stratix 10\u00ae FPGA.

            "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#23-prerequisites","title":"2.3 Prerequisites","text":"

            In addition to server setup and FIM compilation, a few linux packages are needed to setup the oneAPI ASP and develop HLD applications.

            1) Install the following packages:

                sudo dnf install numactl-devel ncurses-compat-libs\n

            2) Ensure that IOMMU is turned on as explained in section Building and Installing the OFS DFL Kernel Drivers from Source in Getting started guides for your target devices:

            • OFS Getting Started User Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
            • OFS Getting Started User Guide: For Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.

            You can verify this setting using cat /proc/cmdline command. The output must have intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200.

            $ cat /proc/cmdline\nBOOT_IMAGE=(hd1,gpt2)/vmlinuz-6.1.41-dfl root=/dev/mapper/rhel-root ro crashkernel=auto resume=/dev/mapper/rhel-swap rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\n

            3) Install HLD development software. Please see Table 2-1 below for download link. Install the latest version. Use sudo privileges to do the installation.

            Table 2-1 Intel HLD Tool and Download Information

            HLD Tool Target Platform for the oneAPI ASP Tool Download Information Intel\u00ae oneAPI Base Toolkit (Base Kit)
            • Intel\u00ae FPGA SmartNIC N6001-PL
            • Intel\u00ae FPGA PAC D5005
              • Download here

              Tool installation guide for your reference:

              • Intel\u00ae oneAPI Toolkits Installation Guide for Linux* OS

              4) Ensure you have all the Quartus patches installed, refer to Table 2-3 for required Quartus version.

              Note: For Intel\u00ae Agilex\u00ae 7 FPGA ensure Quartus patch 0.13, 0.21 and 0.02iofs are installed. You can find them in a tar file under assets in the following link patch-agx7-ofs-2023-3.tar.gz. For Intel\u00ae Stratix 10\u00ae FPGA ensure Quartus patch 0.23 and 0.01iofs are installed. You can find them in a tar file under assets in the following link patch-s10-ofs-2023-3.tar.gz.For quartus patches installation to work properly, you must have Git Large File Storage (LFS) installed when cloning the ofs-fim repository.

              Use following command to check Quartus version and installed patches.

                  quartus_sh -v\n

              5) After completing the tool installation, set the following environment variables required to execute build scripts successfully:

                  # Adding Quartus to PATH\n    export PATH=$PATH:path-to-quartus-installation-dir/bin\n    export QUARTUS_ROOTDIR=path-to-quartus-installation-dir/quartus\n    export QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\n    # Other OFS environment variables\n    export OFS_ROOTDIR=path-to-directory-containing-cloned-ofs-fim-repo/#ofs-agx7-pcie-attach for Intel\u00ae Agilex\u00ae 7 FPGA or ofs-d5005 for Intel\u00ae Stratix 10\u00ae FPGA\n    export WORKDIR=$OFS_ROOTDIR\n    export QUARTUS_HOME=$QUARTUS_ROOTDIR\n    export QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\n    export IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\n    export OPAE_SDK_REPO_BRANCH=release/branch-tag # Refer to Table 2-3, for OPAE SDK branch tag\n    export OFS_PLATFORM_AFU_BBB=path-to-cloned-ofs-platform-afu-bbb-repo\n    export OPAE_PLATFORM_ROOT=path-to-ofs-fim-pr_build_template-directory # (see section 2.2 for more details)\n    export  OFS_ASP_ROOT=path-to-directory-containing-oneapi-asp/oneapi-asp/platform-name #platform-name is n6001 for Intel\u00ae FPGA SmartNIC N6001-PL and d5005 for Intel\u00ae FPGA PAC D5005 \n    export LIBOPAE_C_ROOT=/usr # (OPAE libraries are installed in /usr/lib64 by default if you followed the OPAE SDK steps covered in section 2.1 as is and installed OPAE rpm packages. If you have a different OPAE installation path, please point LIBOPAE_C_ROOT to your OPAE installation location that you specified using -DCMAKE_INSTALL_PREFIX=installation-path in cmake command for building OPAE)\n

              Note: To re-use this environment setting, you can copy the above export statements to a shell script, update the paths to match your tool installations and source this script each time a new shell is started.

              6) Source initialization script for oneAPI, path is shown in table below.

              Table 2-2 Initialization Script for HLD tool

              Tool Command to source initialization script Intel\u00ae oneAPI Base Toolkit (Base Kit) source path-to-intel-oneapi-toolkit-installation-directory/setvars.sh

              Once the environment variables are set, you can check the tool version using the following commands:

              # Printing all (Quartus, OpenCL SDK, GCC) versions for user info\nquartus_sh -v\nicpx --version (for Intel\u00ae oneAPI Base Toolkit (Base Kit))\ngcc --version\n

              Table 2-3 and 2-4 summarize the tool version/Best Known Configurations(BKC).

              Table 2-3 Best Known Configuration(BKC) for Intel\u00ae Agilex\u00ae 7 FPGA OFS

              Component/Tool Version FPGA Platform Intel\u00ae FPGA SmartNIC N6001-PL Operating System RHEL 8.6 , Kernel : 6.1.41-dfl linux-dfl (DFL Kernel Driver) Tag: ofs-2023.3-6.1-3 opae-sdk Branch: release/2.10.0, Tag: 2.10.0-1 ofs-fim Tag: ofs-2023.3-2 oneapi-asp Tag: ofs-2023.3-2 > Note: Cloning and build of this repo is discussed in the section 2.4 Quartus Prime Pro Edition Version 23.3 Pro Edition with patches (0.13, 0.21 and 0.02iofs) under assets on this link [patch-agx7-ofs-2023-3.tar.gz]](https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-1) Intel\u00ae oneAPI Base Toolkit (Base Kit) Latest version GCC 7.4.0 cmake 3.15

              Table 2-4 Best Known Configuration(BKC) for Intel\u00ae Stratix 10\u00ae FPGA

              Component/Tool Version FPGA Platform Intel\u00ae FPGA PAC D5005 Operating System RHEL 8.6 , Kernel : 6.1.41-dfl linux-dfl (DFL Kernel Driver) Tag: ofs-2023.3-6.1-1 opae-sdk Branch: release/2.10.0, Tag: 2.10.0-1 ofs-fim Tag: ofs-2023.3-1 oneapi-asp Tag: ofs-2023.3-2 > Note: Cloning and build of this repo is discussed in the section 2.4 Quartus Prime Pro Edition Version 23.3 Pro Edition with patches( patch 0.23 and 0.01iofs) under assets on this link patch-s10-ofs-2023-3.tar.gz Intel\u00ae oneAPI Base Toolkit (Base Kit) Latest version GCC 7.4.0 cmake 3.15"},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#24-build-and-install-oneapi-asp","title":"2.4 Build and Install oneapi-asp","text":"

              Once all pre-requisites are installed and the environment variables are set, next step is to clone and build oneapi-asp.

              1) Clone oneapi-asp repository and checkout tag matching the BKC for your target platform (see Tables 2-3 and 2-4 for the BKCs).

              Note: You will need a personal access token (use the classic mode) to be used as the password to clone successfully. Please see more information about token authentication requirements for Git operations here.

                  git clone https://github.com/OFS/oneapi-asp.git\n    cd oneapi-asp\n    git checkout tags/ofs-2023.3-1\n

              Ensure the correct tag has ben checked out: 

                  git describe --tags\n
              • Output:
                  ofs-2023.3-1\n

              2) Ensure that OPAE_PLATFORM_ROOT and LIBOPAE_C_ROOT have been set as described in section 2.3. Generate the oneAPI ASP hardware and software using provided build-bsp.sh script. This script clones required repositories and builds the oneAPI ASP libraries required by HLD host application to run successfully.

                  cd path-to-directory-containing-cloned-oneapi-asp-repo/oneapi-asp/platform-name\n    ./scripts/build-bsp.sh\n

              The generated directory structure is shown below. For more details refer to the oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack.

              \noneapi-asp/platform-name\n|--ase/\n|  |--base/\n|  |--hack_ip_files/\n|  |--compile-kernel.sh\n|  |--run-ase.sh\n|  |--setup.sh\n|  |--simulate-aocx.sh\n|--bringup/source/simple-add-buffers/\n|  |--simple-add-buffers.cpp\n|--hardware/\n|  |--ofs_platform-name/\n|  |--ofs_platform-name_iopipes/\n|  |--ofs_platform-name_usm/\n|  |--ofs_platform-name_usm_iopipes/\n|--linux64/libexec/\n|  |--flash\n|  |--initialize\n|  |--install\n|  |--setup_permissions.sh\n|  |--uninstall\n|--scripts/\n|  |--README.txt\n|  |--build-bsp-sw.sh\n|  |--build-bsp.sh\n|  |--build-default-binaries.sh\n|  |--build-mmd.sh\n|  |--build-opae.sh\n|  |--create-tarball.sh\n|  |--dedup-hardware.sh\n|  |--setup-bsp.py\n|--board_env.xml\n|--README.md\n

              3) Once the oneAPI ASP is generated, add the following to LD_LIBRARY_PATH. You can add it to your script for setting environment variables (if you created one as noted in step 5 in section 2.3)

                  export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:path-to-oneapi-asp/platform-name/linux64/lib\n

              4) Check if FPGA Client Driver(FCD) exists for any other version of oneAPI ASP or for any other board. You can check with aocl list-devices command. It is recommended to run aocl list-devices as root user (login with sudo su) to see all installed ASPs on the host.

              A sample output when a oneAPI ASP FCD is installed is shown below:

              --------------------------------------------------------------------\nDevice Name:\nacl0\n\nBSP Install Location:\n/home/ofsuser/oneapi-asp/platform-name\n\nVendor: Intel Corp\n\nPhysical Dev Name   Status            Information\n\nofs_ee00000         Uninitialized     PR slot function not configured \n                                      Need to follow instructions to bind vfio-pci driver to PR slot function\n\nBSP DIAGNOSTIC_PASSED\n--------------------------------------------------------------------\n

              If a oneAPI ASP/BSP is installed, uninstall using aocl uninstall path-to-oneapi-asp-install-location, where path-to-oneapi-asp-install-location is provided under BSP Install Location: in the output of aocl list-devices. If you are prompted with a question to unset the FCD, type Y. If you are prompted with a question to remove OpenCL BSP configuration settings, type Y.

              Sample output for aocl uninstall command:

              $ aocl uninstall /home/ofsuser/oneapi-asp/platform-name\naocl uninstall: Removing the FPGA Client Driver (FCD) from the system\n[sudo] password for ofsuser:\naocl uninstall: Removing the board package /home/ofsuser/oneapi-asp/platform-name from the list of installed packages. This process may require admin privilege\naocl uninstall: Running uninstall from /home/ofsuser/oneapi-asp/platform-name/linux64/libexec\nDo you want to remove oneAPI-ASP configuration settings [Y/n] Y\nDeleting OPAE config files\nRemoving configuration files\nOFS oneAPI-ASP uninstall complete\n

              5) Install FPGA Client Driver(FCD) file for the oneAPI ASP using aocl install path-to-oneapi-asp/platform-name command as shown below. The host program uses FCD to find and link to the platform Memory Mapped Device (MMD) library. For more information about MMD library, refer to oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack.

                  aocl install path-to-directory-containing-oneapi-asp/oneapi-asp/platform-name\n

              Notes: 1. Type Y when prompted to setup FCD at /opt/Intel/OpenCLFPGA/oneAPI/Boards (default location for Intel oneAPI).

              Sample output aocl install command in Intel\u00ae oneAPI Base Toolkit (Base Kit) environment is shown below.

              aocl install: Setting up the FPGA Client Driver (FCD) to the system. This process may require admin privilege\nInstall the FCD file to /opt/Intel/OpenCLFPGA/oneAPI/Boards\n[sudo] password for ofsuser:\naocl install: Adding the board package path-to-oneapi-asp/platform-name to the list of installed packages\nInstalling the board package driver to the system.\naocl install: Running install from path-to-oneapi-asp/platform-name/linux64/libexec\nConfiguring locked memory setting\nConfiguring udev rules for DFL FPGA device permission\nConfiguring system with 1024 2M hugepages\nSetting access permisions of /dev/uio to 666\nFinished setup_permissions.sh script. All configuration settings are persistent.\nIntel OFS oneAPI ASP install complete.\nRun 'aocl diagnose' to list devices or 'aocl initialize <dev_name> <board_variant> to load default image\n
              "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#25-board-initialization","title":"2.5 Board Initialization","text":"

              OFS software stack expects boards to be initialized with a bitstream for the board variant intended to be used for development. An oneAPI sample application, named simple-add-buffers, has been provided in the oneapi-asp repository for generating initialization bitstreams for included board variants. The sample is located in oneapi-asp/platform-name/bringup/source.

              oneapi-asp has four board variants for Intel\u00ae Agilex\u00ae 7 FPGA and two board variants for Intel\u00ae Stratix 10\u00ae FPGA (oneapi-asp/platform-name/hardware has the hardware design files for these). For more details on the architecture of the board variants, please refer to the oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack.

              Table 2-5 oneAPI Sample Applications

              Board Variants Sample Application
              • ofs_platform-name
              • ofs_platform-name_usm
              • ofs_n6001_iopipes
              • ofs_n6001_usm_iopipes
              simple-add-buffers

              Note: platform-name is n6001 for Intel\u00ae Agilex\u00ae 7 FPGA OFS and d5005 for Intel\u00ae Stratix 10\u00ae FPGA OFS

              All samples are located in oneapi-asp/platform-name/bringup/source.

              "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#251-compile-initialization-bitstreams","title":"2.5.1 Compile Initialization Bitstreams","text":"

              A script is provided in repo to compile simple-add-buffers oneAPI sample application. The script is oneapi-asp/platform-name/scripts/build-default-binaries.sh.

              Script usage is as follows:

                  ./build-default-binaries.sh -b name-of-board-variant\n

              Note: name-of-board-variant can be ofs_platform-name, ofs_platform-name_usm, ofs_n6001_iopipes or ofs_n6001_usm_iopipes where platform-name is n6001 for Intel\u00ae Agilex\u00ae 7 FPGA OFS and d5005 for Intel\u00ae Stratix 10\u00ae FPGA OFS. Refer to Table 2-5 for board variants available for each target platform. Compilation will take a few hours to complete.

              The output directory of the sample application is written to oneapi-asp/platform-name/build/bringup. The generated bitstreams are also copied to oneapi-asp/platform-name/bringup/binaries/. These are used in the initialization of the platform.

              Once the bitstreams are generated, create a VF and initialize the board as explained in following section. Ensure that the FIM has been programmed on the board as explained in section 2.2 Clone and Compile FIM

              "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#252-create-vf","title":"2.5.2 Create VF","text":"

              The oneAPI ASP is located in the PR region of the FIM and is accessed through PF/VF Mux. Refer to the FIM Reference Manual for your target platforms for more details about PF/VF mapping.

              • Reference FIM for Intel\u00ae Agilex\u00ae 7 FPGA OFS: VF0 is mapped to PR region and you can create 1 VF when using this FIM. Base_x16 FIM has 5 PF's and minimal FIM just 1 PF. See Technical Reference Manual: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs for diagram showing PF/VF mapping.
              • Reference FIM for Intel\u00ae Stratix 10\u00ae FPGA OFS: VF1 is mapped to PR region and you must create 2 VFs when using this FIM. This FIM has 1 PF. See FIM Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs for diagram showing PF/VF mapping.
              • Create a VF using PCIe ID obtained from the output of fpgainfo fme (PCIe s\\:b\\:d.f output)
                  sudo pci_device s:b:d.f vf num_vf  #num_vf is 1 for Intel\u00ae Agilex\u00ae 7 FPGA and 2 for Intel\u00ae Stratix 10\u00ae FPGA\n
              • Check that the VF is created using sudo opae.io ls command and note the PCIe ID for the VF(s) (the function number in s\\:b\\:d.f will be different for the VF). Sample output for Intel\u00ae Agilex\u00ae 7 FPGA minimal FIM is shown below. Output for base_x16 FIM should display 5 PF's and the PCIe ID for VF0 will be s:b:d.5.
                  $ sudo opae.io ls\n    [0000:b1:00.0] (0x8086, 0xbcce) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n    [0000:b1:00.1] (0x8086, 0xbccf) Intel Acceleration Development Platform N6001 (Driver: None)\n

              Sample output for Intel\u00ae Stratix 10\u00ae FPGA is shown below.

                  $ sudo opae.io ls\n    [0000:d8:00.0] (0x8086, 0xbcce)  Intel FPGA Programmable Acceleration Card D5005 (Driver: dfl-pci)\n    [0000:d8:00.1] (0x8086, 0xbccf)  Intel FPGA Programmable Acceleration Card D5005 (Driver: dfl-pci)\n    [0000:d8:00.2] (0x8086, 0xbccf)  Intel FPGA Programmable Acceleration Card D5005 (Driver: dfl-pci)\n

              Note:sudo opae.io ls will list the accelerators, respective PCIe ID as well as the driver it is currently bound to.

              • Bind the created VF(s) to vfio-pci driver, use the PCIe ID for the VF(s) for this step. Verify you are using the PCIe ID of the VFs you have created. For example:
              • From sample output for Agilex OFS target platform having minimal FIM programmed shown above, s:b:d.vf will be 0000:b1:00.1 in command below. For base_x16 FIM should be s:b:d.5.
                  sudo opae.io init -d s:b:d.vf $USER\n
              • Sample output for Intel\u00ae Agilex\u00ae 7 FPGA OFS target platform minimal FIM. Output for base_x16 FIM should be similar.
                  $ sudo opae.io init -d 0000:b1:00.1 $USER\n    Unbinding (0x8086,0xbccf) at 0000:b1:00.1 from dfl-pci\n    Binding (0x8086,0xbccf) at 0000:b1:00.1 to vfio-pci\n    iommu group for (0x8086,0xbccf) at 0000:b1:00.1 is 319\n    Assigning /dev/vfio/319 to ofsuser\n    Changing permissions for /dev/vfio/319 to rw-rw----\n\n    $ sudo opae.io ls\n    [0000:b1:00.0] (0x8086, 0xbcce) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n    [0000:b1:00.1] (0x8086, 0xbccf) Intel Acceleration Development Platform N6001 (Driver: vfio-pci)\n\n    $ ls -lt /dev/vfio\n    total 0\n    crw-rw----. 1 ofsuser root 509,   0 Dec 3 20:41 319\n    crw-rw-rw-. 1 root    root  10, 196 Dec 3 20:41 vfio\n
              • Sample output for Intel\u00ae Stratix 10\u00ae FPGA OFS target platform:
              $sudo opae.io init -d 0000:12:00.1 $USER\nUnbinding (0x8086,0xbccf) at 0000:12:00.1 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:12:00.1 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:12:00.1 is 149\nAssigning /dev/vfio/149 to ofsuser\nChanging permissions for /dev/vfio/149 to rw-rw----\n\n$sudo opae.io init -d 0000:12:00.2 $USER\nUnbinding (0x8086,0xbccf) at 0000:12:00.2 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:12:00.2 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:12:00.2 is 152\nAssigning /dev/vfio/152 to ofsuser\nChanging permissions for /dev/vfio/152 to rw-rw----\n\n$ sudo opae.io ls\n[0000:12:00.0] (0x8086:0xbcce) Intel FPGA Programmable Acceleration Card D5005 (Driver: dfl-pci)\n[0000:12:00.1] (0x8086:0xbccf) Intel FPGA Programmable Acceleration Card D5005 (Driver: vfio-pci)\n[0000:12:00.2] (0x8086:0xbccf) Intel FPGA Programmable Acceleration Card D5005 (Driver: vfio-pci)\n\n$ls -lt /dev/vfio\ntotal 0\ncrw-rw----. 1 ofsuser  root  235,   3 Dec 3 16:25 149\ncrw-rw----. 1 ofsuser  root  235,   0 Dec 3 16:22 152\ncrw-rw-rw-. 1 root     root   10, 196 Dec 1 07:28 vfio\n

              If the driver fails to bind due to an error related to iommu_group (e.g. `No such file or directory: '/sys/bus/pci/devices/0000:b1:00.5/iommu_group'), ensure IOMMU is turned on as explained in step 2 in Section 2.3 Prerequisites.

              Note: For more information about pci_device and opae.io utilities, refer to the OPAE FPGA tools page here.

              "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#253-initialize-board-and-run-diagnostic-test","title":"2.5.3 Initialize Board and Run Diagnostic Test","text":"
              • Initialize the board with default bitstream using aocl initialize command
                  aocl initialize device-name name-of-board-variant\n

              Note: device-name in aocl initialize command is from the BSP Diagnostics section in the output of aocl list-devices (sample output is shown in Section 2.4). device-name is acl0 if there is only 1 board connected to the server. name-of-the-board-variant can be one of the supported board variants listed in Table 2-5 provided the sample application bitstream using steps in section above.

              Sample output for aocl initialize acl0 ofs_n6001 is shown below. Output for Intel\u00ae Stratix 10\u00ae FPGA and other board variants should be similar.

              $ aocl initialize acl0 ofs_n6001\naocl initialize: Running initialize from path-to-oneapi-asp/n6001/linux64/libexec\nInitializing with default ASP binary ofs_n6001.fpga\nSaving target image to \"ofs_n6001.aocx.0\"\nConfiguring locked memory setting\n[sudo] password for $USER:\nConfiguring udev rules for DFL FPGA device permission\nConfiguring system with 1024 2M hugepages\nSetting access permisions of /dev/uio to 666\nFinished setup_permissions.sh script. All configuration settings are persistent.\nProgram succeed.\n

              Notes: 1. aocl initialize command needs to be executed only one time unless you reboot the server.

              Run aocl diagnose to check that the board Status under BSP Diagnostics is equal to Passed.

              platform-name in command output below is n6001 for Intel\u00ae Agilex\u00ae 7 FPGA OFS and d5005 for Intel\u00ae Stratix 10\u00ae FPGA OFS.

              $ aocl diagnose\n--------------------------------------------------------------------\nICD System Diagnostics\n--------------------------------------------------------------------\n\nUsing the following location for ICD installation:\n        /etc/OpenCL/vendors\n\nFound 3 icd entry at that location:\n        /etc/OpenCL/vendors/intel64.icd\n\nThe following OpenCL libraries are referenced in the icd files:\n        /opt/intel/oneapi/compiler/latest/lib/libintelocl.so\n\nChecking LD_LIBRARY_PATH for registered libraries:\n        /opt/intel/oneapi/compiler/latest/lib/libintelocl.so was registered on the system.\n\n\nChecking LD_LIBRARY_PATH for registered libraries:\n        /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga/host/linux64/lib/libalteracl.so was registered on the system.\n        /opt/intel/oneapi/compiler/latest/linux/lib/x64/libintelocl_emu.so\n\nChecking LD_LIBRARY_PATH for registered libraries:\n        /opt/intel/oneapi/compiler/latest/linux/lib/x64/libintelocl_emu.so was registered on the system.\n\nUsing OCL_ICD_FILENAMES to search for ICD clients, it is set to libintelocl_emu.so:libalteracl.so:/opt/intel/oneapi/compiler/2024.0/lib/libintelocl.so\n\nChecking LD_LIBRARY_PATH for registered libraries specified by OCL_ICD_FILENAMES\n    libintelocl_emu.so was registered on the system at \n    /opt/intel/oneapi/compiler/2024.0/lib\n    libalteracl.so was registered on the system at \n    /opt/intel/oneapi/compiler/2024.0/opt/oclfpga/host/linux64/lib\n    /opt/intel/oneapi/compiler/2024.0/lib/libintelocl.so was registered on the system.\n\nUsing the following location for fcd installations:\n        /opt/Intel/OpenCLFPGA/oneAPI/Boards\n\nFound 1 fcd entry at that location:\n        /opt/Intel/OpenCLFPGA/oneAPI/Boards/ofs_platform-name_shim.fcd\n\nThe following OpenCL libraries are referenced in the fcd files:\n        libopae-c.so\n/home/ofsuser/oneapi-asp/platform-name/linux64/lib/libMPF.so\n/home/ofsuser/oneapi-asp/platform-name/linux64/lib/libintel_opae_mmd.so\n\nChecking LD_LIBRARY_PATH for registered libraries:\n        libopae-c.so was registered on the system at /usr/lib64\n        /home/ofsuser/oneapi-asp/platform-name/linux64/lib/libMPF.so was registered on the system.\n        /home/ofsuser/oneapi-asp/platform-name/linux64/lib/libintel_opae_mmd.so was registered on the system.\n\nNumber of Platforms = 4\n        1. Intel(R) FPGA Emulation Platform for OpenCL(TM)    | Intel(R) Corporation           | OpenCL 1.2 Intel(R) FPGA SDK for OpenCL(TM), Version 20.3\n        2. Intel(R) FPGA SDK for OpenCL(TM)                   | Intel(R) Corporation           | OpenCL 1.0 Intel(R) FPGA SDK for OpenCL(TM), Version 2024.0\n        3. Intel(R) OpenCL                                    | Intel(R) Corporation           | OpenCL 3.0 LINUX\n        4. Intel(R) FPGA SDK for OpenCL(TM)                   | Intel(R) Corporation           | OpenCL 1.0 Intel(R) FPGA SDK for OpenCL(TM), Version 2024.0\n--------------------------------------------------------------------\nICD diagnostics PASSED\n--------------------------------------------------------------------\n--------------------------------------------------------------------\nBSP Diagnostics\n--------------------------------------------------------------------\n--------------------------------------------------------------------\nDevice Name:\nacl0\n\nBSP Install Location:\n/home/ofsuser/oneapi-asp/platform-name\n\nVendor: Intel Corp\n\nPhysical Dev Name   Status            Information\n\nofs_ee00001         Passed            Intel OFS Platform (ofs_ee00001)\n                                    PCIe b1:00.0\n                                    FPGA temperature = 59 degrees C.\n\nBSP DIAGNOSTIC_PASSED\n--------------------------------------------------------------------\n\nCall \"aocl diagnose <device-names>\" to run diagnose for specified devices\nCall \"aocl diagnose all\" to run diagnose for all devices\n

              Run complete diagnostic using aocl diagnose device-name command. device-name is acl0 if you have only 1 board in the server. The test reads and write data to the board to check the interfaces function correctly and report the measured bandwidth. The test must show BSP DIAGNOSTIC_PASSED message at the end.

                  aocl diagnose acl0\n

              Next section you will build and run oneAPI host applications.

              Once you are done with your application testing, you can release the device from vfio-pci driver, the steps for this are provided in Section 2.7 Release VF.

              "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#26-compile-and-run-oneapi-sample-applications","title":"2.6 Compile and Run oneAPI Sample Applications","text":"

              Different code samples are provided for testing different board interfaces. Please refer to table below for more information.

              Table 2-6 DPC++ Sample Application

              Board Variants Sample Application Description Link to Samples Repository Location of the Sample Samples Repository Tag
              • ofs_platform-name
              • ofs_platform-name_usm
              board_test This sample measures kernel clock frequency, kernel launch latency and tests the different interfaces required by the oneAPI kernel to function correctly (e.g. host to kernel interface, kernel to EMIF as well as host to EMIF). oneAPI-samples path-to-oneAPI-samples/oneAPI-samples/DirectProgramming/C++SYCL_FPGA/ReferenceDesigns/board_test oneAPI compiler version you are using, this sample supports usm board variant from 2024.0 oneAPI version.
              • ofs_n6001_iopipes
              • ofs_n6001_usm_iopipes
              io_streaming_one_pipe An FPGA code sample to perform a loopback test using SYCL* input/output (I/O) pipes to stream data through the FPGA I/O. examples-afu path-to-examples-afu/examples-afu/oneapi-samples/io_streaming_one_pipe ofs-2023.3-2

              First clone the repository and then checkout to the corresponding tag. 

                  git clone link-to-samples-repository\n    cd oneAPI-samples # (oneapi-samples for examples-afu repository)\n    git checkout tags/samples-repository-tag\n

              Note: For link-to-samples-repository and samples-repository-tag refer to the table 2-6. To check your oneAPI compiler version use the command:

              \n    icpx --version\n

              Follow steps below to compile and run oneAPI board_test and io_streaming_one_pipe binaries. Use -DFPGA_DEVICE in cmake command to provide path to the oneAPI ASP and name of board variant being compiled. For board_test when targeting USM board variant add -DSUPPORTS_USM=1 flag in cmake command, for more information about this flag see the README file in the location of the sample, for this location refer to the table 2-6.

              Ensure you have sourced setvars.sh script located in the root of your oneAPI installation as explained in Table 2-2 Initialization Script for HLD tool.

                  cd path-to-sample-location\n    mkdir build\n    cd build\n

              Cmake for compiling board_test when targeting USM board variant.

                  cmake -DFPGA_DEVICE=full-path-to-oneapi-asp/platform-name:board_variant -DSUPPORTS_USM=1 ..\n

              Cmake for compiling the rest of the board variants. 

                  cmake -DFPGA_DEVICE=full-path-to-oneapi-asp/platform-name:board_variant ..\n

              Compile the design using the generated Makefile. The following build targets are provided:

              • Generate the optimization report:
                  make report\n
              • Compile for FPGA hardware (takes longer to compile, targets FPGA device):
                  make fpga\n

              Hardware compilation takes several hours to complete. Once complete, you should see sample-name.fpga executable generated, where sample-name could be board_test or io_streaming_one_pipe.

              For more information on additional environment settings required for running io_streaming_one_pipe sample see the README file in the location of the sample, for this location refer to the table 2-6 .

              Run the generated hardware executable as follows:

                 ./sample-name.fpga\n

              Note: If your FPGA compile fails to meet timing requirements, the Intel oneAPI compiler prints an error message, returns an error code and deletes the generated binary. In case of timing failure, *.failing_clocks.rpt and *.failing_paths.rpt files are generated in compiled output directory sample-name.fpga.prj, where sample-name could be board_test or io_streaming_one_pipe. You can recompile with a different seed using -Xsseed option. You can pass this option using USER_HARDWARE_FLAGS=-Xsseed=seed_value in the cmake command above and recompile hardware image.

              To view test details and usage information using the binary, use the -help option.

                  ./sample-name.fpga -help # sample-name could be board_test or io_streaming_one_pipe.\n
              "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#27-release-vf","title":"2.7 Release VF","text":"

              Once you are done with your application testing, you can release the device from vfio-pci driver using following command.

                  $ sudo opae.io release -d s:b:d.vf\n

              Sample output for Intel\u00ae Agilex\u00ae 7 FPGA OFS target platform having programmed minimal FIM is shown below.The output for Intel\u00ae Stratix 10\u00ae FPGA target platform and base_x16 FIM should be similar. For Intel\u00ae Stratix 10\u00ae FPGA you will need to release an extra VF as for this target 2 Vfs were created. 

              $ sudo opae.io release -d 0000:b1:00.1\nReleasing (0x8086,0xbccf) at 0000:b1:00.1 from vfio-pci\nRebinding (0x8086,0xbccf) at 0000:b1:00.1 to dfl-pci\n\n$ sudo opae.io ls\n[0000:b1:00.0] (0x8086, 0xbcce) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n[0000:b1:00.1] (0x8086, 0xbccf) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
              "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#30-further-development","title":"3.0 Further Development","text":"

              Once you have completed running the oneAPI sample application, you can start developing your own applications.

              For more information about developing FPGA applications with Intel oneAPI, refer to Intel\u00ae oneAPI Programming Guide and FPGA Optimization Guide for Intel\u00ae oneAPI Toolkits.

              If you want to customize the oneAPI ASP, you can refer to oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack.

              "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#notices-disclaimers","title":"Notices & Disclaimers","text":"

              Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Performance varies by use, configuration and other factors. Your costs and results may vary. You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications. Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards. \u00a9 Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.

              OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122.

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/","title":"Docker User Guide: Intel\u00ae Open FPGA Stack","text":"

              Last updated: February 03, 2024

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#1-introduction","title":"1 Introduction","text":"

              This document is intended to help you get started in evaluating Open FPGA Stack (Intel\u00ae OFS) using Docker for the Intel\u00ae Platforms. The Intel FPGA platforms can be used as a starting point for evaluation and development. This document covers critical topics related to the initial setup of the Docker solution included with the OFS release.

              After reviewing the document, you shall be able to:

              • Set up the Intel\u00ae Quartus\u2122 Prime Pro Edition Software in a host server
              • Set up the Docker engine
              • Build and load your Docker image to the Docker engine
              • Run a Docker container with OFS preloaded

              The Open FPGA Stack (OFS) Docker image has two main personas:

              • Development: You can develop, simulate, and build any component of the OFS. The Docker image enables you to use your laptop or server without having drivers, FPGA Platform, or specific Linux* distribution installed in your host computer. You can follow the development flow provided to run Docker on Linux.
              • Deployment: You can program, load binaries, or execute real-time testing using the OPAE and OFS. To do so, the host computer must have the specified software distribution and drivers installed.
              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#12-background-information","title":"1.2 Background Information","text":"

              A container is a fully functional and portable cloud or non-cloud computing environment that includes an application, associated libraries, and other dependencies. Docker containers do not require a hardware hypervisor, instead using the application layer of the host computer, which means they tend to be smaller, faster to setup, and require fewer resources when compared to a virtual machine (VM).

              The OFS provides the flexibility to support various orchestration or management systems, including bare metal, VM, and Docker.

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#13-relevant-information","title":"1.3 Relevant information","text":"
              • What is a container?
              • Docker vs. Virtual Machines
              • Does the Docker container have its own Kernel?
                • No, Docker image or Container uses the application layer of the host computer; this functionality is the main reason for docker having lightweight and fast applications.
              • Does Docker run on Linux, macOS, and Windows?
              • Intel Docker Image can use the PCIe card from the host server?
                • Yes, The drivers and additional information could be shared, but this could create potential security concerns (ensure your system is secure).
              • Docker security
              • Docker subscription
              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#20-prerequisites-and-scope","title":"2.0 Prerequisites and Scope","text":"

              The OFS release targeting the compatible OFS Platform's is built upon tightly coupled software and firmware versions. Use this section as a general reference for the versions in this release.

              The following table highlights the hardware that comprises the Best-Known Configuration (BKC) for the OFS release. For a detailed explanation and safety information regarding the setup go to OFS Site select your desired platform and select Getting stated guide. This site walks you through the BIOS configuration changes needed to enable the OFS Platform's.

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#30-development-installation","title":"3.0 Development Installation","text":"

              Docker engines have cross-compatibility with multiple systems, but the host server does not require any specific distribution. However, the Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 requires a specific version. For this guide, Red Hat Linux is used for general instructions.

              The OFS Docker image includes all the libraries and tools required by the OFS and OPAE SDK (Python, Perl, CMake, and so on).

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#31-intel-quartus-prime-software-installation","title":"3.1 Intel Quartus Prime Software Installation","text":"

              Building AFUs with OFS for Intel Agilex FPGA requires the build machine to have at least 64 GB of RAM.

              Go to OFS Site select your desired platform and select Getting stated guide for a list of detailed steps for the Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 installation.

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#32-docker-engine-installation","title":"3.2 Docker Engine installation","text":""},{"location":"hw/common/user_guides/ug_docker/ug_docker/#rhel-86","title":"RHEL 8.6","text":"

              The Docker installation steps for RHEL 8.6 are the following:

              1. Remove old versions; older versions of Docker were called docker or docker-engine. If these are installed, uninstall them, along with associated dependencies. Also, uninstall Podman and the related dependencies if installed already.

                 sudo dnf remove docker \\\n                  docker-client \\\n                  docker-client-latest \\\n                  docker-common \\\n                  docker-latest \\\n                  docker-latest-logrotate \\\n                  docker-logrotate \\\n                  docker-engine \\\n                  podman \\\n                  runc\n

              2. Add the Docker repository to your system:

                sudo dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo\n

              3. Install the latest version of Docker Engine, containerd, and Docker Compose, or go to the next step to install a specific version.

                sudo dnf install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin\n

              4. Start the Docker daemon:

                sudo systemctl start docker\n

              5. Enable the Docker daemon to start on boot:

                sudo systemctl enable --now docker\nsudo systemctl enable --now containerd\n

              6. Verify that Docker is installed and running:

                sudo systemctl status docker\n

                You should see a message indicating that the Docker daemon is active and running.

                Note: If you want to use Docker as a non-root user, you should add your user to the docker group:

                sudo usermod -aG docker your-user\n

                You will need to log out and back in for the changes to take effect.

              7. Ensure your proxies are setup in case you needed

                sudo mkdir -p /etc/systemd/system/docker.service.d \n\nnano /etc/systemd/system/docker.service.d/http-proxy.conf\n\n[Service] \nEnvironment=\"HTTP_PROXY=http://proxy.example.com:80/\"\nEnvironment=\"HTTPS_PROXY=https://proxy.example.com:443/\"\n\n#save and close \n\nsudo systemctl daemon-reload\nsudo systemctl restart docker\n
              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#ubuntu-2204","title":"Ubuntu 22.04","text":"

              The Docker installation steps for Ubuntu are the following:

              1. Remove old versions; older versions of Docker were called docker or docker-engine. If these are installed, uninstall them, along with associated dependencies. 

                sudo apt-get remove docker docker-engine docker.io containerd runc\n

              2. Install packages to allow apt to use a repository

                sudo apt-get update\nsudo apt-get install \\\n    ca-certificates \\\n    curl \\\n    gnupg \\\n    lsb-release\n

              3. Add Docker's official GPG key:

                sudo mkdir -p /etc/apt/keyrings\ncurl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg\n

              4. The following command to set up the repository:

                echo \\\n  \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \\\n  $(lsb_release -cs) stable\" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null\n

              5. Update the package manager index again:

                sudo apt-get update\n

              6. Install Docker:

                sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin\n

              7. Start the Docker daemon:

                sudo systemctl start docker\n

              8. Enable the Docker daemon to start on boot:

                sudo systemctl enable --now docker\nsudo systemctl enable --now containerd\n

              9. Verify that Docker is installed and running:

                sudo systemctl status docker\n

                You should see a message indicating that the Docker daemon is active and running.

                Note: If you want to use Docker as a non-root user, you should add your user to the docker group:

                sudo usermod -aG docker your-user\n

                You will need to log out and back in for the changes to take effect.

              10. Ensure your proxies are setup in case you needed

                sudo mkdir -p /etc/systemd/system/docker.service.d \n\nnano /etc/systemd/system/docker.service.d/http-proxy.conf\n\n[Service] \nEnvironment=\"HTTP_PROXY=http://proxy.example.com:80/\"\nEnvironment=\"HTTPS_PROXY=https://proxy.example.com:443/\"\n\n#save and close \n\nsudo systemctl daemon-reload\nsudo systemctl restart docker\n
              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#33-load-docker-image-installation","title":"3.3 Load Docker Image installation","text":"

              The Dockerfile is released in conjunction with the OFS stack release, and The file needs to be loaded into your host computer to start a docker container.

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#build-the-image","title":"Build the image","text":"

              1. You can download the Dockefile from OFS GitHub Docker.

              2. Inside the Dockerfile folder, you will find the DockerFile edit and modify the following lines:

                ENV no_proxy=   #you could use  github.com here\nENV http_proxy= #setup proxy\nENV https_proxy=  #setup proxy\nENV GITUSER= #setup github user\nENV GITTOKEN= #setup github token\nENV REDUSER= #redhat user \nENV REDPASS= #redhat password\nENV DW_LICENSE_FILE= #DW license\nENV SNPSLMD_LICENSE_FILE= #Synopsys license\nENV LM_LICENSE_FILE= #Quartus License\n

                Save the file

              3. Create and load the image:

                cd Docker_file\ndocker build -t ofs:latest . --no-cache\n

                Note: Never remove --no-cache this could cause issues with your environmental variables inside of the container

              4. Use the following command to ensure the image is loaded correctly:

                sudo docker images\nREPOSITORY    TAG                       IMAGE ID       CREATED          SIZE\nofs           latest                    fc80175d13a0   \u221e seconds ago   2.55GB\n
              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#volumen-creation","title":"Volumen creation","text":"

              1. Docker requires a volume to move data from the host computer (Persistent data) to the docker container and vice versa. To create a docker volume, use the following command:

                docker volume create --name DataOFS\n

                For more information about Docker volume go here.

                Tip: Remember, The docker container has a limited lifecycle; the files and data are lost when the docker is Stopped-> Deleted.

              2. Check where the docker volume is mapped in your host server:

                docker volume inspect DataOFS\n[\n    {\n        \"CreatedAt\": \"xxxxxxxxxx\",\n        \"Driver\": \"local\",\n        \"Labels\": {},\n        \"Mountpoint\": \"/var/lib/docker/volumes/DataOFS/_data\",\n        \"Name\": \"DataOFS\",\n        \"Options\": {},\n        \"Scope\": \"local\"\n    }\n]\n

              3. Inside of your docker container, you can use cp command to copy from your docker to your host:

                cp /atmydocker/myfile.txt /dataofs\n

                The docker container path is /dataofs the host path is /var/lib/docker/volumes/DataOFS/_data.

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#34-create-a-container","title":"3.4 Create a container","text":"

              Now you are ready to start the container, and you should be prepared to run it: 1. First, Let's create the template for the run command, copy your Quartus installation path and paste it under -v (Don't Run the command yet):

              docker run --rm -itd --name myOFS -v=<yourintallationfolder>:/home/intelFPGA_pro/:ro -v=DataOFS:/dataofs ofs:latest /bin/bash\n

              Tip: you can change myOFS with any other value. The value is the given name of the container.

              1. Using the previous example now, you can execute the docker run command. 
                docker run --rm -itd --name myOFS -v=/home/intelFPGA_pro/23.3:/home/intelFPGA_pro/:ro -v=DataOFS:/dataofs ofs:latest /bin/bash\nbdc1289fb0813bb325b55dd11df4eeec252143d6745a6e5772638fbc107d0949\n

              2. Now the docker container should be available.

                # sudo docker ps\nCONTAINER ID   IMAGE                         COMMAND       CREATED          STATUS   PORTS     NAMES\nbdc1289fb081   ofs:latest                    \"/bin/bash\"   46 seconds ago   Up 45 seconds      myOFS\n

              Your Container ID is bdc1289fb081.

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#35-evaluate-ofs-container","title":"3.5 Evaluate OFS container","text":"

              The OFS container has two possible ways to interact with the container:

              • Interactive mode:

                This mode it takes you straight inside the container and uses the command terminal as a regular Linux console.

                1. Enable the interactive mode:

                  docker attach myOFS\n[root@bdc1289fb081 /]#\n

                  The container id is shown when you are in interactive mode [root@bdc1289fb081 /]#.

                2. Now verify the variables and Quartus is appropriately set up and recognized:

                  quartus_syn --version\n\nQuartus Prime Synthesis\nVersion Quartus Prime Pro Version 23.3\n

                3. Everything is set up correctly. Please go to the following link for more information related to the OFS Site select your desired platform and select Getting stated guide.

                  Tip: If you need to de-attach without stopping the container, you can use Ctrl+P or Ctrl+Q. For custom combinations, for example, docker attach --detach-keys=\"ctrl-a\" myOFS and if you press CTRL+A you will exit the container without killing it.

              • De-attach Mode:

                This mode runs your container in the background and allows you to run multiple commands without going inside of the docker container.

                1. The OFS Docker image already includes the evaluation script.

                2. Let's use option 2 - Check versions of Operating System and Quartus Premier Design Suite (QPDS); remember multiple options could not be available if the DFL drivers and the FPGA Platform is not installed, This example uses the Intel\u00ae FPGA SmartNIC N6001-PL . 

                  $ sudo docker exec -it myOFS /home/OFS_BUILD_ROOT/ofs-agx7-pcie-attach_eval.sh 2\n\nGo to selection: 2\n###########################################################################################\n#################### Check versions of Operation System, Quartus ##########################\n###########################################################################################\n\nChecking Linux release\nLinux version 6.1.41-dfl .....\n\n....\n\ncycle complete exiting...\n

                3. The Intel Docker image includes the script ofs_extratool.sh to allow you to change the seed value. 

                  sudo docker exec -it myOFS /home/OFS_BUILD_ROOT/ofs_extratool.sh -s 5\n
                  Now you can control and compile the design. You can use the interactive or de-attach mode.

                4. If you need to save the log file and output files use the following command 

                  sudo docker exec -it myOFS /home/OFS_BUILD_ROOT/ofs_extratool.sh -e\n

                  all the files are saved under the share volume, DataOFS , /var/lib/docker/volumes/DataOFS/_data

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#40-deployment","title":"4.0 Deployment","text":"

              The OFS docker image allows you to connect with your FPGA Platform. The main difference from the development installation process is that you are able to test with real hardware, but you must have a specific requirement to have a fully compatible system.

              Information related to host setup please go to OFS Site select your desired platform and select Getting stated guide.

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#41-installation-of-deployment-server","title":"4.1 Installation of Deployment server","text":"

              Once you ensure the DFL drivers are installed, follow the below steps:

              1. Follow the steps listed in sections 2.1 to 2.3
                • 2.1 Quartus installation
                • 2.2 Docker Engine installation
                • 2.3 Load Docker Image installation
              2. The steps required for DFL driver installation are documented OFS Site select your desired platform and select Getting stated guide.

              Now you should have all the steps required, and you can run the docker image directly.

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#42-create-a-container","title":"4.2 Create a container","text":"

              Now you are ready to start the container, and should be prepared to run it (Note: now we are adding a new flag to allow us to access the PCIe devices \u201c\u2014privileged\u201d) :

              1. First, copy your Quartus installation path and paste it under -v:

                docker run --rm --privileged -itd --name myOFS -v=<yourintallationfolder>:/home/intelFPGA_pro/:ro -v=DataOFS:/dataofs ofs:latest /bin/bash\n

                Example, my Quartus installation is located at \"/home/intelFPGA_pro/23.3\" as a result, my command should be 

                docker run --rm --privileged -itd --name myOFS -v=/home/intelFPGA_pro/23.3:/home/intelFPGA_pro/:ro -v=DataOFS:/dataofs ofs:latest /bin/bash\nbdc1289fb0813bb325b55dd11df4eeec252143d6745a6e5772638fbc107d0949\n

                Tip: you can change myOFS with any other value. The value is the given name of the container.

                Important: The --privileged flag gives all capabilities to the container. When the operator executes docker run --privileged, Docker will enable access to all devices on the host as well as set some configuration in AppArmor or SELinux to allow the container nearly all the same access to the host as processes running outside containers on the host. Additional information about running with --privileged is available on the Docker Blog.

              :warning: Only use --privileged under development infrastructure, never in production!

              1. Execute the docker run command.

                docker run --rm --privileged -itd --name myOFS -v=/home/intelFPGA_pro/23.3:/home/intelFPGA_pro/:ro -v=DataOFS:/dataofs ofs:latest /bin/bash\n25b41eb4d232de9c750b52ddc6b92a3db612200e5993f55733b59068898623d7\n

              2. Now, the docker container should be available.

                # sudo docker ps\nCONTAINER ID   IMAGE                              COMMAND       CREATED     STATUS     PORTS     NAMES\n25b41eb4d232   ofs:latest                        \"/bin/bash\"   13 seconds ago   Up 12 seconds     myOFS\n

              \u200b Your Container ID is 25b41eb4d232.

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#43-evaluate-ofs-container","title":"4.3 Evaluate OFS container","text":"

              The OFS container has two possible ways to interact with the container:

              • Interactive mode:

                This mode it takes you straight inside the container and uses the command terminal as a regular Linux console.

                1. Enable the interactive mode:

                  docker attach myOFS\n[root@25b41eb4d232 /]#\n

                  The container id is shown when you are in interactive mode [root@25b41eb4d232 /]#.

                2. Now verify the variables and Quartus is appropriately setup and recognized:

                  quartus_syn --version\n\nQuartus Prime Synthesis\nVersion 23.3\n

                3. Everything is set up correctly. Please go to the following link for more information related to the OFS Site select your desired platform and select User Guide, Technical Reference Manual, Developer Guide, or Getting Started Guide.

                  Tip: If you need to de-attach without stopping the container you can use Ctrl+P or Ctrl+Q. For custom, combinations use for example docker attach --detach-keys=\"ctrl-a\" myOFS and if you press CTRL+A you will exit the container, without killing it.

              • De-attach Mode:

                This mode runs your container in the background and allows you to run multiple commands without going inside of the docker container.

                1. The OFS Docker image already includes the eval script.

                2. Run the script and make a selection, you can directly execute with the following command:

                  Let's use option 3 - Identify Platform Hardware via PCIe; remember the DFL drivers need be installed. 

                $ sudo docker exec -it myOFS /home/OFS_BUILD_ROOT/ofs-agx7-pcie-attach_eval.sh 3\n\nGo to selection: 3\n\n\nPCIe card detected as\n\n\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nb1:00.1 Processing accelerators: Intel Corporation Device bcce\nb1:00.2 Processing accelerators: Intel Corporation Device bcce\nb1:00.4 Processing accelerators: Intel Corporation Device bcce\n\nHost Server is connected to SINGLE card configuration\n\ncycle complete exiting...\n
                1. The Intel Docker image includes the script ofs_extratool.sh to allow you to change the seed value.
                  ```sh\nsudo docker exec -it myOFS /home/OFS_BUILD_ROOT/ofs_extratool.sh -s 5\n```\n

                Now you can control and compile the design using the interactive or de-attach mode.

              "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#notices-disclaimers","title":"Notices & Disclaimers","text":"

              Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Performance varies by use, configuration and other factors. Your costs and results may vary. You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications. Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards. \u00a9 Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.

              OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122.

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/","title":"Evaluation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach","text":""},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#1-overview","title":"1 Overview","text":""},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#11-about-this-document","title":"1.1 About this Document","text":"

              This document serves as a set-up and user guide for the checkout and evaluation of an Intel\u00ae FPGA SmartNIC N6001-PL and Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) development platform using Open FPGA Stack (OFS). After reviewing the document, you will be able to:

              • Use the script to run through the most common build and simulation flows when using OFS.
              • Run hardware and software tests to evaluate the complete OFS flow
              • Modify and leverage the script to the your environment and design
              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#table-1-2-software-version-summary","title":"Table 1-2: Software Version Summary","text":"Component Version Description FPGA Platform Intel\u00ae FPGA SmartNIC N6001-PL, Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) Intel platform you can use for your custom board development OFS FIM Source Code Branch: https://github.com/OFS/ofs-agx7-pcie-attach, Tag: ofs-2023.3-2 OFS Shell RTL for Intel Agilex FPGA (targeting Intel\u00ae FPGA SmartNIC N6001-PL) AFU Examples Branch: examples-afu , Tag:ofs-examples-https://github.com/OFS/examples-afu/releases/tag/ofs-2023.3-2 Tutorials and simple examples for the Accelerator Functional Unit region (workload region) OPAE SDK Branch: 2.10.0-1, Tag: 2.10.0-1 Open Programmable Acceleration Engine Software Development Kit Kernel Drivers Branch: ofs-2023.3-6.1-3, Tag: ofs-2023.3-6.1-3 OFS specific kernel drivers OPAE Simulation Branch: opae-sim, Tag: 2.10.0-1 Accelerator Simulation Environment for hardware/software co-simulation of your AFU (workload) Intel Quartus Prime Pro Edition Design Software 23.3 [Intel\u00ae Quartus\u00ae Prime Pro Edition Linux] Software tool for Intel FPGA Development Operating System RHEL 8.6 Operating system on which this script has been tested

              A download page containing the release and already-compiled FIM binary artifacts that you can use for immediate evaluation on the Intel\u00ae FPGA SmartNIC N6001-PL can be found on the OFS ofs-2023.3-2 official release drop on GitHub.

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#2-introduction-to-ofs-evaluation-script","title":"2 Introduction to OFS Evaluation Script","text":"

              By following the setup steps and using the OFS evaluation script you can quickly evaluate many features that the OFS framework provides and also leverage this script for your own development.

              NOTE:

              This guide uses the Intel\u00ae FPGA SmartNIC N6001-PL as the platform for all example steps. Additionally, this guide and the example steps can be used with other platforms, such as the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile).

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#21-pre-requisites","title":"2.1 Pre-Requisites","text":"

              This script uses the following set of software tools which should be installed using the directory structure below. Tool versions can vary.

              • Intel Quartus\u00ae Prime Pro Software
              • Synopsys\u00ae VCS Simulator
              • Siemens\u00ae Questa\u00ae Simulator

              Figure 2-1 Folder Hierarchy for Software Tools

              1. You must create a directory named \"ofs-X.X.X\" where the X represents the current release number, for example ofs-2023.3-2.

              2. You must clone the required OFS repositories as per Figure 2-2. Please refer to the BKC table 1-2 for locations. When cloning the FIM repository, please follow the instructions in the following guides

              • FPGA Interface Manager (FIM) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach
              • FPGA Interface Manager (FIM) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach (2xF-tile).

              Additionally, please follow the instructions in the following guides for the BKC software installation.

              • Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL)
              • Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 SoC Attach FPGAs (Intel Agilx 7 FPGA F-Series Development Kit (2xF-Tile))
              1. Once the repositories are cloned, The scripts that go with each user guide are found in the assets tab of the corresponding FIM RTL repository. Download the evaluation script (ofs-agx7-pcie-attach_eval.sh) and copy it to the ofs-2023.3-2 directory location as shown in the example below:

              Figure 2-2 Directory Structure for OFS Project

              ## ofs-2023.3-2\n##  -> examples-afu\n##  -> linux-dfl\n##  -> ofs-agx7-pcie-attach\n##  -> oneapi-asp\n##  -> oneAPI-samples\n##  -> opae-sdk\n##  -> opae-sim\n##  -> ofs-agx7-pcie-attach_eval.sh\n
              1. The README file that accompanies each user guide are found in the assets tab of the corresponding FIM RTL repository. Download the README file (README_ofs-agx7-pcie-attach_eval.txt) and copy it to the ofs-2023.3-2 directory location. The README informs the user which sections to modify in the script prior to building the FIM and running hardware, software and simulation tests.
              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#22-intel-agilex-7-pcie-attach-evaluation-script-modification","title":"2.2 Intel\u00ae Agilex\u00ae 7 PCIe Attach Evaluation Script modification","text":"

              To adapt this script to the user environment please follow the instructions below which explains which line numbers to change in the ofs-agx7-pcie-attach.sh script

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#user-directory-creation","title":"User Directory Creation","text":"

              The user must create the top-level source directory and then clone the OFS repositories

              mkdir ofs-2023.3-2\n

              In the example above we have used ofs-2023.3-2 as the directory name

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#set-up-proxy-server-lines-67-69","title":"Set-Up Proxy Server (lines 67-69)","text":"

              Please enter the location of your proxy server to allow access to external internet to build software packages.

              Note: Failing to add proxy server will prevent cloning of repositories and the user will be unable to build the OFS framework.

              export http_proxy=<user_proxy>\nexport https_proxy=<user_proxy>\nexport no_proxy=<user_proxy>\n
              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#license-files-lines-72-74","title":"License Files (lines 72-74)","text":"

              Please enter the the license file locations for the following tool variables

              export LM_LICENSE_FILE=<user_license>\nexport DW_LICENSE_FILE=<user_license>\nexport SNPSLMD_LICENSE_FILE=<user_license>\n
              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#tools-location-line-87-88-89-90","title":"Tools Location (line 87, 88, 89, 90)","text":"

              Set Location of Quartus, Synopsys, Questasim and oneAPI Tools

              export QUARTUS_TOOLS_LOCATION=/home\nexport SYNOPSYS_TOOLS_LOCATION=/home\nexport QUESTASIM_TOOLS_LOCATION=/home\nexport ONEAPI_TOOLS_LOCATION=/opt\n

              In the example above /home is used as the base location of Quartus, Synopsys and Questasim tools, /opt is used for the oneAPI tools

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#quartus-tools-version-line-95","title":"Quartus Tools Version (line 95)","text":"

              Set version of Quartus

              export QUARTUS_VERSION=23.3\n

              In the example above \"23.3\" is used as the Quartus tools version

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#opae-tools-line-108","title":"OPAE Tools (line 108)","text":"

              change OPAE SDK VERSION

              export OPAE_SDK_VERSION=2.10.0-1\n

              In the example above \"2.10.0-1\" is used as the OPAE SDK tools version

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#pcie-bus-number","title":"PCIe (Bus Number)","text":"

              The Bus number must be entered by the user after installing the hardware in the chosen server, in the example below \"b1\" is the Bus Number for a single card as defined in the evaluation script.

              export OFS_CARD0_BUS_NUMBER=b1\n

              The evaluation script uses the bus number as an identifier to interrogate the card. The command below will identify the accelerater card plugged into a server. 

              lspci | grep acc\n\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)<br>\nb1:00.1 Processing accelerators: Intel Corporation Device bcce<br>\nb1:00.2 Processing accelerators: Intel Corporation Device bcce<br>\nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device<br>\nb1:00.4 Processing accelerators: Intel Corporation Device bcce<br>\n

              The result identifies the card as being assigned \"b1\" as the bus number so the entry in the script changes to

              export OFS_CARD0_BUS_NUMBER=b1\n

              The user can also run the following command on the ofs-agx7-pcie-attach_eval.sh script to automatically change the bus number to b1 in the ofs-agx7-pcie-attach_eval.sh script.

              grep -rli 'b1' * | xargs -i@ sed -i 'b1' @

              if the bus number is 85 for example 

              85:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)<br>\n85:00.1 Processing accelerators: Intel Corporation Device bcce<br>\n85:00.2 Processing accelerators: Intel Corporation Device bcce<br>\n85:00.3 Processing accelerators: Red Hat, Inc. Virtio network device<br>\n85:00.4 Processing accelerators: Intel Corporation Device bcce<br>\n

              the command to change to 85 in the evaluation script would be

              grep -rli 'b1' * | xargs -i@ sed -i '85' @

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#ofs-platform-example-n6000-n6001-fseries-dk-iseries-dk-custom_board-choice-line-173","title":"OFS Platform (Example:= n6000, n6001, fseries-dk, iseries-dk, custom_board) choice (line 173)","text":"

              The script is designed to accomodate many OFS platform choices eg, n6000, n6001, fseries-dk, iseries-dk or a custom_board of your choice. By default the script defaults to n6001 as shown below and is set by a variable on line 173 of the ofs-agx7-pcie-attach_eval.sh script.

                export OFS_PLATFORM=n6001\n

              but the user can switch platform by changing the OFS_PLATFORM variable, so for example if the user wants to switch to the i-series development kit the command would be

                export OFS_PLATFORM=iseries-dk\n

              The ofs-agx7-pcie-attach_eval.sh script has now been modified to the server set-up and the user can proceed to build, compile and simulate the OFS stack

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#3-ofs-agx7-pcie-attach-evaluation-script","title":"3 ofs-agx7-pcie-attach Evaluation Script","text":""},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#31-overview","title":"3.1 Overview","text":"

              The figure below shows a snapshot of the full evaluation script menu showing all 62 options and each one of 11 sub-menus which focus on different areas of evaluation. Each of these menu options are described in the next section.

              Figure 3-1 ofs-agx7-pcie-attach_eval.sh Evaluation Menu

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#311-ofs-tools-menu","title":"3.1.1 OFS TOOLS MENU","text":"

              By selecting \"List of Documentation for OFS Project,\" a list of links to the latest OFS documentation appears. Note that these links will take you to documentation for the most recent release which may not correspond to the release version you are evaluating. To find the documentation specific to your release, ensure you clone the intel-ofs-docs tag that corresponds to your OFS version.

              By selecting \"Check Versions of Operating System and Quartus Premier Design Suite\", the tool verifies correct Operating System, Quartus version, kernel parameters, license files and paths to installed software tools.

              Menu Option Example Output 1 - List of Documentation for OFS PCI Attach Project Project Open FPGA Stack Overview Guides you through the setup and build steps to evaluate the OFS solution https://ofs.github.io 2 - Check versions of Operating System and Quartus Premier Design Suite (QPDS) Checking Linux release Linux version 5.15.52-dfl (guest@hw-rae-svr4-l) (gcc (GCC) 8.5.0 20210514 (Red Hat 8.5.0-4), GNU ld version 2.30-79.el8) #1 SMP Fri Sep 23 17:19:37 BST 2022 Checking RedHat release CentOS Linux release 8.3.2011 Checking Ubuntu release cat: /etc/lsb-release: No such file or directory Checking Kernel parameters BOOT_IMAGE=(hd0,gpt2)/vmlinuz-5.15.52-dfl root=/dev/mapper/cl-root ro crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200 Checking Licenses LM_LICENSE_FILE is set to port@socket number:port@socket number DW_LICENSE_FILE is set to port@socket number:port@socket number SNPSLMD_LICENSE_FILE is set to port@socket number:port@socket number Checking Tool versions QUARTUS_HOME is set to /home/intelFPGA_pro/23.3/quartus QUARTUS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus IMPORT_IP_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../ip QSYS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../qsys/bin Checking QPDS Patches Quartus Prime Shell Version 23.3 Build XXX XX/XX/XXXX Patches X.XX SC Pro Edition Copyright (C) XXXX Intel Corporation. All rights reserved."},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#312-ofs-hardware-menu","title":"3.1.2 OFS HARDWARE MENU","text":"

              Identifies card by PCIe number, checks power, temperature and current firmware configuration.

              Menu Option Example Output 3 - Identify Platform Hardware via PCIe PCIe card detected as b1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01) b1:00.1 Processing accelerators: Intel Corporation Device bcce b1:00.2 Processing accelerators: Intel Corporation Device bcce b1:00.4 Processing accelerators: Intel Corporation Device bcce Host Server is connected to SINGLE card configuration 4 - Identify the Board Management Controller (BMC) Version and check BMC sensors Intel Acceleration Development Platform N6001 Board Management Controller NIOS FW version: 3.2.0 Board Management Controller Build version: 3.2.0 //****** BMC SENSORS ******// Object Id : 0xEE00000 PCIe s:b:d.f : 0000:B1:00.0 Vendor Id : 0x8086 Device Id : 0xBCCE SubVendor Id : 0x8086 SubDevice Id : 0x1771 Socket Id : 0x00 Ports Num : 01 Bitstream Id : 0x50102027135A894 Bitstream Version : 5.0.1 Pr Interface Id : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2 Note: This feature is not supported on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) 5 - Identify the FPGA Management Engine (FME) Version Intel Acceleration Development Platform N6001 Board Management Controller NIOS FW version: 3.2.0 Management Controller Build version: 3.2.0 //****** FME ******// Object Id : 0xEE00000 PCIe s:b:d.f : 0000:B1:00.0 Vendor Id : 0x8086 Device Id : 0xBCCE SubVendor Id : 0x8086 SubDevice Id : 0x1771 Socket Id : 0x00 Ports Num : 01 Bitstream Id : 0x50102027135A894 Bitstream Version : 5.0.1 Pr Interface Id : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2 Boot Page : user1 Factory Image Info : a7c6879683182ce61084c420e51f50b6 User1 Image Info : 8a7440ddff52e0e27dbb989d5eb954f4 User2 Image Info : a7c6879683182ce61084c420e51f50b6 6 - Check Board Power and Temperature Intel Acceleration Development Platform N6001 Board Management Controller NIOS FW version: 3.2.0 Board Management Controller Build version: 3.2.0 //****** POWER ******// Object Id : 0xEE00000 PCIe s:b:d.f : 0000:B1:00.0 Vendor Id : 0x8086 Device Id : 0xBCCE SubVendor Id : 0x8086 SubDevice Id : 0x1771 Socket Id : 0x00 Ports Num : 01 Bitstream Id : 0x50102027135A894 Bitstream Version : 5.0.1 Pr Interface Id : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2 ( 1) VCCRT_GXER_0V9 Voltage : 0.91 Volts etc ...................... Intel Acceleration Development Platform N6001 Board Management Controller NIOS FW version: 3.2.0 Board Management Controller Build version: 3.2.0 //****** TEMP ******// Object Id : 0xEE00000 PCIe s:b:d.f : 0000:B1:00.0 Vendor Id : 0x8086 Device Id : 0xBCCE SubVendor Id : 0x8086 SubDevice Id : 0x1771 Socket Id : 0x00 Ports Num : 01 Bitstream Id : 0x50102027135A894 Bitstream Version : 5.0.1 Pr Interface Id : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2 ( 1) FPGA E-Tile Temperature [Remote] : 33.50 Celsius etc ...................... Note: This feature is not supported on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) 7 - Check Accelerator Port status //****** PORT ******// Object Id : 0xED00000 PCIe s:b:d.f : 0000:B1:00.0 Vendor Id : 0x8086 Device Id : 0xBCCE SubVendor Id : 0x8086 SubDevice Id : 0x1771 Socket Id : 0x00 8 - Check MAC and PHY status Intel Acceleration Development Platform N6001 Board Management Controller NIOS FW version: 3.2.0 Board Management Controller Build version: 3.2.0 //****** MAC ******// Object Id : 0xEE00000 PCIe s:b:d.f : 0000:B1:00.0 Vendor Id : 0x8086 Device Id : 0xBCCE SubVendor Id : 0x8086 SubDevice Id : 0x1771 Socket Id : 0x00 Ports Num : 01 Bitstream Id : 0x50102027135A894 Bitstream Version : 5.0.1 Pr Interface Id : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2 Number of MACs : 255 mac info is not supported Intel Acceleration Development Platform N6001 Board Management Controller NIOS FW version: 3.2.0 Board Management Controller Build version: 3.2.0 //****** PHY ******// Object Id : 0xEE00000 PCIe s:b:d.f : 0000:B1:00.0 Vendor Id : 0x8086 Device Id : 0xBCCE SubVendor Id : 0x8086 SubDevice Id : 0x1771 Socket Id : 0x00 Ports Num : 01 Bitstream Id : 0x50102027135A894 Bitstream Version : 5.0.1 Pr Interface Id : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2 //****** HSSI information ******// HSSI version : 1.0 Number of ports : 8 Port0 :25GbE DOWN Port1 :25GbE DOWN Port2 :25GbE DOWN Port3 :25GbE DOWN Port4 :25GbE DOWN Port5 :25GbE DOWN Port6 :25GbE DOWN Port7 :25GbE DOWN Note: This feature is not supported on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#313-ofs-pfvf-mux-menu","title":"3.1.3 OFS PF/VF MUX MENU","text":"

              This menu reports the number of PF/VF functions in the reference example and also allows you to reduce the number to 1PF and 1VF to reduce resource utilisation and create a larger area for your workload development. This selection is optional and if the user wants to implement the default number of PF's and VF's then option 9, 10 and 11 should not be used. Additionally the code used to make the PF/VF modification can be leveraged to increase or modify the number of PF/VFs in the existing design within the limits that the PCIe Subsystem supports (8PF/2K VFs)

              Menu Option Description 9 - Check PF/VF Mux Configuration This menu selection displays the current configuration of all n6001 ofss files located in the following directory $OFS_ROOTDIR/tools/ofss_config Check n6001 base config OFSS set up [ip] type = ofs [settings] platform = n6001 family = agilex fim = base_x16 part = AGFB014R24A2E2V device_id = 6001 Check n6001 hssi_2x100 OFSS set up [ip] type = hssi [settings] output_name = hssi_ss num_channels = 2 data_rate = 100GCAUI-4 Check n6001 hssi_2x100_caui2 OFSS set up [ip] type = hssi [settings] output_name = hssi_ss num_channels = 2 data_rate = 100GAUI-2 Check n6001 hssi_8x10 OFSS set up [ip] type = hssi [settings] output_name = hssi_ss num_channels = 8 data_rate = 10GbE Check n6001 hssi_8x25 OFSS set up [ip] type = hssi [settings] output_name = hssi_ss num_channels = 8 data_rate = 25GbE Check n6001 IOPLL OFSS set up [ip] type = iopll [settings] output_name = sys_pll instance_name = iopll_0 [p_clk] freq = 470 Check n6001 Memory OFSS set up [ip] type = memory [settings] output_name = mem_ss_fm preset = n6001 Check n6001 PCIe Host OFSS set up [ip] type = pcie [settings] output_name = pcie_ss [pf0] num_vfs = 3 bar0_address_width = 20 vf_bar0_address_width = 20 [pf1] [pf2] bar0_address_width = 18 [pf3] [pf4] Check n6001 PCIe 1pf_1vf OFSS set up [ip] type = pcie [settings] output_name = pcie_ss [pf0] num_vfs = 1 bar0_address_width = 20 vf_bar0_address_width = 20 10 - Modify PF/VF Mux Configuration As an example this menu selection modifies the pcie_host.ofss file to 1 PF located in the following directory $OFS_ROOTDIR/tools/ofss_config This option also displays the the modified pcie_host.ofss file 11 - Build PF/VF Mux Configuration If option 10 is not used then then the default number of PF's and VF's is used to build the FIM, if option 10 is selected then only 1 VF is built to reduce logic utilisation"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#314-ofs-fimpr-build-menu","title":"3.1.4 OFS FIM/PR BUILD MENU","text":"

              Builds FIM, Partial Reconfiguration Region and Remote Signal Tap

              Menu Option Description 12 - Check OFS software version for OFS Project OFS_ROOTDIR is set to /home/user_area/ofs-X.X.X/ofs-agx7-pcie-attach OPAE_SDK_REPO_BRANCH is set to release/X.X.X OPAE_SDK_ROOT is set to /home/user_area/ofs-X.X.X/ofs-agx7-pcie-attach/../opae-sdk LD_LIBRARY_PATH is set to /home/user_area/ofs-X.X.X/ofs-agx7-pcie-attach/../opae-sdk/lib64: 13 - Build FIM for Hardware This option builds the FIM based on the setting for the $OFS_PLATFORM, $FIM_SHELL environment variable. Check this variable in the following file ofs-agx7-pcie-attach_eval.sh 14 - Check FIM Identification of FIM for Hardware The FIM is identified by the following file fme-ifc-id.txt located at $OFS_ROOTDIR/$FIM_WORKDIR/syn/board/$OFS_PLATFORM/syn_top/ 15 - Build Partial Reconfiguration Tree for Hardware This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development and also for the oneAPI build flow 16 - Build Base FIM Identification(ID) into PR Build Tree template This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU and oneAPI workloads 17 - Build Partial Reconfiguration Tree for Hardware with Remote Signal Tap This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development and also for the oneAPI build flow for the Remote Signal Tap flow 18 - Build Base FIM Identification(ID) into PR Build Tree template with Remote Signal Tap This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree for Remote Signal Tap to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU and oneAPI workloads"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#315-ofs-hardware-programmingdiagnostic-menu","title":"3.1.5 OFS HARDWARE PROGRAMMING/DIAGNOSTIC MENU","text":"

              The following submenu allows you to: * Program and check flash * Perform a remote system update (RSU) of the FPGA image into the FPGA * Bind virtual functions to VFIO PCIe driver * Run host exerciser (HE) commands such as loopback to test interfaces VFIO PCI driver binding * Read the control and status registers (CSRs) for bound modules that are part of the OFS reference design.

              Menu Option Description 19 - Program BMC Image into Hardware The user must place a new BMC flash file in the following directory $OFS_ROOTDIR/bmc_flash_files. Once the user executes this option a new BMC image will be programmed. A remote system upgrade command is initiated to store the new BMC image 20 - Check Boot Area Flash Image from Hardware This option checks which location area in FLASH the image will boot from, the default is user1 Boot Page : user1 Note: This feature is not supported on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) 21 - Program FIM Image into user1 area for Hardware This option programs the FIM image \"ofs_top_page1_unsigned_user1.bin\" into user1 area in flash Note: Please refer to the Getting Started Guide for details on flashing images for the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) 22 - Initiate Remote System Upgrade (RSU) from user1 Flash Image into Hardware This option initiates a Remote System Upgrade and soft reboots the server and re-scans the PCIe bus for the new image to be loaded 2022-11-10 11:26:24,307 - [[pci_address(0000:b1:00.0), pci_id(0x8086, 0xbcce)]] performing RSU operation 2022-11-10 11:26:24,310 - [[pci_address(0000:b0:02.0), pci_id(0x8086, 0x347a)]] removing device from PCIe bus 2022-11-10 11:26:24,357 - waiting 10 seconds for boot 2022-11-10 11:26:34,368 - rescanning PCIe bus: /sys/devices/pci0000:b0/pci_bus/0000:b0 2022-11-10 11:26:35,965 - RSU operation complete Note: Please refer to the Getting Started Guide for details on flashing images for the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) 23 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 24 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 25 - Create Virtual Functions (VF) and bind driver to vfio-pci Hardware This option creates vfio-pci driver binding for the PF's and VF's Once the VF's have been bound to the driver the user can select menu option 23 to check that the new drivers are bound 26 - Verify FME Interrupts with hello_events The hello_events utility is used to verify FME interrupts. This tool injects FME errors and waits for error interrupts, then clears the errors 27 - Run HE-LB Test This option runs 5 tests 1) checks and generates traffic with the intention of exercising the path from the AFU to the Host at full bandwidth 2) run a loopback throughput test using one cacheline per request 3) run a loopback read test using four cachelines per request 4) run a loopback write test using four cachelines per request 5) run a loopback throughput test using four cachelines per request 28 - Run HE-MEM Test This option runs 2 tests 1) Checking and generating traffic with the intention of exercising the path from FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host 2) run a loopback throughput test using one cacheline per request 29 - Run HE-HSSI Test This option runs 1 test HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs, and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic 1) Send traffic through the 10G AFU 30 - Run Traffic Generator AFU Test This option runs 3 tests TG AFU has an OPAE application to access & exercise traffic, targeting a specific bank 1) Run the preconfigured write/read traffic test on channel 0 2) Target channel 1 with a 1MB single-word write only test for 1000 iterations 3) Target channel 2 with 4MB write/read test of max burst length for 10 iterations 31 - Read from CSR (Command and Status Registers) for Hardware This option reads from the following CSR's HE-LB Command and Status Register Default Definitions HE-MEM Command and Status Register Default Definitions HE-HSSI Command and Status Register Default Definitions"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#316-ofs-hardware-afu-testing-menu","title":"3.1.6 OFS HARDWARE AFU TESTING MENU","text":"

              This submenu tests partial reconfiguration by building and loading an memory-mapped I/O example AFU/workload, executes software from host, and tests remote signal tap.

              Menu Option Description 32 - Build and Compile host_chan_mmio example This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS (Green Bit Stream) ready for hardware programming 33 - Execute host_chan_mmio example This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test 34 - Modify host_chan_mmio example to insert Remote Signal Tap This option inserts a pre-defined host_chan_mmio.stp Signal Tap file into the OFS code to allow a user to debug the host_chan_mmio AFU example 35 - Build and Compile host_chan_mmio example with Remote Signal Tap This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS(Green Bit Stream) ready for hardware programming with Remote Signal tap enabled 36 - Execute host_chan_mmio example with Remote Signal Tap This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test. The user must open the Signal Tap window when running the host code to see the transactions in the Signal Tap window"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#317-ofs-hardware-afu-bbb-testing-menu","title":"3.1.7 OFS HARDWARE AFU BBB TESTING MENU","text":"

              This submenu tests partial reconfiguration using a hello_world example AFU/workload, executes sw from host

              Menu Option Description 37 - Build and Compile hello_world example This option builds the hello_ world example from the following repo $FPGA_BBB_CCI_SRC/samples/tutorial/afu_types/01_pim_ifc/$AFU_BBB_TEST_NAME, where AFU_BBB_NAME=hello_world. This produces a GBS(Green Bit Stream) ready for hardware programming 38 - Execute hello_world example This option builds the host code for hello_world example and programs the GBS file and then executes the test"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#318-ofs-oneapi-project-menu","title":"3.1.8 OFS ONEAPI PROJECT MENU","text":"

              Builds oneAPI kernel, executes sw from host and runs diagnostic tests

              Menu Option Result 39 - Check oneAPI software versions for Project This option checks the setup of the oneAPI software and adds the relevant oneAPI environment variables to the terminal. This option also informs the user to match the oneAPI software version to the oneAPI-samples version 40 - Build and clone shim libraries required by oneAPI host This option builds the oneAPI directory structure 41 - Install oneAPI Host Driver This option Installs the oneAPI Host driver at the following location /opt/Intel/OpenCLFPGA/oneAPI/Boards/, and requires sudo permission 42 - Uninstall oneAPI Host Driver This option Uninstall's the oneAPI Host driver, and requires sudo permissions 43 - Diagnose oneAPI Hardware This option Checks ICD (Intel Client Driver) and FCD (FPGA Client Driver), oneAPI library locations and detects whether oneAPI BSP is loaded into the FPGA 44 - Build oneAPI BSP Default Kernel (hello_world) This option Builds the oneAPI BSP using simple-add_buffers kernel 45 - Build oneAPI MakeFile Environment This option Builds the oneAPI environment using a Makefile for kernel insertion 46 - Compile oneAPI Sample Application (board_test) for Emulation This option compiles the board_test kernel for Emulation 47 - Run oneAPI Sample Application (board_test) for Emulation This option executes the board_test kernel for Emulation 48 - Generate oneAPI Optimization report for (board_test) This option generates an optimization report for the board_test kernel 49 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 50 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 51 - Create Virtual Function (VF) and bind driver to vfio-pci Hardware This option creates vfio-pci driver binding for the PF's and VF's Once the VF's have been bound to the driver the user can select menu option 45 to check that the new drivers are bound 52 - Program OpenCL BSP Default Kernel (hello_world) This option programs the FPGA with a aocf file based on the hello_world kernel 53 - Compile oneAPI Sample Application (board_test) for Hardware This option compiles the board_test kernel for Hardware 54 - Run oneAPI Sample Application (board_test) for Hardware This option builds the host code for board_test kernel and executes the program running through kernel and host bandwidth tests"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#319-ofs-unit-test-project-menu","title":"3.1.9 OFS UNIT TEST PROJECT MENU","text":"

              Builds, compiles and runs standalone simulation block tests. More unit test examples are found at the following location ofs_n6001/sim/unit_test

              Menu Option Result 55 - Generate Simulation files for Unit Test This option builds the simulation file set for running a unit test simulation 56 - Simulate Unit Test dfh_walker and log waveform This option runs the dfh_walker based on the environment variable \"UNIT_TEST_NAME=dfh_walker\" in the evaluation script. A user can change the test being run by modifying this variable"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#3110-ofs-uvm-project-menu","title":"3.1.10 OFS UVM PROJECT MENU","text":"

              Builds, compiles and runs full chip simulation tests. The user should execute the options sequentially ie 68,69, 70 and 71

              Menu Option Description 57 - Check UVM software versions for Project DESIGNWARE_HOME is set to /home/synopsys/vip_common/vip_Q-2020.03A UVM_HOME is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm VCS_HOME is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel VERDIR is set to /home/user_area/ofs-X.X.X/ofs-agx7-pcie-attach/verification VIPDIR is set to /home/user_area/ofs-X.X.X/ofs-agx7-pcie-attach/verification 58 - Compile UVM IP This option compiles the UVM IP 59 - Compile UVM RTL and Testbench This option compiles the UVM RTL and Testbench 60 - Simulate UVM dfh_walking_test and log waveform This option runs the dfh_walking test based on the environment variable \"UVM_TEST_NAME=dfh_walking_test\" in the evaluation script. A user can change the test being run by modifying this variable 61 - Simulate all UVM test cases (Regression Mode) This option runs the n6001 regression mode, cycling through all UVM tests defined in verification/tests/test_pkg.svh file"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#3111-ofs-build-all-project-menu","title":"3.1.11 OFS BUILD ALL PROJECT MENU","text":"

              Builds the complete OFS flow, good for regression testing and overnight builds

              For this menu a user can run a sequence of tests (compilation, build and simulation) and executes them sequentially. After the script is successfully executed, a set of binary files is produced which a you can use to evaluate your hardware. Log files are also produced which checks whether the tests passed.

              A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 62 from the main menu the script will execute 24 tests ie (main menu options 2, 9, 12, 13, 14, 15, 16, 17, 18, 32, 34, 35, 37, 39, 40, 44, 45, 53, 55, 56, 57, 58, 59 and 60. These 24 menu options are chosen to build the complete OFS flow covering build, compile and simulation.

              Menu Option Result 62 - Build and Simulate Complete n6001 Project Generating Log File with date and timestamp Log file written to /home/guest/ofs-2.3.1/log_files/ofs-agx7-pcie-attach_log_2022_11_10-093649/ofs-agx7-pcie-attach_eval.log"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#definition-of-multi-test-set-up","title":"Definition of Multi-Test Set-up","text":"

              Menu Option 62 above in the evaluation script can be refined to tailor it to the users need and is principally defined by the variable below

              MULTI_TEST[A,B]=C

              where

              A= Total Number of menu options in script B= Can be changed to a number to select the test order C= Menu Option in Script

              Example 1 MULTI_TEST[62,0]=2

              A= 62 is the total number of options in the script B= 0 indicates that this is the first test to be run in the script C= Menu option in Script ie 2- List of Documentation for OFS n6001 Project

              Example 2 MULTI_TEST[62,0]=2 MULTI_TEST[62,1]=9

              In the example above two tests are run in order ie 0, and 1 and the following menu options are executed ie 2- List of Documentation for OFS n6001 Project and 9 - Check OFS software versions for OFS n6001 Project

              The user can also modify the build time by de-selecting options they do not wish to use, see below for a couple of use-case scenarios.

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#default-user-case","title":"Default User Case","text":"

              A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 62 from the main menu the script will execute 24 tests ie (main menu options 2, 9, 12, 13, 14, 15, 16, 17, 18, 32, 34, 35, 37, 39, 40, 44, 45, 53, 55, 56, 57, 58, 59 and 60. All other tests with an \"X\" indicates do not run that test.

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#user-case-for-ofs-fimpr-build-menu","title":"User Case for OFS FIM/PR BUILD MENU","text":"

              In the example below when the user selects option 62 from the main menu the script will only run options from the OFS FIM/PR BUILD MENU (7 options, main menu options 12, 13, 14, 15, 16, 17 and 18). All other tests with an \"X\" indicates do not run that test.

              "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#4-n6001-common-test-scenarios","title":"4 n6001 Common Test Scenarios","text":"

              This section will describe the most common compile build scenarios if a user wanted to evaluate an acceleration card on their server. The Pre-requisite column indcates the menu comamnds that must be run befere executing the test eg To run Test 5 then a user needs to have run option 13, 15 and 16 before running options 23, 24, 25, 32 and 33.

              Test Test Scenario Pre-Requisite Menu Option Menu Option Test 1 FIM Build - 13 Test 2 Partial Reconfiguration Build 13 15, 16 Test 3 Program FIM and perform Remote System Upgrade 13 21, 22 Test 4 Bind PF and VF to vfio-pci drivers - 23, 24, 25 Test 5 Build, compile and test AFU on hardware 13, 15, 16 23, 24, 25, 32, 33 Test 6 Build, compile and test AFU Basic Building Blocks on hardware 13, 15, 16 23, 24, 25, 37, 38 Test 7 Build, compile and test oneAPI on hardware 13, 15, 16 39, 40, 41, 44, 45, 49, 50, 51, 52, 53, 54 Test 8 Build and Simulate Unit Tests - 55, 56 Test 9 Build and Simulate UVM Tests - 57, 58, 59, 60 Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework. In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#notices-disclaimers","title":"Notices & Disclaimers","text":"

              Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Performance varies by use, configuration and other factors. Your costs and results may vary. You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications. Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards. \u00a9 Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.

              OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122.

              "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/","title":"FPGA Developer Journey Guide: Open FPGA Stack","text":"

              Last updated: February 03, 2024

              "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#1-introduction","title":"1 Introduction","text":"

              This document is intended to help you understand the FPGA developer flow using OFS as well as considerations you should take when creating your custom platform.

              After reviewing the document, you shall be able to:

              • Understand how to evaluate the OFS framework for your platform needs.
              • Select a starting shell or FIM to begin your work.
              • Understand what resources are available for guiding you through making modifications for custom design as well as simulation and debug.

              The general development flow is depicted in the diagram below and discussed in more detail in each section of this document.

              Figure 1-1: FPGA Developer Flow

              "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#2-evaluate-ofs","title":"2 Evaluate OFS","text":"

              The repositories in the OFS site are tagged based on year and release number. For example, a tag of 2023.3-2-1 indicates the first release package of the quarter. If there an updates to this release package, the last number will be incremented by 1, such as 2023.3-2-2.

              By clicking on the release link to the right of the RTL repositories, you will find the latest release, the tag number and release notes.

              Figure 2-1: OFS Repository Release Page Link

              By scrolling to the end of the release page, you will find assets attached to the release that you can leverage for quick evaluation of OFS, such as FPGA binary files, POF and SOF as well as a sample AFU and a partial reconfiguration directory for building your own custom workload.

              There are two ways to evaluate OFS depending on your needs:

              Option 1: Setup your card and software in a server using the steps provided in one of the corresponding Getting Started Guides and leverage the appended binaries in the FIM RTL repository release page \"Assets\" tab to preview the software and design functionality the OFS framework provides you out of the box. This step will give you a good high-level overview of OFS. Getting Started Guides are available for the following FIM(shell) designs:

              • Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL)
              • Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile))
              • Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 SoC Attach FPGAs
              • Getting Started Guide: Open FPGA Stack for Intel\u00ae Stratix\u00ae 10 PCIe Attach FPGAs

              Option 2: After your card and software are installed using the steps provided in one of the corresponding Getting Started Guides listed above, use a corresponding Evaluation Guide and provided evaluation script to run through all the capabilities of the OFS framework by selecting one of the choices in the evaluation menu. The evaluation script gives you familiarity of the entire design, build, simulation, programming and test flow for OFS, including a OneAPI flow.

              • Evaluation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach
              • Evaluation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach
              • Evaluation User Guide: OFS for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach

              The scripts that go with each user guide are found in the assets tab of the corresponding FIM RTL repository's latest tag release page.

              To use the full functionality of the script you will want to ensure you clone all of the appropriate repositories below at the same level just under an OFS directory you create, such as ofs-2023.3-2, similar to the figure below.

              Figure 2-2: Directory Structure of Cloned Repositories

              ##|-- ofs-2023.3-2\n##|    |--examples-afu\n##|    |--linux-dfl\n##|    |--ofs-agx7-pcie-attach\n##|    |--ofs-oneapi-asp\n##|    |--opae-sdk\n##|    |--opae-sim\n##|    |--ofs-agx7-pcie-attach_eval.sh\n

              You can access the repositories from ofs.github.io by clicking on the GitHub icon in the right corner of the site.

              Figure 2-3: OFS Site Link from ofs.github.io

              After making your top level directory, initializing the repository and installing git lfs, clone one of the FIM RTL repositories you intend to use:

              #Make top level directory\nmkdir OFS\ncd OFS\n\n#Initialize repository and install git lfs\ngit init\n\ncurl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\nsudo dnf install git-lfs\ngit lfs install\n\n#Select a FIM RTL repository to clone\n#To clone Intel Agilex 7 PCIe Attach FIM RTL repository\n\ngit clone --recurse-submodules https://github.com/OFS/ofs-agx7-pcie-attach.git\ncd ofs-agx7-pcie-attach\ngit checkout --recurse-submodules tags/ofs-2023.3-2\n\n\n#To clone Intel Agilex 7 SoC Attach FIM RTL Repository\ngit clone --recurse-submodules https://github.com/OFS/ofs-f2000x-pl.git\ncd ofs-f2000x-pl\ngit checkout --recurse-submodules tags/ofs-2023.3-1\n\n#To clone Intel Stratix 10 PCIe Attach FIM RTL Repository\ngit clone --recurse-submodules https://github.com/OFS/ofs-d5005.git\ncd ofs-d5005\ngit checkout --recurse-submodules tags/ofs-2023.3-1\n

              After cloning your FIM RTL repository, clone the other necessary repositories:

              #All other repositories below should be cloned to use the evaluation script:\n\ngit clone https://github.com/OFS/linux-dfl\ncd /home/OFS/linux-dfl\ngit checkout tags/ofs-2023.3-6.1-3\n\n#Use this OPAE clone command for all PCIe Attach Cards\ngit clone https://github.com/OFS/opae-sdk\ncd /home/OFS/opae-sdk\ngit checkout tags/2.10.0-1\n\n#Use this OPAE clone command for the SoC Attach Card\ngit clone https://github.com/OFS/opae-sdk\ncd /home/OFS/opae-sdk\ngit checkout tags/2.10.0-1\n\ngit clone https://github.com/OFS/oneapi-asp.git \ncd /home/OFS/oneapi-asp\ngit checkout tags/ofs-2023.3-2\n\ngit clone https://github.com/OFS/ofs-platform-afu-bbb.git\ncd /home/OFS/ofs-platform-afu-bbb\ngit checkout tags/ofs-2023.2-1\n\ngit clone https://github.com/OFS/opae-sim.git\ncd /home/OFS/opae-sim\ngit checkout tags/2.10.0-1\n\ngit clone https://github.com/OFS/examples-afu.git\ncd /home/OFS/examples-afu\ngit checkout tags/ofs-2023.2-1\n

              You will also want to ensure you install the correct version of Intel Quartus Prime Pro as directed in the release notes in addition to any Quartus patches. Note that Quartus Prime Pro software can be downloaded from the downloads tab on intel.com. Quartus Prime Pro patches required are attached to the assets tab at the bottom of the tagged RTL repository release page. Simulator tools as listed corresponding Simulation User Guides:

              • Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach
              • Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach
              • Simulation User Guide: OFS for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach
              "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#3-select-a-starting-shellfim","title":"3 Select a Starting Shell/FIM","text":"

              To begin your development, start with an existing shell that most closely matches your device and end solution. The OFS site has both Intel Intel Agilex 7 and Stratix 10 FPGA reference designs you can use as a starting point for your own design. These designs can be ported to different device OPNs if a different resource utilization is required.

              To begin you may want to learn more about Intel Stratix 10 and Intel Agilex 7 family offerings. Please refer to the following links:

              • Intel Agilex 7 Product Page
              • Intel Stratix 10 Product Page

              Note that each reference design provides an integrated shell, called the FPGA Interface Manager (FIM), which is encompasses in blue in the diagrams below. This shell provides standard AXI interfaces to the Accelerator Functional Unit (AFU) region, shown in green, which depicts a region where a workload or partial reconfiguration slot resides. The regions are not drawn to scale. The figures and tables below give an overview of the available starting points for your design.

              Figure 3-1: OFS FIM Targeting Intel\u00ae Agilex\u00ae7 PCIe Attach (P-tile, E-tile)

              Key Feature Description Target OPN AGFB014R24A2E2V PCIe P-tile PCIe* Gen4x16 Virtualization 5 physical functions/3 virtual functions with ability to expand Memory 5 DDR Channels:* One HPS DDR4 bank, x40 (x32 Data and x8 ECC), 1200 MHz, 1GB each* Four Fabric DDR4 banks, x32 (no ECC), 1200 MHz, 4GB Ethernet 2x4x25GbE, 2x4x10GbE or 2x100GbE Hard Processor System 64-bit quad core Arm\u00ae Cortex\u00ae-A53 MPCore with integrated peripherals. Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration)* Platform Controller Management Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported OneAPI OneAPI Acceleration Support Package (ASP) provided with compiled FIM to support OneAPI Runtime Software Support * Linux DFL drivers targeting OFS FIMs* OPAE Software Development Kit* OPAE Tools Target Board Intel\u00ae FPGA SmartNIC N6001-PL

              Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae PCIe Attach Reference FIM documentation collection.

              Figure 3-2: OFS FIM Targeting Intel\u00ae Agilex\u00ae7 PCIe Attach (2xF-tile)

              Key Feature Description Target OPN AGFB027R24C2E2VR2 PCIe P-tile PCIe* Gen4x16 (currently downtrains to Gen4x8 in the ES version of the development kit) Virtualization 5 physical functions/3 virtual functions with ability to expand Memory 3 DDR Channels:* One HPS DDR4 bank, x40 (x32 Data and x8 ECC), 2400 MHz, 1GB each* Two Fabric DDR4 banks, x64 (no ECC), 2400 MHz, 8GB Ethernet 2x4x25GbE Hard Processor System 64-bit quad core Arm\u00ae Cortex\u00ae-A53 MPCore with integrated peripherals. Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration)* Platform Controller Management Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported OneAPI OneAPI Acceleration Support Package (ASP) provided with compiled FIM to support OneAPI Runtime Software Support * Linux DFL drivers targeting OFS FIMs* OPAE Software Development Kit* OPAE Tools Target Board Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)

              Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae PCIe Attach Reference FIM documentation collection.

              Figure 3-3: OFS FIM Features Targeting Intel\u00ae Agilex\u00ae 7 SoC Attach

              Key Feature Description Device OPN AGFC023R25A2E2VR0 PCIe P-tile PCIe* Gen4x16 to the HostP-tile PCIe* Gen4x16 to the SoC (IceLake-D) Virtualization Host: 2 physical functions SoC: 1 physical function and 3 Virtual functions Memory Four Fabric DDR4 banks, x40 (optional ECC, be configured as x32 and ECC x8 ), 1200 MHz, 4GB Ethernet 2x4x25GbE Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration) * Platform Controller Management Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported Software Support * Linux DFL drivers targeting OFS FIMs * OPAE Software Development Kit * OPAE Tools Target Board Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL

              Note: Source code for BMC RTL/Firmware that works with this reference FIM can be obtained by contacting your Intel Sales Representative.

              Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae SoC Attach Reference FIM documentation collection.

              Figure 3-4: OFS FIM Targeting Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach

              Key Feature Intel Stratix 10 Reference FIM Device OPN 1SX280HN2F43E2VG Ethernet Configuration 1x10GbE example with 2x100GbE capability PCIe Gen3x16 EMIF Up to four DDR channels PF/VF 1 PF/3 VFs Management FPGA Management Engine (FME) with FIM management registers Interface Arm\u00ae AMBA\u00ae4 AXI Interface HLD support oneAPI Software Kernel code upstreamed to Linux.org Target Board Intel\u00ae FPGA Programmable Acceleration Card D5005

              Click here for the OFS Collateral for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach Reference FIM documentation.

              "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#4-review-release-notes","title":"4 Review Release Notes","text":"

              Before beginning your design, read the release notes for each repository you are using by selecting the appropriate release tag found by clicking on the \"Release\" link to the right of the corresponding repository. The release page may also have assets appended such as useful binaries, patches or scripts.

              "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#5-setup-your-environment-for-development","title":"5 Setup Your Environment For Development","text":"

              When you are ready to begin development you will want to ensure you have any other setup requirements satisfied by reviewing instructions in the corresponding FIM Developer Guide and if you are implementing a OneAPI Board Support Package, the oneAPI ASP Getting Started User Guide as well.

              • FPGA Interface Manager (FIM) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach (2xF-tile) * Can be used with Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)
              • FPGA Interface Manager (FIM) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach (P-tile, E-tile) * Can be used with Intel FPGA SmartNIC N6001-PL
              • FPGA Interface Manager (FIM) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach * Can be used with Intel Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL
              • FPGA Interface Manager (FIM) Developer Guide: OFS for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach * Can be used with Intel FPGA PAC D5005

              For oneAPI setup:

              • oneAPI Accelerator Support Package (ASP): Getting Started User Guide * Can be used with: - Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) - Intel FPGA SmartNIC N6001-PL
              "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#6-customize-your-shell-build-and-debug","title":"6 Customize Your Shell, Build and Debug","text":"

              For your own custom design, you may need to:

              • Target the OFS reference design to a different device
              • Modify the pin assignments
              • Modify or remove existing peripheral IP
              • Add new interface IP
              • Repartition the partial reconfiguration region

              The FIM Developer Guides for each reference FIM show you how to make these changes and provide steps on how to run unit simulation or add SignalTap to your design for debugging.

              If you are also interested in testing different examples for the Acceleration Functional Unit (AFU) or workload region of your design or creating your own AFU design, you can refer to the corresponding AFU Developer Guides:

              • Accelerator Functional Unit (AFU) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach
              • Accelerator Functional Unit (AFU) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach
              • [Accelerator Functional Unit (AFU) Developer Guide: OFS for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach]
              "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#7-simulate-and-debug","title":"7 Simulate and Debug","text":"

              Setup and test files to perform system-level Universal Verification Methodology (UVM) testing are provided in each FIM RTL repository. Please refer to the corresponding Simulation User Guide for details on test bench architecture, setup and testing.

              • Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach
              • Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach
              • Simulation User Guide: OFS for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach
              "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#8-optional-build-oneapi-accelerator-support-package-asp","title":"8 Optional: Build OneAPI Accelerator Support Package (ASP)","text":"

              If you are considering providing oneAPI support for your custom board design, you must integrate the oneAPI ASP hardware and software components into your design. Reference the following documents to learn about the architecture and implementation flow for the oneAPI ASP with an OFS design.

              • oneAPI Accelerator Support Package (ASP): Getting Started User Guide
              • oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack
              "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#9-optional-driver-or-ofs-software-tool-modifications","title":"9 Optional: Driver or OFS Software Tool Modifications","text":"

              As you add or remove interfaces to your custom design, you may need to modify or enhance existing drivers that accompany the OFS reference design. Additionally, you may decide you want to create additional utilities or plugins leveraging the OFS software infrastructure. In this case, refer to the OFS Software tab to learn more about the underlying driver and software architecture and how to make modifications.

              Additionally, for guidance on using a Kernel-based Virtual Machine with OFS, refer to our KVM User Guide.

              KVM User Guide: Open FPGA Stack

              "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#10-test-and-deploy","title":"10 Test and Deploy","text":"

              When testing and deploying your platform you are encouraged to modify and tailor the evaluation scripts you used in Section 2 to assist in automating your continuous integration flow. You may also want to refer to our Docker User Guide to understand how to use Docker for Development and Deployment.

              • Docker User Guide: Intel Open FPGA Stack
              "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#notices-disclaimers","title":"Notices & Disclaimers","text":"

              Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Performance varies by use, configuration and other factors. Your costs and results may vary. You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications. Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards. \u00a9 Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.

              OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122.

              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/","title":"Virtual machine User Guide: Open FPGA Stack + KVM","text":"

              Last updated: February 03, 2024

              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#document-scope","title":"Document scope","text":"

              The document describes setting up and configuring a virtual machine to use PCIe devices. Here are the steps that the document may include:

              1. Install the necessary tools, such as virt-manager, on the host machine. This may involve downloading and installing the software from the internet.
              2. Enable the virtualization feature on the host machine. This may involve going into the BIOS settings and enabling hardware-assisted virtualization or using a command-line tool to enable it in the operating system.
              3. Use virt-manager to create a new virtual machine and configure its settings. This may involve choosing a name and operating system for the virtual machine and setting the amount of memory and storage it will use.
              4. Install the OPAE (Open Programmable Acceleration Engine) tool on the virtual machine. This may involve downloading and installing the OPAE software.
              5. Install the DFL (Data Field Level) drivers on the virtual machine. These drivers allow the virtual machine to access and use the PCIe devices on the host machine. This may involve downloading and installing the drivers from the internet.
              6. Once all of the steps have been completed, you should be able to use the virtual machine to access and use the PCIe devices on the host machine. You may need to configure the virtual machine's settings to enable it to use the PCIe devices, such as by assigning a specific device to the virtual machine.
              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#1-modes-of-operation","title":"1. Modes of Operation","text":"

              Our current operational framework stipulates two distinct modes of operation for PF/VF configurations. When using a 2 PF enabled FIM design, both the workload and management ports can be interchangeably passed through to a VM or run on bare-metal.

              1. Management Mode: This mode necessitates the passthrough of only the FME device (use fpgainfo fme to discover your port number, normally .0). The reason for this is that the Open FPGA Stack (OFS) depends on this address for management. Under this mode, the use of the exerciser and virtual functions is not feasible.

              2. Virtual Function Mode: This mode comes into effect when a user needs to utilize the Virtual Functions (VF). The user will convert (example) Physical Function 0 (PF0) to three Virtual Functions (VF). This means the PF will cease to function for management purposes. Once the VFs are set up, they essentially take over the role of the PF in communicating with the Virtual Machines (VMs).

                However, this mode is subject to a limitation. If the user needs to execute 'fpgainfo fme' or 'fpgaupdate', they will need to transition from Virtual Function Mode to Management Mode. Conversely, if the user intends to utilize the Virtual Functions, they would need to switch from Management Mode to Virtual Function Mode. It is imperative to bear this limitation in mind when operating within these modes.

              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#2-enable-virtualization","title":"2. Enable Virtualization","text":"

              To check if virtualization is enabled on a Red Hat system using lscpu and grep, you can use the following command:

              lscpu -e | grep Virtualization\n

              This command will run lscpu with the -e or --extended option, which displays information about the CPU and its available virtualization capabilities. Then, it pipes the output to grep with the search pattern \"Virtualization\". If the system has support for virtualization, the output will show the \"Virtualization\" field and its value, for example:

              Virtualization: VT-x\n

              In this example, the output shows that the system supports Intel VT-x virtualization technology. If the \"Virtualization\" field is empty, the system does not have support for virtualization. Keep in mind that even if the system has support for virtualization, it may not be enabled in the BIOS or the operating system itself.

              Check the following for the bios configuration, Enabling Intel VT-d Technology

              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#3-verify-environment-setup","title":"3. Verify Environment Setup","text":"
              1. Open a terminal window and log in as a user with sudo privileges.
              2. Check if the virtualization kernel modules are loaded by running the following command:
              lsmod | grep kvm\n

              1. If the command outputs a list of modules, the virtualization kernel modules are loaded, and virtualization is enabled on your system.

              2. The virtualization kernel modules are not loaded if the command does not output anything. You can try loading them manually by running the following command:

              sudo modprobe kvm\n
              1. If the kernel modules are not loaded, and you cannot load them manually, it may be because virtualization is not supported or enabled in your system's BIOS or UEFI settings. You must reboot your system and enter the BIOS or UEFI settings menu to enable virtualization. The exact steps for doing this may vary depending on your system's hardware and BIOS/UEFI version, so consult your motherboard or system documentation for specific instructions.
              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#4-install-virtual-machine-manager","title":"4. Install Virtual Machine Manager","text":"

              Virtual Machine Manager (also known as libvirt) can be installed by following the below steps:

              1. Open a terminal window and log in as a user with sudo privileges.

              2. Update your system package index by running the following command:

                • Redhat

                sudo dnf update\n
                * Ubuntu 
                sudo apt update\n

              3. Install the libvirt package and any required dependencies by running the following command:

                • Redhat 
                sudo dnf install @virtualization\n
                • Ubuntu
                sudo apt install qemu-kvm libvirt-bin bridge-utils virt-manager\n

              4. Start the libvirtd service and enable it to start automatically at boot time by running the following commands:

              sudo systemctl start libvirtd\nsudo systemctl enable libvirtd\n
              1. Optional: Install the virt-manager package, which provides a GUI application for managing virtual machines, by running the following command:
              sudo dnf install virt-manager\n
              1. Optional: If you want to be able to run virtual machines as a non-root user, add your user to the libvirt group by running the following command, replacing \"USERNAME\" with your username:
              sudo usermod -a -G libvirt USERNAME\n
              1. You can now launch virt-manager by running the command virt-manager as the non-root user.

              Note: By default, virt-manager will only allow non-root users to create and manage virtual machines with limited resources, such as a limited amount of memory and CPU cores. To allow non-root users to create and manage virtual machines with more resources, you need to edit the /etc/libvirt/qemu.conf configuration file and set the user and group values for the dynamic_ownership option to 1. For example:

              # Set user and group ownership of dynamic /dev/kvm device nodes\ndynamic_ownership = 1\nuser = \"root\"\ngroup = \"root\"\n

              You will also need to restart the libvirtd service for the changes to take effect. You can do this by running the command. 

              sudo systemctl restart libvirtd\n
              1. Reboot your server to apply the changes 
              reboot\n

              After completing these steps, you should be able to use the virt-manager GUI application to manage virtual machines on your system.

              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#5-create-a-vm-using-virt-manager","title":"5. Create a VM Using Virt-Manager","text":"

              Before creating the virtual machine, ensure the DFL drivers are installed in your host machine; the instructions are located here, OFS Site select your desired platform and select Getting stated guide.

              To create a Red Hat 8.2 or Ubuntu 22.04 virtual machine (VM) using virt-manager and share PCI devices with the VM, you will need to perform the following steps:

              1. Start the virt-manager GUI by running the following command:
              sudo virt-manager&\n

              1. Create a new connection from the menu File-> \"Add Connection,\" Use the default options and click \"Connect.\"

              2. In the virt-manager window, click the \"New virtual machine\" button.

              3. In the \"New VM\" wizard, select \"Local install media (ISO image or CDROM)\" as the installation source, and then click \"Forward.\"

                • Get the Red Hat image from the following link.

                  https://developers.redhat.com/content-gateway/file/rhel-8.2-x86_64-dvd.iso

                • Get the Ubuntu image from the following link.

                  https://releases.ubuntu.com/22.04/ubuntu-22.04.1-desktop-amd64.iso

              4. In the next step, Click Browse -> Browse local, select the Red Hat 8.2 ISO image as the installation source and click \"Forward\".

                Note: if the system is not detected, disable \"Automatic detected from the installation media/source\" and type ubuntu and select 19.10 (this should be fine for the 22.04); this step is necessary to copy the default values for the specific OS

              5. In the next step, specify a name and location for the VM, and select the desired memory and CPU configuration. in our case, 16 cores and 64 GB of RAM; Click \"Forward\" to continue.

              6. Select \"enable storage for this virtual machine,\" Select \"Create a new disk for the virtual machine,\" and enter a size for the virtual disk (at least 200~300GB in case you need to compile the design) or create a custom storage.

                1. If you need to create custom storage, select \"Select or Create custom storage\" and click \"Manage.\"

                2. Click on the \"+\" icon (Bottom left) to create the storage pool.

                3. In the \"Create a new storage pool\" dialog, enter a name for the storage pool and select the type of storage pool you want to create; select the Target Path and Click \"Finish.\"

                4. Select the pool and later click on the \"+\" icon (The Icon is on the right side of the Volume label) to create the New Storage Volume.

                5. In the \"Create Storage Volume\" dialog, Define the name and format (keep with the default qcow2) and select the Max Capacity (at least 200~300GB in case you need to compile the design); click \"Finish\" to create the disk.

                6. Once the disk is created, it will appear in your virtual machine's list of storage devices. You can now use this disk just like any other disk. Select from the list and Click \"Choose Volume.\"

              7. In the next step, select the \"Customize configuration before install\" option and click \"Finish.\"

              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#51-passing-devices-to-the-vm","title":"5.1 Passing Devices to the VM","text":"

              In the \"Overview\" tab, select \"Add Hardware,\" choose \"PCI Host Device\" from the drop-down menu and choose the PCI device you want to share with the VM. Click \"Apply\" to apply the changes, and then click \"Finish\" to create the VM.

              Depending on the FIM currently loaded onto your FPGA device, you have access to a few modes of operation. Management Mode and Deployment mode can be used on any FIM that supports a PF/VF split architecture. When using the 2 PF FIM, see 2 PF Mode.

              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#511-management-mode","title":"5.1.1 Management Mode","text":"

              This will only allow you to load the binaries to the FPGA, you only need to add the PF listed at the fpgainfo fme command.

              fpgainfo fme\n\nfpgainfo fme\nIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: xxxx \nBoard Management Controller Build version: xxxx \n//****** FME ******//\nObject Id                        : 0xEE00000\nPCIe s:b:d.f                     : 0000:b1:00.0\n

              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#512-deployment-mode","title":"5.1.2 Deployment Mode","text":"

              The main idea of this mode is enable the Virtual function used by the Agilex PCIe Attach OFS under the Physical Function 0, This option will allow us to use the Host Exercises.

              Note: assigning multiple devices to the same VM on a guest IOMMU, you may need to increase the hard_limit option in order to avoid hitting a limit of pinned memory. The hard limit should be more than (VM memory size x Number of PCIe devices)

              1. Create 3 VFs in the PR region.

                sudo pci_device b1:00.0 vf 3 \n

              2. Verify all 3 VFs were created.

                sh lspci -s b1:00 b1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01) b1:00.1 Processing accelerators: Intel Corporation Device bcce b1:00.2 Processing accelerators: Intel Corporation Device bcce b1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device b1:00.4 Processing accelerators: Intel Corporation Device bcce b1:00.5 Processing accelerators: Intel Corporation Device bccf b1:00.6 Processing accelerators: Intel Corporation Device bccf b1:00.7 Processing accelerators: Intel Corporation Device bccf

              3. Bind all of the PF/VF endpoints to the vfio-pci driver.

                sudo opae.io init -d 0000:b1:00.1 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 187\nAssigning /dev/vfio/187 to DCPsupport\nChanging permissions for /dev/vfio/187 to rw-rw----\n\nsudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188\nAssigning /dev/vfio/188 to DCPsupport\nChanging permissions for /dev/vfio/188 to rw-rw----\n\n...\n\nsudo opae.io init -d 0000:b1:00.7 user:user\nBinding (0x8086,0xbccf) at 0000:b1:00.7 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.7 is 319\nAssigning /dev/vfio/319 to DCPsupport\nChanging permissions for /dev/vfio/319 to rw-rw----\n

              4. Check that the accelerators are present using fpgainfo. Note your port configuration may differ from the below.

                sudo fpgainfo port \n//****** PORT ******//\nObject Id                        : 0xEC00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0xE0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.7\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 4dadea34-2c78-48cb-a3dc-5b831f5cecbb\n//****** PORT ******//\nObject Id                        : 0xC0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.6\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 823c334c-98bf-11ea-bb37-0242ac130002\n//****** PORT ******//\nObject Id                        : 0xA0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.5\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n//****** PORT ******//\nObject Id                        : 0x80B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.4\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 44bfc10d-b42a-44e5-bd42-57dc93ea7f91\n//****** PORT ******//\nObject Id                        : 0x40B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n//****** PORT ******//\nObject Id                        : 0x20B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 3e7b60a0-df2d-4850-aa31-f54a3e403501\n

              The following table contains a mapping between each VF, Accelerator GUID, and component.

              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#table-16-accelerator-pfvf-and-guid-mappings","title":"Table 16: Accelerator PF/VF and GUID Mappings","text":"Component VF Accelerator GUID Intel N6001-PL FPGA SmartNIC Platform base PF XXXX:XX:XX.0 N/A VirtIO Stub XXXX:XX:XX.1 3e7b60a0-df2d-4850-aa31-f54a3e403501 HE-MEM Stub XXXX:XX:XX.2 56e203e9-864f-49a7-b94b-12284c31e02b Copy Engine XXXX:XX:XX.4 44bfc10d-b42a-44e5-bd42-57dc93ea7f91 HE-MEM XXXX:XX:XX.5 8568ab4e-6ba5-4616-bb65-2a578330a8eb HE-HSSI XXXX:XX:XX.6 823c334c-98bf-11ea-bb37-0242ac130002 MEM-TG XXXX:XX:XX.7 4dadea34-2c78-48cb-a3dc-5b831f5cecbb

              1. Ensure you add the desired VF in your PCIE devices list.

              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#513-2-pf-mode","title":"5.1.3 2 PF Mode","text":"

              For FIMs that support the dual PF architecture, you have the option to pass through any number of PFs into the VM. The VM's software will recognize any management / workload ports and probe them appropriately. This assumes you have the OPAE SDK and Linux DFL drivers installed on both the VM and host.

              1. Bind all endpoints you wish to pass through to the VM to the vfio-pci driver on the host.

                sudo opae.io init -d 0000:b1:00.0 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 187\nAssigning /dev/vfio/187 to user\nChanging permissions for /dev/vfio/187 to rw-rw----\nsudo opae.io init -d 0000:b1:00.1 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 187\nAssigning /dev/vfio/187 to user\nChanging permissions for /dev/vfio/187 to rw-rw----\n

              2. Pass through any required hardware endpoints, select \"Add Hardware\" -> \"PCI Host Device\".

              3. Run the following command on the host and VM to allocate hugepages for workload testing:

                echo 4194304 | sudo tee /sys/module/vfio_iommu_type1/parameters/dma_entry_limit\n
              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#52-virt-manager-configuration-changes","title":"5.2 Virt-Manager Configuration Changes","text":"

              1. Edit the XML file for your machine and include the following

                1. < ioapic driver='qemu'/> inside of features:

                  <features>\n  <acpi/>\n  <apic/>\n  <ioapic driver='qemu'/>\n</features>\n

                2. Inside of devices

                  <devices>\n    ........\n    ......\n    <iommu model='intel'>\n      <driver intremap='on' caching_mode='on'/>\n    </iommu>\n</devices>\n

                3. Ensure the hard limit is setup correctly otherwise you can only pass one device:

                  <memtune>\n  <hard_limit unit='G'>64</hard_limit>\n</memtune>\n

                  Note: assigning multiple devices to the same VM on a guest IOMMU, you may need to increase the hard_limit option in order to avoid hitting a limit of pinned memory. The hard limit should be more than (VM memory size x Number of PCIe devices)

                4. Save the changes \"Apply\"

              2. On the host machine append intel_iommu=on to the end of the GRUB_CMDLINE_LINUX line in the grub configuration file.

                nano /etc/default/grub\n......\nGRUB_CMDLINE_LINUX=\"....... ... intel_iommu=on\"\n...\n#Refresh the grub.cfg file for these changes to take effect\n\ngrub2-mkconfig -o /boot/grub2/grub.cfg\nshutdown -r now\n

              3. Ensure your devices are enumerated properly.

                1. Example in you host system should look like this:

                  1. Management Mode:

                  B1:00.0\n

                  2. Deployment Mode:

                   B1:00.5\n

                2. Under the virtual machine (The PCIe Address is an example you could get a different number):

                  1. Management Mode:

                  177:00.0\n

                  2. Deployment Mode:

                   177:00.0\n

              4. Click on \"Begin Installation.\" and follow the wizard installation of the OS.

              5. Once the VM is created, you can start it by selecting it in the virt-manager window and clicking the \"Run\" button. This will boot the VM and start the Red Hat 8.2/Ubuntu installation process. Follow the on-screen instructions to complete the installation.

              6. Under your virtual machine, configure your VM proxy:

                • Redhat How to apply a system-wide proxy?
                • Ubuntu Define proxy settings
                • Configure Git to use a proxy

              7. To include OPAE in your virtual machine, follow the instructions from the following link OFS Site select your desired platform and select Getting stated guide. To install the DFL drivers, please follow the instructions from the following link OFS Site select your desired platform and select Getting stated guide.

              8. Use the OPAE SDK tool opae.io (under your virtual machine) to check default driver binding using your card under test PCIe B:D.F (Management mode).

                sudo fpgainfo fme\n\nIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: xxx \nBoard Management Controller Build version: xxx\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:177:00.0\n

              9. Use the Virtual function (Not supported at management mode)

                1. Ensure the DFL kernel drivers is install in your VM system

                2. Bind VFs to VFIO driver

                  $ sudo opae.io init -d 0000:177:00.0\n[sudo] password for dcpsupport: \nopae.io 0.2.3\nBinding (0x8086,0xbccf) at 0000:177:00.0 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:177:00.0 is 13\n

                3. Verify the binding is correct.

                  $ opae.io ls\nopae.io 0.2.3\n[0000:177:00.0] (0x8086, 0xbccf) Intel N6001 ADP VF (Driver: vfio-pci)\n

                4. Test the HE mem

                   host_exerciser mem\n starting test run, count of 1\n API version: 1\n Frequency of AFU clock unknown. Assuming 350 MHz.\n Allocate SRC Buffer\n Allocate DST Buffer\n Allocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 6737\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1022\n    Bandwidth: 3.405 GB/s\n    Test mem(1): PASS\n

              After the installation, you can use virt-manager to manage and configure the VM to move from Management mode to Deployment or vice versa, including setting up networking, attaching additional storage, and installing additional software. The shared PCI device will be available to the VM, allowing it to use it as if it were connected directly to the physical system.

              "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#notices-disclaimers","title":"Notices & Disclaimers","text":"

              Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Performance varies by use, configuration and other factors. Your costs and results may vary. You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications. Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards. \u00a9 Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.

              OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/","title":"Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach**","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel Max10 or Intel Cyclone10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implemented on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Management Component Transport Protocol MCTP A standardized model for communication with management controllers. Defines the transport protocol carrying PLDM messages through the BMC. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE-SDK The OPAE-SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Level Data Model PLDM A specification for reporting telemetry data to the host, such as board temperature, voltage, and current. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework. In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#1-overview","title":"1 Overview","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#11-about-this-document","title":"1.1 About this Document","text":"

              This document serves as a set-up and user guide for the UVM simulation tool using Open FPGA Stack (OFS) for Intel\u00ae Agilex\u00ae 7 FPGAs PCIe Attach and the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile). After reviewing the document, you will be able to:

              • Set-up the UVM verification tool suite
              • Run pre-existing UVM unit tests and also create new UVM tests for your design

              NOTE:

              This guide uses the Intel\u00ae FPGA SmartNIC N6001-PL as the platform for all example steps. Additionally, this guide and the example steps can be used with other platforms, such as the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile).

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#2-introduction-to-uvm","title":"2 Introduction to UVM","text":"

              The Functional Verification Environment for OFS is UVM (Universal Verification Methodology) compliant and provides configurable setup for verifying various FIM features in simulation.

              The purpose of this document is to demonstrate a full chip level and unit level UVM based verification environment for the current base shell FIM architecture as well as providing examples to extend the setup for different OFS variants by reusing the existing architecture and infrastructure for UVM based verification.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#3-universal-testbench-architecture","title":"3 Universal Testbench Architecture","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#31-overview","title":"3.1 Overview","text":"

              The main idea behind UVM is to develop modular, reusable, and a scalable testbench structure by providing an API framework that can be deployed across multiple projects.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#32-core-verification-concepts","title":"3.2 Core Verification Concepts","text":"

              The following is the list of verification components that will be used to design a UVM testbench architecture:

              \u2022 Sequencer \u2022 Driver \u2022 Monitor \u2022 Scoreboard \u2022 Virtual Sequencer \u2022 Interface \u2022 Verification Environment \u2022 TOP Testbench

              Figure 1 provides the general UVM Testbench and the verification components involved in the top-level architecture.

              Figure 1 Typical UVM Testbench

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4-ofs-testbench-architecture","title":"4 OFS Testbench Architecture","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#41-overview","title":"4.1 Overview","text":"

              OFS (Open FPGA Stack) provides a UVM (Universal Verification Methodology) environment for the FIM with a modular, reusable, and scalable testbench structure via an API framework.

              The framework consists of a FIM Testbench which is UVM compliant and integrates third party VIPs from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this Testbench. UVM RAL(Register Abstraction Level) is used for CSR (Command and Status Registers) verification.

              The qualified verification IPs will help to detect incorrect protocol behavior, help to focus on FIM features and accelerate the verification process.

              Verification components include:

              \u2022 FIM monitor to detect correct design behavior\n\u2022 FIM assertions for signal level integrity testing\n\u2022 Arm AMBA Arm\u00ae AMBA\u00ae 4 AXI4 scoreboards to check data integrity\n\u2022 FIM coverage to collect functional data\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#42-base-fim-dut","title":"4.2 Base FIM DUT","text":"

              The hardware architecture of an Agilex FIM is based on the OFS hardware architecture. The following is the list of features and subsystems supported in the base shell.

              \u2022   PCIe Subsystem\n\u2022   HSSI Subsystem\n\u2022   Memory Subsystem\n\u2022   HPS Subsystem\n\u2022   FME\n\u2022   AFU with PR support\n\u2022   QSFP Controllers\n\u2022   PMCI Controller, MCTP\n

              Figure 2 DUT Base Shell Diagram

              Figure 2 shows the high level architecture of an Agilex Base Shell. It has a Gen4x16, 100G Ethernet Datapath in a 2x4x25G configuration. The Agilex Base Shell is a shell that will enable a user to build other shell variants for a custom configuration. For the N6001 board there is one shell variant

              base_x16

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#43-full-chip-level-verification-archiecture-for-fim","title":"4.3 Full Chip Level Verification Archiecture for FIM","text":"

              Figure 3 shows a graphical representation a full chip testbench that includes major RTL blocks depicted in a OFS Agilex based UVM environment

              Figure 3 OFS FIM Testbench

              The major connection is the interface between the Xeon CPU and the FPGA where the PCIe Verification IP is connected to PCIe Subsystem. Therefore, as a full chip simulation environment, PCIe host VIP is the sole VIP/BFM used. PCIe host VIP connects to PCIe device which resides in FPGA in serial mode.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#431-testbench-components","title":"4.3.1 Testbench components","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4311-tb-top","title":"4.3.1.1 TB TOP","text":"

              TOP is the top level testbench and consists of a FIM DUT instance and top-level UVM Verification Environment instance. It binds RTL inputs with the verification environmnet interfaces to drive stimulus. It also has clock generation and reset driving logic.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4312-fim-verification-environment","title":"4.3.1.2 FIM Verification Environment","text":"

              This is the top most verification environment class and consists of the protocol specific PCI Express and AXI UVM environment VIP instances, Monitors, Scoreboards, Assertions, Functional coverage Modules and other verification components. It instantiates Virtual sequencers to control stimuli for FIM traffic from different sequencers of the VIPs.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4313-synopsys-vips","title":"4.3.1.3 Synopsys VIPs","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#43131-pci-vip-as-host","title":"4.3.1.3.1 PCI VIP as Host","text":"

              This is Synopsys Licensed PCI Express Gen4 VIP and acts as Root Port. The PCI Express link is connected to the DUT using TX-RX lanes. Agent is an active component and includes a sequencer to generate TLPs, Driver to drive it on a PCI Express link and Monitor to check the protocol correctness.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#43132-axi-streaming-vip-monitors","title":"4.3.1.3.2 AXI-Streaming VIP Monitors","text":"

              This is Synopsys Licensed AXI streaming interface Verification IP used as a Passive Agent to monitor AXI-ST links at various points. Please refer to Figure 3 to see all the AXI-ST monitors connected to different modules.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#43133-axi4-memory-mapped-vip-monitors","title":"4.3.1.3.3 AXI4-Memory Mapped VIP Monitors","text":"

              This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 memory mapped interface Verification IP used in passive mode to observe memory requests. For each master-slave pair, the verification environment has a VIP instantiated in passive mode for monitoring protocol violations and performing data integrity checks.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#43134-axi4-lite-vip-monitors","title":"4.3.1.3.4 AXI4-Lite VIP Monitors","text":"

              This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 Lite interface Verification IP used in passive mode to observe MMIO requests. For each master-slave pair, the verification environment has a VIP instantiated in passive mode for monitoring protocol violations and perfoming data integrity checks. Please refer to Figure 3 to see all the Arm\u00ae AMBA\u00ae 4 AXI4-Lite monitors connected to different modules.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#43135-axi4-lite-vip-as-pmci-master","title":"4.3.1.3.5 AXI4-Lite VIP as PMCI Master","text":"

              This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 Lite interface Verification IP used to drive and observe MMIO requests as PMCI master. For each master-slave pair, the verification environment has a VIP instantiated in active mode and includes a sequencer to generate MMIO requests, driver to drive these requests on AXI-4 Lite interface to BPF and a monitor for observing protocol violations and data integrity checks.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4314-axi4-s-scoreboard","title":"4.3.1.4 AXI4-S Scoreboard","text":"

              The Arm\u00ae AMBA\u00ae 4 AXI4-S scoreboard checks data integrity of source and sink components. It has input transactions from VIP monitors and a TLP to AXI converter for PCIe TLP packets. It makes sure the valid TLPs or AXI transactions are not missed while traversing from Host to AFU and reverse. The scoreboard will be disabled for error testing especially for invalid TLP requests and UR responses.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4315-virtual-sequencer","title":"4.3.1.5 Virtual Sequencer","text":"

              The virtual sequencer is the top-level sequencer which controls Enumeration, MMIO Requests, downstream and Upstream traffic as well as HSSI and Memory transactions. It makes sure that the transactions are ordered correctly as per the design specification while running a UVM test simulation.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4316-fim-monitor","title":"4.3.1.6 FIM Monitor","text":"

              The FIM monitor is used for checking the correctness of a specific FIM feature. As an example, a user can add interrupt related checks, error generation and clearing checks, Protocol checker impacts etc. in this component.

              This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4317-fim-assertions","title":"4.3.1.7 FIM Assertions","text":"

              The assertion component is used for checking the signal level integrity of a specific FIM feature or behavior. As an example, we can add interrupt signal triggering and clear assertions, Error detection to error register bit assertions, protocol checker and clearing logic, FLR behavior assertion etc. in this top-level module. There are alos assertions to make sure the inputs are driven correctly to catch errors in a users simulation.

              This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4318-fim-functional-coverage","title":"4.3.1.8 FIM Functional Coverage","text":"

              The FIM functional coverage component will have the functional coverage of each feature like interrupts, CSR's, MMIO requests, Byte align transactions, error reporting etc. to demonstrate a variety of FIM features. This will help us to find holes in the design as well as wide verification coverage to make sure thorough testing of a design. It will also provide a user what the FIM is capable of and how much code coverage they are testing.

              This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#5-uvm-verification-set-up","title":"5 UVM Verification set-up","text":"

              To run the tutorial steps in this guide requires the following development environment:

              Item Version Intel Quartus Prime Pro Intel Quartus Prime Pro 23.3 Simulator Synopsys VCS P-2019.06-SP2-5 or newer for UVM simulation of top level FIM Simulator (Questasim) Questasim 2023.2 or newer for UVM simulation of top level FIM"},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#51-uvm-prerequisite","title":"5.1 UVM Prerequisite","text":"

              Retrieve OFS repositories.

              The OFS FIM source code is included in the OTCShare GitHub repository. Create a new directory to use as a clean starting point to store the retrieved files. The following is a short description of each repository, followed by the git commands for cloning. The instructions section uses the HTTPS git method for cloning repositories. Cloning the repo using the HTTPS method requires a personal access token. Please see this blog post for information about obtaining a personal access token Token authentication requirements for Git operations.

              Navigate to location for storage of OFS source, create the top-level source directory and clone OFS repositories.

              $ mkdir ofs-2023.3-2\n$ cd ofs-2023.3-2\n$ export OFS_BUILD_ROOT=$PWD\n$ git clone --recurse-submodules https://github.com/OFS/ofs-agx7-pcie-attach.git\n\nCloning into 'ofs-agx7-pcie-attach'...'\nUsername for 'https://github.com': <<Enter your git hub username>>\nPassword for 'https://<<Your username>>': <<Enter your personal access token>>\nremote: Enumerating objects:  ....\n...\n...\nResolving deltas  ..., done.\n\n$ cd ofs-agx7-pcie-attach\n$ git checkout tags/ofs-2023.3-2\n

              Verify that the correct tag/branch have been checked out 

              $ git describe --tags\n\n$ ofs-2023.3-2\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#52-license-requirements","title":"5.2 License Requirements","text":"

              The FIM Testbench is UVM compliant and integrates third party VIP's from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this TB. UVM RAL (Register Abstraction Layer) is used for CSR Verification.

              The Qualified Verification IPs will help to detect incorrect protocol behavior easily, help to focus on FIM features and accelerate the verification process.

              \u2022 VCS & DVE\n\u2022 SNPS-Assertions\n\u2022 Verdi\n\u2022 VerdiCoverage\n\u2022 VerdiSimDB\n\u2022 VerdiTransactionDebugUltra\n\u2022 VIP-AMBA-AXI-SVT\n\u2022 VIP-AMBA-STREAM-SVT\n\u2022 VIP-PCIE-SVT\n\u2022 VIP-PCIE-TS-SVT\n\u2022 VIP-PCIE-G3-OPT-SVT\n\u2022 VIP-Ethernet-SVT\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#53-software-tools-requirements","title":"5.3 Software Tools Requirements","text":"

              The following tools are required for successful UVM set-up

              • Python 3.6.8
              • Synopsys PCIE and AMBA AXI UVM VIP Q-2020.03A License
              • Synopsys Verdi R-2020.12-SP2 License Note: Makefile can be modified to use DVE instead of Verdi
              • VCS R-2020.12-SP2 License
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#54-creating-a-software-tools-script","title":"5.4 Creating a Software Tools Script","text":"

              The UVM tool set-up is best done by creating a simple set-up script so all applicable tools are sourced before running the tests.

              The following environment variables can be pasted into a script and used prior to running the UVM verification environment

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#license-files","title":"License Files","text":"
              export LM_LICENSE_FILE=\nexport SNPSLMD_LICENSE_FILE=\n

              The license environment variables LM_LICENSE_FILE and SNPSLMD_LICENSE_FILE can point to a server license on your system.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#general-environment-variables","title":"General Environment Variables","text":"
              export IOFS_BUILD_ROOT=$PWD\nexport OFS_ROOTDIR=<user_path>/ofs-agx7-pcie-attach\nexport WORKDIR=$OFS_ROOTDIR\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#quartus-tools","title":"Quartus Tools","text":"
              export QUARTUS_HOME=<user_path>/intelFPGA_pro/23.3/quartus\nexport QUARTUS_ROOTDIR=$QUARTUS_HOME\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$PATH\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-verification-tools","title":"Synopsys Verification Tools","text":"
              export DESIGNWARE_HOME=<user_path>/synopsys/vip_common/vip_Q-2020.03A\nexport PATH=$DESIGNWARE_HOME/bin:$PATH\nexport UVM_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport PATH=$VCS_HOME/bin:$PATH\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-verification-tools","title":"QuestaSIM Verification Tools","text":"
              export MTI_HOME=<user_path>/mentor/questasim/2023.2/linux64\nexport PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\nexport QUESTA_HOME=$MTI_HOME\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#6-running-a-uvm-simulation-test-and-analysing-results","title":"6 Running a UVM Simulation Test and Analysing Results","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#61-simulation","title":"6.1 Simulation","text":"

              The default simulator used in the simulation script is Synopsys VCS-MX. Users can refer to the options and adopt the options for other simulators. The script is a makefile that calls vlogan, vcs and simv for compilation, elaboration and simulation, respectively.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#62-file-structure","title":"6.2 File Structure","text":"

              After cloning the repo, the verification and ofs-common directories contain all UVM verification related files. The directory structure is shown in Figure 4 below.

              Figure 4 UVM Verification Directory File Structure

              ofs-agx7-pcie-attach/verification/testbench has a testbench, uvm env, virtual sequencer, RAL etc.

              ofs-agx7-pcie-attach/tests contains all uvm tests and sequences.

              Users can run the simulation under \"ofs-agx7-pcie-attach/verification/scripts\" directory and the simulation result is outputted to a \"sim\" directory for Synopsys VCS or sim_msim for Questasim.

              The simulation result folder is named after the test name with increasing suffix number. If user runs the same test multiple times, the suffix is incremented by 1 each time.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#63-uvm-test-suite","title":"6.3 UVM Test Suite","text":"

              The UVM environment contains a variety of tests that have been developed to test out the FIM portion of OFS.

              The table below has four columns which describe the \"Test Name\", \"DUT Scope\", \"Test Scenario\" and the \"Checking Criteria\".

              Tests are located at ofs-agx7-pcie-attach/verification/tests

              Test Name DUT Scope Test Scenario Checking Criteria afu_mmio_flr_pf0_test PF0 FLR Reset Apply FLR Reset for PF0 and deassert. Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF0 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf0_vf0_test PF0_VF0_FLR Reset Apply FLR Reset for PF0_VF0 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf0_vf1_test PF0_VF1_FLR Reset Apply FLR Reset for PF0_VF1 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf0_vf2_test PF0_VF2_FLR Reset Apply FLR Reset for PF0_VF2 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf2_test PF2 FLR Reset Apply FLR Reset for PF2 and deassert. Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF2 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf3_test PF3 FLR Reset Apply FLR Reset for PF3 and deassert. Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF3 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf4_test PF4 FLR Reset Apply FLR Reset for PF4 and deassert. Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF4 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_stress_5bit_tag_test AFU-Stress To check the AFU Stress by sending traffic from all PF/VF. i.e Send traffic on HE-LPK/HE-MEM and initiating MMIO access to other PF/VFs Data checking afu_stress_8bit_tag_test AFU-Stress To check the AFU Stress by sending traffic from all PF/VF. i.e Send traffic on HE-LPK/HE-MEM and initiating MMIO access to other PF/VFs Data checking afu_stress_test AFU-Stress 1. Initiate transactions to all the supported PF/VF from PCIE VIP and ensure that traffic is sent to all blocks of the AFU. 2. Ensure that CE/HE-LB/HE-MEM/HSSI/BPF/FME are seeing traffic. 3. Ensure that HE-LB/HE-MEM/CE sends DMWR/DMRD requests to PCIE VIP. 4. Ensure the Mux/DeMux blocks is able to handle the traffic based on the PF's/VF's and proper muxing/demuxing happens. Data checking bar_32b_test PCIe MMIO Path Set the BAR values to 32bit and test mmio access BAR address, Register Base Offset bar_64b_test PCIe MMIO Path Set the BAR values to 64bit and test mmio access BAR address, Register Base Offset dfh_walking_test DFH DFH walking offset checking, eol checking -> tb emif_csr_test EMIF CSR access data checking fme_csr_test FME CSR CSR accesses data checking fme_err_intr_test Interrupt FME Interrupt request using FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read fme_intr_test Interrupt FME interrupt request using RAS ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read fme_multi_err_intr_test Interrupt FME interrupt request using RAS ERROR and FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests generated from FME and RAS ERROR bits he_hssi_csr_test HE-HSSI CSR accesses data checking he_hssi_rx_lpbk_25G_10G_test HE-HSSI Sending back to back ethernet data traffic with 25G speed on HSSI RX Port0-7 lanes using Ethernet VIPs Enable the loopback mode in HE-HSSI and compare the pkts recived on HSSI TX Port(DATA CHECKING) he_hssi_tx_lpbk_P0_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port0 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P1_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port1 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P2_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port2 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P3_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port3 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P4_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port4 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P5_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port5 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P6_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port6 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P7_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port7 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_lpbk_cont_test HE-LPBK Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking he_lpbk_csr_test HE-LPBK CSR accesses data checking he_lpbk_flr_rst_test HE-LPBK Set HE_LPK in all the modes randomly and iterate the transactions in loop. At the end of every loop assert the Soft reset in the middle of the transactions data checking, counter checking he_lpbk_long_rst_test HE-LPBK Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_lpbk_long_test HE-LPBK Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_lpbk_multi_user_intr_test HE-LPBK generate user HE_LB interrupt interrupt checking he_lpbk_rd_cont_test HE-LPBK Read only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking he_lpbk_rd_test HE-LPBK Read only mode. Randomize num_lines, addresses, req_len counter checking he_lpbk_reqlen16_test HE-LPBK To check the behavior of HE_LPK block when req_length 16 and num_lines set to 1024 Cache lines in Loopback mode data checking, counter checking he_lpbk_reqlen1_test HE-LPBK Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_lpbk_reqlen2_test HE-LPBK Loopback mode. 128 CLs, req_len = 2CL, random addresses data checking, counter checking he_lpbk_reqlen4_test HE-LPBK Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_lpbk_reqlen8_test HE-LPBK Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_lpbk_rst_in_middle_test PCIe MMIO Path Set HE_LPK in all the modes randomly and iterate the transactions in loop. At the end of every loop assert the Soft reset in the middle of the transactions Register Base Offset he_lpbk_test HE-LPBK Loopback mode. Randomize num_lines, addresses, req_len data checking, counter checking he_lpbk_thruput_contmode_test HE-LPBK Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking he_lpbk_thruput_test HE-LPBK generate user HE_LB interrupt counter checking he_lpbk_user_intr_test HE-LPBK Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_lpbk_wr_cont_test HE-LPBK Write only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking he_lpbk_wr_test HE-LPBK Write only mode. Randomize num_lines, addresses, req_len counter checking he_mem_cont_test HE-MEM Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking, counter checking he_mem_csr_test HE-MEM CSR accesses data checking he_mem_lpbk_long_rst_test HE-MEM Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_mem_lpbk_long_test HE-MEM Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_mem_lpbk_reqlen16_test HE-MEM To check the behavior of HE_MEM block when req_length 16 and num_lines set to 1024 Cache lines in Loopback mode data checking, counter checking he_mem_lpbk_reqlen1_test HE-MEM Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_mem_lpbk_reqlen2_test HE-MEM Loopback mode. 128 CLs, req_len = 2CL, random addresses data checking, counter checking he_mem_lpbk_reqlen4_test HE-MEM Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_mem_lpbk_reqlen8_test HE-MEM Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_mem_lpbk_test HE-MEM Loopback mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_multi_user_intr_test Interrupt Back to back multiple User interrupt request from HE MEM Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read for multiple back to back request he_mem_rd_cont_test HE-MEM Read only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_rd_test HE-MEM Read only mode. Randomize num_lines, addresses, req_len counter checking he_mem_thruput_contmode_directed_test HE-MEM Set HE_LPK in thruput mode and send traffic with req len 1 and num_lines set to 40 data checking, counter checking he_mem_thruput_contmode_test HE-MEM Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_thruput_test HE-MEM Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_mem_user_intr_test Interrupt FME interrupt request using RAS ERROR and FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests generated from FME and RAS ERROR bits he_mem_wr_cont_test HE-MEM Write only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_wr_test HE-MEM Write only mode. Randomize num_lines, addresses, req_len counter checkingt he_random_test All HEs Enable all HEs and randomize modes data checking if in lpbk mode, counter checking helb_rd_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Read only mode Measure the performance helb_rd_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Read only mode Measure the performance helb_rd_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Read only mode Measure the performance helb_thruput_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Thruput mode Measure the performance helb_thruput_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Thruput mode Measure the performance helb_thruput_4cl_5bit_tag_test Performance Set HE_LPK in thruput mode and send traffic with req len 4 and num_lines set to 1024. Measure the Read/Write performance Measure the performance helb_thruput_4cl_8bit_tag_test Performance Set HE_LPK in thruput mode and send traffic with req len 4 and num_lines set to 1024. Measure the Read/Write performance Measure the performance helb_thruput_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Thruput mode Measure the performance helb_wr_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Write only mode Measure the performance helb_wr_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Write only mode Measure the performance helb_wr_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Write only mode Measure the performance hssi_ss_test HSSI CSR accesses data checking malformedtlp_pcie_rst_test Protocol Checker generate malformed TLP protocol error and initiate pcie reset to clear the error Check for error malformedtlp_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error. 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear). 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4. 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set. 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one. 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set. 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. 2.After clearing the error register ,check if normal transcation are completing. MaxTagError_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error. 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear). 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4. 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set. 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one. 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set. 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. 2.After clearing the error register ,check if normal transactions are completing. mem_tg_csr_test MEM-TG CSR access data checking mem_tg_traffic_gen_test MEM-TG This test checks the MEM_TG traffic generator flow for 1 bank data checking mini_smoke_test All HEs shorter simpler version of random test for turn-in sanity check data checking if in lpbk mode, counter checking mix_intr_test Interrupt Mix interrupt testcase to send multiple FME and User interrupt request simultaneously Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests through different sources - FME and HE-MEM modules mmio_pcie_mrrs_128B_mps_128B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_128B_mps_256B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_256B_mps_128B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_256B_mps_256B_test PCIe - Max Payload/Max Read Req Size Random length mmio Read Checking valid possible combination of MPS & MRRS mmio_stress_nonblocking_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux with non-blocking MMIO reads data checking mmio_stress_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux data checking mmio_test PCIe MMIO Path MMIO targeting PF0(ST2MM, FME, PMCI, QSFP, HSSI SS), PF1, PF2,PF3, PF4, PF1.VF1, PF1.VF2 data checking mmio_unimp_test PCIe MMIO Path MMIO acccess to unimplemented addresses data checking MMIODataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error. 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear). 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4. 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set. 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one. 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set. 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. 2.After clearing the error register ,check if normal transcation are completing. MMIOInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error. 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear). 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4. 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set. 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one. 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set. 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. 2.After clearing the error register ,check if normal transcation are completing. MMIOTimedout_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error. 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear). 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4. 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set. 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one. 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set. 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. 2.After clearing the error register ,check if normal transcation are completing. pcie_csr_test PCIESS Earlier msix registers were in fme block but now it has moved from fme to pciess. Hence coded a seperate test for msix data checking pcie_pmci_mctp_multi_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM multiple packets received from PMCI controller over APF and BPF to PCIe HIA subsystem pcie_pmci_mctp_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM (Vendor Defined Message) single packet received from PCIe HIA subsystem to PMCI controller over APF and BPF via AXI4-lite memory write request pmci_csr_test PMCI CSR CSR accesses data checking pmci_fme_csr_test PMCI FME CSR CSR accesses data checking pmci_pcie_mctp_multi_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM multiple packets received from PMCI controller over APF and BPF to PCIe HIA subsystem pmci_pcie_mctp_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM single packet received from PMCI controller over APF and BPF to PCIe HIA subsystem pmci_pciess_csr_test PMCI PCIESS CSR CSR accesses data checking pmci_qsfp_csr_test PMCI QSFP CSR CSR accesses data checking port_gasket_csr_test PORT GASKET CSR accesses data checking qsfp_csr_test QSFP CSR accesses data checking TxMWrDataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error. 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear). 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4. 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set. 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one. 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set. 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. 2.After clearing the error register ,check if normal transcation are completing. TxMWrInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error. 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear). 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4. 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set. 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one. 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set. 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. 2.After clearing the error register ,check if normal transcation are completing. uart_intr_test UART Checking Generates UART interrupt Check interrupt UnexpMMIORspErr_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error. 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear). 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4. 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set. 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one. 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set. 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. 2.After clearing the error register ,check if normal transcation are completing. vdm_err_vid_test Vendor ID check generate a packet with undefined Vendor-ID from Host to PMCI_SS ID check

              The next section describes how to compile and build the UVM environment prior to running each UVM test and analyinsg the results in the log files

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#64-ip-compile","title":"6.4 IP Compile","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-vcs","title":"Synopsys VCS","text":"

              To compile all IPs for the Synopsys VCS simulater targetting the Intel\u00ae FPGA SmartNIC N6001-PL:

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk cmplib_adp\n

              To compile all IPs for the Synopsys VCS simulater targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk cmplib_adp FTILE_SIM=1\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-tbd","title":"Questasim (TBD)","text":"

              To compile all IPs for the Questasim simulater targetting the Intel\u00ae FPGA SmartNIC N6001-PL:

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk cmplib_adp\n

              To compile all IPs for the Questasim simulater targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk cmplib_adp FTILE_SIM=1\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#65-rtl-test-bench-compile","title":"6.5 RTL & Test Bench Compile","text":"

              The RTL file list for compilation is located here: verification/scripts/rtl_comb.f

              The TB file list for compilation is located here: verification/scripts/ver_list.f

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-vcs_1","title":"Synopsys VCS","text":"

              To compile RTL and Testbench for the Synopsys VCS simulater targetting the Intel\u00ae FPGA SmartNIC N6001-PL:

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_adp DUMP=1\n

              To compile RTL and Testbench for the Synopsys VCS simulater targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_adp FTILE_SIM=1 DUMP=1\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-tbd_1","title":"Questasim (TBD)","text":"

              To compile RTL and Testbench for the Questasim simulater targetting the Intel\u00ae FPGA SmartNIC N6001-PL:

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_adp DUMP=1\n

              To compile RTL and Testbench for the Questasim simulater targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_adp FTILE_SIM=1 DUMP=1\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#66-ip-and-rtl-test-bench-compile","title":"6.6 IP and RTL & Test Bench Compile","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-vcs_2","title":"Synopsys VCS","text":"

              If the user wants to compile all IPs and RTL Testbench in one command for Synopsys VCS targetting the Intel\u00ae FPGA SmartNIC N6001-PL then follow the procedure below

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_all DUMP=1\n

              If the user wants to compile all IPs and RTL Testbench in one command for Synopsys VCS targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) then follow the procedure below

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_all FTILE_SIM=1 DUMP=1\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-tbd_2","title":"Questasim (TBD)","text":"

              If the user wants to compile all IPs and RTL Testbench in one command for Questasim targetting the Intel\u00ae FPGA SmartNIC N6001-PL then follow the procedure below

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_all DUMP=1\n

              If the user wants to compile all IPs and RTL Testbench in one command for Questasim targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) then follow the procedure below

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_all FTILE_SIM=1 DUMP=1\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-vcs_3","title":"Synopsys VCS","text":"

              To run a simulation for Synopsys VCS targetting the Intel\u00ae FPGA SmartNIC N6001-PL:

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk run TESTNAME=ofs_mmio_test DUMP=1\n

              To run a simulation for Synopsys VCS targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk run TESTNAME=ofs_mmio_test FTILE_SIM=1 DUMP=1\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-tbd_3","title":"Questasim (TBD)","text":"

              To run a simulation for Questasim targetting the Intel\u00ae FPGA SmartNIC N6001-PL:

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=ofs_mmio_test DUMP=1\n

              To run a simulation for Questasim targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):

                  cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=ofs_mmio_test FTILE_SIM=1 DUMP=1\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-vcs_4","title":"Synopsys VCS","text":"

              To dump the waveform, \"DUMP=1\" must be added into the command line for Synopsys VCS build and simulation targetting the Intel\u00ae FPGA SmartNIC N6001-PL:

                  gmake -f Makefile_VCS.mk build_adp DUMP=1\n\n    gmake -f Makefile_VCS.mk run TESTNAME=<test_case_name> DUMP=1\n

              Or

                  gmake -f Makefile_VCS.mk build_run TESTNAME=<test_case_name> DUMP=1\n

              To dump the waveform, \"DUMP=1\" must be added into the command line for Synopsys VCS build and simulation targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):

                  gmake -f Makefile_VCS.mk build_adp FTILE_SIM=1 DUMP=1\n\n    gmake -f Makefile_VCS.mk run TESTNAME=<test_case_name> FTILE_SIM=1 DUMP=1\n

              Or

                  gmake -f Makefile_VCS.mk build_run TESTNAME=<test_case_name> FTILE_SIM=1 DUMP=1\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-tbd_4","title":"Questasim (TBD)","text":"

              To dump the waveform, \"DUMP=1\" must be added into the command line for Questasim build and simulation targetting the Intel\u00ae FPGA SmartNIC N6001-PL: 

                  gmake -f Makefile_MSIM.mk build_adp DUMP=1\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=<test_case_name> DUMP=1\n

              Or

                  gmake -f Makefile_MSIM.mk build_run TESTNAME=<test_case_name> DUMP=1\n

              To dump the waveform, \"DUMP=1\" must be added into the command line for Questasim build and simulation targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):

                  gmake -f Makefile_MSIM.mk build_adp FTILE_SIM=1 DUMP=1\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=<test_case_name> FTILE_SIM=1 DUMP=1\n

              Or

                  gmake -f Makefile_MSIM.mk build_run TESTNAME=<test_case_name> FTILE_SIM=1 DUMP=1\n

              There are some optimizations in the Table below for convenience if you want to bypass some commands for both Synopsys VCS and Questasim:

              Command (Synopsys VCS) for n6001 Command (Questasim) for n6001 Command (Synopsys VCS) for fseries-dk Command (Questasim) for fseries-dk Details gmake -f Makefile_VCS.mk build_all DUMP=1 gmake -f Makefile_MSIM.mk build_all DUMP=1 gmake -f Makefile_VCS.mk build_all FTILE_SIM=1 DUMP=1 gmake -f Makefile_MSIM.mk build_all FTILE_SIM=1 DUMP=1 compile IP + compile RTL gmake -f Makefile_VCS.mk build_run TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk build_run TESTNAME= DUMP=1 gmake -f Makefile_VCS.mk build_run TESTNAME= FTILE_SIM=1 DUMP=1 gmake -f Makefile_MSIM.mk build_run TESTNAME= FTILE_SIM=1 DUMP=1 compile RTL + run test gmake -f Makefile_VCS.mk do_it_all TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk do_it_all TESTNAME= DUMP=1 gmake -f Makefile_VCS.mk do_it_all TESTNAME= FTILE_SIM=1 DUMP=1 gmake -f Makefile_MSIM.mk do_it_all TESTNAME= FTILE_SIM=1 DUMP=1 compile IP, RTL and run test gmake -f Makefile_VCS.mk rundb TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk rundb TESTNAME= DUMP=1 gmake -f Makefile_VCS.mk rundb TESTNAME= FTILE_SIM=1 DUMP=1 gmake -f Makefile_MSIM.mk rundb TESTNAME= FTILE_SIM=1 DUMP=1 run test in sim dir + over-writes content"},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#67-uvm-regression-test","title":"6.7 UVM Regression Test","text":"

              If the user wants to run the complete set of UVM tests in one command for VCS and Questasim targetting the Intel\u00ae FPGA SmartNIC N6001-PL then follow the procedure below

              cd $VERDIR/scripts\n\npython uvm_regress.py -l -n 8 -p adp -k top_pkg -s vcs -c none\n\nFor Regression in VCS with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -p adp -k top_pkg -s vcs -c none\n

              Results are created in a sim directory ($VERDIR/sim) with individual testcase log dir

              For Regression in MSIM with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -p adp -k top_pkg -s msim -c none\n

              Results are created in a sim directory ($VERDIR/sim_msim) with individual testcase log dir

              If the user wants to run the complete set of UVM tests in one command for VCS and Questasim targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) then follow the procedure below

              TBC\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#68-uvm-waveform-and-transcript-analysis","title":"6.8 UVM Waveform and Transcript Analysis","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-vcs_5","title":"Synopsys VCS","text":"

              Running Synopsys VCS UVM tests will generate a ofs-agx7-pcie-attach/verification/sim directory

              \u2022 All build time logs are located at ofs-agx7-pcie-attach/verification/sim\n\n\u2022 Each testcase will have separate directory inside sim ofs-agx7-pcie-attach/verification/sim/<test_case_name>\n

              There are two tracker or log files that are available: runsim.log and trans.log.

              runsim.log is the simulation log file generated from Synopsys VCS. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 5

              Figure 5 runsim.log

              trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 6

              Figure 6 trans.log

              The waveform generated is named as \"inter.vpd\". To open the waveform, go to simulation result directory and run 

                  dve -full64 -vpd inter.vpd &\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-tbd_5","title":"Questasim (TBD)","text":"

              Running Questasim UVM tests will generate a ofs-agx7-pcie-attach/verification/sim_msim directory

              \u2022 All build time logs are at ofs-agx7-pcie-attach/verification/sim_msim\n\n\u2022 Each testcase will have separate directory inside sim ofs-agx7-pcie-attach/verification/sim_msim/<test_case_name>\n

              There are two tracker or log files that are available: runsim.log and trans.log.

              runsim.log is the simulation log file generated from Questasim. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 7

              Figure 7 runsim.log

              trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 8

              Figure 8 trans.log

              The waveform generated is named as \"vsim.wlf\". To open the waveform, go to simulation result directory and run 

                  vsim -view vsim.wlf &\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#69-uvm-coverage-analysis","title":"6.9 UVM Coverage Analysis","text":"

              For Intel\u00ae FPGA SmartNIC N6001-PL

              The following command allows to run a single testcase with coverage enabled

                  gmake -f Makefile_VCS.mk cmplib_adp && gmake -f Makefile_VCS.mk build_adp DUMP=1 DEBUG=1 COV_FUNCTIONAL=1&& gmake -f Makefile_VCS.mk run TESTNAME=<TESTCASE-NAME> DUMP=1 DEBUG=1 COV_FUNCTIONAL=1 &\n

              The following command shows how to merge and generate the coverage report

                  urg -dir <$VERDIR/sim/simv.vdb> <$VERDIR/sim/regression.vdb> -format both -dbname <regression_database_name>\n

              This will generate both urgreport directory and .vdb file Multiple regression.vdb from different regressions can be merged with the same command. 

                  e.g \"urg -dir <path1_till_simv.vdb> <path1_till_regression.vdb> <path2_till_regression.vdb> -report <dir> -format both -dbname <dirname>\"\n

              The following commands shows how to launch DVE and check the coverage reports

              To open DVE of a single regression or testcase, execute:\n\n    dve -full64 -cov -covdir simv.vdb regression.vdb &\n\nTo open DVE of a merged regression test, execute:\n\n    dve -full64 -cov -covdir <dirname.vdb> &\n

              Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)

              TBC\n

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#7-csr-verification-using-uvm-ral","title":"7 CSR Verification using UVM RAL","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#71-overview","title":"7.1 Overview","text":"

              The UVM Register Layer provides a standard base class library that enable users to implement the object-oriented model to access the DUT registers and memories. The UVM Register Layer is also referred to as UVM Register Abstraction Layer (UVM RAL). Design registers can be accessed independently of the physical bus interface. i.e. by calling read/write methods. This is shown in Figure 9 below.

              Figure 9 RAL UVM Testbench

              The RAL register models for different CSR's mimics the design registers. All RAL files are located here.

                  ofs-agx7-pcie-attach/verification/testbench/ral\n

              The RAL model is generated through the Synopsys RALGEN tool and is used for CSR verification.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#72-ral-integration","title":"7.2 RAL Integration","text":"

              For UVM RAL model integration to the environment, adapters for each CSR is implemented and integrated into the Testbench Environment. It is used to convert the PCIe bus sequence items into uvm_reg_bus_op and vice versa. The CSR test cases pick up all the registers from the respective CSR blocks and perform a default value, wr/rd check on them.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#73-ral-model-generation","title":"7.3 RAL Model Generation","text":"

              Steps for RAL model generation

              Excel(xls) file containing the registers is required. Make sure there are separate xls files for each CSR and the worksheet name must contain the name \"reg_fields\".

              Excel sheet snapshot example below for EMIF_CSR.xls located at /ipss/mem/rtl/adp

              \u2022 Navigate to ofs-agx7-pcie-attach/ofs-common/verification/common/scripts/ral\n\u2022 Copy the excel file (xls) to the above area\n\u2022 In the bash terminal run mk_ral.sh <Excel sheet name without extension > <output *.sv file name without ral_  prepended >\n\u2022 The above steps generate two ral *.sv files. File with _cov suffix is a coverage enabled ral model. \n\u2022 Copy *.sv files to ofs-agx7-pcie-attach/verification/testbench/ral\n

              \u2022 As an example to generate ral_ac_ce.sv from AC_CE_CSR.xls file the command is\n\n    mk_ral.sh AC_CE_CSR ac_ce\n

              This generates two ral models (ral_ac_ce.sv and ral_ac_ce_cov.sv)

              To add new registers

              \u2022 To create new registers, copy existing ones and modify the cells in the xls. Make sure the last line is also a copied blank line\n\u2022 Follow all the steps of RAL model generation\n

              To Generate a RAL model when a new xls sheet is created for a new component

              \u2022 Copy the relevant xls sheet to ofs-agx7-pcie-attach/ofs-common/verification/common/scripts/ral\n\u2022 Follow all the steps of RAL model generation\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#74-top-level-verification-architecture-for-csr-testing","title":"7.4 Top Level Verification Architecture for CSR testing","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#741-testbench-components","title":"7.4.1 Testbench components","text":"

              The testbench components for RAL are defined below

              \u2022 ral_reg_iofs_* (uvm_reg) generated by the steps as mentioned in section 5.3\n

              The uvm register class is written by extending the uvm_reg. A register represents a set of fields that are accessible as a single entity Each register contains any number of fields, which mirror the values of the corresponding elements in hardware

              \u2022 ral_block_iofs_* (uvm_block) generated in the same register file\n

              A register model is an instance of a register block, which may contain any number of registers, register files, memories, and other blocks

              \u2022 ral_block_ofs (uvm_block) \u2013 Contains all the CSR block instantiations\n\u2022 Reg2vip_*_adapter (uvm_reg_adapter) \u2013 This class defines an interface for converting between uvm_reg_bus_op and a specific bus transaction. For each CSR a respective adapter is present\n

              All the components are defined in ofs-agx7-pcie-attach/ofs-common/verification/testbench

              Integration of components in testbench

              \u2022 The RAL blocks and adapters for each CSR is instantiated in tb_env\n\u2022 The bar range for each CSR is also set in the tb_env\n

              Sample Environment Integration snippets

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#8-modifying-uvm-testbench","title":"8 Modifying UVM Testbench","text":"

              The next sections describe what needs to be considered when modifying the UVM, adding a new interface to the testbench and creating a new UVM test for a customised OFS Accelerator platform.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#81-modifying-uvm-environment-for-new-shell-variant","title":"8.1 Modifying UVM environment for new Shell Variant","text":"

              OFS n6001 comprises a shell based on PCIe Gen4x16 and is named base_x16

              This base_x16 shell is described by an RTL file list, IP File lists and setup scripts to complete the build flow

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#82-modifying-uvm-environment-and-setting-up-compile-and-run-flow","title":"8.2 Modifying UVM environment and setting up Compile and Run flow","text":"

              All the variants can mostly reuse the existing UVM infrastructure to setup the build and run flow

              \u2022 Create directory under $OFS_BUILD_ROOT new variant e.g ofs-n9000\n\u2022 Change directory to $OFS_BUILD_ROOT/ofs-n9000/verification/scripts\n\u2022 modify Makefile it to point to the new RTL, IP and script files.\n

              Following these three steps above will enable the build and sim flow to run the existing UVM TB and tests with new IOFS n9000 variant.

              Adding a new interface requires signal connections in the testbench. An additional BFM or verification IP is needed to drive the new interface. The main testbench file tb_top.sv is found at the following location

                  ofs-agx7-pcie-attach/verification/testbench\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#83-adding-a-new-ral-directory","title":"8.3 Adding a new RAL directory","text":"

              In case the design has many register changes and the user decides to generate all the new RAL models instead of reusing from existing base RAL models, the following steps will help to create and integrate a new RALDIR in the UVM environment.

              \u2022 Generate the new RAL files in desired directory. Preferably under the \"ofs-agx7-pcie-attach/verification/common/testbench\" \n\u2022 By default, the Makefile points to base FIM RAL so set the RALDIR path in the Makefile to the new generated RAL file directory\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#84-modifying-tbtest-files-for-new-variant","title":"8.4 Modifying TB/Test files for new Variant","text":"

              Create a define for the new variant. e.g 'define FIM_NEW. If you are modifying common files then add new logic under this define so other projects will not get affected with variant specific change.

              If there are more changes, please create separate \"testbench\" and \"test\" folders under this new variant. 

              \u2022 Extend variant specific env file from base env file to add variant specific changes.\n\u2022 Create new test/seq lib files in \"tests\" folder.\n\u2022 Create new variant package, add new TB/Tests/Sequence lib files and also import the base package files.\n

              If you are adding new files then make sure it's included in Makefile for the build+run flow.

              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#85-uvm-pcie-drivers","title":"8.5 UVM PCIe Drivers","text":"

              The \"svt_pcie_driver_app_transaction_base_sequence\" is part of Synopsys PCIe VIP library. You can find the sequence definition in the following two directories

              \u2022 Navigate to \"$DESIGNWARE_HOME/vip/svt/pcie_svt/Q-2020.03/sverilog/src/vcs/svt_pcie_driver_app_transaction_sequence_collection.svp\" file. All the base and PCIe sequences are defined in this file.\n\n\u2022 When the OFS UVM build command is executed, it creates \"vip\" directory under \"$OFS_BUILD_ROOT/ofs-agx7-pcie-attach/verification\". You can also find the same sequence file at \"$IOFS_BUILD_ROOT/ofs-agx7-pcie-attach/verification/vip/pcie_vip/src/sverilog/vcs/svt_pcie_driver_app_transaction_sequence_collection.sv\"\n
              "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#notices-disclaimers","title":"Notices & Disclaimers","text":"

              Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Performance varies by use, configuration and other factors. Your costs and results may vary. You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications. Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards. \u00a9 Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.

              OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122.

              "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/","title":"Accelerator Functional Unit Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae","text":""},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#1-introduction","title":"1. Introduction","text":"

              This document is a design guide for creating an Accelerator Functional Unit (AFU) using Open FPGA Stack (OFS) for Intel\u00ae Stratix 10\u00ae FPGA. The AFU concept consists of separating the FPGA design development process into two parts, the FIM and AFU, as shown in the diagram below:

              This diagram shows the FPGA board interface development separation from the internal FPGA workload creation. This separation starts with the FPGA Interface Manager (FIM), which consists of the external interfaces and board management functions. The FIM is the base system layer typically provided by board vendors. The FIM interface is specific to a particular physical platform. The AFU uses the external interfaces with user-defined logic to perform a specific application. Separating the lengthy and complicated process of developing and integrating external interfaces for an FPGA into a board allows the AFU developer to focus on their workload needs. Intel\u00ae OFS for Intel\u00ae Stratix 10\u00ae FPGA provides the following tools for rapid AFU development:

              • Scripts for both compilation setup
              • Integration with Open Programmable Acceleration Engine (OPAE) SDK for rapid software development for your AFU application

              Please notice that the AFU region consists of both static and PR logic in the above block diagram. Creating AFU logic for the static region is described in FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs. This guide covers logic in the AFU Main (PR) region.

              "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#11-document-organization","title":"1.1 Document Organization","text":"

              This document is organized as follows:

              • Description of design flow
              • Interfaces and functionality provided in the Intel\u00ae FPGA PAC D5005 FIM
              • Downloading and installing Intel\u00ae OFSand OPAE SDK
              • Hardware/Software co-simulation using ASE
              • Testing the AFU example in Intel\u00ae FPGA PAC D5005
              • Debugging an AFU with Remote Signal Tap

              This guide provides theory followed by tutorial steps to solidify your AFU development knowledge.

              This guide uses the Intel\u00ae FPGA PAC D5005 as the platform for all tutorial steps. Additionally, this guide and the tutorial steps can be used with other platforms; However, please consult the board and FIM supplier of other platforms for specific instructions on the use of custom FIM to develop AFU design.

              If you have worked with previous Intel\u00ae Programmable Acceleration products, you will find OFS for Intel\u00ae Stratix 10\u00ae FPGA is similar; however, there are differences, and you are advised to carefully read and follow the tutorial steps to understand the design tools and flow fully.

              "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#12-prerequisite","title":"1.2 Prerequisite","text":"

              This guide assumes you understand the following FPGA logic design-related knowledge and skills:

              • FPGA compilation flows, including the Intel\u00ae Quartus\u00ae Prime Pro Edition design flow.
              • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition software, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on critical timing paths.
              • RTL and coding practices to create synthesized logic.
              • High-level synthesis (HLS) and Platform Designer design entry tools are supported.
              • RTL simulation tools.
              • Signal Tap Logic Analyzer tool in the Intel\u00ae Quartus\u00ae Prime Pro Edition software.
              "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#121-development-environment","title":"1.2.1 Development Environment","text":"

              To run the tutorial steps in this guide requires this development environment:

              Item Version Operating System RHEL 8.6 Python 3.6.8 cmake 3.15 GCC 7.4.0 git 1.8.3.1 perl 5.8.8

              Verify your development has the above tools installed.

              The following server and Intel\u00ae PAC card are required to run the examples in this guide:

              1. Intel\u00ae FPGA PAC D5005 with root entry hash erased (Please contact Intel\u00ae for root entry hash erase instructions). The standard Intel\u00ae FPGA PAC D5005 card is programmed only to allow the FIM binary files signed by Intel\u00ae to be loaded. The root entry hash erases process will allow unsigned FIM binary files to be loaded.
              2. Qualified Server Models see Qualified Servers.
              3. Intel\u00ae FPGA PAC D5005 installed in the qualified server following instructions in OFS Getting Started User Guide: For Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.
              "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#13-acceleration-functional-unit-afu-development-flow","title":"1.3 Acceleration Functional Unit (AFU) Development Flow","text":"

              OFS Stack provides a rapid design methodology for creating complex FPGA applications. In addition, you are provided with the following:

              • Hardware shell layer, known as FIM
              • Software stack including tools for debug/diagnostics
              • FPGA design flow with full-stack simulation support
              • AFU code samples demonstrating all interfaces

              For any non-Intel\u00ae platform, contact your board vendor for the above components specific to the platform. To start with AFU development, the first step should be to understand your platform capabilities. For example, what interface is the FPGA connected to the Host machine over PCI-E, if it is AXI like the Intel\u00ae Stratix 10\u00ae FPGA Platform, or CCIP or CXL. Does the platform provide an External Memory Interface or the HSSI interface? Once you know what the platform offers, you can develop your AFU requirements and architecture as the next step. This document will cover example AFU architecture and things that will help build AFU for Intel\u00ae Stratix 10\u00ae FPGA reference platform and others coming in the future. In addition, this knowledge can be relatively applied for AFU development on other vendor-provided platforms.

              The figure below shows a typical AFU development process independent of the platform used. 

              flowchart  TB;\n    A[Understand platform capabilities with OFS]-->B[Review AFU requirements and code samples provided];\n    B[Review AFU requirements and code samples provided]-->C[Define AFU architecture];\n    C[Define AFU architecture]-->D[Design AFU hardware];\n    D[Design AFU hardware]-->E[Develop AFU software to control hardware];\n    E[Develop AFU software to control hardware]-->F{\"Simulate in AFU Simulation Enviroment (ASE)\"};\n    F:::if -- Pass --> H[\"Compile AFU for synthesis, place & route and timing (uses Quartus)\"];\n    H[\"Compile AFU for synthesis, place & route and timing (uses Quartus)\"] --> I[\"Analyze Quartus Compile reports\"];\n    I --> J{\"Quartus reports clean? (e.g. timing closed)\"};\n    J:::if -- No --> P2;\n    J -- Yes --> K[\"Run/Validate design on OFS Platform\"];\n    K --> L{\"Hardware validation pass?\"};\n    L == Yes ==> M[\"AFU ready to deploy\"];\n    L -- No --> N[\"Debug on hardware using traditional FPGA tools (e.g. SignalTab\"];\n    N --> P2[\"Fix AFU design (e.g Design changes, timing closure constraints)\"];\n    P2 --> O{\"Need functional validation?\"};\n    O:::if -- Yes -->P[\"Fix AFU design (e.g Functional design changes, bug fixes)\"];\n    O -- No -->H;    \n    F -- Fail --> P;\n    P -->D;      \n\n    classDef default color:#fff,fill:#0071c5,stroke:#71c5,stroke-width:1px\n    classDef if color:#0071c5,fill:#fff,stroke:#0071c5,stroke-width:2px
              "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#131-high-level-data-flow","title":"1.3.1. High Level Data Flow","text":"

              The OFShigh-level data flow is shown below: The control and data are composed of the following:

              • Host Interface Adapter (PCIe)
              • Low-Performance Peripherals
                • Slow speed peripherals (I2C, Smbus, etc)
                • Management peripherals (FME)
              • High-Performance Peripherals
                • Memory peripherals
                • Acceleration Function peripherals
                • HPS Peripheral
              • Fabrics
                • Peripheral Fabric (multi-drop)
                • AFU Streaming fabric (point to point)

              Peripherals are connected using AXI or Avalon:

              • Via the peripheral fabric (AXI4-Lite, multi-drop)
              • Via the AFU streaming fabric (AXI-S, point to point)

              Peripherals are presented to software as:

              • OFS managed peripherals that implement DFH CSR structure.
              • Native driver managed peripherals (i.e., Exposed via an independent PF, VF)

              The peripherals connected to the peripheral fabric are primarily OPAE managed resources, whereas the peripherals connected to the AFU are \"primarily\" driven by native OS drivers. The word \"primarily\" is used since the AFU is not mandated to expose all its peripherals to Intel\u00ae OPAE. Instead, it can be connected to the peripheral fabric but can choose to expose only a subset of its capability to OPAE.

              OFS uses a defined set of CSRs to expose the functionality of the FPGA to the host software. These registers are described in Open FPGA Stack Reference Manual - MMIO Regions section.

              If you make changes to the FIM that affect the software operation, Intel\u00ae OFS provides a mechanism to communicate that information to the proper software driver. The Device Feature Header (DFH) structure provides a mechanism to maintain compatibility with OPAE software. Please see FPGA Device Feature List (DFL) Framework Overview for an excellent description of DFL operation from the driver perspective.

              When planning your address space for your FIM updates, please be aware OFS FIM targeting Intel\u00ae FPGA PAC D5005, 256KB of MMIO region is allocated for external FME features, and 128kB of MMIO region is given for external port features. Each external feature must implement a feature DFH, and the DFH needs to be placed at the 4KB boundary. The last feature in the external feature list must have the EOL bit in its DFH set to 1 to mark the end of the external feature list. Since the FPGA address space is limited, consider using an indirect addressing scheme to conserve address space.

              "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#132-considerations-for-pim-usage","title":"1.3.2. Considerations for PIM Usage","text":"

              When creating an AFU, a designer needs to decide what type of interfaces the platform (FIM) should provide to the AFU. The FIM can provide the native interfaces (i.e. PCIe TLP commands) or standard memory mapped interfaces (i.e. AXI-MM or AVMM) by using the PIM. The PIM is an abstraction layer consisting of a collection of SystemVerilog interfaces and shims to enable partial AFU portability across hardware despite variations in hardware topology and native interfaces. The PIM adds a level of logic between the AFU and the FIM converting the native interfaces from the FIM to match the interfaces provided by the AFU.

              The following resources are available to assist in creating an AFU:

              PIM Core Concepts provides details on using the PIM and its capabilities.

              Connecting an AFU to a Platform using PIM guides you through the steps needed to connect a PIM Based AFU to the FIM.

              The AFU Tutorial provides several AFU examples. These examples can be run with the current OFS FIM package. There are three AFU types of examples provided (PIM based, hybrid and native). Each example provides the following:

              • RTL, which includes the following interfaces:
                • Host Channel:
                  • Host memory, providing a DMA interface.
                  • MMIO, providing a CSR interface.
                • Local Memory
              • Host software example interfacing to the CSR interface and host memory interface, using the OPAE C API.
              • Accelerator Description File .json file
              • Source file list
              "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#133-afu-interfaces-included-with-intel-fpga-pac-d5005","title":"1.3.3 AFU Interfaces Included with Intel\u00ae FPGA PAC D5005","text":"

              The figure below shows the interfaces available to the AFU in this architecture. It also shows the design hierarchy with module names from the FIM (top.sv) to the PR region AFU (afu_main.sv). One of the main differences from the previous Intel\u00ae Stratix 10\u00ae FPGA OFS architecture is a static port gasket region (port_gasket.sv) that has components to facilitate the AFU and also consists of the GBS region (afu_main.sv) via the PR slot. The Port Gasket contains all the PR -specific modules and logic, e.g., PR slot reset/freeze control, user clock, remote STP etc. Architecturally, a Port Gasket can have multiple PR slots to which user workload can be programmed. However, only one PR slot is supported for Intel\u00ae OFS Release for Intel\u00ae Stratix 10\u00ae FPGA. Therefore, everything in the Port Gasket until the PR slot should be provided by the FIM developer. The task of the AFU developer is to add their desired application in the afu_main.sv module by stripping out unwanted logic and instantiating the target accelerator. As shown in the figure below, here are the interfaces connected to the AFU (highlighted in green) via Intel\u00ae FPGA PAC D5005 FIM:

              • AXI Streaming (AXI-S) interface to the Host via PCIe Gen3x16
              • Avalon Memory-Mapped Channels (4) to the DDR4 EMIF interface
              • AXI Streaming (AXI-S) interface to the HSSI 10G Ethernet

              "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#134-platform-capabilities","title":"1.3.4. Platform Capabilities","text":"

              The FIM targets operation in the Intel\u00ae FPGA PAC D5005 card. The block diagram of the Intel\u00ae FPGA PAC D5005 is shown below:

              The key Intel\u00ae FPGA PAC D5005 FPGA interfaces are:

              • Host interface - PCIe Gen3 x 16
              • Network interface
                • 2 - QSFP28 cages
                • Current FIM supports 1 x 10 GbE, other interfaces can be created
              • External Memory
                • 2 or 4 channels of DDR4-2400 to RDIMM modules
                • RDIMM modules = 8GB organized as 1 Gb X 72
              • Board Management
                • SPI interface
                • FPGA configuration
              "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#135-top-level-fpga","title":"1.3.5. Top Level FPGA","text":"

              The internal FPGA architecture is shown below:

              "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2-set-up-afu-development-environment","title":"2. Set Up AFU Development Environment","text":"

              This section covers:

              • Development environment set up
              • Retrieving and installing OFS, OPAE SDK
              • Building theIntel\u00ae FPGA PAC D5005 FIM
              • Building a relocatable AFU tree
              • Compiling the host_chan_mmio example AFU

              Additionally, this section includes steps to demonstrate loading and running the host_chan_mmio example AFU in an Intel\u00ae FPGA PAC D5005 equipped Linux server.

              "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#21-prepare-afu-development-environment","title":"2.1. Prepare AFU development environment","text":"

              Typical development and hardware test environments consist of a development server or workstation with installed FPGA development tools and a separate server installed with the target OFS-compatible FPGA PCIe card. The typical usage and flow of data between these two servers are shown below:

              Please refer to Unit Level Simulation if you would like to make any simulation Unit Level Simulation.

              Note that both development and hardware testing can be performed on the same server if desired.

              This guide uses Intel\u00ae FPGA PAC D5005 as the target OFS-compatible FPGA PCIe card platform for demonstration steps. The Intel\u00ae FPGA PAC D5005 must be fully installed following OFS Getting Started User Guide: For Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs. If using a different OFS FPGA PCIe card, contact your supplier for instructions on how to install and operate a user-developed AFU.

              NOTE:

              The following chapters assume you use the same server for development and Deployment (Run the FIM/AFU/SW over the Intel\u00ae FPGA PAC D5005):

              Development: Modify the FIM/AFU/SW run simulation and compile the design (Generate the binaries). Deployment: Program the binaries under the Intel\u00ae FPGA PAC D5005 and exercise the Hardware and Sw with real hardware

              "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#211-installation-of-quartus-and-ofs","title":"2.1.1. Installation of Quartus and OFS","text":"

              Building AFU with OFS forIntel\u00ae Stratix 10\u00ae FPGA requires the build machine to have at least 64 GB of RAM.

              The following is a summary of the steps to set up for AFU development:

              1. Install Intel\u00ae Quartus\u00ae Prime Pro Edition 23.3 Linux with Intel\u00ae Stratix 10\u00ae FPGA device support.
              2. Make sure support tools are installed and meet version requirements.
              3. Clone the repository.
              4. Review the files provided in the repository.
              5. Build a relocatable PR tree - this will be the base FIM for your AFU.

              Intel\u00ae Quartus\u00ae Prime Pro Edition version 23.3 is the currently verified version of Intel\u00ae Quartus\u00ae Prime Pro Edition 23.3 used for building the AFU images. The recommended Best Known Configuration (BKC) OFS Version 2023.3:

              Item Version Intel\u00ae Quartus\u00ae Prime Pro Edition 23.3 Operating System RHEL 8.6 OPAE SDK 2.10.0-1 OFS Release ofs-2023.3-2 Python 3.6.8 cmake 3.15 GCC 7.4.0 git 1.8.3.1 perl 5.8.8"},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2111-installation-of-quartus","title":"2.1.1.1 Installation of Quartus","text":"

              Intel Quartus Prime Pro Version 23.3 is verified to work with the latest OFS release ofs-2023.3. However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.

              Use RedHatEnterprise Linux\u00ae (RHEL) for compatibility with your development flow and also testing your FIM design in your platform.

              Prior to installing Quartus:

              1. Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.

                • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.

              2. Perform the following steps to satisfy the required dependencies.

                $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n

                Apply the following configurations.

                $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n

              3. Create the default installation path: /intelFPGA_pro/, where is the default path of the Linux workstation, or as set by the system administrator and is your Quartus version number.

                The installation path must satisfy the following requirements:

                • Contain only alphanumeric characters
                • No special characters or symbols, such as !$%@^&*<>,
                • Only English characters
                • No spaces

              4. Download your required Quartus Prime Pro Linux version here.

              5. Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.23.

              6. After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:

                export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n

                For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:

                export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n

              7. Verify, Quartus is discoverable by opening a new shell:

                $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                8. "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2112-install-ofs","title":"2.1.1.2. Install OFS","text":"

                1. Retrieve OFS repositories:

                  The Intel\u00ae OFS FIM source code is included in the OFS GitHub repository. First, create a new directory to store the retrieved files as a clean starting point. The following is a short description of each repository, followed by the git commands for cloning. The instructions section uses the HTTPS git method for cloning repositories.

                2. Navigate to the location for storage of OFS source, create the top-level source directory, and clone OFS repositories.

                mkdir ofs_fim_build_root\ncd ofs_fim_build_root\n
                export OFS_BUILD_ROOT=$PWD\n
                git clone --recurse-submodules  https://github.com/OFS/ofs-d5005.git\n

                Console Output: 

                Cloning into 'ofs-d5005' ...\nUsername for 'https://github.com': <<Enter your git hub username>>\nPassword for 'https://<<Your username>>': <<Enter your personal access token>>\nremote: Enumerating objects:  ....\n...\n...\nResolving deltas  ..., done.\n

                Edit your bashrc file ~/.bashrc to add the following lines: 

                export OFS_ROOTDIR=$OFS_BUILD_ROOT/ofs-d5005\nexport WORKDIR=$OFS_ROOTDIR\nexport VERDIR=$OFS_ROOTDIR/verification\n
                cd ofs-d5005\n

                Select the latest OFS Release 

                git checkout tags/ofs-2023.3-2\n

                Console Output: ```sh You are in 'detached HEAD' state. You can look around, make experimental changes and commit them, and you can discard any commits you make in this state without impacting any branches by performing another checkout.

                If you want to create a new branch to retain commits you create, you may do so (now or later) by using -b with the checkout command again. Example:

                git checkout -b HEAD is now at 7e4dc70 ofs-2023.3-2"},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2113-directory-structure-of-ofs","title":"2.1.1.3. Directory Structure of OFS","text":"

                Verify the following directories in the $OFS_BUILD_ROOT directory with the following command. 

                cd  $OFS_ROOTDIR\nls\n

                Console Output: 

                 eval_scripts ipss ofs-common license  LICENSE.md  README.md  sim  src  syn  verification\n

                The directories are arranged as shown below:

                \u251c\u2500\u2500 eval_scripts\n\u2502   \u251c\u2500\u2500 ofs_d5005_eval.sh\n\u2502   \u251c\u2500\u2500 README_ofs_d5005_eval.txt\n|\n\u251c\u2500\u2500 ofs-common\n\u2502   \u251c\u2500\u2500 scripts\n\u2502   \u251c\u2500\u2500 src\n\u2502   \u251c\u2500\u2500 verification\n|   \u251c\u2500\u2500 LICENSE.txt   \n\u2502   \u2514\u2500\u2500 README.md\n\u251c\u2500\u2500 ipss        **Directory ipss consists of Platform Designer subsystems used in FIM**\n\u2502   \u251c\u2500\u2500 hssi\n\u2502   \u251c\u2500\u2500 mem\n\u2502   \u251c\u2500\u2500 pcie\n|   \u251c\u2500\u2500 pmic \n|   \u251c\u2500\u2500 spi  \n\u2502   \u2514\u2500\u2500 README.md\n\u251c\u2500\u2500 license\n\u2502   \u2514\u2500\u2500 quartus-0.0-0.01iofs-linux.run    ** Quartus Patch with IP licenses.  \n\u2502                                         ** Note, these licenses are not used for Intel\u00ae FPGA PAC D5005** \n\u251c\u2500\u2500 sim             **Unit level simulation files**\n\u2502   \u251c\u2500\u2500 unit_test\n\u2502   \u251c\u2500\u2500 scripts\n\u2502   \u251c\u2500\u2500 bfm\n\u2502   \u251c\u2500\u2500 rp_bfm   \n\u2502   \u2514\u2500\u2500 readme.txt     \n\u2502    \n\u251c\u2500\u2500 LICENSE.txt\n\u251c\u2500\u2500 README.md\n|\n\u251c\u2500\u2500 src             **Source RTL files**\n\u2502   \u251c\u2500\u2500 afu_top\n\u2502   \u251c\u2500\u2500 includes\n\u2502   \u251c\u2500\u2500 pd_qsys\n\u2502   \u251c\u2500\u2500 top\n|   \u2514\u2500\u2500 README.md\n\u251c\u2500\u2500 external\n\u2502   \u2514\u2500\u2500 ofs-platform-afu-bbb\n|\n\u251c\u2500\u2500 syn              **Quartus compilation settings**\n\u2502   \u251c\u2500\u2500 scripts\n\u2502   \u251c\u2500\u2500 setup\n\u2502   \u251c\u2500\u2500 syn_top\n\u2502   \u251c\u2500\u2500 readme.txt\n\u2502   \u2514\u2500\u2500 README\n
                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2114-license-installation-for-ofs","title":"2.1.1.4 License Installation for OFS","text":"

                The required setup Intel\u00ae OFS License quartus-0.0-0.01iofs-linux.run, follow the following steps :

                cd $OFS_ROOTDIR/license\nchmod +x quartus-0.0-0.01iofs-linux.run\nsudo ./quartus-0.0-0.01iofs-linux.run\n# Confirm the license instaltion using below command.\nquartus_syn --version\n
                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2115-retrieve-pim-files","title":"2.1.1.5. Retrieve PIM Files","text":"

                The ofs-platform-afu-bbb repository contains the PIM files and example AFU that can be used for testing and demonstration purposes. This guide will use the host_chan_mmio example in the remaining sections to demonstrate OFS capabilities.

                cd $OFS_BUILD_ROOT\n
                git clone https://github.com/OFS/ofs-platform-afu-bbb.git\n
                Edit your bashrc file ~/.bashrc to add the following lines: 
                export OFS_PLATFORM_AFU_BBB=$OFS_BUILD_ROOT/ofs-platform-afu-bbb\n

                Verify the following directories are present in $OFS_BUILD_ROOT directory. 

                cd $OFS_PLATFORM_AFU_BBB\n
                ls\n

                Console Output: 

                 COPYING  plat_if_develop  plat_if_release  plat_if_tests  README.md\n

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#212-compiling-the-ofs-fim","title":"2.1.2. Compiling the OFS FIM","text":"

                Intel\u00ae OFS provides a build script with the following FPGA image creation options:

                • Flat compile, which combines the FIM and AFU into one FPGA image loaded into the entire FPGA device as a static image.
                • A PR compile that creates an FPGA image consisting of the FIM that is loaded into the static region of the FPGA and a default AFU that is loaded into dynamic region. Additional AFU may be loaded into the dynamic region using partial reconfiguration.

                The build scripts included with OFS are verified to run in a bash shell. Other shells have not been tested. Each build script step will take several hours to complete. Building in Quartus GUI is not supported - you must build with the provided scripts.

                The following sections describe how to set up the environment and build the provided FIM with a relocatable tree supporting PR . You will use this relocatable PR tree for all example AFU simulation and compilation steps in this guide.

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2121-setting-up-required-environment-variables","title":"2.1.2.1. Setting Up Required Environment Variables","text":"

                Set required environment variables as shown below. These environment variables must be set before simulation or compilation tasks, so creating a simple script to set these variables saves time.

                Edit your bashrc file ~/.bashrc to add the following lines: 

                export OPAE_SDK_REPO_BRANCH=release/2.10.0\n

                Check point : Ensure you file ~/.bashrc have all the following lines:

                export QUARTUS_MAINPATH=<Quartus install directory>\nexport QUARTUS_ROOTDIR=$QUARTUS_MAINPATH/quartus\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ipexport\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport INTELFPGAOCLSDKROOT=$QUARTUS_MAINPATH/hld\nexport QSYS_ROOTDIR=$QUARTUS_MAINPATH/qsys/bin\nexport PATH=$PATH:$QUARTUS_ROOTDIR/bin\nexport OFS_BUILD_ROOT=<root location> ** Here should be located your ofs-d5005 and ofs-platform-afu-bbb\nexport OFS_ROOTDIR=$OFS_BUILD_ROOT/ofs-d5005\nexport WORKDIR=$OFS_ROOTDIR\nexport VERDIR=$OFS_ROOTDIR/verification\nexport OFS_PLATFORM_AFU_BBB=$OFS_BUILD_ROOT/ofs-platform-afu-bbb\nexport OPAE_SDK_REPO_BRANCH=release/2.10.0\n

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2122-compiling-your-base-fim","title":"2.1.2.2. Compiling Your Base FIM","text":"

                The usage of the compile build script is shown below:

                ofs-common/scripts/common/syn/build_top.sh [-p] target_configuration work_dir \n\n      * target_configuration - Specifies the project  \n         For example: d5005\n\n      * work_dir - Work Directory for this build in the form a directory name. It is created in the <local repo directory>/ofs-d5005/<work_dir> \n          - NOTE: The directory name must start with \"work\". If the working directory exists, the script stops and asks if you want to overwrite the directory.\n            - e.g.\n                - ofs-common/scripts/common/syn/build_top.sh d5005 work_d5005\n\n                work directory as a name will be created in <local repo directory>/ofs-d5005/work_d5005\n\n\n                The obmission of <work_dir> results in a default work directory (<local repo  directory>/ofs-d5005/work)\n\n        - compile reports and artifacts (.rpt, .sof, etc) are stored in <work_dir>/syn/<OFS_PROJECT>/<OFS_FIM>/<OFS_BOARD>/syn_top/output_files\n\n        - There is a log file created in ofs-d5005 directory.  \n        - [-p]  Optional switch for creating a relocatable PR  build tree supporting the creation of a PR -able AFU workload.   \n        The \"-p\" switch invokes generate_pr_release.sh at the end of the FIM build and writes the PR  build tree to the top of the working directory. More information on this option is provided below. \n
                In the following example, you will build the provided example design using a flat, non-PR build flow. If you use the -p, you could avoid the section.

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#21221-relocatable-pr-directory-tree","title":"2.1.2.2.1. Relocatable PR Directory Tree.","text":"

                Build the provided base example design:

                cd $OFS_ROOTDIR\n
                ofs-common/scripts/common/syn/build_top.sh d5005 work_d5005\n

                Console Output: 

                    ... build takes ~5 hours to complete\n\nCompile work directory:     <$OFS_BUILD_ROOT>/work_d5005/syn/syn_top\nCompile artifact directory: <$OFS_BUILD_ROOT>/work_d5005/syn/syn_top/output_files\n\n\n***********************************\n***\n***        OFS_PROJECT: d5005\n***        Q_PROJECT:  d5005\n***        Q_REVISION: d5005\n***        SEED: 03\n***        Build Complete\n***        Timing Passed!\n***\n

                Pro Tip: if the timing report fails, try to go into the iofs_pr_afu.qsf and modify the seed number from 3 to 4, it will create multiple seed/starting points of your design to find the best timing/fit. /home/<myuser>/<mainfolderforOFS>/ofs-d5005/work_d5005/syn/syn_top/iofs_pr_afu.qsf 

                set_global_assignment -name SEED 0 #0-4\n

                The build script copies the ipss, sim, src, and syn directories to the specified work directory, and then these copied files are used in the Quartus compilation process. Therefore, do not edit the files in the work directory; these files are copies of source files.

                Some of the critical output files are described below:

                $OFS_ROOTDIR//syn/syn_top 

                \u251c\u2500\u2500 syn_top                    //Intel\u00ae FPGA PAC D5005 Quartus build area with Quartus files used this build\n\u2502  \u251c\u2500\u2500 d5005.ipregen.rpt       // IP regeneration report states the output of IP upgrade\n\u2502  \u251c\u2500\u2500 d5005.qpf               // Quartus Project File (qpf) mentions about Quartus version and project revision\n\u2502  \u251c\u2500\u2500 d5005.qsf               // Quartus Settings File (qsf) lists current project settings and entity level assignments\n\u2502  \u251c\u2500\u2500 d5005.stp               // Signal Tap file included in the d5005.qsf. This file can be modified as required\n\u2502  \u251c\u2500\u2500 fme_id.mif              // the fme id hex value is stored in a mif file format\n\u2502  \u251c\u2500\u2500 iofs_pr_afu.json        // PR JSON file\n\u2502  \u251c\u2500\u2500 iofs_pr_afu.qsf                // PR AFU qsf file\n\u2502  \u251c\u2500\u2500 iofs_pr_afu_sources.tcl        // AFU source file list\n

                $OFS_ROOTDIR//syn/syn_top/output_files == Directory with build reports and FPGA programming files.

                The programming files consist of the Quartus generated d5005.sof and d5005.pof. The Intel\u00ae FPGA PAC D5005 board hardware provides a 2 Gb flash device to store the FPGA programming files and a BMC CARD that reads this flash and programs the Intel\u00ae FPGA PAC D5005 Intel\u00ae Stratix 10\u00ae FPGA. The ./ofs-common/scripts/common/syn/build_top.sh script runs script file ./ofs-common/scripts/common/syn/build_top.sh which takes the Quartus generated d5005.sof and creates binary files in the proper format to be loaded into the 2 Gb flash device. You can also run build_flash.sh by itself if needed.

                The build script will run PACSign and create an unsigned FPGA programming file for both user1 and user2 locations of the Intel\u00ae FPGA PAC D5005 flash. Please note, if the Intel\u00ae FPGA PAC D5005 has the root entry hash key loaded, then PACsign must be run to add the proper key to the FPGA binary file. Please see Security User Guide: Intel\u00ae Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs for details on the security aspects of Intel\u00ae Open FPGA Stack and refer to Board Management User Guide for Flash partition.

                The following table provides further detail on the generated bin files.

                File Description d5005.sof This is the Quartus generated programming file created by Quartus synthesis and place and route. This file can be used to program the FPGA using a JTAG programmer. This file is the source file for the binary files used to program the FPGA flash. d5005.bin This is an intermediate raw binary image of the FPGA d5005_page1.bin This is the binary file created from the input file, d5005.sof. This file is used as the input file to the PACSign utility to generate d5005_page1_unsigned.bin binary image file. d5005_page1_unsigned.bin This is the unsigned PACSign output which can be programmed into the FPGA flash of an unsigned Intel\u00ae FPGA PAC D5005 using the OPAE SDK utility fpgasupdate mfg_d5005_reversed.bin A particular programming file for a third-party device used in board manufacturing. This file is typically not used.

                build/output_files/timing_report == Directory containing clocks report, failing paths and passing margin reports

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#213-relocatable-pr-directory-tree","title":"2.1.3. Relocatable PR Directory Tree","text":"

                If you are developing FIM to be used by another team developing the AFU workload, scripts are provided that create a relocatable PR directory tree. ODM and board developers will use this capability to enable a broad set of AFU to be loaded on a board using PR . The relocatable PR directory contains the Quartus *.qdb file that goes the FIM.

                Creating the relocatable PR directory tree requires a clone of the Intel\u00ae Basic Building Blocks (BBB) repository. The OFS_PLATFORM_AFU_BBB environment variable must point to the repository. If not done previously, clone the Intel\u00ae Basic Building Blocks repository and create OFS_PLATFORM_AFU_BBB environment variable.

                cd $OFS_BUILD_ROOT\n
                git clone https://github.com/OFS/ofs-platform-afu-bbb.git\n
                export OFS_PLATFORM_AFU_BBB=$OFS_BUILD_ROOT/ofs-platform-afu-bbb\n
                cd $OFS_ROOTDIR\n

                You can create this relocatable PR directory tree by either:

                • Build FIM and AFU using ofs-common/scripts/common/syn/build_top.sh followed by running /syn/common/scripts/generate_pr_release.sh (section 2.1.3. Relocatable PR Directory Tree)
                • Build FIM and AFU using ofs-common/scripts/common/syn/build_top.sh with optional -p switch included

                The generate_pr_release.sh has the following command structure:

                ofs-common/scripts/common/syn/generate_pr_release.sh -t <path to generated release tree> *Board Build Target* <work dir from build_top.sh>\nWhere:\n-t <path to generated release tree> = location for your relocatable PR  directory tree\n*Board Build Target* is the name of the board target/FIM e.g. d5005\n<work dir from build_top.sh> \n
                Here is an example of running the generate_pr_release.sh script in user mode:

                ofs-common/scripts/common/syn/generate_pr_release.sh -t work_d5005/build_tree d5005  work_d5005\n

                Console Output: 

                **********************************\n********* ENV SETUP **************\nFIM Project:\n  OFS_PROJECT = d5005\n  OFS_FIM     = .\n  OFS_BOARD   = .\n  Q_PROJECT   = d5005\n  Q_REVISION  = d5005\n  Fitter SEED = 03\nFME id\n  BITSTREAM_ID = 040100022c164db1\n  BITSTREAM_MD = 0000000002212053\n\n...\n...\n

                The resulting relocatable build tree has the following structure:

                .\n\u251c\u2500\u2500 bin\n\u2502   \u251c\u2500\u2500 afu_synth\n\u2502   \u251c\u2500\u2500 build_env_config\n\u2502   \u251c\u2500\u2500 run.sh -> afu_synth\n\u2502   \u2514\u2500\u2500 update_pim\n\u251c\u2500\u2500 hw\n\u2502   \u251c\u2500\u2500 blue_bits\n\u2502   \u2502   \u251c\u2500\u2500 d5005_page1_unsigned.bin\n\u2502   \u2502   \u2514\u2500\u2500 d5005.sof -> ../lib/build/syn/syn_top/   output_files/d5005.sof\n\u2502   \u2514\u2500\u2500 lib\n\u2502       \u251c\u2500\u2500 build\n\u2502       \u251c\u2500\u2500 fme-ifc-id.txt\n\u2502       \u251c\u2500\u2500 fme-platform-class.txt\n\u2502       \u2514\u2500\u2500 platform\n\u251c\u2500\u2500 README\n

                Edit your bashrc file ~/.bashrc to add the following line: 

                export OPAE_PLATFORM_ROOT=$OFS_ROOTDIR/work_d5005/build_tree\n

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#214-programing-the-fim","title":"2.1.4. Programing the FIM","text":"
                1. Run the following command to find the PCIe address for your card.
                sudo fpgainfo fme\n

                Console Output: 

                Board Management Controller, MAX10 NIOS FW version: 2.0.14\nBoard Management Controller, MAX10 Build version: 2.0.8\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nDevice Id                        : 0xBCCE\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 288511863935352239\nBitstream Version                : 4.0.1\nPr Interface Id                  : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98\nBoot Page                        : user\n

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2141-load-fim-into-the-flash-of-the-intel-fpga-pac-d5005","title":"2.1.4.1. Load FIM into the Flash of the Intel\u00ae FPGA PAC D5005","text":"

                The base FIM used in AFU compilation must be loaded on the board. In this step, you will load the generated FIM binary into the Intel\u00ae FPGA PAC D5005 FPGA flash. By performing this step, subsequent AFU developed in this guide will use this base FIM and allow your newly created AFU to match the base FIM loaded on the board.

                More information related to fpgaupdate is located OFS Getting Started User Guide: For Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.

                Run fpgasupdate to load the image into the user location of the Intel\u00ae FPGA PAC D5005 FPGA flash and the RSU command to reboot the PCIE Card:

                sudo fpgasupdate $OFS_ROOTDIR/work_d5005/syn/syn_top/output_files/d5005_page1_unsigned.bin 3b:00.0\n
                Run rsu command to re-configure FPGA on Intel\u00ae FPGA PAC D5005. 
                sudo rsu bmcimg 3b:00.0\n

                sudo fpgainfo fme\n

                Console Output: ```sh

                Board Management Controller, MAX10 NIOS FW version: 2.0.14 Board Management Controller, MAX10 Build version: 2.0.8 //****** FME ******// Object Id : 0xF000000 PCIe s:b:d.f : 0000:3b:00.0 Device Id : 0xBCCE Socket Id : 0x00 Ports Num : 01 Bitstream Id : 288511863935352239 Bitstream Version : 4.0.1 Pr Interface Id : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98 Boot Page : user

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#30-opae-software-development-kit","title":"3.0 OPAE Software Development Kit","text":"

                The OPAE SDK software stack sits in user space on top of the Intel\u00ae OFS kernel drivers. It is a common software infrastructure layer that simplifies and streamlines the integration of programmable accelerators such as FPGAs into software applications and environments. OPAE consists of a set of drivers, user-space libraries, and tools to discover, enumerate, share, query, access, manipulate, and re-configure programmable accelerators. OPAE is designed to support a layered, common programming model across different platforms and devices. To learn more about OPAE, its documentation, code samples, an explanation of the available tools, and an overview of the software architecture, please visit the OPAE.io page.

                The OPAE SDK source code is contained within a single GitHub repository hosted at the OPAE GitHub. This repository is open source and should not require any permissions to access.

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#31-opae-sdk-build-environment-setup","title":"3.1 OPAE SDK Build Environment Setup","text":"

                This installation process assumes the user has access to an internet connection to pull specific GitHub repositories and satisfy package dependencies. If an offline install process is required, please reach out to your Intel\u00ae representative.

                1. Before OPAE SDK installation, the user must remove any prior OPAE frameworks. To remove these packages:

                sudo dnf remove opae*\n

                2. The user must enable the following repository changes in order to install all dependencies on CentOS 8.3:

                sudo dnf config-manager --set-enabled powertools\nsudo dnf install epel-release\n

                3. The following package dependencies must be satisfied by the user. Double check that all packages have been found and installed:

                sudo dnf install autoconf automake bison boost boost-devel cmake doxygen dwarves elfutils-libelf-devel \\\nflex gcc gcc-c++ git hwloc-devel json-c-devel libarchive libedit libedit-devel libpcap libpng12 libuuid libuuid-devel libxml2 libxml2-devel make ncurses  \\\nncurses-devel ncurses-libs openssl-devel python2-pip python3-devel python3-jsonschema rsync tbb-devel libudev-devel\n

                All steps in this installation will use a generic top-level directory at $OFS_BUILD_ROOT. If the user has created a different top-level directory, replace this path with the user's custom path.

                4. Initialize an empty git repository and clone the tagged OPAE SDK source code:

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#32-install-opae-sdk","title":"3.2 Install OPAE SDK","text":"

                Perform the following steps to install OPAE SDK:

                cd $OFS_BUILD_ROOT\ngit clone https://github.com/OFS/opae-sdk.git\ncd opae-sdk\ngit checkout tags/2.10.0-1 -b release/2.10.0\n
                Verify proper branch is selected

                git describe\n  2.10.0-1\n\ngit branch\n  master\n  * release/2.10.0\n
                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#33-building-and-installing-the-opae-sdk","title":"3.3 Building and Installing the OPAE SDK","text":"

                1. Build the OPAE SDK source code, and pack it into several local RPM packages. Building the code into packages allows for easier installation and removal. This build script can use multiple processors to parallelize the build process. Display how many processors are available with the nproc command, and then specify how many make threads to utilize with the -j option. Note that the number of threads can exceed the number of processors. In this case, the number of threads is set to the number of processors in the system.

                cd $OFS_BUILD_ROOT/opae-sdk\nmkdir install-opae-sdk\ncd install-opae-sdk\ncmake .. -DCPACK_GENERATOR=RPM -DOPAE_BUILD_FPGABIST=ON -DOPAE_BUILD_PYTHON_DIST=ON -DCMAKE_BUILD_PREFIX=/install-opae-sdk \nmake -j `nproc`\nmake -j `nproc` package_rpm\n

                The install-opae-sdk directory location was selected for ease of use. If the user wishes to build the OPAE SDK in a different location, they will need to replace the '..' in the above command with the direct or relative path to the opae-sdk repository.

                2. After a successful compile, there should be eight packages present:

                cd $OFS_BUILD_ROOT/opae-sdk/install-opae-sdk\nls | grep rpm\nopae-2.10.0-1.x86_64.rpm                                                                                                    \nopae-PACSign-2.10.0-1.x86_64.rpm                                                                                            \nopae-devel-2.10.0-1.x86_64.rpm                                                                                              \nopae-libs-2.10.0-1.x86_64.rpm                                                                                               \nopae-opae.admin-2.10.0-1.x86_64.rpm                                                                                         \nopae-packager-2.10.0-1.x86_64.rpm                                                                                           \nopae-tests-2.10.0-1.x86_64.rpm                                                                                              \nopae-tools-2.10.0-1.x86_64.rpm                                                                                              \nopae-tools-extra-2.10.0-1.x86_64.rpm\n

                3. Install the OPAE SDK packages:

                cd $OFS_BUILD_ROOT/opae-sdk/install-opae-sdk\nsudo dnf localinstall -y opae*.rpm\n

                4. check that all packages have been installed:

                rpm -qa | grep opae\nopae-devel-2.10.0-1.x86_64                                                                                                  \nopae-packager-2.10.0-1.x86_64                                                                                               \nopae-2.10.0-1.x86_64                                                                                                        \nopae-tools-2.10.0-1.x86_64                                                                                                  \nopae-PACSign-2.10.0-1.x86_64                                                                                                \nopae-tools-extra-2.10.0-1.x86_64                                                                                            \nopae-opae.admin-2.10.0-1.x86_64                                                                                             \nopae-tests-2.10.0-1.x86_64                                                                                                  \nopae-libs-2.10.0-1.x86_64\n

                5. Setup required environment variables

                export PATH=$PATH:$OFS_BUILD_ROOT/opae-sdk/install-opae-sdk/bin\nexport LIBRARY_PATH=$OFS_BUILD_ROOT/opae-sdk/install-opae-sdk/lib\nexport LD_LIBRARY_PATH=$OFS_BUILD_ROOT/opae-sdk/install-opae-sdk/lib64\n
                cd ../lib/python*/site-packages\nexport PYTHONPATH=$PWD\n

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#4-compiling-an-afu","title":"4. Compiling An AFU","text":"

                This section will use the FIM build tree created in the previous steps to compile an example AFU. This section will continue the work with the host_chan_mmio AFU.. You can perform the build steps listed below to demonstrate the ease in building and running a real example on the Intel\u00ae FPGA PAC D5005.

                To run the steps in this section, you must complete all steps in section 2. Set Up AFU Development Environment, and ensure the OPAE_PLATFORM_ROOT \"environment variable that points to the directory of the PR build tree generated previously.

                Ensure your bashrc file ~/.bashrc have the following line: 

                export OPAE_PLATFORM_ROOT=$OFS_ROOTDIR/work_d5005/build_tree\n

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#41-set-afu-synthesis-environment","title":"4.1. Set AFU Synthesis Environment","text":"

                Here, you will create the synthesis environment to build the host_chan_mmio example. The PIM flow includes the synthesis environment creation script afu_synth_setup for this task. The usage of afu_synth_setup is shown below:

                usage: afu_synth_setup [-h] -s SOURCES [-p PLATFORM] [-l LIB] [-f] dst\nGenerate a Quartus build environment for an AFU. A build environment is\ninstantiated from a release and configured for the specified AFU. AFU\nsource files are specified in a text file parsed by rtl_src_config,\nwhich is part of the OPAE base environment.\npositional arguments:\n  dst                   Target directory path (directory must not exist).\noptional arguments:\n  -h, --help            show this help message and exit\n  -s SOURCES, --sources SOURCES\n                        AFU source specification file that will be passed to\n                        rtl_src_config. See \"rtl_src_config --help\" for the\n                        file's syntax. rtl_src_config translates the source\n                        list into either Quartus or RTL simulator syntax.\n  -p PLATFORM, --platform PLATFORM\n                        FPGA platform name.\n  -l LIB, --lib LIB     FPGA platform release hw/lib directory. If not\n                        specified, the environment variables OPAE_FPGA_HW_LIB\n                        and then BBS_LIB_PATH are checked.\n  -f, --force           Overwrite target directory if it exists.\n

                Execute afu_synth_setup \"as follows to create the synthesis environment for a host_chan_mmio \"AFU that fits the Intel\u00ae FPGA PAC D5005 FIM previously constructed.

                cd $OFS_ROOTDIR/work_d5005/\nafu_synth_setup -s $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/test_mmio_axi1.txt build_d5005_x16\n

                Now, execute the afu_synth command that resides inside the $OFS_ROOTDIR/work_d5005/build_tree/bin directory, to actually build the host_chan_mmio AFU.

                cd $OFS_ROOTDIR/work_d5005/build_d5005_x16\n$OPAE_PLATFORM_ROOT/bin/afu_synth\n...\n...\nWrote host_chan_mmio.gbs\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n Design meets timing\n===========================================================================\n
                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#411-loading-and-running-the-host_chan_mmio-example-afu","title":"4.1.1. Loading and Running the host_chan_mmio example AFU","text":"

                Once the compilation completes successfully, load the new bitstream file, host_chan_mmio.gbs, into the partial reconfiguration region of the target Intel\u00ae FPGA PAC D5005. Keep in mind, that the loaded image is dynamic - this image is not stored in flash, and if the card is power cycled, then the PR region is re-loaded with the default AFU.

                To load the image, perform the following steps:

                cd $OFS_ROOTDIR/work_d5005/build_d5005_x16\nsudo fpgasupdate host_chan_mmio.gbs 3b:00.0\n[sudo] password for <<Your username>>: \n[WARNING ] Update starting. Please do not interrupt.\n[INFO    ] \nPartial Reconfiguration OK\n[INFO    ] Total time: 0:00:01.88\n

                Determine the BDF of the Intel\u00ae FPGA PAC D5005.

                The PCIe BDF address is initially determined when the server powers on. The user can determine the addresses of all Intel\u00ae FPGA PAC D5005 using lspci:

                lspci -d :bcce\n3b:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n

                Set up your board to work with the newly loaded host_chan_mmio.gbs

                1. Create the Virtual Functions (VFs): 

                  sudo pci_device 3b:00.0 vf 3\n

                2. Verify that all three VFs have been created.

                $ lspci -s 3b:00\n3b:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n3b:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\n3b:00.2 Processing accelerators: Intel Corporation Device bcce (rev 01)\n3b:00.3 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                1. Bind the 3 VFs to the vfio-pci driver.

                sudo opae.io init -d , e.g. 

                $ sudo opae.io init -d 0000:3b:00.1 user:user\nBinding (0x8086,0xbccf) at 0000:3b:00.1 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:3b:00.1 is 142\nAssigning /dev/vfio/142 to <local user>\nChanging permissions for /dev/vfio/142 to rw-rw----\n\n$ sudo opae.io init -d 0000:3b:00.2 user:user\nBinding (0x8086,0xbccf) at 0000:3b:00.2 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:3b:00.2 is 143\nAssigning /dev/vfio/143 to <local user>\nChanging permissions for /dev/vfio/143 to rw-rw-----\n\n$ sudo opae.io init -d 0000:3b:00.3 user:user\nBinding (0x8086,0xbccf) at 0000:3b:00.3 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:3b:00.3 is 144\nAssigning /dev/vfio/144 to <local user>\nChanging permissions for /dev/vfio/144 to rw-rw----\n
                1. Verify the new AFU is loaded. The host_chan_mmio AFU GUID is 76d7ae9c-f66b-461f-816a-5428bcebdbc5.
                $ fpgainfo port\n//****** PORT ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0x603B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : d15ab1ed-0000-0000-0210-000000000000\n//****** PORT ******//\nObject Id                        : 0x403B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : 76d7ae9c-f66b-461f-816a-5428bcebdbc5\n//****** PORT ******//\nObject Id                        : 0x203B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n

                Run the host_chan_mmio software application to demonstrate the newly loaded AFU image. You navigate to $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw and compile the software application and then run.

                If OPAE SDK libraries were not installed in the default systems directory /usr/lib64/ \", define the OPAE_LOC environment variable to point to the directory where the OPAE SDK libraries were installed.

                $ export OPAE_LOC=/usr\n$ export LIBRARY_PATH=$OPAE_LOC/lib64:$LIBRARY_PATH\n$ export LD_LIBRARY_PATH=$OPAE_LOC/lib64:$LD_LIBRARY_PATH\n
                cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw\nmake \n./host_chan_mmio\n

                Console Output: 

                AFU ID:  76d7ae9cf66b461f 816a5428bcebdbc5\nAFU MMIO interface: AXI Lite\nAFU MMIO read bus width: 64 bits\n512 bit MMIO write supported: yes\nAFU pClk frequency: 250 MHz\n\nTesting 32 bit MMIO reads:\n  PASS - 4 tests\n\nTesting 32 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 64 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 512 bit MMIO writes:\n  PASS\n

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#412-loading-and-running-the-hello_world-example-afu","title":"4.1.2. Loading and running the hello_world example AFU","text":"

                The platform-independent example AFUs repository provides some interesting examples AFU's. In this section, you will compile and execute the PIM-based hello_world AFU. The RTL of the hello_world AFU receives from the host application an address via memory-mapped I/O (MMIO) write and generates a DMA write to the memory line at that address. The content written to memory is the string \"Hello world!\". The host application spins, waiting for the memory line to be updated. Once available, the software prints out the string.

                The hello_world example AFU consists of the following files. 

                hello_world\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u251c\u2500\u2500 ccip\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_ccip.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u2514\u2500\u2500 hello_world.json\n\u251c\u2500\u2500 README.md\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 hello_world.c\n    \u2514\u2500\u2500  Makefile\n
                The hw directory contains the RTL to implement the hardware functionality using CCIP, Avalon, and AXI interfaces. However, this guide will use the AXI version of the AFU to go through the compilation steps. The sw directory of the AFU contains the source code of the host application that communicates with the AFU hardware.

                The following instructions can be used to compile other AFU samples accompanying this repository.

                1. If not done already, download and clone the repository.
                   cd $OFS_BUILD_ROOT \n   git clone https://github.com/OFS/examples-afu.git\n
                1. Make sure to set the next environment variables.
                  # OPAE and MPF libraries must either be on the default linker search paths or on both LIBRARY_PATH and LD_LIBRARY_PATH.  \n  $ export OPAE_LOC=/usr\n  $ export LIBRARY_PATH=$OPAE_LOC/lib:$LIBRARY_PATH\n  $ export LD_LIBRARY_PATH=$OPAE_LOC/lib64:$LD_LIBRARY_PATH\n\n  # OPAE_PLATFORM_ROOT points to a release tree that has been configured with the Platform Interface Manager (PIM).  \n  $ export OPAE_PLATFORM_ROOT=$OFS_ROOTDIR/work_d5005/build_tree\n

                1. Compile the hello_word sample AFU. 

                    $ cd $OFS_ROOTDIR/work_d5005\n  $ afu_synth_setup -s $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/hw/rtl/axi/sources.txt hello_world_synth\n  $ cd hello_world_synth\n  $ ${OPAE_PLATFORM_ROOT}/bin/afu_synth\n\n\n.\n.\n.\nInfo (19538): Reading SDC files took 00:00:06 cumulatively in this process.\nWrote hello_world.gbs\n\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'hello_world.gbs'\n Design meets timing\n===========================================================================\n

                2. To test the AFU in actual hardware, load the hello_world.gbs to the Intel\u00ae FPGA PAC D5005 card. For this step to be successful, the Intel\u00ae FPGA PAC D5005 FIM must have already been loaded to the Intel\u00ae FPGA PAC D5005 card following the steps described in Section 2 of this document.

                  $ cd $OFS_ROOTDIR/work_d5005/hello_world_synth\n  $ sudo fpgasupdate hello_world.gbs 3b:00.0\n  [sudo] password for <<Your username>>: \n[2022-12-06 13:25:10.22] [WARNING ] Update starting. Please do not interrupt.\n[2022-12-06 13:25:12.06] [INFO    ] \nPartial Reconfiguration OK\n[2022-12-06 13:25:12.06] [INFO    ] Total time: 0:00:01.83\n

                Set up your Intel\u00ae FPGA PAC D5005 board to work with the newly loaded hello_world.gbs file.

                #  Create the Virtual Functions (VFs):\n\n $ sudo pci_device 3b:00.0 vf 3\n\n # Verify:\n $ lspci -s 3b:00\n3b:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n3b:00.1 Processing accelerators: Intel Corporation Device bccf (rev 01)\n3b:00.2 Processing accelerators: Intel Corporation Device bccf (rev 01)\n3b:00.3 Processing accelerators: Intel Corporation Device bccf (rev 01)\n\n # Bond VFs to VFIO driver.  Enter <<Your username>>\n\nsudo opae.io init -d 0000:3b:00.1 <Your username>\n Unbinding (0x8086,0xbcce) at 0000:3b:00.1 from dfl-pci\n Binding (0x8086,0xbccf) at 0000:3b:00.1 to vfio-pci\n iommu group for (0x8086,0xbccf) at 0000:3b:00.1 is 142\n Assigning /dev/vfio/142 to <Your username>\n Changing permissions for /dev/vfio/142 to rw-rw----\n\nsudo opae.io init -d 0000:3b:00.2 <Your username>\n Unbinding (0x8086,0xbccf) at 0000:3b:00.2 from dfl-pci\n Binding (0x8086,0xbccf) at 0000:3b:00.2 to vfio-pci\n iommu group for (0x8086,0xbccf) at 0000:3b:00.2 is 143\n Assigning /dev/vfio/143 to <Your username>\n Changing permissions for /dev/vfio/143 to rw-rw----\n\nsudo opae.io init -d 0000:3b:00.3 <Your username>\n Unbinding (0x8086,0xbccf) at 0000:3b:00.3 from dfl-pci\n Binding (0x8086,0xbccf) at 0000:3b:00.3 to vfio-pci\n iommu group for (0x8086,0xbccf) at 0000:3b:00.3 is 144\n Assigning /dev/vfio/144 to <Your username>\n Changing permissions for /dev/vfio/144 to rw-rw----\n\n# < Verify the new AFU is loaded.  The hello_world AFU GUID is \"c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\".\n\n$ fpgainfo port\n//****** PORT ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0x603B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\n//****** PORT ******//\nObject Id                        : 0x403B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\n//****** PORT ******//\nObject Id                        : 0x203B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n
                1. Compile and execute the host application of the hello_world AFU. You should see the application outputs the \"Hello world!\" message in the terminal.
                  # Move to the sw directory of the hello_world AFU and run the following commands in user mode\n   cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/sw/\n\n   make\n\n  # Launch the host application\n   ./hello_world\n   Hello world TLP!\n
                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#413-modify-the-afu-user-clocks-frequency","title":"4.1.3. Modify the AFU user clocks frequency","text":"

                An OPAE compliant AFU specifies the frequency of the uclk_usr and uclk_usr_div2 clocks through the JSON file for AFU configuration located under the <afu_example>/hw/rtl directory of an AFU design. For instance, the AFU configuration file of the host_chan_mmio example is $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/host_chan_mmio.json.

                The AFU specifies the frequency for uClk_usr in its platform configuration file using the following key:value pairs:

                  \"clock-frequency-high\": [<float-value>|\u201dauto\u201d|\u201dauto-<float-value>\u201d]\n  \"clock-frequency-low\": [<float-value>|\u201dauto\u201d|\u201dauto-<float-value>\u201d]\n

                These key:value tuples are used to configure the PLL of the target platform that provides the user clocks through the AFU clocks interface. In addition, the specified frequency affects the timing closure process on the user clocks during AFU compilation.

                Setting the value field to a float number (e.g., 315.0 to specify 315 MHz) drives the AFU generation process to close timing within the bounds set by the low and high values and sets the AFU's JSON metadata to specify the user clock PLL frequency values.

                The following example shows the JSON file of the host_chan_mmio to set the AFU uClk to 300 MHz and uClk_div2 to 150 MHz.

                {\n   \"version\": 1,\n   \"afu-image\": {\n      \"power\": 0,\n      \"clock-frequency-high\": 300,\n      \"clock-frequency-low\": 150,\n      \"afu-top-interface\":\n         {\n            \"class\": \"ofs_plat_afu\"\n         },\n      \"accelerator-clusters\":\n         [\n            {\n               \"name\": \"host_chan_mmio\",\n               \"total-contexts\": 1,\n               \"accelerator-type-uuid\": \"76d7ae9c-f66b-461f-816a-5428bcebdbc5\"\n            }\n         ]\n   }\n}\n

                Save the changes to host_chan_mmio.json file, then execute the afu_synth_setup script to create a new copy of the AFU files with the modified user clock settigns.

                  $ cd $OFS_ROOTDIR/work_d5005\n  $ afu_synth_setup -s $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/test_mmio_axi1.txt build_d5005_afu_clks\n\nLoading platform database: /home/<Your username>/<Your localpath>/ofs-d5005/work_d5005/build_tree/hw/lib/platform/platform_db/ofs_d5005.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting platform/platform_afu_top_config.vh\nWriting platform/platform_if_addenda.qsf\nWriting ../hw/afu_json_info.vh\n
                Compile the host_chan_mmio AFU with the new frequency values.

                   cd $OFS_ROOTDIR/work_d5005/build_d5005_afu_clks\n   $OFS_ROOTDIR/work_d5005/build_tree/bin/afu_synth\n

                During the compilation phase, you will observe the Timing Analyzer uses the specified user clock frequency values as the target to close timing.

                AFU developers must ensure the AFU hardware design meets timing. The compilation of an AFU that fails timing shows a message similar to the following.

                .\n.\n.\n\nWrote host_chan_mmio.gbs\n\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n\n  *** Design does not meet timing\n  *** See build/syn/syn_top/output_files/timing_report\n\n===========================================================================\n

                The previous output indicates the location of the timing reports for the AFU designer to identify the failing paths and perform the necessary design changes. Next, is a listing of the timing report files from a host_chan_mmio AFU that fails to meet timing after modifying the user clock frequency values.

                  $ cd $OFS_ROOTDIR/work_d5005/build_d5005_afu_clks\n  $ ls build/syn/syn_top/output_files/timing_report\n\nclocks.rpt  clocks.sta.fail.summary  clocks.sta.pass.summary iofs_pr_afu_2_slow_900mv_0c_recovery.rpt\niofs_pr_afu_2_slow_900mv_0c_setup.rpt\niofs_pr_afu_2_slow_900mv_100c_recovery.rpt\niofs_pr_afu_2_slow_900mv_100c_setup.rpt\niofs_pr_afu_2_slow_vid2_0c_recovery.rpt\niofs_pr_afu_2_slow_vid2_0c_setup.rpt\niofs_pr_afu_2_slow_vid2_100c_recovery.rpt\niofs_pr_afu_2_slow_vid2_100c_setup.rpt\niofs_pr_afu_MIN_fast_900mv_0c_recovery.rpt\niofs_pr_afu_MIN_fast_900mv_0c_setup.rpt\niofs_pr_afu_MIN_fast_900mv_100c_recovery.rpt\niofs_pr_afu_MIN_fast_900mv_100c_setup.rpt\n

                Warning: AFU developers must inform software developers of the maximum operating frequency (Fmax) of the user clocks to avoid any unexpected behavior of the accelerator and potentially of the overall system.

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#5-simulating-an-afu-using-ase","title":"5. Simulating an AFU using ASE","text":"

                The AFU Simulation Environment (ASE) is a hardware/software co-simulation environment for your AFU. See diagram below illustrating ASE operation:

                ASE uses the simulator Direct Programming Interface (DPI) to provide HW/SW connectivity. The PCIe connection to the AFU under testing is emulated with a transactional model.

                The following list describes ASE operation:

                • Attempts to replicate the transactions that will be seen in real system.
                • Provides a memory model to AFU, so illegal memory accesses can be identified early.
                • Not a cache simulator.
                • Does not guarantee synthesizability or timing closure.
                • Does not model system latency.
                • No administrator privileges are needed to run ASE. All code is user level.

                The remainder of this section is a tutorial providing the steps on how to run ASE with either VCS or QuestaSim using an example AFU and the AFU build tree previously created in this guide.

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#51-set-up-steps-to-run-ase","title":"5.1. Set Up Steps to Run ASE","text":"

                In this section you will set up your server to support ASE by independently downloading and installing OPAE SDK and ASE. Then, set up the required environment variables.

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#511-install-opae-sdk","title":"5.1.1. Install OPAE SDK","text":"

                Follow the instructions documented in the Getting Started Guide: Intel\u00ae Open FPGA Stack for Intel\u00ae FPGA PAC D5005, section 5.0 OPAE Software Development Kit to build and install the required OPAE SDK for the Intel\u00ae FPGA PAC D5005 PAC card.

                The Intel\u00ae FPGA PAC D5005 PAC card requires opae-2.10.0-1. Follow the instructions provided in the Getting Started Guide: Intel\u00ae Open FPGA Stack for Intel\u00ae FPGA PAC D5005 section 5.0 OPAE Software Development Kit. However, just make sure to check out the cloned repository to tag 2.10.0-1 and branch release/2.10.0.

                git checkout tags/2.10.0-1 -b release/2.10.0\n
                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#512-install-ase-tools","title":"5.1.2 Install ASE Tools","text":"

                ASE is an RTL simulator for OPAE-based AFUs. The simulator emulates both the OPAE SDK software user space API and the AFU RTL interface. The majority of the FIM as well as devices such as PCIe and local memory are emulated with simple functional models.

                ASE must be installed separatedly from the OPAE-SDK. However, the recommendation is to install it in the same target directory as OPAE-SDK.

                1. If not done already, set the environment variables as described in section, Set Up AFU Development Environment.

                2. Clone the ase-sim repository.

                  $ cd $OFS_BUILD_ROOT\n  $ git clone https://github.com/OFS/opae-sim.git\n  $ cd opae-sim  \n
                2. Building ASE requires the include file mock/opae_std.h. If the OPAE-SDK was installed under the default system directories, the C_INCLUDE_PATH variable must be set as follows. 

                export C_INCLUDE_PATH=\"/usr/src/debug/opae-2.10.0-1.el8.x86_64/tests/framework\"\n
                1. Create a build directory and build ASE to be installed under the default system directories along with OPAE SDK.
                   mkdir build\n   cd build\n   cmake  -DCMAKE_INSTALL_PREFIX=/usr ..\n   make\n

                Optionally, if the desire is to install ASE binaries in a different location to the system's default, provide the path to CMAKE through the CMAKE_INSTALL_PREFIX switch, as follows.

                   cmake -DCMAKE_INSTALL_PREFIX=<</some/arbitrary/path>> ..  \n
                1. Install ASE binaries and libraries under the system directory /usr.
                   sudo make install  \n
                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#513-setup-required-ase-environment-variables","title":"5.1.3. Setup Required ASE Environment Variables","text":"

                The values set to the following environment variables assume the OPAE SDK and ASE were installed in the default system directories below /usr. Setup these variables in the shell where ASE will be executed. You may wish to add these variables to the script you created to facilitate configuring your environment.

                   cd /usr/bin\n   export PATH=$PWD:$PATH\n   cd ../lib/python*/site-packages\n   export PYTHONPATH=$PWD\n   cd /usr/lib\n   export LIBRARY_PATH=$PWD\n   cd /usr/lib64\n   export LD_LIBRARY_PATH=$PWD\n   cd $OFS_BUILD_ROOT/ofs-platform-afu-bbb\n   export OFS_PLATFORM_AFU_BBB=$PWD\n   cd $OFS_ROOTDIR/work_d5005/build_tree\n   export OPAE_PLATFORM_ROOT=$PWD\n\n  ## For VCS, set the following:\n\n   export VCS_HOME=<Set the path to VCS installation directory>\n   export PATH=$VCS_HOME/bin:$PATH\n\n  ## For QuestaSIM, set the following:\n   export MTI_HOME=<path to Modelsim installation directory>\n   export PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\n
                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#52-simulating-the-host_chan_mmio-afu","title":"5.2. Simulating the host_chan_mmio AFU","text":"

                The $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio is a simple example demonstrating both hardware and software access to an AFU. The host_chan_mmio example AFU consists of the following files:

                host_chan_mmio\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon_from_ccip.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon_from_axi.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_avalon.sv\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_axi.sv\n\u2502       \u251c\u2500\u2500 host_chan_mmio.json\n\u2502       \u251c\u2500\u2500 test_mmio_avalon0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon1.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon2_512rw.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi1.txt\n\u2502       \u2514\u2500\u2500 test_mmio_axi2_512rw.txt\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 main.c\n    \u251c\u2500\u2500 Makefile\n

                This example AFU contains examples using both Avalon and AXI interface buses. This guide will use the AXI version of the host_chan_mmio AFU.

                ASE uses client-server application architecture to deliver hardware/software co-simulation. You require one shell for the hardware based simulation and another shell where the software application is running. The hardware is started first with a simulation compilation and simulator startup script, once the simulator has loaded the design, it will wait until the software process starts. Once the software process starts, the simulator proceeds. Transaction logging and waveform capture is performed.

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#521-set-up-and-run-the-hw-simulation-process","title":"5.2.1 Set Up and Run the HW Simulation Process","text":"

                You will run the afu_sim_setup script to create the scripts for running the ASE environment. The afu_sim_setup script has the following usage:

                usage: afu_sim_setup [-h] -s SOURCES [-p PLATFORM] [-t {VCS,QUESTA,MODELSIM}]\n                     [-f] [--ase-mode ASE_MODE] [--ase-verbose]\n                     dst\n\nGenerate an ASE simulation environment for an AFU. An ASE environment is\ninstantiated from the OPAE installation and then configured for the specified\nAFU. AFU source files are specified in a text file that is parsed by\nrtl_src_config, which is also part of the OPAE base environment.\n\npositional arguments:\n  dst                   Target directory path (directory must not exist).\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -s SOURCES, --sources SOURCES\n                        AFU source specification file that will be passed to\n                        rtl_src_config. See \"rtl_src_config --help\" for the\n                        file's syntax. rtl_src_config translates the source\n                        list into either Quartus or RTL simulator syntax.\n  -p PLATFORM, --platform PLATFORM\n                        FPGA Platform to simulate.\n  -t {VCS,QUESTA,MODELSIM}, --tool {VCS,QUESTA,MODELSIM}\n                        Default simulator.\n  -f, --force           Overwrite target directory if it exists.\n  --ase-mode ASE_MODE   ASE execution mode (default, mode 3, exits on\n                        completion). See ase.cfg in the target directory.\n  --ase-verbose         When set, ASE prints each CCI-P transaction to the\n                        command line. Transactions are always logged to\n                        work/ccip_transactions.tsv, even when not set. This\n                        switch sets ENABLE_CL_VIEW in ase.cfg.\n

                Run afu_sim_setup to create the ASE simulation environment for the host_chan_mmio example AFU. The '-t VCS' option indicates to prepare the ASE simulation environment for VCS.

                cd $OFS_ROOTDIR/work_d5005/\n\nafu_sim_setup -s $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/test_mmio_axi1.txt -t VCS host_chan_mmio_sim\n\nCopying ASE from /opae-sdk/install-opae-sdk/share/opae/ase...\nCopying ASE from /usr/share/opae/ase...\n#################################################################\n#                                                               #\n#             OPAE Intel(R) Xeon(R) + FPGA Library              #\n#               AFU Simulation Environment (ASE)                #\n#                                                               #\n#################################################################\n\nTool Brand: VCS\nLoading platform database: /home/<Your username>/<Your localpath>/ofs-d5005/work_d5005/build_tree/hw/lib/platform/platform_db/ofs_d5005.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting rtl/platform_afu_top_config.vh\nWriting rtl/platform_if_addenda.txt\nWriting rtl/platform_if_includes.txt\nWriting rtl/ase_platform_name.txt\nWriting rtl/ase_platform_config.mk and rtl/ase_platform_config.cmake\nASE Platform: discrete (FPGA_PLATFORM_DISCRETE)\n

                The afu_sim_setup creates the ASE scripts in the directory host_chan_mmio_sim where the afu_sim_setup script was run. Start the simulator as shown below in user mode:

                   cd host_chan_mmio_sim\n   make\n   make sim\n

                This process launches the AFU hardware simulator. Before moving to the next section, pay attention to the simulator output highlighted in the image below.

                The simulation artifacts are stored in host_chan_mmio/work and consist of:

                log_ase_events.tsv\nlog_ofs_plat_host_chan.tsv \nlog_ofs_plat_local_mem.tsv \nlog_pf_vf_mux_A.tsv \nlog_pf_vf_mux_B.tsv \n
                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#522-set-up-and-run-the-sw-process","title":"5.2.2 Set Up and Run the SW Process","text":"

                Open an additional shell to build and run the host application that communicates with the actual AFU hardware. Set up the same environment variable you have set up in the shell you have been working on until this point.

                Additionally, as indicated by the hardware simulator output that is currently executing in the \"simulator shell\", copy and paste the line \"export ASE_WORKDIR=...\", into the new \"software shell\". See the last image of the previous section.

                   export ASE_WORKDIR= <<as directed in HW simulation shell>>\n
                Then, go to the sw directory of the host_chan_mmio AFU example to compile the host application.

                cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw  \nmake\n\nafu_json_mgr json-info --afu-json=../hw/rtl/host_chan_mmio.json --c-hdr=obj/afu_json_info.h\nWriting obj/afu_json_info.h\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c main.c -o obj/main.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c test_host_chan_mmio.c -o obj/test_host_chan_mmio.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/connect.c -o obj/connect.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/csr_mgr.c -o obj/csr_mgr.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/hash32.c -o obj/hash32.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/test_data.c -o obj/test_data.o\ncc -o host_chan_mmio obj/main.o obj/test_host_chan_mmio.o obj/connect.o obj/csr_mgr.o obj/hash32.o obj/test_data.o  -z noexecstack -z relro -z now -pie -luuid -lopae-c\n

                Now, launch the host application to exercise the AFU hardware running on the simulator shell. The next image shows the AFU hardware simulation process on the left side shell. The right hand shell shows the host application's output of a successful simulation.

                Finally, on the hardware simulation shell, you can view the wave forms by invoking the following command.

                   make wave\n

                This brings up the VCS simulator GUI and loads the simulation waveform files. Use the Hierarchy window to navigate to the afu instance located under, ase_top | ase_top_plat | ofs_plat_afu | afu , as shown below.

                Right click on the afu (afu) entry to display the drop-down menu. Then, click on Add to Waves | New Wave View to display the following waveforms window.

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#53-simulating-the-hello_world-afu","title":"5.3 Simulating the hello_world AFU","text":"

                In this section, you will quickly simulate the PIM-based hello_world sample AFU accompanying the example_afu repository.

                1. Set the environment variables as described in section 5.1. Set Up Steps to Run ASE.

                2. Prepare an RTL simulation environment for the AXI version of the hello_world AFU.

                  Simulation with ASE requires two software processes, one to simulate the AFU RTL and the other to run the host software that exercises the AFU. To construct an RTL simulation environment under the directory $OFS_ROOTDIR/work_d5005, execute the following.

                    $ cd $OFS_ROOTDIR/work_d5005\n  $ afu_sim_setup -s $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/hw/rtl/axi/sources.txt -t VCS hello_world_sim\n\nCopying ASE from /usr/local/share/opae/ase...\n#################################################################\n#                                                               #\n#             OPAE Intel(R) Xeon(R) + FPGA Library              #\n#               AFU Simulation Environment (ASE)                #\n#                                                               #\n#################################################################\n\nTool Brand: VCS\nLoading platform database: /home/<Your username>/<Your localpath>/ofs-d5005/work_d5005/build_tree/hw/lib/platform/platform_db/ofs_d5005.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting rtl/platform_afu_top_config.vh\nWriting rtl/platform_if_addenda.txt\nWriting rtl/platform_if_includes.txt\nWriting rtl/ase_platform_name.txt\nWriting rtl/ase_platform_config.mk and rtl/ase_platform_config.cmake\nASE Platform: discrete (FPGA_PLATFORM_DISCRETE)\n

                  The afu_sim_setup script constructs an ASE environment in the hello_world_sim subdirectory. If the command fails, confirm that the path to the afu_sim_setup is on your PATH environment variable (in the OPAE SDK bin directory) and that your Python version is at least 3.7.

                3. Build and execute the AFU RTL simulator in user mode.

                     cd $OFS_ROOTDIR/work_d5005/hello_world_sim\n   make\n   make sim  \n

                  The previous commands will build and run the VCS RTL simulator, which prints a message saying it is ready for simulation. The simulation process also prints a message instructing you to set the ASE_WORKDIR environment variable in a second shell.

                  1. Open a second shell where you will build and execute the host software. In this new \"software shell\", set up the environment variables you have set up so far in the \"hardware simulation\" shell.

                  2. Also, set the ASE_WORKDIR environment variable following the instructions given in the \"hardware simulation\" shell.

                  export ASE_WORKDIR=$OFS_ROOTDIR/work_d5005/hello_world_sim/work\n
                  6. Then, move to the sw directory of the hello_world AFU sample to build the host software.

                  cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/sw\n   make      \n
                  1. Run the hello_world host application to resume the work of the RTL simulation. The host software process and the RTL simulation execute in lockstep. If successful, you should see the Hello world! output.
                    $ with_ase ./hello_world\n\n  [APP]  Initializing simulation session ...\nHello world!\n  [APP]  Deinitializing simulation session\n  [APP]         Took 43,978,424 nsec\n  [APP]  Session ended\n

                  The image below shows the simulation of the AFU hardware and the execution of the host application side-by-side.

                  1. Finally, on the hardware simulation shell, you can view the wave forms by invoking the following command.
                     make wave\n

                  This brings up the DVE GUI and loads the simulation waveform files. Use the Hierarchy window to navigate to the AFU instance located under, ase_top | ase_top_plat | ofs_plat_afu, as shown below.

                  Right click on the ofs_plat_afu entry to display the drop-down menu. Then, click on Add to Waves | New Wave View to display the following waveforms window.

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#6-adding-remote-signal-tap-logic-analyzer-to-debug-the-afu","title":"6. Adding Remote Signal Tap Logic Analyzer to debug the AFU","text":"

                The OPAE SDK provides a remote Signal Tap facility. It also supports the following in system debug tools included with the Intel\u00ae Intel\u00ae Quartus\u00ae Prime Pro Edition:

                • In-system Sources and Probes
                • In-system Memory Content Editor
                • Signal Probe
                • System Console

                This section is a short guide on adding remote Signal Tap instances to an AFU for in-system debugging. In order of execution, you can follow the steps in the following sections to create an instrumented AFU. The host_chan_mmio AFU is used in this guide as the target AFU to be instrumented.

                You need a basic understanding of Signal Tap. Please see the Signal Tap Logic Analyzer: Introduction & Getting Started Web-Based Training for more information.

                You will run with a Signal Tap GUI running locally on the server with the Intel\u00ae FPGA PAC D5005 as shown below:

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#61-adding-rstp-to-the-host_chan_mmio-afu","title":"6.1. Adding RSTP to the host_chan_mmio AFU","text":"

                RSTP is added to an AFU by:

                1. Defining signals to be instrumented in Signal Tap. Create a new *.stp file.
                2. Modify ofs_top.qpf to include the new *.stp file
                3. Modify ofs_top.qsf
                4. Modify ofs_pr_afu.qsf
                5. Re-run afu_synth_setup to update project settings
                6. Re-run $OPAE_PLATFORM_ROOT/bin/afu_synth to build the PR -able image containing the RSTP instance

                The following steps use the previously built host_chan_mmio AFU example. You can use these detailed steps to instrument your AFU.

                1. Navigate to host_chan_mmio AFU Quartus project and open the project using Quartus GUI.

                   cd $OFS_ROOTDIR/work_d5005/build_d5005_x16/build/syn/syn_top\n $ quartus d5005.qpf &\n

                2. Once the project is loaded in Quartus, review the project hierarchy as shown in the Project Navigator. This example will add Signal Tap probe points to the AFU region. Reviewing the code will give insight into the function of this block. You can up the code in the Project Navigator by expanding afu_top - port_gasket - pr_slot - afu_main - ofs_plat_afu, then select instance afu, right-click, select Locate Node - Locate in Design File as shown below.

                3. Bring up Signal Tap to create the *.stp file. In the Quartus GUI, go to Tools - Signal Tap Logic Analyzer. Click Create to accept the default template in the New File from Template pop-up. The Signal Tap Logic Analyzer window comes up.

                4. Set up the clock for the Signal Tap logic instance by clicking ... button as shown below:

                  5. The Node Finder comes up, and you will click ... as shown below to bring up the hierarchy navigator or copy-paste the following location at Look in:

                iofs_top|afu_top|port_gasket|pr_slot|afu_main|ofs_plat_afu|afu\n

                1. In the Select Hierarchy Level, navigate to top - afu_top - port_gasket - pr_slot - afu_main - ofs_plat_afu, then select instance afu and click Ok.

                2. Enter *clk* in the Named: box and click Search. This brings up matching terms. Click mmio64_if.clk and >. Verify your Node Finder is as shown below and then click Ok:

                3. Double click the Double-click to add nodes and once again, click ... and navigate to top - afu_top - port_gasket - pr_slot - afu_main - ofs_plat_afu, then select instance afu and click Ok. Enter mmio64 then click >> to add these signals to the STP instance as shown below:

                  Then click Insert and Close.

                4. Save the newly created STP by clicking File - Save As in the save as navigate to $OFS_ROOTDIR/work_d5005/build_d5005_x16/build/syn/syn_top and save the STP file as host_chan_mmio.stp as shown below:

                5. Edit ofs_top.qsf to add host_chan_mmio.stp file and enable STP. Open $OFS_ROOTDIR/work_d5005/build_d5005_x16/build/syn/syn_top/d5005.qpf in an editor and modify lines as shown below:
                set_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE host_chan_mmio.stp\nset_global_assignment -name SIGNALTAP_FILE host_chan_mmio.stp\n

                Save the d5005.qpf and close Quartus.

                1. Edit iofs_pr_afu.qsf to add host_chan_mmio.stp file and enable STP. Open $OPAE_PLATFORM_ROOT/hw/lib/build/syn/syn_top/iofs_pr_afu.qsf in an editor and ensure the lines are included as below (note: the verilog macro INCLUDE_REMOTE_STP will already be present), also copy and paste the file host_chan_mmio.stp in this location:

                The updated lines are: 

                set_global_assignment -name VERILOG_MACRO \"INCLUDE_REMOTE_STP\"\nset_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE host_chan_mmio.stp\nset_global_assignment -name SIGNALTAP_FILE host_chan_mmio.stp\n
                Save the iofs_pr_afu.qsf and ensure Quartus is closed.

                1. The afu_synth script is run to create a new copy of AFU files. In your original build shell, enter the following commands:
                    $ cd $OFS_ROOTDIR/build_d5005_x16\n    $ afu_synth_setup -s $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/test_mmio_axi1.txt build_d5005_x16_stp\n\n    Notice that your previous build_d5005_x16_stp directory is preserved, and a new build_d5005_x16_stp directory is created. You will use build_d5005_x16_stp to build the STP-enabled image.\n\n    $ cd build_d5005_x16_stp\n    $ $OPAE_PLATFORM_ROOT/bin/afu_synth\n\n...\n...\nWrote host_chan_mmio.gbs\n\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n  Design meets timing\n===========================================================================\n
                1. Once compilation completes, the new host_chan_mmio.gbs file that contains the Signal Tap instance can be loaded.
                $ sudo fpgasupdate host_chan_mmio.gbs 3b:00.0\n[sudo] password for <myuser>: \n[WARNING ] Update starting. Please do not interrupt.\n [INFO    ] \nPartial Reconfiguration OK\n[INFO    ] Total time: 0:00:01.87\n
                1. Use the OPAE SDK mmlink tool to create a TCP/IP connection to your Intel\u00ae Stratix 10\u00ae FPGA card under test. The mmlink command has the following format:
                Usage:\nmmlink\n<Segment>             --segment=<SEGMENT NUMBER>\n<Bus>                 --bus=<BUS NUMBER>           OR  -B <BUS NUMBER>\n<Device>              --device=<DEVICE NUMBER>     OR  -D <DEVICE NUMBER>\n<Function>            --function=<FUNCTION NUMBER> OR  -F <FUNCTION NUMBER>\n<Socket-id>           --socket-id=<SOCKET NUMBER>  OR  -S <SOCKET NUMBER>\n<TCP PORT>            --port=<PORT>                OR  -P <PORT>\n<IP ADDRESS>          --ip=<IP ADDRESS>            OR  -I <IP ADDRESS>\n<Version>             -v,--version Print version and exit\n

                ProTip:

                Open a new shell session for mmlink; this console needs to remain open to allow mmlink connection.

                Enter the command below to create a connection using port 3333:

                $ sudo mmlink -P 3333 -B 0x3b\n\n ------- Command line Input START ----\n\n Socket-id             : -1\n Port                  : 3333\n IP address            : 0.0.0.0\n ------- Command line Input END   ----\n\nPORT Resource found.\nServer socket is listening on port: 3333\n

                Leave this shell open with the mmlink connection.

                1. In this step, you will open a new shell and enable JTAG over protocol. You must have Quartus Prime Pro \u00ae 23.3 Programmer loaded on the Intel\u00ae FPGA PAC D5005 server for local debugging.
                $ jtagconfig --add JTAG-over-protocol sti://localhost:0/intel/remote-debug/127.0.0.1:3333/0\n\nVerify connectivity with jtagconfig --debug\n\n$ jtagconfig --debug\n1) JTAG-over-protocol [sti://localhost:0/intel/remote-debug/127.0.0.1:3333/0]\n   (JTAG Server Version 23.3.0 Build 104 09/14/2022 SC Pro Edition)\n  020D10DD   VTAP10 (IR=10)\n    Design hash    D41D8CD98F00B204E980\n    + Node 00406E00  Virtual JTAG #0\n\n  Captured DR after reset = (020D10DD) [32]\n  Captured IR after reset = (155) [10]\n  Captured Bypass after reset = (0) [1]\n  Captured Bypass chain = (0) [1]\n
                1. Start Quartus Signal Tap GUI, connect to target, load stp file by navigating to $OPAE_PLATFORM_ROOT/hw/lib/build/syn/syn_top/ . The Quartus Signal Tap must be the same version of Quartus used to compile the host_chan_mmio.gbs. Quartus Prime Pro \u00ae 23.3 Pro is used in the steps below:
                cd $OPAE_PLATFORM_ROOT/hw/lib/build/syn/syn_top/\nquartus_stpw host_chan_mmio.stp\n

                This command brings up Signal Tap GUI. Connect to the Signal Tap over protocol by selecting the Hardware button on the right side of the GUI and clicking the \"Please Select\" pull-down as shown below:

                JTAG over protocol selected:

                This connection process will take approximately 2-3 minutes for the Signal Tap instance to indicate \"Ready to acquire\".

                8) Set the trigger condition for a rising edge on signal valid signal. 9) In the Signal Tap window, enable acquisition by pressing key F5. The Signal Tap GUI will indicate \"Acquisition in progress\". Run the hello_world application and observe that the Signal Tap instance has triggered. You should see signals being captured in the Signaltap GUI.

                See captured image below:

                To end your Signal Tap session, close the Signal Tap GUI, then in the mmlink shell, enter ctrl c to kill the mmlink process.

                "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#notices-disclaimers","title":"Notices & Disclaimers","text":"

                Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Performance varies by use, configuration and other factors. Your costs and results may vary. You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications. Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards. \u00a9 Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.

                OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122.

                "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/","title":"Intel\u00ae FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA","text":""},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#1-introduction","title":"1. Introduction","text":""},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#11-about-this-document","title":"1.1. About This Document","text":"

                This document serves as a design guide for FPGA developers, system architects and hardware developers using OFS as a starting point for the creating the FPGA Interface Manager (FIM) for a custom FPGA acceleration board or Platform with Intel FPGAs.

                This document uses the Intel\u00ae FPGA PAC D5005 as an example platform to illustrate key points and demonstrate how to extend the capabilities provided in OFS (Open FPGA Stack) to custom platforms. The demonstration steps serves as a tutorial for the development of your OFS knowledge.

                This document covers OFS architecture lightly. For more details on the OFS architecture, please see [Open FPGA Stack Technical Reference Manual].

                You are encouraged to read [OFS AFU Development Guide] to fully understand how AFU Developers will use your newly developed FIM.

                Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#12-introduction","title":"1.2. Introduction","text":"

                Open FPGA Stack (OFS) addresses the scalability for FPGA acceleration boards and workloads by providing a powerful and systematic methodology for the rapid development of FPGA-based Acceleration systems. This methodology addresses the key challenges of hardware, software and workload developers by providing a complete FPGA project consisting of RTL and simulation code, build scripts and software. The FPGA project released in OFS can be rapidly customized to meet new market requirements by adding new features, custom IPs and Intel interface subsystems IPs.

                A high-level overview of the OFS Intel\u00ae Stratix 10\u00ae FPGA hardware architecture on the Intel\u00ae Stratix 10\u00ae FPGA reference platform, Intel\u00ae FPGA PAC D5005 is shown in the below figure. The provided FPGA architecture is divided into two main components

                - The outer area in white, the FPGA Interface manager (or FIM) - The inner area in green, the Acceleration Function Unit or AFU Region.

                The outer area, the FIM, provides the core infrastructure and interfaces within the FPGA. The AFU region is where a user\u2019s custom logic would reside for their specific workload.

                * FPGA external interfaces and IP cores (e.g. Ethernet, DDR-4, PCIe, etc) * PLLs/resets * FPGA - Board management infrastructure * Interface to Acceleration Function Unit (AFU)

                The AFU region has both static and dynamic partial reconfiguration regions enabling a lot of customization.

                * Uses the FIM interfaces to perform useful work inside the FPGA * Contains logic supporting partial reconfiguration * Remote Signal Tap core for remote debugging of workload

                Outside of the FPGA is the Board Management Controller which provides board management, root of trust, board monitoring, and remote system updates.

                The overall architecture is built to be very composable and modular in blocks that can be modified while leaving the rest of the infrastructure intact so you may only need to modify a few of these blocks.

                "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#12-release-capabilities","title":"1.2. Release Capabilities","text":"

                This release of OFS FIM supports the following key features:

                • 1 - Host channel interface via PCIe Gen 3 x 16 SRIOV (1PF, 3 VF, AXI-S TLP packets)
                • DDR4 SDRAM External memory interface (AXI-M)
                • 1 - 10G Ethernet interfaces (1x10G)
                • MSI-X Interrupts (PF, VF)
                • 1 - AFU
                • Exercisers demonstrating PCIe, external memory and Ethernet interfaces
                • Port, FME CSR
                • Remote Signal Tap

                OFS is extensible to meet the needs of a broad set of customer applications, however not all use cases are easily served. The general uses cases listed below are examples where the OFS base design can be easily re-used to build a custom FIM: 1. Use OFS reference design as-is - Porting the code to another platform that is identical to the OFS reference platform only changing target FPGA device and pinout - Change I/O assignments without changing design 2. Update the configuration of peripheral IP in OFS reference design, not affecting FIM architecture - External memory settings - HSSI analog settings 3. Remove/update peripheral feature in OFS reference design, not affecting FIM architecture - External memory speed/width change - Change 10G Ethernet to 25 or 100G Ethernet IP - Change number of VFs supported 4. Add new features as an extension to OFS reference design, not affecting FIM architecture - Add/remove external memory interface to the design - Add/remove user clocks for AFU - Add/remove IP to the design with connection to AFU

                More advanced use cases requiring changes or additions to the host PCIe channel are not easily supported with this release of the OFS FIM.

                Reuse of the provided host management FPGA logic and software is the fastest and most simple approach to FIM customization.

                "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#13-prerequisites","title":"1.3. Prerequisites","text":""},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#131-base-knowledge-and-skills-prerequisites","title":"1.3.1. Base Knowledge and Skills Prerequisites","text":"

                OFS is an advanced application of FPGA technology. This guide assumes you have the following FPGA logic design-related knowledge and skills:

                • FPGA compilation flows using Intel\u00ae Quartus\u00ae Prime Pro Edition design flow.
                • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
                • RTL and coding practices for FPGA implementation.
                • RTL simulation tools.
                • Intel\u00ae Quartus\u00ae Prime Pro Edition Signal Tap Logic Analyzer tool software.
                "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#132-development-environment","title":"1.3.2. Development Environment","text":"

                To run the tutorial steps in this guide requires this development environment:

                Item Version Intel Quartus Prime Pro Intel Quartus Prime Pro 23.3 (with license patch) Target D5005 Sever Operating System RHEL 8.6 OPAE SDK 2.10.0-1 Linux DFL ofs-2023.3-6.1-2 Python 3.6.8 cmake 3.15 GCC 7.4.0 perl 5.8.8

                The following server and Intel PAC card are required to run the examples in this guide:

                1. Qualified Intel Xeon \u00ae server see Qualified Servers.
                2. Intel\u00ae FPGA PAC D5005 with root entry hash erased (Please contact Intel for root entry hash erase instructions). The standard Intel\u00ae FPGA PAC D5005 card is programmed to only allow the FIM binary files signed by Intel to be loaded. The root entry hash erase process will allow newly created, unsigned FIM binary files to be loaded.
                3. Intel\u00ae FPGA PAC D5005 installed in the qualified server following instructions in [OFS Getting Started User Guide].

                The steps included in this guide have been verified in the Dell R740 and HPE ProLiant DL380 Gen10 servers.

                "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#2-high-level-description","title":"2. High Level Description","text":"

                The FIM targets operation in the Intel\u00ae FPGA PAC D5005 card. The block diagram of the D5005 is shown below:

                The key D5005 FPGA interfaces are:

                • Host interface - PCIe Gen3 x 16
                • Network interface
                  • 2 - QSFP28 cages
                  • Current FIM supports 1 x 10 GbE, other interfaces can be created
                • External Memory
                  • 2 or 4 channels of DDR4-2400 to RDIMM modules
                  • RDIMM modules = 8GB organized as 1 Gb X 72
                • Board Management
                  • SPI interface
                  • FPGA configuration
                "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#21-fpga-interface-manager-overview","title":"2.1. FPGA Interface Manager Overview","text":"

                The FPGA Interface Manager architecture is shown in the below diagram:

                The FIM consists of the following components - PCIe Subsystem - Memory Subsystem - HSSI Subsystem - Platform Management Component Intercommunications (PMCI) - Board Peripheral Fabric (BPF) - AFU Peripheral Fabric (APF) - Port Gasket - AXI-S PF/VF Demux/Mux - Host Exerciser Modules - HE-MEM, HE-LB, HE-HSSI - FPGA Management Engine (FME)

                "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#22-fim-fpga-resource-usage","title":"2.2. FIM FPGA Resource Usage","text":"

                The FIM uses a small portion of the available FPGA resources. The table below shows resource usage for a base FIM built with 2 channels of external memory, a small AFU instantiated that has host CSR read/write, external memory test and Ethernet test functionality.

                Entity ALMs Used % ALMS Used M20Ks % M20Ks used DSP Blocks Pins IOPLLs OFS_top 125009.4 13.0% 661 5.4% 0 630 15 afu_top 70522.7 7.0% 228 2.4% 0 0 1 auto_fab_0 1305.7 0.0% 9 0.1% 0 0 0 bpf_rsv_5_slv 0.6 0.0% 0 0.0% 0 0 0 bpf_rsv_6_slv 0.6 0.0% 0 0.0% 0 0 0 bpf_rsv_7_slv 0.4 0.0% 0 0.0% 0 0 0 bpf 241.9 0.0% 0 0.0% 0 0 0 emif_top_inst 10508.6 1.0% 0 0.0% 0 0 12 eth_ac_wrapper 6024.8 0.5% 9 0.1% 0 0 0 fme_top 615.5 0.2% 7 0.1% 0 0 0 pcie_wrapper 35424.7 3.5% 348 2.9% 0 0 1 pmci_top 318.5 0.1% 0 0.0% 0 0 0 rst_ctrl 40.2 0.0% 0 0.0% 0 0 0 sys_pll 0.5 0.0% 0 0.0% 0 0 1 Total ALMS 933,120 Total M20Ks 11,721 Summary FPGA Resource Utilization Logic utilization (in ALMs) 124,092 / 933,120 ( 13 % ) Total dedicated logic registers 282822 Total pins 630 / 912 ( 69 % ) Total block memory bits 3,425,120 / 240,046,080 ( 1 % ) Total RAM Blocks 661 / 11,721 ( 6 % ) Total DSP Blocks 0 / 5,760 ( 0 % ) Total eSRAMs 0 / 75 ( 0 % ) Total HSSI P-Tiles 17 / 48 ( 35 % ) Total HSSI E-Tile Channels 17 / 48 ( 35 % ) Total HSSI HPS 0 / 1 ( 0 % ) Total HSSI EHIPs 0 / 2 ( 0 % ) Total PLLs 36 / 104 ( 35 % )"},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#23-ofs-directory-structure","title":"2.3. OFS Directory Structure","text":"

                The OFS Git OFS repository ofs-d5005 directory structure is shown below:

                \u251c\u2500\u2500 eval_script\n|   \u251c\u2500\u2500 ofs_d5005_eval.sh\n|   \u2514\u2500\u2500 README_ofs_d5005_eval.txt\n\u251c\u2500\u2500 ipss\n\u2502   \u251c\u2500\u2500 hssi\n|   \u251c\u2500\u2500 mem\n|   \u251c\u2500\u2500 pcie\n|   \u251c\u2500\u2500 pmci\n|   \u251c\u2500\u2500 spi\n|   \u2514\u2500\u2500 README.md\n\u251c\u2500\u2500 license\n\u2502   \u2514\u2500\u2500 quartus-0.0-0.01Intel OFS-linux.run\n\u251c\u2500\u2500 ofs-common\n|   \u251c\u2500\u2500 scripts\n|   \u251c\u2500\u2500 src\n|   \u251c\u2500\u2500 verification\n|   \u251c\u2500\u2500 LICENSE.txt\n|   \u2514\u2500\u2500 README.md\n\u251c\u2500\u2500 sim\n|   \u251c\u2500\u2500 bfm\n|   \u251c\u2500\u2500 rp_bfm\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 scripts\n|   \u251c\u2500\u2500 unit_test \n\u2502\u00a0\u00a0 \u2514\u2500\u2500 readme.txt\n\u251c\u2500\u2500 src\n\u2502   \u251c\u2500\u2500 afu_top\n\u2502   \u251c\u2500\u2500 includes\n\u2502   \u251c\u2500\u2500 pd_qsys\n\u2502   \u251c\u2500\u2500 README.md\n\u2502   \u2514\u2500\u2500 top\n\u251c\u2500\u2500 syn\n\u2502   \u251c\u2500\u2500 scripts\n\u2502   \u251c\u2500\u2500 setup\n\u2502   \u251c\u2500\u2500 syn_top\n\u2502   \u251c\u2500\u2500 readme.txt\n\u2502   \u2514\u2500\u2500 README\n\u251c\u2500\u2500 LICENSE.txt\n\u2514\u2500\u2500 README.md\n

                The contents of each directory are described below:

                Eval Script - Contains scripts for evaluation of OFS for D5005 including compiling FIM/AFU from source, unit level test. Also includes resources to report and setup D5005 development environment

                ipss - Contains the code and supporting files that define or set up the IP subsystems (HSSI, PCIe, memory, PMCI, SPI, etc...) contained in the D5005 FPGA Interface Manager (FIM).

                license - License file for the Low Latency 10Gbps Ethernet MAC (6AF7 0119) IP core.

                ofs-common - This directory contains resources that may be used across the board-specific repositories. This directory is referenced via a link within each of the FPGA-specific repositories.

                sim - Contains the testbenches and supporting code for all the unit test simulations. - Bus Functional Model code is contained here. - Scripts are included for automating a myriad of tasks. - All of the individual unit tests and their supporting code is also located here.

                src - SystemVerilog source and script files - Contains all of the structural and behavioral code for the FIM. - Scripts for generating the AXI buses for module interconnect. - Top-level RTL for synthesis is located in this directory. - Accelerated Functional Unit (AFU) infrastructure code is contained in this directory.

                syn - This directory contains all of the scripts, settings, and setup files for running synthesis on the FIM.

                "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#3-description-of-sub-systems","title":"3. Description of Sub-Systems","text":""},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#31-host-control-and-data-flow","title":"3.1. Host Control and Data Flow","text":"

                The host control and data flow are shown in the diagram below:

                The control and data flow is composed of the following:

                • Host Interface Adapter (PCIe)
                • Low Performance Peripherals
                  • Slow speed peripherals (I2C, Smbus, etc)
                  • Management peripherals (FME)
                • High Performance Peripherals
                  • Memory peripherals
                  • Acceleration Function peripherals
                  • HPS Peripheral
                • Fabrics
                  • Peripheral Fabric (multi drop)
                  • AFU Streaming fabric (point to point)

                Peripherals are connected to one another using AXI:

                • Via the peripheral fabric (AXI4-Lite, multi drop)
                • Via the AFU streaming fabric (AXI-S, point to point)

                Peripherals are presented to software as:

                • OFS managed peripherals that implement DFH CSR structure.
                • Native driver managed peripherals (i.e. Exposed via an independent PF, VF)

                The peripherals connected to the peripheral fabric are primarily Intel OPAE managed resources, whereas the peripherals connected to the AFU are \u201cprimarily\u201d managed by native OS drivers. The word \u201cprimarily\u201d is used since the AFU is not mandated to expose all its peripherals to Intel OPAE. It can be connected to the peripheral fabric, but can choose to expose only a subset of its capability to Intel OPAE.

                OFS uses a defined set of CSRs to expose the functionality of the FPGA to the host software. These registers are described in Open FPGA Stack Reference Manual - MMIO Regions section.

                If you make changes to the FIM that affect the software operation, then OFS provides a mechanism to communicate that information to the proper software driver. The Device Feature Header (DFH) structure provides a mechanism to maintain compatibility with OPAE software. Please see FPGA Device Feature List (DFL) Framework Overview for an excellent description of DFL operation from the driver perspective.

                When you are planning your address space for your FIM updates, please be aware that the OFS FIM targeting Intel\u00ae FPGA PAC D5005, 256KB of MMIO region is allocated for external FME features and 128kB of MMIO region is allocated for external port features. Each external feature must implement a feature DFH, and the DFH needs to be placed at 4KB boundary. The last feature in the external feature list must have the EOL bit in its DFH set to 1 to mark the end of external feature list. Since the FPGA address space is limited, consider using an indirect addressing scheme to conserve address space.

                "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#4-fim-development-flow","title":"4. FIM Development Flow","text":"

                OFS provides a framework of FPGA synthesizable code, simulation environment, and synthesis/simulation scripts. FIM designers can take the provided code and scripts and modify existing code or add new code to meet their specific product requirements.

                FIM development for a new acceleration card consists of the following steps:

                1. Installation of OFS and familiarization with scripts and source code
                2. Development of high-level block diagram with your specific functionality
                  1. Determination of requirements and key performance metrics
                  2. Selection of IP cores
                  3. Selection of FPGA device
                  4. Software memory map
                3. Selection and implementation of FIM Physical interfaces including:
                  1. External clock sources and creation of internal PLL clocks
                  2. General I/O
                  3. Transceivers
                  4. External memories
                  5. FPGA programming methodology
                4. Device physical implementation
                  1. FPGA device pin assignment
                  2. Inclusion of logic lock regions
                  3. Creation of timing constraints
                  4. Create Quartus FIM test project and validate:
                    1. Placement
                    2. Timing constraints
                    3. Build script process
                    4. Review test FIM FPGA resource usage
                5. Select FIM to AFU interfaces and development of PIM
                6. FIM design implementation
                  1. RTL coding
                  2. IP instantiation
                  3. Development of test AFU to validate FIM
                  4. Unit and device level simulation
                  5. Timing constraints and build scripts
                  6. Timing closure and build validation
                7. Creation of FIM documentation to support AFU development and synthesis
                8. Software Device Feature discovery
                9. Hardware/software integration, validation and debugging
                10. High volume production preparation

                The FIM developer works closely with the hardware design of the target board, software development and system validation.

                Understanding how the AFU developer utilizes the FIM is important for FIM development success. Please read [OFS AFU Development Guide] for a detailed description of AFU development.

                "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#41-installation-of-ofs","title":"4.1. Installation of OFS","text":"

                In this section you set up a development machine for compiling the OFS FIM. These steps are separate from the setup for a deployment machine where the FPGA acceleration card is installed. Typically, FPGA development and deployment work is performed on separate machines, however, both development and deployment can be performed on the same server if desired. Please see [OFS Getting Started User Guide] for instructions on installing software for deployment of your FPGA FIM, AFU and software application on a server.

                Building the OFS FIM requires the development machine to have at least 64 GB of RAM.

                The following is a summary of the steps to set up for FIM development:

                1. Install Quartus Prime Pro 23.3 Linux and setup environment
                2. Clone the github ofs-d5005 repository
                3. Test installation by building the provided FIM

                Intel Quartus Prime Pro version 23.3 is the currently verified version of Quartus used for building the FIM and AFU images for this release. Porting to newer versions of Quartus may be performed by developers. Download Quartus Prime Pro Linux version 23.3 from Intel\u00ae Quartus\u00ae Prime Pro Edition Linux.

                Intel Quartus Prime Pro Version 23.3 is verified to work with the latest OFS release ofs-2023.3. However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.

                Use RedHatEnterprise Linux\u00ae (RHEL) for compatibility with your development flow and also testing your FIM design in your platform.

                Prior to installing Quartus:

                1. Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.

                  • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                  • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.

                2. Perform the following steps to satisfy the required dependencies.

                  $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n

                  Apply the following configurations.

                  $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n

                3. Create the default installation path: /intelFPGA_pro/, where is the default path of the Linux workstation, or as set by the system administrator and is your Quartus version number.

                  The installation path must satisfy the following requirements:

                  • Contain only alphanumeric characters
                  • No special characters or symbols, such as !$%@^&*<>,
                  • Only English characters
                  • No spaces

                4. Download your required Quartus Prime Pro Linux version here.

                5. Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.23.

                6. After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:

                  export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n

                  For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:

                  export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n

                7. Verify, Quartus is discoverable by opening a new shell:

                  $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n

                  8. After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:

                  export PATH=$PATH:<Quartus install directory>/quartus/bin\n

                  For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:

                  export PATH=$PATH:/home/intelFPGA_pro/23.3/quartus/bin\n

                  Verify, Quartus is discoverable by opening a new shell:

                  which quartus\n## Output\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                  Note, for some Linux distributions such as RHEL 8.6, Quartus requires installation of the following libraries: 
                  sudo dnf install libnsl\nsudo dnf install ncurses-compat-libs\nsudo ln -s /usr/bin/python3 /usr/bin/python\n

                  You will need to obtain a license for Intel Quartus Prime Pro version 23.3 to compile the design. This license is obtained from Intel. Additionally, OFS for Intel\u00ae Stratix 10\u00ae FPGA requires a license for the Low Latency 10Gbps Ethernet MAC (6AF7 0119) IP core. This license is required to generate a programming file using the provided OFS source code. The Low Latency 10Gbps Ethernet MAC (6AF7 0119) IP core license patch installer is provided in the ofs-d5005 git repository in the /license directory. After cloning the OFS release in step 4 below, you can install this IP license.

                  1. Install git and install git lfs to extract large files within the repository that are compressed with git lfs. Please note, for proper operation of files retrieved from OFS repository, you will require git lfs. 
                  sudo dnf install git\n
                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#install-git-lfs","title":"Install git lfs:","text":"
                  curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\nsudo dnf install git-lfs\ngit lfs install\n
                  1. Retrieve OFS repositories:

                  \u200b The OFS FIM source code is included in the GitHub repository. Create a new directory to use as a clean starting point to store the retrieved files.

                  1. Navigate to location for storage of OFS source, create the top-level source directory and clone OFS repositories.

                  mkdir OFS_fim_build_root\ncd OFS_fim_build_root\nexport OFS_BUILD_ROOT=$PWD\ngit clone --recurse-submodules  https://github.com/OFS/ofs-d5005.git\ncd ofs-d5005\ngit checkout tags/ofs-2023.3-1\n
                  Verify proper tag is selected:

                  git describe --tags\nofs-2023.3-1\n
                  2. Install the Low Latency 10Gbps Ethernet MAC (6AF7 0119) IP license by running provided license installer.

                  cd license\nchmod +x quartus-0.0-0.01Intel OFS-linux.run\nsudo ./quartus-0.0-0.01Intel OFS-linux.run\n
                  1. Verify patch installed 
                    quartus_sh --version\n##Output\nQuartus Prime Shell\nVersion 23.3 Pro Edition\n
                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#42-compiling-ofs-fim","title":"4.2. Compiling OFS FIM","text":"

                  OFS provides a build script with the following FPGA image creation options:

                  • Flat compile which combines the FIM and AFU into one FPGA image that is loaded into the entire FPGA device
                  • A PR compile which creates a FPGA image consisting of the FIM that is loaded into the static region of the FPGA and a default AFU that is loaded into dynamic region. Additional AFUs maybe loaded into the dynamic region using partial reconfiguration.

                  The build scripts included with OFS are verified to run in a bash shell. Other shells have not been tested. Each build script step will take several hours to completes, Please note, building directly in Quartus GUI is not supported - you must build with the provided scripts.

                  The following sections describe how to set up the environment and build the provided FIM and AFU. Follow these steps as a tutorial to learn the build flow. You will use this environment and build scripts for the creation of your specialized FIM.

                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#421-setting-up-required-environment-variables","title":"4.2.1. Setting Up Required Environment Variables","text":"

                  Set required environment variables as shown below. These environment variables must be set prior to simulation or compilation tasks so creating a simple script to set these variables saves time.

                  cd $OFS_BUILD_ROOT/ofs-d5005\nexport OFS_ROOTDIR=$PWD\n\n##   Note, OFS_ROOTDIR is the directory where you cloned the repo, e.g. /home/MyProject/ofs-d5005 *\n\nexport WORKDIR=$OFS_ROOTDIR\nexport VERDIR=$OFS_ROOTDIR/verification\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\n\n##   Note, QUARTUS_ROOTDIR is your Quartus installation directory, e.g. $QUARTUS_ROOTDIR/bin contains Quartus executuable*\n\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport OPAE_SDK_REPO_BRANCH=release/2.10.0\n
                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#422-compiling","title":"4.2.2. Compiling","text":"

                  The usage of the compile build script is shown below: 

                  ofs-common/scripts/common/syn/build_top.sh [-p] target_configuration work_dir \nUsage: ofs-common/scripts/common/syn/build_top.sh [-k] [-p] <build target> [<work dir name>]\n\n  Build a FIM instance specified by <build target>. The target names an FPGA architecture, board and configuration.\n\n  The FIM is built in <work dir name>. If not specified, the target is ${OFS_ROOTDIR}/work.\n\n  The -k option preserves and rebuilds within an existing work tree instead of overwriting it.\n\n  When -p is set, if the FIM is able then a partial reconfiguration template tree is generated at the end of the FIM build. The PR template tree is located in the top of the work directory but is relocatable\n  and uses only relative paths. See ofs-common/scripts/common/syn/generate_pr_release.sh for details.\n\n  The -e option runs only Quartus analysis and elaboration.\n\n      * target_configuration - Specifies the project  \n         For example: d5005\n\n      * work_dir - Work Directory for this build in the form a directory name. It is created in the <local repo directory>/ofs-d5005/<work_dir> \n          - NOTE: The directory name must start with \"work\".  If the work directory exists, then the script stops and asks if you want to overwrite the directory.\n            - e.g.\n                - ofs-common/scripts/common/syn/build_top.sh d5005 work_d5005\n\n                work directory as a name will be created in <local repo directory>/ofs-d5005/work_d5005\n\n\n                The obmission of <work_dir> results in a default work directory (<local repo  directory>/ofs-d5005/work)\n\n        - compile reports and artifacts (.rpt, .sof, etc) are stored in <work_dir>/syn/syn_top/output_files\n\n        - There is a log file created in ofs-d5005 directory.  \n        - [-p]  Optional switch for creation of a relocatable PR build tree supporting the creation of a PR-able AFU workload.   \n        The \"-p\" switch invokes generate_pr_release.sh at the end of the FIM build and writes the PR build tree to the top of the work directory.  More information on this option is provided below. \n
                  In the next example, you will build the provided example design using a flat, non-PR build flow.

                  Build the provided base example design:

                  ```bash cd $OFS_BUILD_ROOT/ofs-d5005

                  ofs-common/scripts/common/syn/build_top.sh d5005 work_d5005 ```

                  ```bash ... build takes ~5 hours to complete

                  Compile work directory: <$OFS_BUILD_ROOT>/work_d5005/syn/syn_top Compile artifact directory: <$OFS_BUILD_ROOT>/work_d5005/syn/syn_top/output_files

                  *** OFS_PROJECT: d5005 *** Q_PROJECT: d5005 *** Q_REVISION: d5005 *** SEED: 03 *** Build Complete *** Timing Passed!

                  The build script copies the ipss, sim, src and syn directories to the specified work directory and then these copied files are used in the Quartus compilation process.  Do not edit the files in the work directory, these files are copies of source files.\n\nSome of the key files are described below:\n\n<work_dir>/syn/syn_top == \n```bash\n\u251c\u2500\u2500 syn_top                    // D5005 Quartus build area with Quartus files used this build\n\u2502\u00a0\u00a0\u251c\u2500\u2500 d5005.ipregen.rpt       // IP regeneration report states the output of IP upgrade\n\u2502\u00a0\u00a0\u251c\u2500\u2500 d5005.qpf               // Quartus Project File (qpf) mentions about Quartus version and project revision\n\u2502  \u251c\u2500\u2500 d5005.qsf               // Quartus Settings File (qsf) lists current project settings and entity level assignments\n\u2502\u00a0\u00a0\u251c\u2500\u2500 d5005.stp               // Signal Tap file included in the d5005.qsf. This file can be modified as required if you need to add an Signal Tap instance\n\u2502\u00a0\u00a0\u251c\u2500\u2500 fme_id.mif              // the fme id hex value is stored in a mif file format\n\u2502  \u251c\u2500\u2500 Intel OFS_pr_afu.json        // PR JSON file\n\u2502\u00a0\u00a0\u251c\u2500\u2500 Intel OFS_pr_afu.qsf                // PR AFU qsf file\n\u2502\u00a0\u00a0\u251c\u2500\u2500 Intel OFS_pr_afu_sources.tcl        // AFU source file list\n\u2502\u00a0\u00a0\u251c\u2500\u2500 ip_upgrade_port_diff_reports   // IP upgrade report files for reference\n
                  /syn/syn_top/output_files == Directory with build reports and FPGA programming files.

                  The programming files consist of the Quartus generated d5005.sof and d5005.pof. The D5005 board hardware provides a 2 Gb flash device to store the FPGA programming files and a MAX10 BMC that reads this flash and programs the D5005 Intel\u00ae Stratix 10\u00ae FPGA FPGA. The syn/build_top.sh script runs script file syn/syn_top/build_flash/build_flash.s which takes the Quartus generated d5005.sof and creates binary files in the proper format to be loaded into the 2 Gb flash device. You can also run build_flash.sh by yourself if needed. The build_flash script runs PACSign (if installed) to create an unsigned FPGA programming file that can be stored in the D5005 FPGA flash. Please note, if the D5005 has the root entry hash key loaded, then PACsign must be run with d5005_page1.bin as the input with the proper key to create an authenticated FPGA binary file. Please see [Security User Guide: Intel\u00ae Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA] for details on the security aspects of Intel\u00ae Open FPGA Stack.

                  The following table provides further detail on the generated bin files.

                  File Description d5005.sof This is the Quartus generated programming file created by Quartus synthesis and place and route. This file can be used to programming the FPGA using a JTAG programmer. This file is used as the source file for the binary files used to program the FPGA flash. d5005.bin This is an intermediate raw binary image of the FPGA d5005_page1.bin This is the binary file created from input file, d5005.sof. This file is used as the input file to the PACSign utility to generate d5005_page1_unsigned.bin binary image file. d5005_page1_unsigned.bin This is the unsigned PACSign output which can be programmed into the FPGA flash of an unsigned D5005 usign the OPAE SDK utility fpgasupdate mfg_d5005_reversed.bin A special programming file for a third party programming device used in board manufacturing. This file is typically not used.

                  build/output_files/timing_report == Directory containing clocks report, failing paths and passing margin reports

                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#423-relocatable-pr-directory-tree","title":"4.2.3. Relocatable PR Directory Tree","text":"

                  If you are developing a FIM to be used by another team developing the AFU workload, scripts are provided that create a relocatable PR directory tree. ODM and board developers will make use of this capability to enable a broad set of AFUs to be loaded on a board using PR. The relocatable PR directory contains the Quartus *.qdb file that goes the FIM.

                  The creation of the relocatable PR directory tree requires a clone of the Intel Basic Building Blocks (BBB) repository. The OFS_PLATFORM_AFU_BBB environment variable must point to the repository, for example.

                  cd $OFS_BUILD_ROOT\ngit clone https://github.com/OPAE/ofs-platform-afu-bbb\ncd ofs-platform-afu-bbb\nexport OFS_PLATFORM_AFU_BBB=$PWD\ncd $OFS_ROOTDIR\n

                  You can create this relocatable PR directory tree by either:

                  • Build FIM and AFU using /syn/build_top.sh followed by running./ofs-common/scripts/common/syn/generate_pr_release.sh
                  • Build FIM and AFU using /syn/build_top.sh with optional -p switch included

                  The generate_pr_release.sh has the following command structure:

                  ./ofs-common/scripts/common/syn/generate_pr_release.sh -t <path to generated release tree> *Board Build Target* <work dir from build_top.sh>\n\nWhere:\n\n-t <path to generated release tree> = location for your relocatable PR directory tree\n*Board Build Target* is the name of the board target/FIM e.g. d5005\n<work dir from build_top.sh> \n
                  Here is an example of running the generate_pr_release.sh script:

                  ofs-common/scripts/common/syn/generate_pr_release.sh -t work_d5005/build_tree d5005  work_d5005\n
                  **********************************\n********* ENV SETUP **************\n\nFIM Project:\n  OFS_PROJECT = d5005\n  OFS_FIM     = .\n  OFS_BOARD   = .\n  Q_PROJECT   = d5005\n  Q_REVISION  = d5005\n  Fitter SEED = 03\nFME id\n  BITSTREAM_ID = 04010002c7cab852\n  BITSTREAM_MD = 0000000002204283\n\n...\n...\n
                  The resulting relocatable build tree has the following structure: 
                  .\n\u251c\u2500\u2500 bin\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 afu_synth\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 build_env_config\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 run.sh -> afu_synth\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 update_pim\n\u251c\u2500\u2500 hw\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 blue_bits\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u251c\u2500\u2500 d5005_page1_unsigned.bin\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u2514\u2500\u2500 d5005.sof -> ../lib/build/syn/syn_top/output_files/d5005.sof\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 lib\n\u2502\u00a0\u00a0     \u251c\u2500\u2500 build\n\u2502\u00a0\u00a0     \u251c\u2500\u2500 fme-ifc-id.txt\n\u2502\u00a0\u00a0     \u251c\u2500\u2500 fme-platform-class.txt\n\u2502\u00a0\u00a0     \u2514\u2500\u2500 platform\n

                  This build tree can be moved to a different location and used for AFU development of a PR capable AFU to be used with this board.

                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#424-unit-level-simulation","title":"4.2.4. Unit Level Simulation","text":"

                  Unit level simulation of key components is provided. These simulations provide verification of the following areas:

                  • HSSI
                  • PCIe
                  • External Memory
                  • FIM management

                  These simulations use the Synopsys VCS simulator. Each simulation contains a readme file explaining how to run the simulation. Refer to [Simulation User Guide: Open FPGA Stack for Intel Intel\u00ae Stratix 10\u00ae FPGA] for details of simulation examples. Your simulation shell requires Python, Quartus, and VCS to run. To run a simulation of the dfh_walker that simulates host access to the internal DFH registers, perform the following steps:

                  Before running unit simulation, you must set environment variables as described below:

                  cd $OFS_BUILD_ROOT/ofs-d5005\nexport OFS_ROOTDIR=$PWD\n\n##   *Note, OFS_ROOTDIR is the directory where you cloned the repo, e.g. /home/MyProject/ofs-d5005 *\n\nexport WORKDIR=$OFS_ROOTDIR\nexport VERDIR=$OFS_ROOTDIR/verification\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\n\n##   *Note, QUARTUS_ROOTDIR is your Quartus installation directory, e.g. $QUARTUS_ROOTDIR/bin contains Quartus executuable*\n\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport OPAE_SDK_REPO_BRANCH=release/2.10.0\n

                  To compile all IPs:

                  To Generate Simulation Files & compile all IPs, run the following command:

                  cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\nsh gen_sim_files.sh d5005\n
                  The RTL file list for unit_test is located here: $OFS_ROOTDIR/sim/scripts/rtl_comb.f

                  The IPs are generated here: 

                  $OFS_ROOTDIR/sim/scripts/qip_gen\n
                  The IP simulation filelist is generated here: 

                  $OFS_ROOTDIR/sim/scripts/ip_flist.f\n
                  Once the IPs are generated, they can be used for any unit test.

                  To run the simulation, run the following command:

                  cd $OFS_ROOTDIR/sim/unit_test/<Unit Test Name>/scripts\nsh run_sim.sh VCS=1\n
                  Simulation files are located in the sim/unit_test//sim directory.

                  To view simulation waveform:

                  cd $OFS_ROOTDIR/sim/unit_test/<test_name>/script/sim/unit_test/<test_name>/scripts/sim_vcs\ndve -full64 -vpd vcdplus.vpd &\n
                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#4241-dfh-walking-unit-simulation-output","title":"4.2.4.1. DFH Walking Unit Simulation Output","text":"
                  ********************************************\n Running TEST(0) : test_fme_dfh_walking\n********************************************\nREAD64: address=0x00000000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x4000000010000000\n\nFME_DFH\n   Address   (0x0)\n   DFH value (0x4000000010000000)\nREAD64: address=0x00001000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000020000001\n\nTHERM_MNGM_DFH\n   Address   (0x1000)\n   DFH value (0x3000000020000001)\nREAD64: address=0x00003000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000010000007\n\nGLBL_PERF_DFH\n   Address   (0x3000)\n   DFH value (0x3000000010000007)\nREAD64: address=0x00004000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x30000000c0001004\n\nGLBL_ERROR_DFH\n   Address   (0x4000)\n   DFH value (0x30000000c0001004)\nREAD64: address=0x00010000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x300000010000000e\n\nSPI_DFH\n   Address   (0x10000)\n   DFH value (0x300000010000000e)\nREAD64: address=0x00020000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000100000020\n\nPCIE_DFH\n   Address   (0x20000)\n   DFH value (0x3000000100000020)\nREAD64: address=0x00030000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x300000010000100f\n\nHSSI_DFH\n   Address   (0x30000)\n   DFH value (0x300000010000100f)\nREAD64: address=0x00040000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000500000009\n\nEMIF_DFH\n   Address   (0x40000)\n   DFH value (0x3000000500000009)\nREAD64: address=0x00090000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000010001005\n\nFME_PR_DFH\n   Address   (0x90000)\n   DFH value (0x3000000010001005)\nREAD64: address=0x00091000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x4000000010001001\n\nPORT_DFH\n   Address   (0x91000)\n   DFH value (0x4000000010001001)\nREAD64: address=0x00092000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000010000014\n\nUSER_CLOCK_DFH\n   Address   (0x92000)\n   DFH value (0x3000000010000014)\nREAD64: address=0x00093000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x30000000d0002013\n\nPORT_STP_DFH\n   Address   (0x93000)\n   DFH value (0x30000000d0002013)\nREAD64: address=0x000a0000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000010000002010\n\nAFU_INTF_DFH\n   Address   (0xa0000)\n   DFH value (0x3000010000002010)\nMMIO error count matches: x\n\nTest status: OK\n\n********************\n  Test summary\n********************\n   test_fme_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n

                  The simulation transcript is displayed while the simulation runs. The transcript is saved to the file transcript.out for review after the simulation completes. The simulation waveform database is saved as vcdplus.vpd for post simulation review. You are encouraged to run the additional simulation examples to learn about each key area of the OFS shell.

                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#43-compiling-the-ofs-fim-using-eval-script","title":"4.3. Compiling the OFS FIM using Eval Script","text":"

                  The Evaluation Script provides resources to setup and report D5005 development environment. You can use the evaluation script to compile and simulate the FIM. Refer to README_ofs_d5005_eval.txt for details of using the evaluation script.

                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#44-debugging","title":"4.4. Debugging","text":"

                  For debugging issues within the FIM, Signal Tap can be used to gain internal visibility into your design. This section describes the process of adding a Signal Tap instance to your FIM design

                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#441-signal-tap-prerequisites","title":"4.4.1. Signal Tap Prerequisites","text":"

                  To use Signal Tap with OFS, you will need the following:

                  • Understanding of Signal Tap fundamentals - please review Quartus Prime Pro Edition User Guide: Debug Tools. section 2. Design Debugging with the Signal Tap Logic Analyzer.
                  • The Intel\u00ae FPGA PAC D5005 has a built in Intel FPGA Download Cable II allowing JTAG access to the S10 FPGA. You can access the D5005 built in Intel FPGA Download Cable II by connecting your server to the Micro USB connector as shown below:

                  • If you are using a custom board without a built-in Intel FPGA Download Cable then an external Intel FPGA Download Cable II (see Download Cables for more information) can be used for Signal Tap access. The custom board must have JTAG access to the target FPGA for the Intel FPGA Download Cable II.
                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#442-adding-signal-tap","title":"4.4.2. Adding Signal Tap","text":"

                  The following steps guide you through the process of adding a Signal Tap instance to your design. The added Signal Tap instance provides hardware to capture the desired internal signals and connect the stored trace information via JTAG. Please be aware, the added Signal Tap hardware will consume FPGA resources and may require additional floorplanning steps to accommodate these resources. Some areas of the FIM use logic lock regions and these regions may need to be re-sized. These steps assume the use of the Intel\u00ae FPGA PAC D5005.

                  1. Perform a full compile using the script build_top.sh.
                  2. Once the compile completes open the Quartus GUI using the FIM project. The Quartus project is named d5005 and is located in the work directory syn/syn_top/d5005.qpf. Once the project is loaded, go to Tools > Signal Tap Logic Analyzer to bring up the Signal Tap GUI.

                  1. Accept the \"Default\" selection and click \"Create\".

                  1. This brings up Signal Tap Logic Analyzer window as shown below:

                  1. Set up clock for STP instance. In this example, the EMIF CSR module is being instrumented. If unfamiliar with code, it is helpful to use the Quartus Project Navigator to find the specific block of interest and open the design instance for review. For example, see snip below using Project Navigator to open emif_csr block:

                  1. After reviewing code, assign clock for sampling Signal Tap instrumented signals of interest. Note, the clock selected and the signals you want to view should be the same for best trace fidelity. Different clocks can be used however there maybe issues with trace inaccuracy due sampling time differences. In the middle right of the Signal Tap window under Signal Configuration, Clock: select \"\u2026\" as shown below:

                  1. After reviewing code, assign clock for sampling Signal Tap instrumented signals of interest. In the middle right of the Signal Tap window under Signal Configuration, Clock: select \"\u2026\" as shown below: This brings up the Node Finder tool. Input \"emif_csr\" into Named and select \"Search\". This brings up all nodes from the pre-synthesis view. Expand, \"mem\" and \"emif_csr\" and scroll through this list to become familiar with nodes, and then select csr_if.clk and click \">\" to select this clock as shown below and click \"OK\":

                  1. Update the sample depth and other Signal Tap settings as needed for your debugging criteria.

                  1. In the Signal Tap GUI add nodes to be instrumented by double clicking on \"Double-click to add nodes\".

                  1. This brings up the Node Finder. Add signals to be traced by the Signal Tap instance. Click \"Insert\" to add the signals.
                  2. To provide a unique name for your Signal Tap instance, select \"auto signaltap_0\", right click and select rename instance and provide a descriptive name for your instance.
                  3. Save the newly created Signal Tap file and click \"Yes\" to add the new Signal Tap file to the project.
                  4. Compile the project with the Signal Tap file added to the project.
                  5. Once the compile successfully completes with proper timing, you can load the generated d5005.sof using the Intel FPGA Downloader cable.
                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#443-signal-tap-trace-acquisition","title":"4.4.3. Signal Tap trace acquisition","text":"

                  To acquire signals using SignalTap, first load the Signal Tap instrumented SOF file into your target board, open the STP file in the Signal Tap GUI and start the signal acquisition.

                  Avoid system hang during programming the sof file, mask AER regsiter using below steps

                  Find Root complex - End Point mapping using the below command

                  lspci -vt\n
                  +-[0000:3a]-+-00.0-[3b-3c]----00.0  Intel Corporation Device bcce\n |           +-05.0  Intel Corporation Sky Lake-E VT-d\n |           +-05.2  Intel Corporation Sky Lake-E RAS Configuration Registers\n |           +-05.4  Intel Corporation Sky Lake-E IOxAPIC Configuration Registers\n |           +-08.0  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-09.0  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-0a.0  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-0a.1  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-0a.2  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-0a.3  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-0a.4  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-0a.5  Intel Corporation Sky Lake-E LM Channel 1\n

                  Use the bus information from the lspci logs to mask the AER (Advanced Error Reporting) register

                  sudo su\n\nsetpci -s 0000:3b:00.0 ECAP_AER+0x08.L=0xFFFFFFFF \nsetpci -s 0000:3b:00.0 ECAP_AER+0x14.L=0xFFFFFFFF\nsetpci -s 0000:3a:00.0 ECAP_AER+0x08.L=0xFFFFFFFF\nsetpci -s 0000:3a:00.0 ECAP_AER+0x14.L=0xFFFFFFFF\necho \"1\" > /sys/bus/pci/devices/0000:3b:00.0/remove\n\nexit\n
                  1. The SOF file is located in the work directory work_d5005/syn/syn_top/output_files/d5005.sof. If the target FPGA is on a different server, then transfer d5005.sof and STP files to the server with the target FPGA. Load the SOF using the Intel\u00ae FPGA PAC D5005 built-in Intel FPGA Download Cable II. 
                  sudo su\necho \"1\" > /sys/bus/pci/rescan\n
                  1. Make sure D5005 is present by checking expected bitstream ID using command:
                  sudo fpgainfo fme\nIntel FPGA Programmable Acceleration Card D5005\nBoard Management Controller, MAX10 NIOS FW version: 2.0.13 \nBoard Management Controller, MAX10 Build version: 2.0.8 \n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x40100022C164DB1\nBitstream Version                : 4.0.1\nPr Interface Id                  : 210a4631-18bb-57d1-879f-2c3d59b26e37\nBoot Page                        : user\n
                  1. Once the SOF file is loaded, start the Quartus Signal Tap GUI.

                  quartus_stpw\n
                  The Signal Tap GUI comes up.

                  1. In the Signal Tap GUI, Hardware: selection box select the cable \"Stratix10 Darby Creek [ JTAG cable number ]\" as shown below:

                  1. In File open your STP file. Your STP file settings will load. If not already set, you can create the trigger conditions, then start analysis with F5.

                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#5-fim-modification-example","title":"5. FIM Modification Example","text":"

                  An example of FIM modification is provided in this section. This example can be used in your specific application as a starting point. This example shows the basic flow and listing of files that are to be changed.

                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#51-hello-fim-example","title":"5.1. Hello FIM example","text":"

                  If you intend to add a new module to the FIM area, then you will need to inform the host software of the new module. The FIM exposes its functionalities to host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). This set of CSR registers and their operation is described in FIM MMIO Regions.

                  See FPGA Device Feature List (DFL) Framework Overview for a description of the software process to read and process the linked list of Device Feature Header (DFH) CSRs within a FPGA.

                  This example adds a simple DFH register set to the FIM. You can use this example as the basis for adding a new feature to your FIM.

                  The steps to add this simple DFH register are described below.

                  1. Review current design documentation: OFS Tech Ref MMIO Regions
                  2. Understand FME and Port regions, DFH walking, DFH register structure
                  3. Run unit level simulations and review output: i. sim/unit_test/dfh_walker
                  4. Note DFH link list order, see DFH Walker Unit Level Simulation Output
                  5. Make code changes to top level FIM file to instantiate new DFH register
                  6. The DFH registers follow a link list. This example inserts the hello_fim DFH register after the EMIF DFH register, so the emif_csr.sv parameters are updated to insert the hello_fim DFH register as next register in the link list.
                  7. Create the new hello_fim SystemVerilog files.
                  8. Update and run the dfh_walker unit simulation files
                  9. Update synthesis files to include the new hello_fim source files
                  10. Build and test the new FIM

                  The following sections describe changes to add the hello_fim DFH example to the Intel provided FPGA design.

                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#511-srctopiofs_topsv","title":"5.1.1. src/top/iofs_top.sv","text":"
                  1. Edit top level design module: src/top/iofs_top.sv
                    1. Instantiate new hello_fim module in OFS_top.sv at line 294
                  //*******************************\n// FME\n//*******************************\n\n  fme_top \n  fme_top(\n          .clk               (clk_1x                    ),\n          .rst_n             (rst_n_d_1x       ),\n          .pwr_good_n        (ninit_done                ),\n          .i_pcie_error      ('0                        ),\n\n          .axi_lite_m_if     (bpf_fme_mst_if            ),\n          .axi_lite_s_if     (bpf_fme_slv_if            )\n         );\n\n\n`ifdef INCLUDE_HELLO_FIM\n\n        hello_fim_top \n           hello_fim_top (\n        .clk   (clk_1x),\n\n       .rst_n                      (rst_n_d_1x),\n       .csr_lite_if             (bpf_rsv_5_slv_if)\n\n\n        );\n`endif\n\n//*******************************\n// AFU\n//*******************************\n

                  You will connect the Hello_FIM DFH register to the existing BPF reserved link 5. The provided OFS reference design includes 3 reserved BPF interfaces available for custom usage such as new OPAE controlled modules. The address map of BPF is shown below:

                  Address Size (Byte) Feature Master 0x00000 \u2013 0x0FFFF 64K FME (FME, Error, etc) Yes 0x10000 \u2013 0x1FFFF 64K PMCI Proxy (SPI Controller) Yes 0x20000 \u2013 0x2FFFF 64K PCIe CSR 0x30000 \u2013 0x3FFFF 64K HSSI CSR 0x40000 \u2013 0x4FFFF 64K EMIF CSR 0x50000 \u2013 0x5FFFF 64K Reserved 0x60000 \u2013 0x6FFFF 64K Reserved 0x70000 \u2013 0x7FFFF 64K Reserved

                  The BPF reserved link 5 is connected to a dummy connection to prevent this link from being optimized out the design during synthesis. You will add a compiler `define that will cause this dummy connection to be removed when the variable INCLUDE_HELLO_FIM is defined by editing line 575 if iofs_top.sv as shown below:

                      // Reserved address response\n   `ifndef INCLUDE_HELLO_FIM\n    bpf_dummy_slv\n    bpf_rsv_5_slv (\n        .clk            (clk_1x),\n        .dummy_slv_if   (bpf_rsv_5_slv_if)\n    );\n    `endif\n
                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#512-ipssmememif_csrsv","title":"5.1.2. ipss/mem/emif_csr.sv","text":"

                  The Hello_FIM DFH is inserted in the DFH link list after the EMIF CSR DFH and before the FME_PR DFH. The file ipss/d5005/emif/emif_csr.sv contains a parameter defining the next address for the next DFH in in the link list chain. You will change the next address offset to be 0x10000 so the reveserved BPF AXI lite link connected to the Hello_FIM DFH register is next in the DFH link list.

                  module emif_csr #(\n  parameter NUM_LOCAL_MEM_BANKS = 1,\n  parameter END_OF_LIST         = 1'b0,\n  `ifndef INCLUDE_HELLO_FIM\n  parameter NEXT_DFH_OFFSET     = 24'h05_0000\n  `else\n  parameter NEXT_DFH_OFFSET     = 24'h01_0000//New for Hello_FIM, next offset now at 0x50000\n  `endif\n)\n
                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#513-srchello_fimhello_fim_topsv","title":"5.1.3. src/hello_fim/hello_fim_top.sv","text":"

                  Create hello_fim_top.sv, and store it in src/hello_fim directory. The main purpose of this RTL is to convert AXI4-Lite interface to a simple interface to interface with the registers in hello_fim_com.sv. This register sets the DFH feature ID to 0xfff which is undefined. Since this for test purposes, using an undefined feature ID will result in no driver being used. Normally, a defined feature ID will be used to associate a specific driver with the FPGA module.

                  // ***************************************************************************\n//                               INTEL CONFIDENTIAL\n//\n//        Copyright (C) 2021 Intel Corporation All Rights Reserved.\n//\n// The source code contained or described herein and all  documents related to\n// the  source  code  (\"Material\")  are  owned  by  Intel  Corporation  or its\n// suppliers  or  licensors.    Title  to  the  Material  remains  with  Intel\n// Corporation or  its suppliers  and licensors.  The Material  contains trade\n// secrets  and  proprietary  and  confidential  information  of  Intel or its\n// suppliers and licensors.  The Material is protected  by worldwide copyright\n// and trade secret laws and treaty provisions. No part of the Material may be\n// used,   copied,   reproduced,   modified,   published,   uploaded,  posted,\n// transmitted,  distributed,  or  disclosed  in any way without Intel's prior\n// express written permission.\n//\n// No license under any patent,  copyright, trade secret or other intellectual\n// property  right  is  granted  to  or  conferred  upon  you by disclosure or\n// delivery  of  the  Materials, either expressly, by implication, inducement,\n// estoppel or otherwise.  Any license under such intellectual property rights\n// must be express and approved by Intel in writing.\n//\n// You will not, and will not allow any third party to modify, adapt, enhance, \n// disassemble, decompile, reverse engineer, change or create derivative works \n// from the Software except and only to the extent as specifically required by \n// mandatory applicable laws or any applicable third party license terms \n// accompanying the Software.\n//\n// -----------------------------------------------------------------------------\n// Engineer     : \n// Create Date  : Nov 2021\n// Module Name  : hello_fim_top.sv\n// Project      : OFS\n// -----------------------------------------------------------------------------\n//\n// Description: \n// This is a simple module that implements DFH registers and \n// AVMM address decoding logic.\n\n\nmodule hello_fim_top  #(\n   parameter ADDR_WIDTH  = 12, \n   parameter DATA_WIDTH = 64, \n   parameter bit [11:0] FEAT_ID = 12'hfff,\n   parameter bit [3:0]  FEAT_VER = 4'h0,\n   parameter bit [23:0] NEXT_DFH_OFFSET = 24'h04_0000,\n   parameter bit END_OF_LIST = 1'b0\n)(\n   input  logic    clk,\n   input  logic    rst_n,\n// -----------------------------------------------------------\n//  AXI4LITE Interface\n// -----------------------------------------------------------\n   ofs_fim_axi_lite_if.slave   csr_lite_if\n);\n\nimport ofs_fim_cfg_pkg::*;\nimport ofs_csr_pkg::*;\n\n//-------------------------------------\n// Signals\n//-------------------------------------\n   logic [ADDR_WIDTH-1:0]              csr_waddr;\n   logic [DATA_WIDTH-1:0]              csr_wdata;\n   logic [DATA_WIDTH/8-1:0]            csr_wstrb;\n   logic                               csr_write;\n   logic                               csr_slv_wready;\n   csr_access_type_t                   csr_write_type;\n\n   logic [ADDR_WIDTH-1:0]              csr_raddr;\n   logic                               csr_read;\n   logic                               csr_read_32b;\n   logic [DATA_WIDTH-1:0]              csr_readdata;\n   logic                               csr_readdata_valid;\n   logic [ADDR_WIDTH-1:0]              csr_addr;\n\n   logic [63:0]                        com_csr_writedata;\n   logic                               com_csr_read;\n   logic                               com_csr_write;\n   logic [63:0]                        com_csr_readdata;\n   logic                               com_csr_readdatavalid;\n   logic [5:0]                         com_csr_address;\n\n// AXI-M CSR interfaces\nofs_fim_axi_mmio_if #(\n   .AWID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .AWADDR_WIDTH (ADDR_WIDTH),\n   .WDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH),\n   .ARID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .ARADDR_WIDTH (ADDR_WIDTH),\n   .RDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH)\n) csr_if();\n\n// AXI4-lite to AXI-M adapter\naxi_lite2mmio axi_lite2mmio (\n   .clk       (clk),\n   .rst_n     (rst_n),\n   .lite_if   (csr_lite_if),\n   .mmio_if   (csr_if)\n);\n\n//---------------------------------\n// Map AXI write/read request to CSR write/read,\n// and send the write/read response back\n//---------------------------------\nofs_fim_axi_csr_slave #(\n   .ADDR_WIDTH (ADDR_WIDTH),\n   .USE_SLV_READY (1'b1)\n\n   ) csr_slave (\n   .csr_if             (csr_if),\n\n   .csr_write          (csr_write),\n   .csr_waddr          (csr_waddr),\n   .csr_write_type     (csr_write_type),\n   .csr_wdata          (csr_wdata),\n   .csr_wstrb          (csr_wstrb),\n   .csr_slv_wready     (csr_slv_wready),\n   .csr_read           (csr_read),\n   .csr_raddr          (csr_raddr),\n   .csr_read_32b       (csr_read_32b),\n   .csr_readdata       (csr_readdata),\n   .csr_readdata_valid (csr_readdata_valid)\n);\n\n// Address mapping\nassign csr_addr             = csr_write ? csr_waddr : csr_raddr;\nassign com_csr_address      = csr_addr[5:0];  // byte address\nassign csr_slv_wready       = 1'b1 ;\n// Write data mapping\nassign com_csr_writedata    = csr_wdata;\n\n// Read-Write mapping\nalways_comb\nbegin\n   com_csr_read             = 1'b0;\n   com_csr_write            = 1'b0;\n   casez (csr_addr[11:6])\n      6'h00 : begin // Common CSR\n         com_csr_read       = csr_read;\n         com_csr_write      = csr_write;\n      end   \n      default: begin\n         com_csr_read       = 1'b0;\n         com_csr_write      = 1'b0;\n      end\n   endcase\nend\n\n// Read data mapping\n   always_comb begin\n      if (com_csr_readdatavalid) begin\n         csr_readdata       = com_csr_readdata;\n         csr_readdata_valid = 1'b1;\n      end\n      else begin\n         csr_readdata       = '0;\n         csr_readdata_valid = 1'b0;\n      end\n   end\n\n   hello_fim_com  #(\n   .FEAT_ID          (FEAT_ID),\n   .FEAT_VER         (FEAT_VER),\n   .NEXT_DFH_OFFSET  (NEXT_DFH_OFFSET),\n   .END_OF_LIST      (END_OF_LIST)\n) hello_fim_com_inst (\n   .clk                   (clk                     ),\n   .rst_n                 (rst_n                  ),\n   .writedata             (com_csr_writedata       ),\n   .read                  (com_csr_read            ),\n   .write                 (com_csr_write           ),\n   .byteenable            (4'hF                    ),\n   .readdata              (com_csr_readdata        ),\n   .readdatavalid         (com_csr_readdatavalid   ),\n   .address               (com_csr_address         )\n   );\n\n\nendmodule\n
                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#514-srchello_fimhello_fim_comsv","title":"5.1.4. src/hello_fim/hello_fim_com.sv","text":"

                  Create hello_fim_com.sv, and store it in src/hello_fim directory. This is the simple RTL to implement the Hello FIM registers. You may use this set of registers as the basis for your custom implementation.

                  // ***************************************************************************\n//                               INTEL CONFIDENTIAL\n//\n//        Copyright (C) 2021 Intel Corporation All Rights Reserved.\n//\n// The source code contained or described herein and all  documents related to\n// the  source  code  (\"Material\")  are  owned  by  Intel  Corporation  or its\n// suppliers  or  licensors.    Title  to  the  Material  remains  with  Intel\n// Corporation or  its suppliers  and licensors.  The Material  contains trade\n// secrets  and  proprietary  and  confidential  information  of  Intel or its\n// suppliers and licensors.  The Material is protected  by worldwide copyright\n// and trade secret laws and treaty provisions. No part of the Material may be\n// used,   copied,   reproduced,   modified,   published,   uploaded,  posted,\n// transmitted,  distributed,  or  disclosed  in any way without Intel's prior\n// express written permission.\n//\n// No license under any patent,  copyright, trade secret or other intellectual\n// property  right  is  granted  to  or  conferred  upon  you by disclosure or\n// delivery  of  the  Materials, either expressly, by implication, inducement,\n// estoppel or otherwise.  Any license under such intellectual property rights\n// must be express and approved by Intel in writing.\n//\n// You will not, and will not allow any third party to modify, adapt, enhance, \n// disassemble, decompile, reverse engineer, change or create derivative works \n// from the Software except and only to the extent as specifically required by \n// mandatory applicable laws or any applicable third party license terms \n// accompanying the Software.\n//\n// -----------------------------------------------------------------------------\n// Engineer     : \n// Create Date  : Nov 2021\n// Module Name  : hello_fim_com.sv\n// Project      : IOFS\n// -----------------------------------------------------------------------------\n//\n// Description: \n// This is a simple module that implements DFH registers and \n// AVMM address decoding logic.\n\n\nmodule hello_fim_com #(\n    parameter bit [11:0] FEAT_ID = 12'h001,\n    parameter bit [3:0]  FEAT_VER = 4'h1,\n    parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n    parameter bit END_OF_LIST = 1'b0\n )(\n input clk,\n input rst_n,\n input [63:0] writedata,\n input read,\ninput write,\ninput [3:0] byteenable,\noutput reg [63:0] readdata,\noutput reg readdatavalid,\ninput [5:0] address\n);\n\n\nreg [63:0] rdata_comb;\nreg [63:0] scratch_reg;\n\nalways @(posedge clk)  \n   if (!rst_n) readdata[63:0] <= 64'h0; else readdata[63:0] <= rdata_comb[63:0];\n\nalways @(posedge clk)\n   if (!rst_n) readdatavalid <= 1'b0; else readdatavalid <= read;\n\nwire wr = write;\nwire re = read;\nwire [5:0] addr = address[5:0];\nwire [63:0] din  = writedata [63:0];\nwire wr_scratch_reg = wr & (addr[5:0]  == 6'h30)? byteenable[0]:1'b0;\n\n// 64 bit scratch register\nalways @( posedge clk)\n   if (!rst_n)  begin\n      scratch_reg <= 64'h0;\n   end\n   else begin\n   if (wr_scratch_reg) begin \n      scratch_reg <=  din;  \n   end\nend\n\nalways @ (*)\nbegin\nrdata_comb = 64'h0000000000000000;\n   if(re) begin\n      case (addr)  \n    6'h00 : begin\n        rdata_comb [11:0]   = FEAT_ID ;  // dfh_feature_id  is reserved or a constant value, a read access gives the reset value\n        rdata_comb [15:12]  = FEAT_VER ;  // dfh_feature_rev    is reserved or a constant value, a read access gives the reset value\n        rdata_comb [39:16]  = NEXT_DFH_OFFSET ;  // dfh_dfh_ofst is reserved or a constant value, a read access gives the reset value\n        rdata_comb [40]     = END_OF_LIST ;        //dfh_end_of_list\n        rdata_comb [59:40]  = 20'h00000 ;  // dfh_rsvd1     is reserved or a constant value, a read access gives the reset value\n        rdata_comb [63:60]  = 4'h3 ;  // dfh_feat_type  is reserved or a constant value, a read access gives the reset value\n    end\n    6'h30 : begin\n        rdata_comb [63:0]   = scratch_reg; \n    end\n        6'h38 : begin\n                rdata_comb [63:0]       = 64'h6626_0701_5000_0034;\n        end\n    default : begin\n        rdata_comb = 64'h0000000000000000;\n    end\n      endcase\n   end\nend\n\nendmodule\n
                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#515-unit-level-simulations","title":"5.1.5. Unit Level Simulations","text":"

                  To run a unit level simulation test for the updated RTL files, make modifications to your cloned /my_ofs_project/ofs-d5005/sim/d5005/unit_test/dfh_walker files. The following simulation files are updated to test the new hello_fim.

                  1. Edit sim/unit_test/dfh_walker/testbench/test_csr_defs.sv

                    1. Update enum line 38
                         typedef enum {\n      FME_DFH_IDX,\n      THERM_MNGM_DFH_IDX,\n      GLBL_PERF_DFH_IDX,\n      GLBL_ERROR_DFH_IDX,\n      SPI_DFH_IDX,\n      PCIE_DFH_IDX,\n      HSSI_DFH_IDX,\n      EMIF_DFH_IDX,\n      HELLO_FIM_DFH_IDX,//New for HELLO_FIM\n      FME_PR_DFH_IDX,\n      PORT_DFH_IDX,\n      USER_CLOCK_DFH_IDX,\n      PORT_STP_DFH_IDX,\n      AFU_INTF_DFH_IDX,\n      MAX_FME_DFH_IDX\n   } t_fme_dfh_idx;\n

                  1. Edit function dfh_name line 78

                  function automatic dfh_name[MAX_FME_DFH_IDX-1:0] get_fme_dfh_names();\n      dfh_name[MAX_FME_DFH_IDX-1:0] fme_dfh_names;\n\n      fme_dfh_names[FME_DFH_IDX]         = \"FME_DFH\";\n      fme_dfh_names[THERM_MNGM_DFH_IDX]  = \"THERM_MNGM_DFH\";\n      fme_dfh_names[GLBL_PERF_DFH_IDX]   = \"GLBL_PERF_DFH\";\n      fme_dfh_names[GLBL_ERROR_DFH_IDX]  = \"GLBL_ERROR_DFH\";\n      fme_dfh_names[SPI_DFH_IDX]         = \"SPI_DFH\";\n      fme_dfh_names[PCIE_DFH_IDX]        = \"PCIE_DFH\";\n      fme_dfh_names[HSSI_DFH_IDX]        = \"HSSI_DFH\";\n      fme_dfh_names[EMIF_DFH_IDX]        = \"EMIF_DFH\";\n      fme_dfh_names[HELLO_FIM_DFH_IDX]        = \"HELLO_FIM_DFH\";//New for HELLO_FIM\n      fme_dfh_names[FME_PR_DFH_IDX]      = \"FME_PR_DFH\";\n      fme_dfh_names[PORT_DFH_IDX]        = \"PORT_DFH\";\n      fme_dfh_names[USER_CLOCK_DFH_IDX]  = \"USER_CLOCK_DFH\";\n      fme_dfh_names[PORT_STP_DFH_IDX]    = \"PORT_STP_DFH\";\n      fme_dfh_names[AFU_INTF_DFH_IDX]    = \"AFU_INTF_DFH\";\n\n      return fme_dfh_names;\n   endfunction\n

                  1. Update get_fme_dfh_values

                    function automatic [MAX_FME_DFH_IDX-1:0][63:0] get_fme_dfh_values();\n      logic[MAX_FME_DFH_IDX-1:0][63:0] fme_dfh_values;\n\n      fme_dfh_values[FME_DFH_IDX]        = 64'h4000_0000_1000_0000;\n      fme_dfh_values[THERM_MNGM_DFH_IDX] = 64'h3_00000_002000_0001;\n      fme_dfh_values[GLBL_PERF_DFH_IDX]  = 64'h3_00000_001000_0007;\n      fme_dfh_values[GLBL_ERROR_DFH_IDX] = 64'h3_00000_00C000_1004;  \n      fme_dfh_values[SPI_DFH_IDX]        = 64'h3_00000_010000_000e;  \n      fme_dfh_values[PCIE_DFH_IDX]       = 64'h3_00000_010000_0020;  \n      fme_dfh_values[HSSI_DFH_IDX]       = 64'h3_00000_010000_100f;  \n      fme_dfh_values[EMIF_DFH_IDX]       = 64'h3_00000_010000_0009; //Update to link to Hello_FIM \n      fme_dfh_values[HELLO_FIM_DFH_IDX]  = 64'h3_00000_040000_0FFF;  //New for Hello_FIM\n      fme_dfh_values[FME_PR_DFH_IDX]     = 64'h3_00000_001000_1005;  \n      fme_dfh_values[PORT_DFH_IDX]       = 64'h4000_0000_1000_1001;\n      fme_dfh_values[USER_CLOCK_DFH_IDX] = 64'h3_00000_001000_0014;\n      fme_dfh_values[PORT_STP_DFH_IDX]   = 64'h3_00000_00D000_2013;\n      fme_dfh_values[AFU_INTF_DFH_IDX]   = 64'h3_00001_000000_2010; \n\n      return fme_dfh_values;\n   endfunction\n
                  1. Update verification/scripts/Makefile_VCS.mk to set macro for INCLUDE_HELLO_FIM starting at line 56 to add +define+INCLUDE_HELLO_FIM 
                    VLOG_OPT += +define+SIM_MODE +define+VCS_S10 +define+RP_MAX_TAGS=64 +define+INCLUDE_DDR4 +define+INCLUDE_SPI_BRIDGE +define+INCLUDE_USER_CLOCK +define+INCLUDE_HSSI +define+SIM_USE_PCIE_DUMMY_CSR +define+INCLUDE_HELLO_FIM\n
                  2. Update sim/scripts/rtl_comb.f to add the path to your new hello_fim_top and hello_top_com SystemVerilog files. The update is shown below as the new line - 329 below:
                  $WORKDIR/src/hello_fim/hello_fim_com.sv\n$WORKDIR/src/hello_fim/hello_fim_top.sv\n

                  After making these changes, run the unit level simulation using sim/unit_test/dfh_walker test. Before running, ensure your shell has the environment variables set properly as defined in Setting Up Required Environment Variables.

                  cd verification/scripts\ngmake -f Makefile_VCS.mk cmplib\ngmake -f Makefile_VCS.mk build run [DUMP=1]\n

                  Expected output: 

                   ********************************************\n Running TEST(0) : test_fme_dfh_walking\n********************************************\nREAD64: address=0x00000000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x4000000010000000\n\nFME_DFH\n   Address   (0x0)\n   DFH value (0x4000000010000000)\nREAD64: address=0x00001000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000020000001\n\nTHERM_MNGM_DFH\n   Address   (0x1000)\n   DFH value (0x3000000020000001)\nREAD64: address=0x00003000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000010000007\n\nGLBL_PERF_DFH\n   Address   (0x3000)\n   DFH value (0x3000000010000007)\nREAD64: address=0x00004000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x30000000c0001004\n\nGLBL_ERROR_DFH\n   Address   (0x4000)\n   DFH value (0x30000000c0001004)\nREAD64: address=0x00010000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x300000010000000e\n\nSPI_DFH\n   Address   (0x10000)\n   DFH value (0x300000010000000e)\nREAD64: address=0x00020000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000100000020\n\nPCIE_DFH\n   Address   (0x20000)\n   DFH value (0x3000000100000020)\nREAD64: address=0x00030000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x300000010000100f\n\nHSSI_DFH\n   Address   (0x30000)\n   DFH value (0x300000010000100f)\nREAD64: address=0x00040000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000100000009\n\nEMIF_DFH\n   Address   (0x40000)\n   DFH value (0x3000000100000009)\nREAD64: address=0x00050000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000400000fff\n\nHELLO_FIM_DFH\n   Address   (0x50000)\n   DFH value (0x3000000400000fff)\nREAD64: address=0x00090000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000010001005\n\nFME_PR_DFH\n   Address   (0x90000)\n   DFH value (0x3000000010001005)\nREAD64: address=0x00091000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x4000000010001001\n\nPORT_DFH\n   Address   (0x91000)\n   DFH value (0x4000000010001001)\nREAD64: address=0x00092000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000010000014\n\nUSER_CLOCK_DFH\n   Address   (0x92000)\n   DFH value (0x3000000010000014)\nREAD64: address=0x00093000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x30000000d0002013\n\nPORT_STP_DFH\n   Address   (0x93000)\n   DFH value (0x30000000d0002013)\nREAD64: address=0x000a0000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000010000002010\n\nAFU_INTF_DFH\n   Address   (0xa0000)\n   DFH value (0x3000010000002010)\nMMIO error count matches: x\n\nTest status: OK\n\n********************\n  Test summary\n********************\n   test_fme_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n

                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#516-synsyn_topd5005qsf","title":"5.1.6. syn/syn_top/d5005.qsf","text":"

                  1. Edit syn/syn_top/d5005.qsf

                    1. Add new macro \"INCLUDE_HELLO_FIM\" line 107 

                              set_global_assignment -name VERILOG_MACRO \"INCLUDE_HELLO_FIM\"\n

                    2. Add new line 211 to source TCL script with new hello_fim files 

                              set_global_assignment -name SOURCE_TCL_SCRIPT_FILE ../../../syn/setup/hello_fim_design_files.tcl\n

                  Create \"hello_fim_design_files.tcl\" file and store in the syn/setup directory. This tcl file is called from d5005.qsf.

                  # Copyright 2021 Intel Corporation.\n#\n# THIS SOFTWARE MAY CONTAIN PREPRODUCTION CODE AND IS PROVIDED BY THE\n# COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED\n# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE\n# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR\n# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\n# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR\n# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,\n# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE\n# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,\n# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# Hello FIM Files\n#--------------------\nset_global_assignment -name SYSTEMVERILOG_FILE src/hello_fim/hello_fim_com.sv\nset_global_assignment -name SYSTEMVERILOG_FILE src/hello_fim/hello_fim_top.sv\n
                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#517-synsetuphello_fim_design_filestcl","title":"5.1.7. syn/setup/hello_fim_design_files.tcl","text":""},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#518-build-hello_fim-example","title":"5.1.8. Build hello_fim example","text":"

                  With the preceding changes complete, build the new hello_fim example using the following steps:

                  cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh d5005 work_d5005_hello_fim\n

                  Verify the design successfully compiled and timing closure is achieved by checking work_d5005_hello_fim/syn/syn_top/output_files/timing_report/clocks.sta.fail.summary - this file should be empty. If there are timing failures, then this file will list the failing clock domain(s).

                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#519-test-the-hello_fim-on-a-d5005","title":"5.1.9. Test the hello_fim on a D5005","text":"

                  Load the built FPGA binary file using an unsigned image. The FPGA image will be in work_d5005_hello_fim/syn/syn_top/output_files/d5005_page1_unsigned.bin

                  Provide the file d5005_page1_unsigned.bin on the server with the Intel\u00ae FPGA PAC D5005.

                  sudo fpgasupdate d5005_page1_unsigned.bin <D5005 PCIe B:D.F>\nsudo rsu bmcimg <D5005 PCIe B:D.F>\n
                  Verify FPGA image is loaded. 
                  sudo fpgainfo fme\n## Output\nIntel FPGA Programmable Acceleration Card D5005\nBoard Management Controller, MAX10 NIOS FW version: 2.0.13 \nBoard Management Controller, MAX10 Build version: 2.0.8 \n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x40100022C164DB1\nBitstream Version                : 4.0.1\nPr Interface Id                  : 7d91e0d0-4dcd-58c3-a93d-b9295e6e29b0\nBoot Page                        : user\n

                  Use the OPAE SDK tool opae.io to check default driver binding using your card under test PCIe B:D.F. The steps below will use 0000:12:00.0 as the card under test PCIe B:D.F.

                   sudo opae.io init -d 0000:12:00.0 $USER\n ##Output\n [0000:12:00.0] (0x8086, 0xbcce) Intel D5005 ADP (Driver: dfl-pci)\n
                  The dfl-pci driver is used by OPAE SDK fpgainfo commands. The next steps will bind the card under test to the vfio driver to enable access to the registers.

                   sudo opae.io init -d 0000:12:00.0 $USER\n ##Output\n opae.io 0.2.3\nUnbinding (0x8086,0xbcce) at 0000:12:00.0 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:12:00.0 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:12:00.0 is 35\nAssigning /dev/vfio/35 to $USER\n
                  Confirm the vfio driver is bound to the card under test.

                  opae.io ls\n## Output\nopae.io 0.2.3\n[0000:12:00.0] (0x8086, 0xbcce) Intel D5005 ADP (Driver: vfio-pci)\n
                  Run the following command to walk DFH link list. The new hello_fim register is located at offset 0x50000.

                  opae.io walk -d 0000:12:00.0\n## Output\nopae.io 0.2.3\noffset: 0x0000, value: 0x4000000010000000\n    dfh: id = 0x0, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x4\noffset: 0x1000, value: 0x3000000020000001\n    dfh: id = 0x1, rev = 0x0, next = 0x2000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x3000, value: 0x3000000010000007\n    dfh: id = 0x7, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x4000, value: 0x30000000c0001004\n    dfh: id = 0x4, rev = 0x1, next = 0xc000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x10000, value: 0x300000010000000e\n    dfh: id = 0xe, rev = 0x0, next = 0x10000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x20000, value: 0x3000000100000020\n    dfh: id = 0x20, rev = 0x0, next = 0x10000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x30000, value: 0x300000010000100f\n    dfh: id = 0xf, rev = 0x1, next = 0x10000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x40000, value: 0x3000000100000009\n    dfh: id = 0x9, rev = 0x0, next = 0x10000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x50000, value: 0x3000000400000fff\n    dfh: id = 0xfff, rev = 0x0, next = 0x40000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x90000, value: 0x3000000010001005\n    dfh: id = 0x5, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x91000, value: 0x4000000010001001\n    dfh: id = 0x1, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x4\noffset: 0x92000, value: 0x3000000010000014\n    dfh: id = 0x14, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x93000, value: 0x30000000d0002013\n    dfh: id = 0x13, rev = 0x2, next = 0xd000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0xa0000, value: 0x3000010000002010\n    dfh: id = 0x10, rev = 0x2, next = 0x0, eol = 0x1, reserved = 0x0, feature_type = 0x3\n
                  Read the default values from the hello_fim registers:

                  $ opae.io -d 0000:12:00.0 -r 0 peek 0x50000\nopae.io 0.2.3\n0x3000000400000fff\n$ opae.io -d 0000:12:00.0 -r 0 peek 0x50030\nopae.io 0.2.3\n0x0\n$ opae.io -d 0000:12:00.0 -r 0 peek 0x50038\nopae.io 0.2.3\n0x6626070150000034\n
                  Write the scratchpad register at 0x50030

                  $ opae.io -d 0000:12:00.0 -r 0 poke 0x50038 0x123456789abcdef\nopae.io 0.2.3\n$ opae.io -d 0000:12:00.0 -r 0 peek 0x50038\nopae.io 0.2.3\n0x6626070150000034\n$ opae.io -d 0000:12:00.0 -r 0 poke 0x50030 0x123456789abcdef\nopae.io 0.2.3\n$ opae.io -d 0000:12:00.0 -r 0 peek 0x50030\nopae.io 0.2.3\n0x123456789abcdef\n$ opae.io -d 0000:12:00.0 -r 0 poke 0x50030 0xfedcba9876543210\nopae.io 0.2.3\n$ opae.io -d 0000:12:00.0 -r 0 peek 0x50030\nopae.io 0.2.3\n0xfedcba9876543210\n$ opae.io -d 0000:12:00.0 -r 0 poke 0x50030 0x55550000aaaaffff\nopae.io 0.2.3\n$ opae.io -d 0000:12:00.0 -r 0 peek 0x50030\nopae.io 0.2.3\n0x55550000aaaaffff\n

                  Release the card under test from the vfio driver to re-bind to the dfl-pci driver:

                  sudo opae.io release -d 0000:12:00.0\n## Output\nopae.io 0.2.3\nReleasing (0x8086,0xbcce) at 0000:12:00.0 from vfio-pci\nRebinding (0x8086,0xbcce) at 0000:12:00.0 to dfl-pci\n$ sudo opae.io ls\nopae.io 0.2.3\n[0000:12:00.0] (0x8086, 0xbcce) Intel D5005 ADP (Driver: dfl-pci)\n
                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#52-memory-subsystem-modification","title":"5.2. Memory Subsystem Modification","text":"

                  OFS enables modifications on the different subsystems that encompass the FIM. To customize the Memory Subsystem follow these instructions.

                  1. Set up the environment variables as described in section 4.2.1. Setting Up Required Environment Variables

                  2. Modify the NUM_MEM_CH parameter in src/afu_top/mux/top_cfg_pkg.sv Change NUM_MEM_CH from 4 to 2 as shown in below code 

                  //=========================================================================================================================\n//                         OFS Configuration Parameters                                                                 \n//=========================================================================================================================\n     parameter NUM_MEM_CH     = 2                                                 ,// Number of Memory/DDR Channel         \n               NUM_HOST       = 1                                                 ,// Number of Host/Upstream Ports        \n               NUM_PORT       = 4                                                 ,// Number of Functions/Downstream Ports \n               DATA_WIDTH     = 512                                               ,// Data Width of Interface              \n               TOTAL_BAR_SIZE = 20                                                ,// Total Space for APF/BPF BARs (2^N) \n           //------------+-------------+-------------+-----------------+           //--------------------------------------\n           // VF Active  |     PF #    |     VF #    |  Mux Port Map   |           //  PF/VF Mapping Parameters            \n           //------------+-------------+-------------+-----------------+           //--------------------------------------\n             CFG_VA = 0  , CFG_PF = 0  , CFG_VF =  0 ,  CFG_PID = 3    ,           //  Configuration Register Block        \n             HLB_VA = 1  , HLB_PF = 0  , HLB_VF =  0 ,  HLB_PID = 0    ,           //  HE Loopback Engine                  \n             PRG_VA = 1  , PRG_PF = 0  , PRG_VF =  1 ,  PRG_PID = 1    ,           //  Partial Reconfiguration Gasket      \n             HSI_VA = 1  , HSI_PF = 0  , HSI_VF =  2 ,  HSI_PID = 2    ;           //  HSSI interface \n

                  Compile a new FIM that incorporates the newly configured Memory Subsystem. 

                  cd $OFS_BUILD_ROOT/ofs-d5005\nofs-common/scripts/common/syn/build_top.sh d5005 work_d5005_mem_2channel\n
                  ***********************************\n***\n***        OFS_PROJECT: d5005\n***        Q_PROJECT:  d5005\n***        Q_REVISION: d5005\n***        SEED: 03\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n

                  Program d5005_page1_unsigned.bin file using below command

                  sudo fpgasupdate d5005_page1_unsigned.bin 3b:00.0\n

                  Run rsu command 

                  sudo rsu bmcimg 3b:00.0\n

                  Check if binary was loaded correctly 

                  fpgainfo fme\n## Output\nIntel FPGA Programmable Acceleration Card D5005\nBoard Management Controller, MAX10 NIOS FW version: 2.0.14\nBoard Management Controller, MAX10 Build version: 2.0.8\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 288511863935352239\nBitstream Version                : 4.0.1\nPr Interface Id                  : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98\nBoot Page                        : user\n

                  Run Host Excersiser to check Memory Subsystem performance

                  sudo host_exerciser mem\n## Output\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5365\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.054 GB/s\n    Test mem(1): PASS\n

                  Verify Memory controller placement in syn/syn_top/output_files/d5005.fit.place.rpt file. Open fitter place stage report in any text editor of your choice, find keyword emif in the file. You should see emif[0] & emif[1] for Memory channel 0 & 1 respectively.

                  |emif[0].ddr4_pr_freeze_sync|                ; 0.4 (0.0)            ; 0.5 (0.0)                        ; 0.1 (0.0)                            ;\n|resync_chains[0].synchronizer_nocut|        ; 0.4 (0.4)            ; 0.5 (0.5)                        ; 0.1 (0.1)                            ;\n|emif[0].ddr4_softreset_sync|                ; 0.5 (0.0)            ; 0.7 (0.0)                        ; 0.2 (0.0)                            ;\n|resync_chains[0].synchronizer_nocut|        ; 0.5 (0.5)            ; 0.7 (0.7)                        ; 0.2 (0.2)                            ;\n|emif[0].pr_frz_afu_avmm_if|                 ; 647.5 (647.5)        ; 917.3 (917.3)                    ; 272.8 (272.8)                        ;\n|emif[1].ddr4_pr_freeze_sync|                ; 0.4 (0.0)            ; 0.8 (0.0)                        ; 0.4 (0.0)                            ;\n|resync_chains[0].synchronizer_nocut|        ; 0.4 (0.4)            ; 0.8 (0.8)                        ; 0.4 (0.4)                            ;\n|emif[1].ddr4_softreset_sync|                ; 0.4 (0.0)            ; 1.0 (0.0)                        ; 0.6 (0.0)                            ;\n|resync_chains[0].synchronizer_nocut|        ; 0.4 (0.4)            ; 1.0 (1.0)                        ; 0.6 (0.6)                            ;\n|emif[1].pr_frz_afu_avmm_if|                 ; 641.1 (641.1)        ; 914.0 (914.0)                    ; 272.9 (272.9)                        ;\n|p[0].pr_frz_fn2mx_a_port|                   ; 435.4 (0.0)          ; 476.2 (0.0)                      ; 40.8 (0.0)                           ;\n|r.axis_pl_stage[0].axis_reg_inst|           ; 435.4 (435.4)        ; 476.2 (476.2)                    ; 40.8 (40.8)                          ;\n|p[0].pr_frz_fn2mx_b_port|                   ; 434.6 (0.0)          ; 494.3 (0.0)                      ; 59.6 (0.0)                           ;\n
                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#6-conclusion","title":"6. Conclusion","text":"

                  Using the OFS reference design and OPAE SDK enables the rapid creation of market leading FPGA based Acceleration systems. OFS facilitates customization of the FIM area for your custom board or platforms.

                  "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#notices-disclaimers","title":"Notices & Disclaimers","text":"

                  Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Performance varies by use, configuration and other factors. Your costs and results may vary. You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications. Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards. \u00a9 Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.

                  OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122.

                  "},{"location":"hw/d5005/doc_modules/Glossary/","title":"Glossary","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/","title":"FPGA Interface Manager Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA","text":""},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1-overview","title":"1 Overview","text":""},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#11-about-this-document","title":"1.1 About this Document","text":"

                  This document describes the hardware architecture of the\u200b Open FPGA Stack (OFS) targeting the Intel\u00ae Stratix 10 FPGA. After reviewing this document you should understand the features and functions of the components that comprise the FPGA Interface Manager (FIM), also known as the \"shell.\"

                  Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#12-introduction-to-the-open-fpga-stack","title":"1.2 Introduction to the Open FPGA Stack","text":"

                  The Open FPGA Stack (OFS) is a modular collection of hardware platform components, open source upstreamed software, and broad ecosystem support that enables an efficient path to develop a custom FPGA platform. OFS Provides a framework of FPGA synthesizable code, simulation environment and synthesis/simulation scripts. The key components of OFS include: - Target development platforms such as Intel-branded Programmable Acceleration Cards (PACs), Acceleration Development Platforms (ADPs) and third-party platforms.

                  • Board Management Controller RTL and firmware that supports telemetry monitoring, remote configuration updates and most importantly a root of trust for the platform.
                  • Source accessible, modular FPGA Interface manager (FIM) RTL with unit tests that can be leveraged for your own custom FIM design
                  • Basic building blocks for interconnect and PF/VF translation and arbitration; Platform Interface Manager (PIM) which provides Avalon\u00ae bus compliant interfaces.
                  • AFU examples both in the git repository and workload examples provided by 3rd party vendors
                  • The OneAPI shim provides a layer that is used by the OneAPI runtime to communicate with the kernel.
                  • OPAE software development kit (APIs, upstreamed Linux drivers and software tools)
                  • Support for other frameworks to be built on top of the OPAE such as DPDK

                  The OFS hardware repository supports hardware development and simulation. Repositories for OFS high level design support and board management controller RTL and firmware source code are also provided. These repositories can be found in the Intel Opensource Technology GitHub location, which requires entitlement access. To request access, please contact your local Intel sales representative.

                  Table 1-2 OFS GitHub Repositories (https://github.com/OFS/)

                  Repository Contains ofs-fim-common Contains common modules shared by all OFS designs. This repository is a submodule of each platform repository. ofs-d5005 Contains FIM or shell RTL design, automated compilation scripts, unit tests.

                  The OPAE software GitHub site is fully opensource and contains resources for both software and workload developers.

                  Table 1-3 OPAE Public Git Repositories (https://github.com/OFS)

                  OPAE Git Repository Folder Contains linux-dfl Contains OFS Linux drivers that are being upstreamed to the Linux kernel. linux-dfl-backport Backport versions of the linux-dfl to older kernel versions. opae-sdk Contains the files for building and installing OPAE SDK from source. opae-sim Contains an AFU/Workload simulator for software/hardware co-simulation. examples-afu Contains simple AFU tutorials.

                  Providing the hardware and software source code and supporting test frameworks in a GitHub repository allows you to easily customize your own designs with the latest versions.

                  Most hardware and software ingredients are available in our OFS GitHub location. For access to the board management controller firmware and RTL or our security guide for OFS, please contact a local Intel sales representative.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#13-ofs-features","title":"1.3 OFS Features","text":"

                  The OFS architecture within the FPGA comprises two partitions:

                  • FPGA Interface Manager (FIM)
                  • Accelerator Functional Unit (AFU)

                  The FIM or shell provides platform management functionality, clocks, resets and interface access to the host and peripheral features of the acceleration platform. The FIM architecture along with the supporting OPAE software supports features such as partial reconfiguration and virtualization. The FIM provides a standard Arm* AMBA* 4 AXI4 datapath interface. The FIM resides in the static region of the FPGA.

                  The AFU partition is provided for custom acceleration workloads and may contain both static and partial reconfiguration regions.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#131-fpga-interface-manager-fim","title":"1.3.1 FPGA Interface Manager (FIM)","text":"

                  The updated OFS architecture for Intel\u00ae Stratix 10\u00ae FPGA devices improves upon the modularity, configurability and scalability of the first release of the OFS architecture while maintaining compatibility with the original design. The primary components of the FPGA Interface Manager or shell of the reference design are:

                  • PCIe Subsystem
                  • HSSI Subsystem
                  • Memory Subsystem
                  • Reset Controller
                  • FPGA Management Engine
                  • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                  • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                  • SPI Interface to BMC controller

                  The AFU Region provides design space for custom workloads and contains both static and partial reconfiguration regions. Partial reconfiguration allows you to update your specific logic blocks or entire workload while the rest of your static design is still in operation.

                  Note that as discussed previously, the BMC RTL and firmware, the OFS OPAE software stack and support for building your own customer board support package are also provided in separate OFS repositories.

                  Figure 1-2 OFS for Intel Stratix 10 Block Diagram

                  The table below details the features of the OFS release targeting the Intel\u00ae Stratix 10\u00ae FPGA .

                  Table 1-4 Features

                  Key Feature OFS Update Comments PCIe H-tile PCIe Gen3x16 Interface Integrates PCIe TLP adapter for new data mover packet format. MSI-X vector and PBA tables are located in the PCIe subsystem. Interrupts from FME as well as four user interrupts coming from PF0.VF1 are supported. Memory Two Avalon Memory Mapped channels provided as default with capability to compile design with four channels support. - HSSI 1 Arm* AMBA* 4 AXI4-Stream channel of 10G Ethernet, using the low latency Ethernet 10G MAC Intel FPGA IP interfacing to an E-tile PHY. - Manageability SPI interface to Board Management Controller targeting Intel FPGA PAC D5005 - CoreFIM Flexible configuration support using Arm* AMBA* 4 AXI4-Stream Physical Function/Virtual Function (PF/VF) Demux/Mux and AFU Peripheral Fabric (APF) and Board Peripheral (BPF) Fabric Interconnects. APF and BPF fabrics are Platform Designer generated IPs. The Arm* AMBA* 4 AXI4-Stream PF/VF Demux/Mux is a new component. Physical Function/Virtual 1 PF/3VF configuration is provided as an example but the architecture now supports full virtualization with the ability to expand to whatever the PCIe tile supports. - Partial Reconfiguration 1 Partial Reconfiguration region supported in hardware and software - Sample test PR AFUs Host exerciser modules provided to exercise interfaces. These modules are provided in both the flat and PR AFU examples. - OneAPI Yes Available Q1 2023 Software Support OFS software stack with support for full virtualization. -"},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#fpga-management-engine-fme","title":"FPGA Management Engine (FME)","text":"

                  The FIM contains only one FME, regardless of the number of host interfaces to the FIM. The FME provides management features for the platform and controls reset and loading of the AFU into the partial reconfiguration region of the FPGA.

                  Each FME feature exposes its capability to host software drivers through a device feature header (DFH) register found at the beginning of its control status register (CSR) space. The FME CSR maps to physical function 0 (PF0) Base address register 0 (BAR0) so that software can access it through a single PCIe link. For more information about DFHs, refer to the Device Feature Header (DFH) structure.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#streaming-datapath","title":"Streaming Datapath","text":"

                  The FIM implements an AXI4-Stream bus protocol for data transfer in the FIM. AXI4-Stream channels send data packets to and from the host channel IP without data abstraction. Memory-mapped I/O (MMIO) CSR accesses are routed to the ST2MM module which converts the AXI4-Stream to an AXI4 memory mapped protocol.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#virtualization","title":"Virtualization","text":"

                  This design supports virtualization by making use of the virtualization functionality in the PCIe Hard IP and mapping packets to the appropriate physical or virtual function through a PF/VF multiplexer. This reference FIM supports 1 PF and 3 VFs as an example; however, you may extend your configuration to whatever the PCIe Hard IP can support or what your application requires.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#132-afu","title":"1.3.2 AFU","text":"

                  An AFU is an acceleration workload that interfaces to the FIM. The AFU boundary in this design comprises both static and partial reconfiguration (PR) regions. You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region. A port gasket within the design provides all the PR specific modules and logic required partial reconfiguration. Only one partial reconfiguration region is supported in this design.

                  Similar to the FME, the port gasket exposes its capability to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space. The port is part of the FPGA Interface Unit (FIU) that resides in the FIM.

                  You can compile your design in one of the following ways: * Your entire AFU resides in a partial reconfiguration region of the FPGA * The AFU is part of the static region and is compiled a flat design

                  In this design, PF0.VF1 and PF0.VF2 map to host exerciser modules (HEM) that map to HE-LB and HE-HSSI respectively.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#133-platform-interface-manager","title":"1.3.3 Platform Interface Manager","text":"

                  The PIM provides a way to abstract the AXI4-Stream interface to the AFU by providing a library of shims that convert the host channel native packet into other protocols such as CCI-P, AXI4 memory-mapped, Avalon\u00ae streaming (Avalon-ST) or Avalon\u00ae memory-mapped (Avalon-MM). The FPGA or AFU developer implement these interface abstractions in the AFU region of the design.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#134-opae-sdk-fpga-platform-feature-discovery","title":"1.3.4 OPAE SDK FPGA Platform Feature Discovery","text":"

                  The OPAE C library in the OPAE software development kit is built on top of the OPAE Intel FPGA driver stack that abstracts the hardware and operating system specific details of the platform to the host. The FIM implements a DFH linked list to allow an FPGA platform driver running on the host to discover FME, port and AFU features. This model is similar to how PCIe enumeration occurs. You must implement a 64-bit DFH Device Feature Header register at the beginning (first 8B aligned address) of the feature CSR space for a new feature to be discovered or enumerated by a driver.

                  A driver starts the traversing by reading the DFH of the first feature from the first address on PF0 BAR0. Based on the information in the DFH, a driver can determine the CSR address range of the feature and other associated details of the feature. The end of the DFH contains a \"next DFH offset\" field that points the driver to the DFH of the next feature. The software must continue traversing the linked list until it sees the EOL (End-Of-List) bit set to 1 in the \"next DFH offset\" field it is inspecting. A 1 indicates this is the last feature in the feature set. Figure below gives a simple illustration of the feature discovery by traversing the DFH registers.

                  Figure 1-3 Device Feature Header Linked List Traversal

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#135-ofs-reference-design","title":"1.3.5 OFS Reference Design","text":"

                  OFS provides FIM designs you can use as a starting point for your own custom design. These designs target a specific programmable acceleration card or development kit and exercise key FPGA device interfaces. The Intel Stratix\u00ae 10 code line for OFS targets the Intel FPGA PAC D5005. FIM designs are released to OFS D5005 FIM Github Branch for evaluation and use.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#136-fim-simulation","title":"1.3.6 FIM Simulation","text":"

                  OFS provides a UVM environment for the FIM and a framework for new feature verification. UVM provides a modular, reusable, and scalable testbench structure by providing an API framework that can be deployed across multiple projects. The FIM testbench is UVM compliant and integrates third-party verification IPs from Synopsys that require license to use. Verification components include:

                  • FIM monitor to detect correct design behavior
                  • FIM assertions for signal level integrity testing
                  • Arm AMBA AXI4 scoreboards to check data integrity
                  • FIM coverage to collect functional data

                  The verification infrastructure can be in the verification folder here OFS D5005 FIM Github Branch for evaluation and use.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#2-ofs-high-level-architecture","title":"2 OFS High Level Architecture","text":"

                  OFS provides distinct datapaths that simplifies the design and integration process for add or for removing interface modules:

                  • High Bandwidth datapath for AFU-attached high performance peripherals (HSSI, Memory, HPS, workload).
                  • Low Bandwidth datapath for OFS management and slow peripheral components (JTAG, I2C, SMBus).
                  • AFU Peripheral Fabric (APF) to Board Peripheral Fabric (BPF) path to communicate with interface control and status registers (CSRs) and board components.
                  • Peer-to-peer datapath between AFU components.
                  • Peer-to-peer datapath between BPF components.

                  Depending on your design goals, you can present peripherals to software as:

                  • OFS managed peripherals with a device feature header that is part of a device feature list.
                  • Native driver managed peripherals that are exposed through an independent physical function or virtual function.

                  Figure 2-1 OFS Datapath Structure

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#3-pcie-interface","title":"3 PCIe Interface","text":"

                  The FIM's H-tile PCIe* hard IP is a Gen3x16 design. The IP supports SR-IOV and is configured to provide one PF and three VFs. Native PCIe TLP packets are sent through the PCIe using Arm AMBA 4 AXI-4 Stream Protocol. Before they reach the AFU, however, the packets go through an adapter that converts any headers to a data mover format that is forward compatible with Intel Agilex FPGA devices and beyond.

                  Figure 3-1 OFS FIM RX-TX Datapath

                  Some key features of the PCIe interface are:

                  Feature OFS for Intel Stratix 10 Configuration Mode PCIe Gen3x16 Port Mode Native Endpoint SR-IOV 1 PF, 3 VFs MSI-X Support Yes Functional Mode Data Mover Profile Virtual+ TLP Bypass No Header Packing Scheme Simple Data Width 512-bit (64-byte) PLD Clock Frequency 250 MHz Tags Supported 128 Reordering No reordering of requests, no completion reordering Maximum Payload Size 256 Bytes Memory Requests Supported 1CL, 2CL, 4CL MMIO transaction Size 4B, 8B"},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#31-receiver-datapath","title":"3.1 Receiver Datapath","text":"

                  The path of data received by the FIM is as follows:

                  1. The PCIe Hard IP receives TLP packets from the host. Host response types can be:

                    * MMIO read or response * Memory write request * Interrupt response * Memory read request or response

                  2. The PCIe IP routes the TLP packets to the PCIe bridge interface in the FIM, where they get buffered into the RX FIFO.

                  3. The TLP checker in the bridge examines the packets and filters erroneous packets including packets with unsupported requests or fields. Errors are sent to the error logger and are logged as advance error reporting (AER). Error status is sent to the PCIe hard IP and to the FIM error status registers. Appropriate action is taken for the errors as described in the Reliability, Accessibility, Serviceability (RAS) and Error Handling section. The TLP checker also maintains RX buffer credits for TLP completions with data (CplD) that are received from the host in response to a memory read request (MRd request sent by the AFU). The TLP checker notifies the PCIe TX bridge when there are not enough RX buffer credits available in the PCIe RX bridge. If there are not enough credits, the PCIe TX bridge pauses MRd requests from the AFU to Host until there is availability.

                  4. The TLP checker forwards packets that pass TLP checking to an AXI4 adapter which moves the packets into an Arm AMBA 4 AXI4-Stream bus.

                  5. AXI4-Stream packets are sent downstream to datamover AXI4-Stream adapter to be modified into the new datamover packet format.

                  6. Datamover packets are sent to the PF/VF Mux in the AFU.

                  7. The AXI4-Stream to Memory Mapped (ST2MM) module in the AFU region routes MMIO requests targeting FIM CSRs (FME, peripherals and AFU).

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#32-transmit-datapath","title":"3.2 Transmit Datapath","text":"

                  The Transmit (TX) datapath refers to all TLP packet types that originate in the FPGA. A single Arm AMBA 4 AXI4-Stream channel at the port interface boundary of the FIM carries PCIe TLP packets and interrupt requests upstream from the AFU.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#33-data-handshaking","title":"3.3 Data Handshaking","text":"

                  The Arm AMBA 4 AXI4 interfaces to the AFU use the VALID and READY signal for handshaking and backpressure management. The FIM holds the DATA and VALID asserted until the receiver asserts the READY signal. The AFU accepts the data when both the VALID and READY signals are asserted.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#34-arm-amba-4-axi4-stream-interface","title":"3.4 Arm AMBA 4 AXI4-Stream Interface","text":"

                  The table below shows the high-level signal mapping of the channels for the OFS for Intel Stratix 10 FPGA. If you have previously used the OFS EA architecture, that mapping is provided as a comparison as well in this table.

                  Table 3-1 AXI4-Stream RX Channel

                  AXI4-Stream Signal Source OFS Stratix 10 Mapping OFS Early Access Mapping ACLK Clock Source PCLK = 250 MHz PCLK = 250 MHz AResetn Reset Source System Reset System Reset TVALID Master Data Valid Data Valid TREADY Slave Ready Ready TDATA Master Width=512 bits When packet includes a header (32 bytes) and data the packing scheme is: 16B data mover header: 4B prefix PF NumberVM Number VF Active Slot Number Memory Mapped Number8B Address/Metadata Data: 32B If the packet is only data then all 64 bytes comprise data [TLP_CH] [8n] n=49 (392 bits)TLP_CH=2 (2TLP data streams)Mapping of each TLP data stream [391:136] payload (32-byte data payload) [135:8]: hdr (16 byte header) [7:3]: rsvd0 (reserved bits)[2]: end of packet (eop) [1]: start of packet (sop) [0]: valid (TLP packet on data stream is valid) TLAST Master Set to 1'b1 when end of packet is sent; otherwise TLAST is 1'b0 Set to 1'b1 when end of packet is sent; otherwise TLAST is 1'b0 TKEEP Byte Qualifier Signal indicates whether content of the associated byte is valid. Invalid bytes are allowed only during TLAST cycle. Valid bytes always start from Byte 0. Signal indicates whether content of the associated byte is valid. Invalid bytes are allowed only during TLAST cycle. Valid bytes always start from Byte 0. TUSER Master WIDTH = 10 Bit 0 is always equal to 1'b1 to indicate data mover header format [4:1] - Indicates header position on the TDATA bus and is always equal to 4'b0001 indicating that the header starts from Byte0.All other bits of TUSER are unused. [TLP_CH][u-1:0] u=21 Sideband of each TLP data stream[20]: ummio_rd (Unsupported MMIO request)[19:0]: destination routing ID, where:[19:17]= BAR offset[2:0][16:4]=VF number[12:0][3:1]=PF number[2:0][0]=vf_active, indicating if the virtual function feature is enabled

                  Figure 3-3 AXI4-Stream RX Request Cycle Header Format

                  All Host requests sent to the AFU are memory-mapped I/O requests. Of the fields below, the following are not supported in the design: * Prefix * Slot number (set to 0) * Local Address

                  Figure 3-4 AXI4-Stream RX Completion Header Format

                  All completions in the RX direction are data completions. Of the fields below, the following are not supported in the design: * Prefix * MM mode * Slot number (set to 0) * Meta Data

                  Note that: * VF Active, VF Num and PF Num are obtained from TUSER. * Data packet responses (for memory read requests from the AFU) from the PCIe may come out of order when the size is greater than 64 bytes.

                  Table 3-2 AXI4-Stream TX Channel

                  AXI4-Stream Signal Source OFS Stratix 10 Mapping OFS Early Access Mapping ACLK Clock Source PCLK = 250 MHz PCLK = 250 MHz AResetn Reset Source System Reset System Reset TVALID Master Data Valid Data Valid TREADY Slave Ready ReadyOnly 1 ready signal for the two channels TDATA Master Width=512 bits When packet includes a header (32 bytes) and data the packing scheme is: 16B data mover header: 4B prefix PF NumberVM Number VF Active Slot Number Memory Mapped Number8B Address/Metadata Data: 32B If the packet is only data then all 64 bytes comprise data [TLP_CH] [8n] n=49 (392 bits)TLP_CH=2 (2TLP data streams)Mapping of each TLP data stream [391:136] payload (32 byte data payload) [135:8]: hdr (16 byte header) [7:3]: rsvd0 (reserved bits)[2]: end of packet (eop) [1]: start of packet (sop) [0]: valid (TLP packet on data stream is valid) TLAST Master Per protocol Set to 1'b1 TKEEP Byte Qualifier Signal indicates whether content of the associated byte is valid. Invalid bytes are allowed only during TLAST cycle. Valid bytes always start from Byte 0. Signal indicates whether content of the associated byte is valid. Invalid bytes are allowed only during TLAST cycle. Valid bytes always start from Byte 0. TUSER Master WIDTH = 10 Bit 0 \u2013 Indicates Header Format: 0 \u2013 Power user mode header format 1 \u2013 Data mover header format Bit [4:1] - Indicates header position on the TDATA bus and is always equal to 4'b0001 indicating that the header starts from Byte0.All other bits of TUSER are unused. [TLP_CH][u-1:0] u=2Sideband of each TLP data stream[0]=vf_active, indicating if the virtual function feature is enabled[0]: afu_irq (AFU interrupt)

                  Table 3-3 Interrupt Response Channel

                  AXI4-Stream Signal Source Mapping ACLK Clock Source PCLK AResetn Reset Source System Reset TVALID Master Valid TREADY Slave Ready TDATA [8n-1:0] n=3 (24 bits) Master [23:16]: 8-bit interrupt ID [15:0]: Requester ID

                  Figure 3-5: AXI4-Stream TX Request Cycle Header Format

                  All requests in the TX direction are Memory Read/Write. The requester ID does not come from the AFU; the AXI-Stream adapter supplies it. The tag must come from the AFU. Of the fields below, the following are not used in the H-Tile PCIe subsystem design: * Prefix * MM Mode * Slot number (set to 0) * Local Address

                  Note that VF Active, VF Num and PF Num are obtained from the header packet.

                  Figure 3-4 AXI4-Stream TX Completion Header Format

                  All completions in the TX direction are for MMIO. Of the fields below, the following are not supported in the design: * Prefix * MM mode * Slot number (set to 0) * Meta Data

                  Note that: * VF Active, VF Num and PF Num are obtained from TUSER.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#4-platform-interface-manager","title":"4 Platform Interface Manager","text":"

                  The FIM interfaces to an AFU through AXI4-Stream channels. This format allows the AFU to access the host channel's raw interface without any translation. As a FIM developer, you have the option to provide the raw data format associated with the host interface channel to the workload or AFU developer or you can provide an intermediate protocol using Platform Interface Manager Components or your own custom interface. If you expose the raw AXI4-Stream interface of the FIM, workload developers also have the option to convert to a desired protocol using the PIM resources as well.

                  Refer to https://github.com/OPAE/ofs-platform-afu-bbb for more information on options for implementing the PIM.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#5-afu-interface-handler","title":"5 AFU Interface Handler","text":"

                  The AFU Interface Handler resides inline between the PCIe AXI4-Stream Adapter and the AXI4-Stream PF/VF Demux/Mux logic. Its main function is to provide: * Unique PCIe tags \u2013 Each PCIe transaction shares the 128 tags across all VFs in the AFU region * AFU error logging for all VFs in the AFU region

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#51-afu-error-handling","title":"5.1 AFU Error Handling","text":"

                  In this OFS design, the AFU Interface Handler handles error logging for all VFs in the AFU. Errors handled are as follows

                  Checker Field Description AFU protocol checker (PCIe TLP) TxReqCounterOverflow Pending memory write or memory read requests exceed the predefined limit TxFifoOverflow Tx FIFO in the port that buffers TLP packets from AFU is overflow Malformed TLP AFU PCIe TLP contains unsupported format type MaxPayloadError AFU memory write payload size exceeds max_payload_length limit MaxReadReqSizeError AFU memory read payload size exceeds max_read_request_size limit MaxTagError AFU memory read request tag value exceeds the maximum supported tag count TagOccupiedErr AFU sends out memory read request using a tag that is already used for a pending memory read request UnalignedAddrErr The address field in AFU memory write/read request TLP is not DW-aligned. UnexpMMIOResp AFU is sending a MMIO read response with no matching MMIO read request. MMIOTimedOutAFU is not responding to a MMIO read request within the pre-defined response timeout period. MMIODataPayloadOverrunThe number of data payload sent by AFU for a MMIO response (cplD) is more than the data length specified in the response. MMIOInsufficientDataThe number of data payload sent by AFU for a MMIO response (cplD) is less than the data length specified in the response. TxMWrDataPayloadOverrun The number of data payload sent by AFU for a memory write request is more than the data length specified in the request. TxMWrInsufficientData The number of data payload sent by AFU for a memory write request is less than the data length specified in the request. AFU Protocol Checker (AXI4-Stream)TxValidViolationThree checkers are implemented in the FIM to catch errors and protocol violations.

                  To view the CSR space for the AFU interface handle, go to the src/afu_top/AFU_INTF_CSR.xls file OFS D5005 FIM Github Branch.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#6-interconnect-fabric","title":"6 Interconnect Fabric","text":"

                  There are three types of interconnect fabric in the OFS FIM design: * AXI4-Stream mux/demux fabric * AFU Periheral Fabric (APF) * Board Peripheral Fabric (BPF)

                  Figure 6-1 Interonnect Fabric Diagram

                  TLP packets sent from upstream PCIe Subsystem on AXI4-Stream channel are demultiplexed in the AXI4-Stream PF/VF mux/demux fabric and routed to the respective PF/VF function based on the PF/VF information in the TLP header, such as vf_active or the PF/VF number. On the opposite direction, TLP packets from downstream PF/VF function are muxed in the fabric and sent to PCIe subsystem over AXI4-Stream channel.

                  All host MMIO requests targeting PF0 BAR0 are routed to the ST2MM module. The ST2MM converts MMIO TLP packets into AXI-Lite memory requests and places the requests onto AFU Peripheral Fabric (APF). AFU peripherals, such as OFS managed AFU features and ST2MM) and Board Peripheral Fabric (BPF) are interconnected by APF. The BPF is the interconnect fabric one hiearchy below APF which connects all the board peripherals. Both APF and BPF allow multiple AXI4-Lite master and slave interconnect topology.

                  The following table summarizes the mechanism for configuring PF/VF functions:

                  Table 6-1 Interconnect Configuration Methods

                  InterconnectConfiguration Mechanism APF or BPFEither:Use Platform Designer (PD) to generate the fabrics directly, or Specify desired cfg in iofs_dfl.txt and run dfh2tcl.pl script to generate the necessary HW TCL scripts as needed by PD. This PERL script also takes care of invoking PD in script mode to generate the end result of RTL design Verilog files. This is the preferred method for generation. AXI-S PF/VF Demux/MuxUpdate parameters in these RTL files: * src/includes/top_cfg_pkg.sv * src/common/pf_vf_mux.sv Then make the corresponding update to AFU top level instantiation and connections: * src/FIMs/.../afu_top.sv

                  Note:

                  In all cases, you must configure and regenerate the PCIe IP to match the new PF/VF configuration if it deviates from the reference FIM provided.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#61-fabric-generation-flow","title":"6.1 Fabric Generation Flow","text":"

                  You have two options to generate APF/BPF IPs.

                  • Use Quartus Platform Designer. This method requires familiarity with Quartus Platform Designer. With this option, you manually enter each address space and define the associated master and slave interface in a table provided by the Platform Designer. The parameters and attributes such as data width, address width, number of outstanding cycles, \u2026 etc. are also set to the desired values. After this is completed, you then connect each master and slave interface in the table. Platform Designer then generates the RTL files according to the table. For more details, please refer to the Intel Quartus Prime Pro Edition User Guide.
                  • Use the APF/BPF script that reads in iofs_dfl.txt and automatically generates APF/BPF IPs. Both APF and BPF are generated from Platform Designer using hardware TCL scripts. To provide a more user friendly experience to OFS Rel 1 and AC ADP customers, a perl script dfh2tcl.pl has been developed to provide a higher level of abstraction. The below figure illustrates the high level flow:

                  Figure 6-2 APF/BPF Generation

                  Note that the only input required is the iofs_dfl.txt text file which allows you to specify how many ports, which fabric, type of AXI4-Lite port (master, slave, or both), port addresses and sizes. Using iofs_dfl.txt and dfh2tcl.pl to generate the APF and BPF is the preferred method.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#62-iofs_dfltxt-format","title":"6.2 iofs_dfl.txt Format","text":"

                  The following table describes the format of the iofs_dfl.txt input file:

                  Table 6-2 iofs_dfl.txt Format Types

                  ColumnDescription REGISTERName of the port on the APF/BPF fabrics. Note that each entry must be unique. FABRICAllows the user to specify which fabric this port is connected to, and also the type of AXI4-Lite port (master, slave, or both). The format is FABRIC-PORT_TYPE, where: * FABRIC = APF or PBF * PORT_TYPE = MST, SLV, or BID (Master, Slave, or Bi-Directional) BASE_ADDRSpecifies the address offset of the AXI4-Lite port. BAR_SIZE=2^N Specifies the size of the port. For simplicity reasons, all AXI4-Lite ports are configured as 64KB. i.e. 2^16

                  Additional AXI4-Lite ports can be easily created by adding more rows in the iofs_dfl.txt file.

                  Note that there are several reserved ports in both APF and BPF so it might not be necessary for you to regenerate the fabrics if the design does not exceed the number of ports as implemented in the APF/BPF in the reference FIMs.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#63-afu-peripheral-fabric-apf","title":"6.3 AFU Peripheral Fabric (APF)","text":"

                  The AFU Peripheral Fabric (APF) is a 64-bit AXI4-lite compliant interconnect fabric that connects AFU peripheral modules to board peripheral modules through the Board Peripheral Fabric (BPF). The fabric is clocked by clk_csr and has a read allowance and write allowance of 1, i.e. only 1 active write/read is allowed in the fabric at any single time.

                  The address mapping for components interconnected by the APF is listed below. All components are mapped to PF0 BAR0 and implement AXI-lite slave interface. The Master column indicates if a component also implements AXI4-lite master interface which can send request to APF.

                  Table 6-3 APF Address Mapping

                  AddressSize (Byte)FeatureMaster 0x00000 \u2013 0x7FFFF512KBoard Peripherals (See BPF address mapping) No AFU Peripherals 0x80000 \u2013 0x8FFFF64KST2MMYes (Send MMIO request to all the peripherals) 0x90000 \u2013 0x9FFFF64KPort GasketYes 4KPR Control & Status 4KUser clock 16KRemote STP 0xA0000 \u2013 0xAFFFF64KAFU Interface HandlerNo 0xB0000 \u2013 0xBFFFF64KRSV_b_DFHAvailable for customer useMay be programmed as master if used. By default this is a reserved base address. 0xC0000 \u2013 0xCFFFF64KRSV_c_DFHAvailable for customer useMay be programmed as master if used. By default this is a reserved base address. 0xD0000 \u2013 0xDFFFF64KRSV_d_DFHAvailable for customer useMay be programmed as master if used. By default this is a reserved base address. 0xE0000 \u2013 0xEFFFF64KRSV_e_DFHAvailable for customer useMay be programmed as master if used. By default this is a reserved base address. 0xF0000 \u2013 0xFFFFF64KRSV_f_DFHAvailable for customer useMay be programmed as master if used. By default this is a reserved base address.

                  The five reserved regions have associated ports available in the APF for customer use.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#64-board-peripheral-fabric-bpf","title":"6.4 Board Peripheral Fabric (BPF)","text":"

                  The Board Peripheral Fabric is the 64-bit AXI4-Lite compliant interconnect fabric that connects board peripheral modules to APF. The fabric is clocked by clk_csr and has a read allowance and write allowance of 1, i.e. only 1 active write/read is allowed in the fabric at any single time.

                  The address mapping for components interconnected by BPF is listed below. All components are mapped to PF0 BAR0 and implement AXI4-lite slave interface. The Master column indicates if a component also implements AXI4-lite master interface which can send request to BPF.

                  Table 6-4 BPF Address Mapping

                  AddressSize (Byte)FeatureMaster 0x00000 \u2013 0x0FFFF64KFME (FME, Error, etc)Yes 0x10000 \u2013 0x1FFFF64KSPI ControllerYes 0x20000 \u2013 0x2FFFF64KPCIe CSR- 0x30000 \u2013 0x3FFFF64KHSSI CSR- 0x40000 \u2013 0x4FFFF64KEMIF CSR- 0x50000 \u2013 0x5FFFF64KReservedAvailable for customer use.Dependent on user programming 0x60000 \u2013 0x6FFFF64KReservedAvailable for customer use.Dependent on user programming 0x70000 \u2013 0x7FFFF64KReservedAvailable for customer use.Dependent on user programming

                  The three reserved regions have associated ports available in the BPF for customer use.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#65-axi4-stream-pfvf-muxdemux","title":"6.5 AXI4-Stream PF/VF Mux/Demux","text":"

                  The AXI4-Stream PF/VF Mux/Demux routes the PCIe TLP packets from the PCIe subsytem AXI4-Stream RX channel to downstream PF/VF based on the pf_num and vf_num information in the PCIe TLP header.

                  The AXI4-Stream PF/VF mux arbitrates PCIe TLP packets from downstream PF/VF to the PCIe SS AXI-S TX channel. The PF/VF Mux/Demux is an M x N switch that allows any M port to target any N port, and any N port to target any M port, where M is the number of host/upstream ports, and N is the numbers functions/downstream ports. M and N values are parameterized in the RTL for adding, removing, or remapping of FPGA functional units/modules to PF/VF.

                  The fpga top package file, found in the src/includes/ofs_fim_cfg_pkg.sv file OFS D5005 FIM Github Branch contains these parameters as well as the mapping of N port\u2019s PF/VF.

                  Structurally, M x N switch is composed of M number of N:1 mux, and N number of M:1 mux. Each mux output has an arbiter that perform round robin priority arbitration of its inputs. At the mux output is a FIFO with depth greater than the handshake round trip delay. The FIFO allows the switch to arbitrarily insert pipeline/register stages for timing.

                  Note that M x N switch is design for AXI streaming, but it can be easily converted to AVST. The protocol signals pass through switch intact \u2013 only ready, valid, and last (common between AVST and AXI) effect switch operation. The data width of the switch is also parameterized in the src/includes/ofs_fim_cfg_pkg.sv file OFS D5005 FIM Github Branch.

                  The default mapping is shown below:

                  Table 6-5 PF/VF Mapping

                  DevicePhysical Function #Virtual Function #Switch Port ID APF/BPF0x3 HE Loopback000 Port Gasket011 HSSI022

                  For information on how to modify the PF/VF mapping for your own design, refer to the [Intel\u00ae FPGA Interface Manager Developer Guide: Open Stack for Intel\u00ae Stratix 10\u00ae].

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#66-unified-tag-remapping","title":"6.6 Unified Tag Remapping","text":"

                  When a FPGA function sends out a read cycle, it allocates a unique tag which is subsequently used to identify the read completion. The tag is considered busy; it cannot be assigned to another read cycle until read completion. While a tag may be unique within a unit, two different units could unknowingly send out two read cycles of the same tag. The PCIe subsystem requires unique tags for all read cycles irrespective of their origins. Therefore, a mechanism is needed to uniquify tag globally across different units.

                  OFS contains a tag remapper (tag_remap) that intercepts the read cycle, finds a globally unique tag, and replaces the original tag value. It also restores the original tag value when returning completion to the read requester. tag_remap is placed between the AXI4-Stream interface of the PCIE subsystem and the PF/VF Mux/Demux.

                  The logic is described as follows:

                  1. A sub-module (ofs_fim_tag_pool) maintains a pool of available tags.
                  2. TX read requests are held until a tag is available from the pool by setting tvalid=0 to the host, and tready=0 to the PF/VF Mux/Demux.
                  3. When a TX read is dispatched, the tag is marked busy in the pool.
                  4. The original tag is stored in tag_reg, so it can be recovered when returning a completion to the unit/function.
                  5. Since completion to a read request can split into multiple smaller transfer sizes, responses are monitored and the final completion is detected using PCIe TLP rules.
                  6. Tags are released in the pool only when all requested data are transferred.
                  7. When the completion returns, the original tag is restored from tag_reg.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#67-tlp-to-axi4-lite-memory-mapped-bridge-st2mm","title":"6.7 TLP to AXI4-Lite Memory Mapped Bridge (ST2MM)","text":"

                  ST2MM implements the following key features: * Host MMIO bridge * Maps MMIO TLP packets received from the PCIe Subsystem over streaming interface to AXI4-Lite memory-mapped request. The memory-mapped request is sent to AFU or Board peripherals over APF and BPF. * Maps AXI4-lite MM response received from AFU or Board peripherals to TLP packets and send the packets over ST streaming channel to host HIA subsystem. * Sends MMIO response of all 0\u2019s for MMIO read to unused BAR region. * Interrupt * Sends interrupt packets to the PCIe subsystem when interrupt requests are received from the peripherals. Interrupts can be requested by a peripheral through a memory write to interrupt CSR registers in the ST2MM.

                  Figure 6-2 APF/BPF Generation

                  ST2MM implements both AXI4-lite master and slave interfaces that are connected to the designated slave and master port on APF. Host memory requests are sent on the ST2MM master interface to AFP where the requests are routed to the targeted peripherals.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#7-mmio-regions","title":"7 MMIO Regions","text":"

                  The FIM and AFU expose their functionalities to the host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). An MMIO region is an address space within a base address register (BAR) region to which features are memory mapped. For example, when a feature is mapped to an MMIO region, the CSR registers of that feature are located within the address range of that region. There can be multiple MMIO regions within a BAR region.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#71-base-address-register-bar-layout","title":"7.1 Base Address Register (BAR) Layout","text":"

                  The function, BAR and external feature region starting address are put into a platform specific parameter SystemVerilog package file src/includes/ofs_fim_cfg_pkg.sv file OFS D5005 FIM Github Branch.

                  You can modify the parameterization according to your platform requirements, however you must ensure the corresponding software driver is also updated to align with the new assignment.

                  Table 7-1 BAR Layouts

                  PF VF Feature BAR BAR Size PF0 - OFS Managed Peripherals BAR 0 512K AFU PeripheralsBoard Peripherals 256K256K VF0 HE-LB BAR0 4K VF1 HE-MEM in PR slot BAR0 4K VF2 HE-HSSI BAR0 4K"},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#72-feature-region","title":"7.2 Feature Region","text":"

                  A group of related CSRs can be categorized as a feature region. For example, a DMA engine has queue management function and quality of service (QoS) function; these are two different features of the DMA engine. A feature region is contained within a single PCIe BAR and cannot span across two BAR region boundaries. You can view the PF0 BAR0 MMIO mapping by referencing thesrc/common/fme/fme_csr_pkg.sv file OFS D5005 FIM Github Branch file.

                  A Device Feature Header (DFH) register marks the start of the feature region and sub-feature region, and you must place it at the first address of the region. Each DFH starts at 4KB boundary. A DFH register contains information that OPAE software requires to enumerate the feature. It also has an offset field that points to the next DFH in a feature list. OPAE software traverses the linked list of DFHs in each BAR region to discover all the features implemented on the platform. The EOL field in a DFH marks the end of a DFH list and is only set in the DFH of the last feature in the feature list. The feature type field in the DFH is used to differentiate between the different types of feature region. Basic building blocks (BBB) and private features are always a child of an AFU or FPGA Interface Unit (FIU) and must be contained within an AFU or FIU, respectively.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#721-device-feature-header-dfh-structure","title":"7.2.1 Device Feature Header (DFH) Structure","text":"

                  All DFHs must follow the following structure to be compatible with OPAE software.

                  Table 7-2: DFH Structure

                  Bitfield Name Range Access Description FeatureType 63:60 RO 4\u2019b0000 \u2013 Reserved 4\u2019b0001 \u2013 AFU4\u2019b0010 \u2013 BBB4\u2019b0011 \u2013 Private Feature4'b0100 \u2013 FIU/FIM Reserved 59:41 Rsvd Reserved EOL 40 RO End of DFH List1'b0=No other feature header beyond this one1'b1=This is the last feature header NextDFHByteOffset 39:16 RO Next DFH byte offsetNext DFH Address= Current DFH address + Next DFH byte offset. You can also use this value as an indication of the maximum size of the MMIO region occupied by this feature. FeatureRev 15:12 RO For AFU Feature type= AFU major version number that is user defined.All other feature types= Feature revision number FeatureID 11:0 RO For AFU feature type= CoreFIM version numberFor BBB feature type= Intel defined ID for BBBFor private feature type= User-defined ID to identify within an AFU For FIU type=ID for FIU unit (ex. 0 for FME, 1 for Port)

                  You must increment a feature revision number if a feature changes. This change requires a corresponding change in the software to detect the new version and report mismatches between the hardware and software revision number.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#73-control-and-status-registers","title":"7.3 Control and Status Registers","text":"

                  All the Control and Status Registers (CSRs) in the FIM are 64-bit registers with the following MMIO write and MMIO read support.

                  Table 7-3: CSR MMIO Read and Write Support

                  Request Memory Attribute Payload size Memory Ordering MMIO Write UC 4B or 8B Strongly ordered MMIO Read UC 4B or 8B Strongly ordered

                  The FIM does not reorder the MMIO requests or responses. For MMIO writes, there is no reordering of requests in FIM, and UC ordering rules are followed. Similarly, for MMIO reads, there is no re-ordering of requests or responses in the FIM. An AFU may opt to re-order the MMIO read responses but the FIM does not enforce read response ordering.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#731-software-access-to-registers","title":"7.3.1 Software Access to Registers","text":"
                  • Software accesses 64-bit registers as aligned quadwords. For example, to modify a field (bit or byte) in a 64-bit register, the entire quadword is read, the appropriate field(s) are modified, and the entire quadword is written back.
                  • When updating registers through multiple accesses (whether in software or due to hardware disassembly), certain registers may have specific requirements on how the accesses must be ordered for proper behavior. These are documented as part of the respective register descriptions.
                  • For compatibility with future extensions or enhancements, software must assign the last read value to all \u201cReserved and Preserved\u201d (RsvdP) fields when written. In other words, any updates to a register must be read so that the appropriate merge between the RsvdP and updated fields occurs. Also, software must assign a value of zero for \u201cReserved and Zero\u201d (RsvdZ) fields when written.
                  • PCIe locked operations to FPGA hardware registers are not supported. Software must not issue locked operations to access FPGA hardware registers.

                  In the following two cases, the FIM terminates MMIO Read requests by sending a completion with the data (CplD) specified below: * MMIO Timeout: This occurs when the AFU does not respond within a set timeout. The timeout value is currently configured to 512 pclks (clk_2x). In this case, the FIM returns all 1s.

                  • Illegal MMIO Accesses: This occurs when the read is accessing undefined registers in the FIM or if an AFU access violation. An example of an access violation is when a PF attempts to access the AFU when it is set to VF mode, or when a VF attempts to access the AFU when it is set to PF mode. In this case, the FME will return all 0s.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#732-register-attribute-definition","title":"7.3.2 Register Attribute Definition","text":"

                  Table 7-4: OFS Register Attribute Definitions

                  Attribute Expansion Description RW Read/Write This bit can be read or written by software. RO Read Only The bit is set by hardware only. Software can only read this bit. Writes do not have any effect. RW1C Read/ Write 1 to Clear Software can read or clear this bit. The software must write 1 to clear this bit. Writing zero to RW1C bit has no effect. Note that a multi-bit RW1C field may exist. In this case, all bits in the field are cleared if a 1 is written to any of the bits. RW1S Read/ Write 1 to Set Software can read this bit. Writing a 1 to the bit sets it to 1. Writing a 0 has no effect. It is not possible for software to set this bit to 0. The 1 to 0 transition can only be performed by HW. RW1CS Read/Write 1 to Clear Sticky Software can read and clear this bit. Writing a 1 to a bit clears it, while writing a 0 to a bit has no effect. This bit is only reinitialized to its default value by a power-on reset. RWD Read/Write Sticky across Hard Reset The bit can be read or written by SW. This bit is sticky or unchanged by any reset type, including Hard Reset. The bit gets cleared only with power on. *S Sticky across Soft Reset The bit will be sticky or unchanged by soft reset. These bits are only re-initialized to their default value by a power-on reset. *D Sticky across Hard Reset The bit is sticky or unchanged by or unchanged by any reset type, including hard reset. The bit gets cleared only with power on. Rsvd Reserved Reserved for future definitions. Currently don\u2019t care bits. RsvdP Reserved and Protected Reserved for future RW implementations. The software must preserve the value of this bit by read modify write. RsvdZ Reserved and Zero Reserved for future RW1C implementations. The software must write zero to this bit."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#733-csr-offset-in-bars","title":"7.3.3 CSR Offset in BARs","text":"

                  The table below captures the FIM and AFU features in the supported BAR regions. The offset highlighted in red indicates the first DFH in the DFH list of a BAR region where device driver starts the DFH traversal.

                  Table 3-6: PF0 BAR0 Features

                  Offset Feature CSR set 0x00000 FME 0x03000 Global Performance 0x04000 Global Error 0x10000 SPI Controller 0x20000 PCIe CSR Interface 0x30000 HSSI CSR Interface 0x40000 EMIF CSR Interface 0x80000 Reserved for ST2MM Bridge 0x90000 PR Control & Status (Port Gasket) 0x91000 Port CSRs (Port Gasket) 0x92000 User Clock (Port Gasket) 0x93000 Remote SignalTap (Port Gasket) 0xA000 AFU Errors (AFU Interface Handler)

                  Table 3-7: PF0 BAR4 Features

                  Offset Feature CSR set 0x02000 MSI-X 0x03000 MSI-X PBA Tables

                  Table 3-8: PF0-VF0 BAR0 Features

                  Offset Feature CSR set 0x00000 HE-LBK

                  Table 3-9: PF0-VF1 BAR0 Features

                  Offset Feature CSR set 0x00000 HE-MEM

                  Table 3-10: PF0-VF2 BAR0 Features

                  Offset Feature CSR set 0x00000 HE-HSSI"},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#8-fim-clocks","title":"8 FIM Clocks","text":"

                  The following table provides the clocks available in the OFS reference design that targets the Intel FPGA PAC D5005. Clocks that the high speed serial interface (HSSI) or external memory interface provide to the FIM may be different depending on if you modify your external features with different components.

                  Table 8-1: External Clock Source

                  Clock Frequency Description SYS_RefClk 100 MHz Reference clock to system IOPLL (sys_pll) which provides FIM system clocks. qsfp*_644_53125_clk 644.5312 5MHz HSSI reference clocks ddr4_mem*.ref_clk 150 MHz Reference clocks to DDR4 interfaces PCIE_REFCLK 100MHz PCIe reference clock

                  Table 8-2: Internal Clocks

                  Clock Frequency Description clk_1x 250 MHz Generated by the system IOPLL (sys_pll). This clock drives CoreFIM datapath and the AFU interface. clk_div2 125 MHz Generated by the system IOPLL, synchronous to clk_1x. This clock drives IM datapath and AFU interface. clk_100 100 MHz Generated by the system IOPLL, synchronous to clk_1x. This clock also supplies the HSSI reconfiguration clock. avl_clk 250 MHz PCIe H-tile Hard IP clock output. This clock is not synchronous to Clk_* DDR4x_USERCLK 299.76 MHz Each of the four DDR interfaces generates one of these clocks, which provides the the clock to the DDR4* Avalon Memory Mapped interfaces. Your memory clock output may be different depending on the memory interface you are implementing in your design. uclk_usr User defined Provides an AFU user clock running at a user specified frequency. Generated by user IOPLL. Not synchronous to clk_*. uclk_usr_div2 uclk_usr/2 Second user clock to AFU running at half the frequency of uclk_usr. Synchronous to uclk_usr. Generated by user IOPLL. hssi[*].f2a_tx_parallel_clk_x1 156.2 MHz 1x TX clock generated from fPLL in the HSSI module in FIM, used to clock the HSSI TX datapath in AFU. hssi[*].f2a_tx_parallel_clk_x2 312.5 MHz 2x TX clock generated from fPLL in the HSSI module in FIM, used to clock the HSSI TX datapath in AFU. hssi[*].f2a_rx_clkout 322.265625MHz RX parallel clock from the HSSI PHY channels, used to clock the HSSI RX data datapath in AFU."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#9-reset","title":"9 Reset","text":""},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#91-reset-signals","title":"9.1 Reset Signals","text":"

                  The system reset of OFS reference platform is driven by nPERST pin, pcie_reset_status signal from the PCIe hard IP, the INIT_DONE and nCONFIG pins of the FPGA, and the locked signal of the SYS IOPLL that provides system clocks to FIM.

                  Upon power-on, the reset module in the FIM holds the FIM in reset until all the reset conditions are de-activated:

                  • nPERST signal is asserted.
                  • The INIT_DONE pin is driven high to indicate core configuration is complete.
                  • The SYS IOPLL is locked.
                  • The reset status from PCIe hard IP is de-asserted indicating the IP is ready for transfer.

                  The reset module places the FIM back into reset if any of these conditions becomes active again. The only way to invoke a system reset to the FIM after power-up is to deassert the nPERST pin either by performing a warm reboot or through PCIe driver intervention. There are soft reset signals set aside to allow software to reset the Port, AFU and partial reconfiguration IP.

                  Table 9-1: FIM System Resets

                  Reset Description nPERST pin Active low PCIe reset pin that serves as the system reset pin on the platform. nCONFIG pin Active low input to the FPGA that causes the FPGA to lose its configuration data, enter a reset state, and tri-state all I/O pins. Host software must reload the FPGA FIM after nCONFIG is activated. ninit_done Active low signal derived from the INIT_DONE pin which indicates the FPGA core configuration is complete and has entered usermode. pcie_reset_status Active high reset status from PCIe hard IP. When driven high, this signal indicates that the PCIe IP core is not ready for usermode. pll_locked Active high SYS IOPLL locked signal

                  Table 9-2: Soft Resets

                  Soft Reset Bitfield Register Description PortSoftReset PORT_CONTROL[0] Resets Port and AFU. FlrPortReset PORT_CONTROL[3] PCIe function level reset that resets Port and AFU when SR-IOV is enabled. PRReset FME_PR_CTRL[0] Resets the partial reconfiguration (PR) controller."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#92-platform-power-up-sequence","title":"9.2 Platform Power up Sequence","text":"

                  Upon power up, the HSSI interfaces of the FIM go through an internal reset and calibration sequence. After the nPERST is de-activated, the PCIe interface and EMIF interfaces are first released from reset, followed by SYS IOPLL when the ninit_done is de-asserted. The rest of the FIM logic is still being hold in reset. The nPOR signal to the PCIe hard IP de-activates following nPERST assertion, which releases PCIe hard IP from reset. The PCIe hard IP asserts pld_clk_inuse to indicate to the application layer that the HIP transaction layer is using the pld_clk as its clock and is ready for operation with the Application layer (pld_clk is stable). Finally, reset_status from PCIe IP is de-asserted. When SYS IOPLL is locked and the reset_status from the PCIe interface is de-asserted, the FIM is released from reset while the Port and AFU is still held in reset by the PortSoftReset register bit (PORT_CONTROL[0]) that is held high until software writes a 0 to this bit to de-activate port reset. At this point, the platform is fully released from reset and PCIe link is going through link training and PCIe enumeration. Once the platform is successfully enumerated, driver can then be loaded to start the device feature discovery process and provision the platform for AFU application usage.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#10-interrupts","title":"10 Interrupts","text":"

                  The OFS platform supports interrupt through MSI-X feature. The OFS reference platform supports at least 4 FME interrupts (PF only) and 4 AFU interrupts (PF and VF).

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#101-msi-x","title":"10.1 MSI-X","text":"

                  In the default implementation the MSI-X feature that handles FME and AFU interrupts is inside the PCIe Subsystem. The MSI-X vector table and Pending Bit Array (PBA) table for PF0 and PF0/VF1 are provided as an example. FME interrupts are primarily used to notify the host of error events occurring in the FIM.

                  All interrupt requests arrive inband through the AXI4-Stream interface to the TX AXI4-stream adapter inside the PCIe Subsystem.

                  An AFU sends an interrupt to the MSI-X module on the AXI interrupt request channel. After the interrupt request is serviced, the MSI-X module sends an interrupt response back to the AFU on the AXI interrupt response channel. The AFU has the flexibility to define the use of each AFU interrupt.

                  The following interrupts are supported: PF supports 7 interrupt vectors: - 0-3: User AFU triggered - 4: Port error triggered - 6: FME error triggered

                  VF supports 5 interrupt vectors: - 0-3: User AFU triggered - 4: Port error triggered

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#11-external-memory-interface-emif","title":"11 External Memory Interface (EMIF)","text":"

                  There are four DDR4 external memory interfaces on the OFS EA FIM that targets the Intel FPGA PAC D5005 FIM for each of the four DDR4 banks (DDR4a, DDR4b, DDR4c, and DDR4d). Two of the DDR4 external memory interfaces and the associated clocks are directly exposed to AFU except for two Avalon Memory Mapped pipeline bridges to facilitate timing closure across PR boundary. The Avalon Memory Mapped interfaces of each external memory interface are connected to an Avalon-MM pipeline bridge (avmm_bridge) in the FIM, which is then connected to another Avalon-MM pipeline bridge in the PR or AFU region. An AFU should use the USER_CLK associated with a memory interface when interfacing with the memory interface for better timing performance.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#111-emif-csr","title":"11.1 EMIF CSR","text":"

                  The CSR for the EMIF feature is memory mapped to the FME BAR region. Following table captures the EMIF CSR registers.

                  Table 9-1: EMIF CSR Registers

                  EMIF_DFH 0x40000 0x3000000050000009 EMIF Management DFH FIELD NAMERANGEACCESSDEFAULT DESCRIPTION FeatureType [63:60] RO 0x3 Feature Type = Private Feature Reserved40 [59:40] RsvdZ 0x0 Reserved NextDfhByteOffset [39:16] RO 0x050000 Next DFH Byte offset FeatureRev [15:12] RO 0x0 Feature Revision FeatureID [11:0] RO 0x9 Feature Id EMIF_STAT 0x40008 0x0000000000000000 EMIF Status FIELD NAMERANGEACCESSDEFAULT DESCRIPTION Reserved [63:16] RsvdZ 0x0 Reserved Reserved [15:12] RsvdZ 0x0 Reserved EmifCalFail [11:8] RO 0x0 EMIF PHY Calibration Failure (1 bit per interface) Reserved [7:4] RsvdZ 0x0 Reserved EmifCalSuccess [3:0] RO 0x0 EMIF PHY Calibration Successful (1 bit per interface)

                  \u200b \u200b

                  EMIF_CTRL 0x40010 0x0000000000000000 EMIF Control FIELD NAMERANGEACCESSDEFAULT DESCRIPTION Reserved [63:0] RsvdZ 0x0 Reserved"},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#92-afu-emif-interface","title":"9.2 AFU EMIF Interface","text":"

                  The FIM exposes 576-bits of Avalon Memory-Mapped data to the AFU, with 512-bit data and additional 64 bits that can either be used for additional metadata, parity or ECC. The AFU has the flexibility to decide the use of the extra 64 bits of data. The ECC soft IP is not enabled in the EMIF IP to allow for the afore-mentioned flexibility. AFU developers can implement the ECC logic in the AFU by making use of the extra 64-bit of data. Avalon Memory-Mapped is the native interface protocol used by Intel EMIF IP. AFU developers who desire other interface protocol in their designs over Avalon Memory-Mapped, such as AXI4 Memory-Mapped, can leverage the bridge in the PIM library.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#12-hssi-subsystem","title":"12 HSSI Subsystem","text":""},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#121-hssi-subsystem-overview","title":"12.1 HSSI Subsystem Overview","text":"

                  The high speed serial interface (HSSI) subsystem architecture provides portability to different Ethernet configurations across platforms and generations and reusability of the hardware framework and software stack. This reference FIM contains the Low Latency Ethernet 10G MAC IP and provides a Linux driver that can be leveraged for customization.

                  The HSSI design is leveraged from our OFS EA release so prior customers can easily maintain compatibility with past designs.

                  A host exerciser, named he-hssi, is provided in the pr_slot of the AFU partition. The Ethernet interface to the AFU has an AXI4-Stream data and sideband interface. The HSSI control and status registers in the FIM are accessible by the AXI4-Stream memory mapped interface.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#122-ofs-hssi-subsystem-interfaces","title":"12.2 OFS HSSI Subsystem Interfaces","text":""},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1221-hssi-subsystem-fim-interfaces","title":"12.2.1 HSSI Subsystem FIM Interfaces","text":"

                  There are three interfaces to the HSSI Subsystem that is part of the FIM:

                  1. AXI4 Memory Mapped to access HSSI-CSR (to FIM)
                  2. AXI4-Stream Ethernet Data Interface (from FIM)
                  3. AXI4-Stream Ethernet Sideband Interface (from FIM)

                  The PCIe subystem uses AXI Memory Mapped accesses to read and write HSSI Control and Status Registers in the FIM. The Ethernet MAC interface typically has a data streaming interface which is mapped to standard AXI4-Stream.

                  Figure 12-1: HSSI Subsystem

                  Additionally, the Ethernet MAC interface has an interface for status and flow control. Status and flow control signals vary across IPs and operating modes so for this design we group the signals into a AXI4-Stream sideband interface which provides a standard interface to the AFU along with platform customizations if needed. The Avalon to Arm AMBA 4 AXI4 bridge (av_axi_st_bridge) converts native Avalon interfaces to an AXI4 Stream interface.

                  The following flow control are implemented in the Ethernet MAC: * IEEE 802.3 flow control: this flow control implements the IEEE 802.3 Annex 31B standard to manage congestion. When the Low Latency Ethernet 10G MAC IP experiences congestion, the core sends a pause frame to request its link partner to suspend transmission for a given period of time. This flow control is a mechanism to manage congestion at the local or remote partner. When the receiving device experiences congestion, it sends an XOFF pause frame to the emitting device to instruct the emitting device to stop sending data for a duration specified by the congested receiver. Data transmission resumes when the emitting device receives an XON pause frame (pause quanta = zero) or when the timer expires.

                  \u2022 Priority-based flow control (PFC): this flow control implements the IEEE 802.1Qbb standard. PFC manages congestion based on priority levels. It supports up to 8 priority queues. When the receiving device experiences congestion on a priority queue, it sends a PFC frame requesting the emitting device to stop transmission on the priority queue for a duration specified by the congested receiver. When the receiving device is ready to receive transmission on the priority queue again, it sends a PFC frame instructing the emitting device to resume transmission on the priority queue

                  To use flow control, set the following registers:

                  On the TX datapath: 1. Set tx_pfc_priority_enable[7:0] (Address :0x0046 -> 11A0) to 0 to disable the PFC. The rest of the bits are unused. 2. Set tx_pauseframe_enable[0] (Address :0x0044 -> 1142) to 1 to enable the flow control.

                  On the RX datapath:

                  \u2022 Set rx_pfc_control[7:0] (Address :0x00C0 -> 818) to 1 to disable the PFC. The rest of the bits are mostly unused. \u2022 Set the IGNORE_PAUSE (Address :0x00AC -> 800) bit in the rx_frame_control register to 0 to enable the flow control.

                  To use Priority-Based Flow Control Follow these steps to use the priority-based flow control (PFC): 1. Enable the Priority-based flow control (PFC) parameter and specify the number of priority levels using the Number of PFC priorities parameter. You can specify between 2 to 8 PFC priority levels. 2. Set the following registers.

                  On the TX datapath: * Set tx_pauseframe_enable (Address :0x0044 -> 1142) to 0 to disable the flow control. * Set tx_pfc_priority_enable[n] (Address :0x0046 -> 11A0) to 1 to enable the PFC for priority queue n.

                  On the RX datapath: * Set the IGNORE_PAUSE bit in the rx_frame_control (Address :0x00AC -> 800) register to 1 to disable the flow control. * Set the rx_pfc_control[7:0] (Address :0x00C0 -> 818) register bits to 0 to enable the PFC. Most of the rest of the bits are unused. * Set PFC Quanta bit for the appropriate queue. Eg: pfc_pause_quanta_0 (0x048 -> 1180) for queue0 and so on. * Connect the avalon_st_tx_pfc_gen_data signal to the corresponding RX client logic and the avalon_st_rx_pfc_pause_data signal to the corresponding TX client logic. * You have the option to configure the MAC RX to forward the PFC frame to the client by setting the rx_pfc_control[16] register to 1. By default, the MAC RX drops the PFC frame after processing it

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1222-hssi-subsystem-afu-interfaces","title":"12.2.2 HSSI Subsystem AFU Interfaces","text":"

                  The HSSI subsystem provides the following interfaces to the AFU region:

                  1. AXI4-Memory Mapped access to the HSSI CSR (to FIM)
                  2. AXI4-Stream Ethernet Data Interface (from FIM)
                  3. AXI4-Stream Ethernet Sideband Interface (from FIM)
                  4. Ethernet Clock Interface (eth_clock) (from FIM)

                  The he-hssi uses the APF interface for HSSI CSR (MMIO) accesses. The AXI4-Stream Ethernet data and side band interface along with Ethernet clocks communicate directly to the he-hssi module in the AFU region through platform independent data structures provided by the PIM. Even if you implement a different MAC you typically can leverage these data structures defined in the hssi/inc/ofs_fim_eth_avst_if.sv file here without modification.

                  While the platform-independent interfaces in ofs_fim_eth_if.sv are convenient containers for passing data streams through the design hierarchy, both the MAC and AFU traffic generator require platform-specific data types. The payloads of the streams in ofs_fim_eth_if.sv are defined in platform-specific structures, with fields that are MAC-specific. In this 10GbE reference design, the payload datatypes are defined in the ipss/hssi/s10/includes/ofs_fim_eth_plat_if_pkg.sv file hhere. Implementers connecting a new MAC should generally edit only ofs_fim_eth_plat_if_pkg.sv when defining payloads.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1223-hssi-sideband-interface","title":"12.2.3 HSSI Sideband Interface","text":"

                  The AXI4-Stream sideband interface has been been defined to allow for time sensitive status and flow control information. It does not have the optional tready signal and assumes the slave always accepts information. The Ethernet sideband interface varies widely across IPs and operating modes and device generations. In OFS Stratix 10 FIM, the Ethernet sideband signals are mapped to the AXI4-Stream interface and interface variations can be accommodated by customizing the tdata signals of the sideband interface in platform specific interface packages using the PIM.

                  As an example, please refer to ofs_fim_eth_sideband_tx_axis_if interface in the ipss/hssi/inc/ofs_fim_eth_if.sv found OFS D5005 FIM Github Branch.

                  The t_axis_eth_sideband_tx and t_axis_eth_sideband_rx structures are found in the ipss/hssi/s10/includes/ofs_fim_eth_plat_if_pkg.sv file OFS D5005 FIM Github Branch.

                  Platform specific details for the 10GbE example are from the ipss/hssi/s10/includes/ofs_fim_eth_plat_if_pkg.sv file OFS D5005 FIM Github Branch.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1224-reconfiguration-interfaces","title":"12.2.4 Reconfiguration Interfaces","text":"

                  The reconfiguration interface in the OFS EA design consists of abstracted and consolidated memory-mapped transceiver reconfiguration interfaces that are exposed to the HSSI CSRs. The reconfiguration interface directly exposes the address space of the MAC and PHY IPs in order to allow a software driver to perform dynamic reconfiguration of those IPs (i.e. read/write access to the Native PHY CSRs). Therefore, to use this interface you must be familiar with the CSR memory maps of the corresponding Intel IP cores.

                  The table below summarizes all the ports associated with Reconfiguration Interfaces.

                  Name Width Domain Description i_xcvr_reconfig_cmd 2 i_reconfig_clk Command port used to specify a read or write access operation to a MAC/PHY CSRs on a selected CSR_interface i_xcvr_reconfig_addr 20 i_reconfig_clkCSR access address of MAC/PHY IP. Note that only the lower 16 bits are used, i_xcvr_reconfig_addr[15:0]. The remaining upper bits, i_xcvr_reconfig_addr[20:16], should be set to zero. i_xcvr_reconfig_writedata 32 i_reconfig_clk CSR data to be written on a write command. o_xcvr_reconfig_readdata 32 i_reconfig_clk CSR data that was read on a read command. o_xcvr_reconfig_ack 1 i_reconfig_clk Reconfiguration acknowledgement bit. Asserted when Reconfiguration Controller is finished performing an Avalon-MM read or write request (i.e. waitrequest is low). Deasserted when i_xcvr_reconfig_cmd is all-zero (i.e. command is invalidated). Note that the controller assumes a valid command when i_xcvr_reconfig_cmd is non-zero."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#12241-reconfiguration-sequence","title":"12.2.4.1 Reconfiguration Sequence","text":"

                  The diagram below explains the sequence of operation and handshaking between software and a memory-mapped dynamic reconfiguration interface.

                  Figure 12-2: Sequence of Operation and Handshaking between Software and a Memory-Mapped Dynamic Reconfiguration Interface.

                  (0) Idle state \u2013 command and address buses are cleared (all-zero). 1. Software sets a desired non-zero command and address to initiate reconfiguration.

                  a.  Memory-Mapped Reconfiguration Controller (MM CTRL) converts the command and address to a single Avalon Memory Mapped read or write request and handles Avalon Memory Mapped protocol details.\n\nb. MM CTRL completes the Avalon Memory Mapped transaction when the `waitrequest` signal of a given Avalon Memory Mapped interface is deasserted.\n\nc. MM CTRL sets the reconfiguration acknowledgment bit and `readdata` (in case of a read command) back to the FME and waits for command and address ports to be cleared by software. \n1.\n

                  1. Meanwhile, software continuously polls the reconfiguration acknowledgment bit and waits for it to get asserted. Assertion of the acknowledgment bit confirms that the MM CTRL has completed the current Avalon Memory Mapped read/write request.

                  2. Software reads readdata from the HSSI_RCFG_DATA CSR that was returned by the MM CTRL (in case of a read command).

                  3. Software clears the command and address buses to communicate back to the MM CTRL that the operation is finished from the CPU\u2019s perspective.

                    a. MM CTRL gets cleared (all-zero) command and address signals from the HSSI_CSR. b. MM CTRL clears (deasserts) the reconfiguration acknowledgment bit back to the HSSI_CSR and is finished / back to idle state. 5. Meanwhile, software continuously polls the reconfiguration acknowledgment bit and waits for it to get deasserted. Deassertion of the acknowledgment bit confirms that the MM CTRL has completed its handshake and is now back to idle state.

                  NOTE

                  Reads and writes cannot be performed at the same time. Remember that when multiple CSRs are at the same address, a Read-Modify-Write operation may be required to change the desired CSR without changing the CSRs in the same address.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1225-hssi-control-and-status-register-csr-map","title":"12.2.5 HSSI Control and Status Register (CSR) Map","text":"

                  The HSSI CSR Map structure is designed to scale according to IP capabilities.

                  • HSSI_DFH allows for identifying HSSI as an external FME feature.
                  • HSSI_CAPABILITY register exposes design capabilities and provides direction to SW/Driver. The fields num_channels, Num_channels_CSR interface, Num_CSR_interface indicate to the software how many CSR interfaces are exposed by the design and how to use mailbox registers (HSSI_RCFG_CMD and HSSI_RCFG_DATA). The number of mailbox registers(HSSI_RCFG_CMD and HSSI_RCFG_DATA) must scale to the number of CSR interfaces exposed by the design. This implementation facilitates flexibility in the design and reuse of the software stack. If you are modifying the HSSI interface, you must update these CSR fields according to HW configuration .

                  Example: If Num_CSR_interface=2 & Num_channels_CSR_interface=2 then channel(0,1) are behind CSR interface 0 handled by HSSI_RCFG_CMD0/DATA0 , channel (2,3) are behind CSR interface 1 handled by HSSI_RCFG_CMD1/DATA1 HSSI_CTRL, HSSI_STATUS0, HSSI_STATUS1 provide control and status information and can scale upto 8 channels. HSSI_RCFG_CMD0, HSSI_RCFG_DATA0 registers are for the reconfiguration interface, and additional mailbox registers could be added depending on the number of CSR interfaces the design exposes.

                  The HSSI CSR Map can be found in the ipss/hssi/s10/hssi_ss_csr.xls file OFS D5005 FIM Github Branch.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1226-hssi-host-exercisier-he-hssi","title":"12.2.6 HSSI Host Exercisier (HE-HSSI)","text":"

                  HE-HSSI is an Ethernet AFU that handles client side ethernet traffic. The reference HE-HSSI has following features:

                  • HE-HSSI wraps the 10G Ethernet AFU that was provided in the OFS EA FIM with a wrapper that provides an E-tile compatible interface with OFS for Intel Stratix 10 and Intel Agilex FPGAs.
                  • Includes 10GbE traffic generator and checker (monitor)
                  • Provides pause signals to the HSSI subsystem for XON and XOFF generation
                  • It can generate traffic or incoming traffic that can be looped back into transmit path by enabling loopback mode, which will bypass traffic generator
                  • At the HE-HSSI interface boundary the Ethernet data interface is AXI4-Stream with 64-bit data at eth_clk clock
                  • An AXI4-Stream to Avalon-ST bridge converts Avalon-ST Ethernet traffic from the HE-HSSI traffic generator to AXI4-Stream traffic and AXI4-Stream RX data from the FIM to Avalon-ST for the HE-HSSI traffic checker. The data width for all the interfaces in this bridge is 64 bits at eth_clk clock.
                  • The Traffic generator and checker modules have a 64-bit data interface at eth_clk clock.
                  • The traffic generator supports the following modes: * Fixed length or Random Length * Incremental pattern or Random pattern
                  • The traffic checker does a 32-bit CRC check
                  • The CSR of this AFU is accessible through AXI4-Stream PCIe TLP interfac
                  • The PCIe TLP to CSR Interface Conversion module converts PCIe TLPs into simple CSR interface
                  • The CSR space of the traffic generator and checker modules are accessed in an indirect way using mailbox registers
                  • Though the default configuration for this reference HE-HSSI is 1x10GbE, it can be scaled up to eight 10G ethernet traffic generators and checkers in one HE-HSSI
                  • If used for more than one channel, each channel has a separate traffic generator and traffic checker with separate CSR space.
                  • Reads and Writes to individual traffic controller CSR spaces can be done by selecting that particular channel using channel select register.

                  The HE-HSSI Ethernet block diagram is below.

                  Figure 12-6: HE-HSSI Block Diagram Block Diagram

                  Figure 12-7: 10G Ethernet AFU Clock Domains

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#12261-he-hssi-csr-map","title":"12.2.6.1 HE-HSSI CSR Map","text":"

                  The reference HSSI AFU contains the following registers and a similar arrangement of register space can be implemented for other usecase specific HSSI AFUs. * AFU DFH Register: Device feature header for the AFU (AFU_DFH) * AFU ID Registers: 128-bit UUID for the AFU which occupies two 64-bit registers (AFU_ID_L, AFU_ID_H) * Mailbox Registers: Command and Data register for traffic controller register access. It follows the standard access method defined for OFS. Access method and implementation is same as Reconfiguration Interface defined for the HSSI FIM. (TRAFFIC_CTRL_CMD, TRAFFIC_CTRL_DATA) * Channel Select Register: Channel select register for traffic controller mailbox access. It is used in cases where more than one channel is in the AFU, else it defaults to zero, meaning channel-0 is selected. (TRAFFIC_CTRL_PORT_SEL) * Scratchpad Register: Scratchpad register for CSR access checking. (AFU_SCRATCHPAD)

                  The CSR excel for the 10G HSSI reference AFU can be found ipss/hssi/s10/hssi_ss_csr.xls OFS D5005 FIM Github Branch.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#123-hssi-software","title":"12.3 HSSI Software","text":"

                  There are two pieces of software related to running the HSSI Subsystem and the HE-HSSI host exerciser: The Linux* dfl network driver and a user space application.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1231-hssi-linux-driver","title":"12.3.1 HSSI Linux Driver","text":"

                  The HSSI subystem is exposed as a feature in the PCIe PF BAR0 region. It has a Device Feature Header (DFH) indicating the HSSI interface. The feature ID in the DFH causes the following driver to be instantiated for the HSSI interface: drivers/net/ethernet/intel/s10hssi.c Kernel Driver Branch

                  The primary functionality of the driver is to interact with the ethernet MAC and PHY through an indirect register access mailbox implemented by the HSSI_RCFG_CMD0, HSSI_RCFG_DATA0 registers described above. To aid in RTL bringup, the driver provides a debugfs interface directly to the indirect register access mailbox. For each HSSI interface in the system there would be a directory with the following form containing two files, regaddr and regval: /sys/kernel/debug/dfl-fme.X.Y

                  To read a register offset in the MAC/PHY write the offset to the regaddr file as a C hex string (e.g. 0x04) and then read the value as string out of regval file. To write a register offset in the MAC/PHY write the offset to the regaddr file as a C hex string (e.g. 0x04) and then write the value as a C hex string to regval file.

                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1232-hssi-user-space-tool","title":"12.3.2 HSSI User Space Tool","text":"

                  The HSSI user space application exports a control interface to the HSSI AFU's packet generator logic. Context-sensitive help is given by the --help option, doc/src/fpga_tools/hssi/hssi.md, OPAE SDK Branch.

                  $ hssi --help\n\n
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#124-user-guidelines","title":"12.4 User Guidelines","text":"
                  You can either leverage Ethernet example designs from platform designer or use your own custom IP\u2019s. However below recommendations would help leverage the infrastructure of the OFS stack:\n* Follow the Ethernet-GBS interface standard, customize platform specific sideband and clock intefaces.\n* Follow the reconfiguration interface example and reuse the hssi_csr block by modifying the memory map (change address decoder map accordingly as well.)
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#13-partial-reconfiguration","title":"13 Partial Reconfiguration","text":"
                  Partial Reconfiguration (PR) is an Intel FPGA technology that allows user to reconfigure parts of the FPGA device dynamically, while the remainder of the device continues to operate. In a non-partial reconfiguration flow, any change to the design requires full reprogramming of the entire configuration RAM (CRAM) arrays in the device. With partial reconfiguration, you can dynamically reprogram one or more CRAM frames. A partial reconfiguration design has a static region, and one or more PR regions, which can be modified to implement new logic. The portion of the CRAM on the chip to be reconfigured is contained within a PR region.\nFor the PR flow, the design should be partitioned into static region and reconfigurable region. The static region is the area of your FPGA that is not reconfigured without reprogramming the entire FPGA. An area of the chip that you plan to partially reconfigure is a PR region. 
                  \n
                  The Port Gasket contains all the PR specific modules and logic, such as PR slot reset/freeze control, user clock, remote STP etc. For this reference example only one PR slot is supported.\nThe following figure depicts the high level view of the Port Gasket:
                  \n
                  Figure 13-1 Partial Reconfiguration Gasket\n
                  \n
                  \n
                  The isolation logic is provided on the output signals of the PR region to ensure they don\u2019t glitch and affect the interfacing logic in the Static Region (SR). The isolation logic is controlled by the PR Freeze logic during PR operation.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#14-reliability-accessibility-serviceability-ras-and-error-handling","title":"14 Reliability, Accessibility, Serviceability (RAS) and Error Handling","text":"
                    \n
                  1. Downstream AFU checker: Identifies AFU violations.  For example, this checker flags violations of the interface specification. 
                    2. \n
                  2. Upstream software or PCIe link checker: Identifies invalid traffic from PCIe that violates either FIM specifications or PCIe specifications. For example, this checker flags an application sending traffic if it violates the FIM specification or creates a PCIe link issue by causing completion timeout or malformed TLP. 
                    3. \n
                  3. FIM - Checks for bugs in the FIM fabric.
                    4. \n
                  \n
                  Errors reported by the checker are logged in either the FME error registers or Port error registers, or both, as shown in the table below.  For more details on each of the registers, please refer to src/common/protocol_checker/protocol_checker_csr.xml file OFS FIM_COMMON Github Branch or the SystemVerilog file src/common/fme/xls/d5005/FME_CSR.xls found OFS FIM_COMMON Github Branch .
                  \n
                  Table 14-1: Error Registers
                  \n\n\n\nMMIO Region\nArea\nRegister\nDescription\n\n\n\n\nFME\nCoreFIM\nFME_ERROR\nFME Error Status Register 0.  Registers parity errors, underflow or overflow errors and access mismatches.\n\n\nFME\nCoreFIM\nFME_ERROR0_MASK\nFME Error Mask Register 0.  Write a 0 to mask errors in the FME Error Status Register 0.\n\n\nFME\nExternal\nPCIE0_ERROR\nPCIe0 Error Status Register.\n\n\nFME\nExternal\nPCIE0_ERROR_MASK\nPCIe0 Error Mask Register 0.  Write a 0 to mask errors in the PCIe0 Error Status Register 0.\n\n\nFME\nCoreFIM\nFME_FIRST_ERROR\nFirst FME Error Register.\n\n\nFME\nCoreFIM\nFME_NEXT_ERROR\nFME Next Error Register.\n\n\nFME\nCoreFIM\nRAS_NOFAT_ERR_STAT\nReliability/Accessibility/Serviceability (RAS) Non-Fatal Error Status Register.\n\n\nFME\nCoreFIM\nRAS_NOFAT_ERR_MASK\nRAS Non-Fatal Error Mask Register. Write 0 to mask error fields in RAS_NOFAT_ERR_STAT Register.\n\n\nFME\nCoreFIM\nRAS_CATFAT_ERR_STAT\nRAS Catastrophic and Fatal Errors Status Register.\n\n\nFME\nCoreFIM\nRAS_CATFAT_ERR_MASK\nRAS Catastrophic and Fatal Errors Mask Register. Write 0 to mask error fields in the RAS_CATFAT_ERR_STAT Register.\n\n\nFME\nCoreFIM\nRAS_ERROR_INJ\nRAS error Injection Register.\n\n\nPORT\nCoreFIM\nPORT_ERROR\nPort Error Status Register.\n\n\nPORT\nCoreFIM\nPORT_FIRST_ERROR\nPort First Error Register .\n\n\nPORT\nCoreFIM\nPORT_MALFORMED_REQ0\nPort Malformed Request Register 0.  Provides malformed request header LSBs.\n\n\nPORT\nCoreFIM\nPORT_MALFORMED_REQ1\nPort Malformed Request Register 1.  Provides malformed request header MSBs."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#141-fme-errors","title":"14.1 FME Errors","text":""},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1411-fme_error0","title":"14.1.1   FME_ERROR0","text":"
                  The FME_ERROR0 register flags CoreFIM FME errors in the Global Error (GLBL_ERROR) private feature. The error bits in this register are sticky bits. You can only clear these bits through software or through a system reset. Writing a 1 to the error field in the register clears the corresponding error bit. Writing a 1 to the corresponding bit in FME_ERROR0_MASK register masks the error.
                  \n
                  Table 14-2: FME Error Types
                  \n\n\n\nError Type\nDescription\n\n\n\n\nFabric errors\nFIFO underflow/overflow condition in CoreFIM. These errors only occur if you have introduced bugs into the FIM or during very rare single event upset (SEU) or SEU-like events.\n\n\nInvalid port access\nA port can either be mapped to a PF or a VF, and not both. The checker reads the AfuAccessCtrl field in the FME CSR PORT0_OFFSET register to determine the access type of the Port. If it finds a PF is trying to access a port that is mapped to a VF or vice-versa, an error will be reported.\n\n\nInvalid AFU access\nAn AFU can either be mapped to a PF or a VF, and not both. The checker reads the AfuAccessCtrl field in the FME CSR PORT0_OFFSET register to determine the access type of the AFU associated with the Port. If it finds a PF is trying to access an AFU that is mapped to a VF or vice-versa, an error is reported and a fake response is sent back to the requester to avoid a completion timeout on the host."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1412-pcie0_error","title":"14.1.2 PCIE0_ERROR","text":"
                  The PCIe Avalon-ST to AXI4-Stream bridge monitors the PCIe link for errors and logs any such errors in the PCIE0_ERROR register (in PCIE0 feature region) and PCIE0_ERROR register in the GLBL_ERR private feature.    The error bits in the PCIE0_ERROR register are sticky bits that you can only clear through software or through a system reset. Writing a 1 to the error field in the register clears the corresponding error bit. Writing a 1 to the corresponding bit in PCIE0_ERROR0_MASK masks the error.  
                  \n
                  If you have other external FME features, you can add similar _ERROR registers to this space. Please refer to the following spreadsheet, bbs/csr/stratix10/pac_d5005/fme_csr_pcie.xls,  OFS D5005 FIM Github Branch or the SystemVerilog file the SystemVerilog file src/common/fme/xls/d5005/FME_CSR.xls found OFS FIM_COMMON Github Branch for more details on this register. \n\n
                  NOTE
                  \n
                  The PCIE0_ERROR register is located in both the Global Error external feature memory space and a separate PCIe external feature memory space.  OPAE software supports the PCIe external feature memory space beginning at offset 0x40000 for OFS EA and going forward.  PCIe registers beginning at 0x4000 in the Global Error external feature memory space is there for backward compatibility to the Intel FPGA PAC D5005 v2.0.1 Acceleration Stack.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1413-fme_first_error-fme_next_error","title":"14.1.3   FME_FIRST_ERROR, FME_NEXT_ERROR","text":"
                  The FME_FIRST_ERROR register flags which of the FME error reporting registers, such as FME_ERROR0, PCIE0_ERROR0, has reported the first error occurrence. The error fields of the first error register are then continuously logged into the FME_FIRST_ERROR register until a system reset or software clears all the errors in that first error register.\nLikewise, the FME_NEXT_ERROR indicates which of the FME error reporting registers (except the first error register) has reported the next occurrence of error after the first error register. The error fields of the next error register are continuously logged into the FME_NEXT_ERROR register until a system reset or software clears all the errors in the second error register.
                  \n
                  Please refer tobbs/csr/stratix10/pac_d5005/s10_iofs_csr_map_update_fme_ral_.xls, OFS D5005 FIM Github Branch for individual register field descriptions or the SystemVerilog file: src/fme/fme_csr.sv, OFS D5005 FIM Github Branch
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1414-reliability-accessibility-serviceability-ras-error-status","title":"14.1.4   Reliability, Accessibility, Serviceability (RAS) Error Status","text":"
                  The RAS feature in CoreFIM labels errors as non-fatal, fatal or catastrophic based on their impact to the system. \n* A non-fatal error usually originates from software or an AFU.  With a non-fatal error, the user application may still be able to recover from the error by performing a soft reset on the AFU, fixing the user application software if necessary, and clearing the error. On the other hand, a fatal or catastrophic error is non-recoverable and requires the platform to be reset.\n* Non-fatal errors are logged in the RAS_NOFAT_ERR_STAT register and fatal or catastrophic errors are logged in the RAS_CATFAT_ERR_STAT register.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#14141-non-fatal-errors","title":"14.1.4.1    Non-Fatal Errors","text":"
                  The RAS_NOFAT_ERR_STAT is a read-only status register that is specifically added for the RAS feature. It logs the high-level status of non-fatal errors in the hardware.  Unlike the error bits in the PCIE0_ERROR and FME_ERROR0 registers which are RW1C (software can write a 1 to clear the error), the error bits in this register are read-only and can only be cleared by system reset. Software has an option to mask the error using RAS_NOFAT_ERR_MASK.\nPlease refer to bbs/csr/stratix10/pac_d5005/s10_iofs_csr_map_update_fme_ral_.xls, OFS D5005 FIM Github Branch for individual register field descriptions or the SystemVerilog file: src/fme/fme_csr.sv, OFS D5005 FIM Github Branch.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#14142-catastrophic-fatal-errors","title":"14.1.4.2    Catastrophic & Fatal Errors","text":"
                  The RAS_CATFAT_ERR_STAT is a read-only status register that is specifically added for the RAS feature. It captures the high-level status of errors that can only be recovered with a system reset. Therefore, the error bits in the RAS_CATFAT_ERR_STAT register are read-only and can only be cleared by system reset or masked through RAS_CATFAT_ERR_MASK.\nPlease refer to bbs/csr/stratix10/pac_d5005/s10_iofs_csr_map_update_fme_ral_.xls, OFS D5005 FIM Github Branch for individual register field descriptions or the SystemVerilog file: src/fme/fme_csr.sv OFS D5005 FIM Github Branch.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1415-ras-error-injection","title":"14.1.5   RAS Error Injection","text":"
                  For software testing purposes, you can inject non-fatal, fatal and catastrophic errors into the platform through the RAS_ERROR_INJ register.  These errors are reported in the RAS_CATFAT_ERR_STAT and RAS_NOFAT_ERR_STAT registers.\nPlease refer to bbs/csr/stratix10/pac_d5005/s10_iofs_csr_map_update_fme_ral_.xls OFS D5005 FIM Github Branch for individual register field descriptions or the SystemVerilog file: src/fme/fme_csr.sv OFS D5005 FIM Github Branch.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1416-fme-error-interrupts","title":"14.1.6   FME Error Interrupts","text":"
                  In an event of an FME error, the MSI-X module in the FIM generates an interrupt so the host can decide on the next course of action. The FIM does not stall upstream and downstream traffic after the FME error. However, a port error triggered by invalid request from a user AFU stalls all the traffic going from AFU to PCIe.\nThe interrupt capability is discoverable by querying the NumbSuppInterrupt field of the PORT_CAPABILITY register in the Port private feature.  The MSI-X vector number is recorded in the InterruptVectorNumber field of the GLBL_ERROR_CAPABILITY register of the Global Error external feature.
                  \n
                  An FME error interrupt is generated in response to the FME_ERROR0, PCIE0_ERROR0, RAS_NOFAT_ERR_STAT or RAS_CATFAT_ERR_STAT registers recording a new, unmasked, error per the rules defined by CoreFIM interrupt feature.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#14161-msi-x-masking-pending-bit-array-pba-clearing","title":"14.1.6.1    MSI-X Masking & Pending Bit Array (PBA) Clearing","text":"
                  If the MSI-X vector corresponding to the FME error interrupt is masked, events are recorded in the PBA array.  Clearing the FME error status registers clears the corresponding MSI-X PBA entries.  If only some events are cleared, the normal interrupt triggering rules apply and a new pending interrupt is registered.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1417-fme-error-handling","title":"14.1.7   FME Error Handling","text":"
                  When the host receives an FME error interrupt, it must take the recommended actions described below to bring the system back to its normal operation.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#14171-catastrophicfatal-error","title":"14.1.7.1    Catastrophic/Fatal Error","text":"
                  A system reset is mandatory for any catastrophic or fatal error.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#14172-non-fatal-error","title":"14.1.7.2    Non-Fatal Error","text":"
                  When software receives a non-fatal error interrupt which does not require a system reset, it can take the following steps to clear the error after software handles the error:\n1.  Set the *_ERROR_MASK register to all 1\u2019s to mask all errors\n2. Clear the *_FIRST_ERROR register\n3. Clear the *_ERROR register\n4. Set *_ERROR_MASK register to all 0\u2019s to enable all errors
                  \n
                    \n
                  • Result: The *_ERROR & *_FIRST_ERROR registers begin capturing new errors.
                    • \n
                  \n\n
                  NOTE
                  \n
                  A system reset can only clear RAS Error status registers.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#142-mmio-requests","title":"14.2   MMIO Requests","text":"
                  The FIM is designed to gracefully handle MMIO request scenarios.  
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1421-unsupported-functions-and-bars","title":"14.2.1 Unsupported Functions and BARs","text":"
                  The OFS FIM EA has only one PCIe link and all MMIO requests from the host are sent through this link. The PCIe hard IP in the FIM guarantees that only TLP packets for the functions and BARs supported by the FIM (as configured in PCIe HIP IP instantiation) are sent to the FIM on the PCIe Avalon Streaming interface.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1422-mmio-request-decoding","title":"14.2.2   MMIO Request Decoding","text":"
                  The packet router and memory decoder in the FIM ensure that only legal MMIO requests are forwarded to the targeted MMIO region. Full address and BAR decoding is done both in the packet router and the memory decoder to ensure the requests are forwarded to the designated CSR region as defined in the MMIO Regions chapter.  Any unsolicited/illegal MMIO request is dropped, and an error is reported back to host through the FME error register.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1423-unused-fmeport-csr-regions","title":"14.2.3   Unused FME/Port CSR Regions","text":"
                  All the CSR slaves in FIM which are mapped to the FME or Port CSR regions must always respond to MMIO read requests targeting its associated CSR region. A CSR slave must return all 0s for MMIO reads to its unused CSR region such as a reserved space or a region where no CSR register is implemented for the address.\nThe FIM ensures MMIO reads to FME or Port CSR regions that are not mapped to any CSR slave always gets a response of all 0s. The memory decoder module and fake responder module in the FIM provide this guaranteed response.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1424-unsupported-mmio-request","title":"14.2.4   Unsupported MMIO Request","text":"
                  Any MMIO request targeting FME or Port CSR regions with a length or address alignment that are not  supported by the FIM is dropped, and an error is logged in PCIE0_ERROR register. The MMIO checker module in the FIM guarantees this response. When an unsupported MMIO read request to the FIM CSR region is detected, the FIM sends back a CPL (completion without data) with error status (UR) back to host.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1425-afu-access-violation","title":"14.2.5   AFU Access Violation","text":"
                  AFU access violations refer to the scenarios where a PF is attempting to access the MMIO region of an AFU bound to a VF (virtualization enabled), or when a VF is trying to access the MMIO region of an AFU bound to a PF (virtualization disabled). When such a violation is detected, the FIM drops the request and logs the error in the FME_ERROR0 register. If the request is an MMIO read request, the FIM returns a fake response to the host.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1426-afu-mmio-response-timeout","title":"14.2.6   AFU MMIO Response Timeout","text":"
                  An AFU MMIO Response timeout functions in the same manner described in the MMIO Response Timeout section.
                  "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#15-design-guidance","title":"15 Design Guidance","text":"
                  The OFS FIM is designed with configurability and scalability in mind.  At a high level, these are the necessary steps for a user to customize the design.  Please refer to the FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel Stratix 10
                  \n
                  Table 15-1 Features
                  \n\n    \n        \n            Step\n            Description\n            Comments  \n        \n    \n    \n        \n            1\n            Re-configure PCIe HIP for additional VFs (if necessary)\n            * PF0 is mandatory for running OPAE software* Only modification of the VFs (added or subtracted) by the user is recommended.\n\u2022   Default configuration supports 1 PF and 3 VFs.\n        \n        \n        \n            2\n            Update AXI4-Stram PF/VF MUX-DEMUX configuration (if necessary)\n\n            * The PF/VF MUX-DEMUX is parameterized for flexibility.  You can change, add or delete PF or VF functions by updating the top_cfg_pkg.sv file.\n* You also have the option of keeping the default configuration and tying off the unused VFs if needed.\n\n        \n \n            3\n            Update top level and AFU level as necessary\n\n     * If you integrating additional external interfaces, make the edits at the top level (iofs_top.sv) and propagate the interface down to the AFU level (afu_top.sv)\n \n            4\n            Add user implemented function(s) in AFU\n            * All of your implemented functions must have the required AXI4-Stream interface for both the data path and the MMIO control path to CSRs. * All CSRs in the user implemented function must have the required DFH layout. * See host exerciser CSRs for reference.\n   \n        \n\n\nFor more information on modifying the FIM, refer to the [Open FPGA Stack Technical Reference Manual].\n\n\n\n## Notices & Disclaimers\n\nIntel\u00ae technologies may require enabled hardware, software or service activation.\nNo product or component can be absolutely secure. \nPerformance varies by use, configuration and other factors.\nYour costs and results may vary. \nYou may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein.\nNo license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document.\nThe products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request.\nIntel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade.\nYou are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards. \n\u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. \n\nOpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. \n\n\n\n\n\n\n\n*[AFU]: Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region\n*[BBB]: Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID.\n*[BKC]: Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against.\n*[BMC]: Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors.\n*[DFL]: Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\n*[FIM]: FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring.\n*[FME]: FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform.\n*[HEM]:  Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc.\n*[JTAG]:  Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology.\n*[OFS]: Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs.\n*[OPAE SDK]: Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE.\n*[PIM]: Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols.\n*[PR]: Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page.\n*[RSU]: Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration.\n*[UVM]: Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework.\n*[TB]: Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output.\n*[Intel VT-d]: Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization.\n*[SR-IOV]: Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance.\n*[MMIO]: Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators.\n*[VFIO]: Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace.\n*[IOCTL]: Input/Output Control, System calls used to manipulate underlying device parameters of special files.\n*[AER]: Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting.\n*[PAC]: Programmable Acceleration Card: FPGA based Accelerator card"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/","title":"Platform Evaluation Script: Open FPGA Stack for Intel Stratix 10 FPGA","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#1-overview","title":"1 Overview","text":""},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#11-about-this-document","title":"1.1 About this Document","text":"
                  This document serves as a set-up and user guide for the checkout and evaluation of an Intel\u00ae FPGA PAC D5005 development platform using Open FPGA Stack (OFS). After reviewing the document, you will be able to:
                   
                     
                  • Set-up and modify the script to your environment
                    •  
                   
                     
                  • Compile and simulate an OFS reference design
                    •  
                   
                     
                  • Run hardware and software tests to evaluate the complete OFS flow
                    •  
                  "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#12-table-software-version-summary","title":"1.2 Table : Software Version Summary","text":"Component Version Description FPGA Platform Intel\u00ae FPGA PAC D5005 Intel platform you can use for your custom board development OFS FIM Source Code Branch: ofs-d5005, Tag: release/ofs-2023.3 OFS Shell RTL for Intel Stratix 10 FPGA (targeting Intel\u00ae FPGA PAC D5005) OFS FIM Common Branch: ofs-2023.3-2, Tag: ofs-2023.3-2 Common RTL across all OFS-based platforms AFU Examples Branch: examples-afu , Tag: ofs-examples-ofs-2023.3-1 Tutorials and simple examples for the Accelerator Functional Unit region (workload region) OPAE SDK Branch: 2.10.0-1, Tag: 2.10.0-1 Open Programmable Acceleration Engine Software Development Kit Kernel Drivers Branch: ofs-2023.3-6.1-2, Tag: ofs-2023.3-6.1-2 OFS specific kernel drivers OPAE Simulation Branch: opae-sim, Tag: 2.10.0-1 Accelerator Simulation Environment for hardware/software co-simulation of your AFU (workload) Intel Quartus Prime Pro Edition Design Software 23.3 Intel\u00ae Quartus\u00ae Prime Pro Edition Linux Software tool for Intel FPGA Development Operating System RHEL 8.6 Operating system on which this script has been tested 
                  A download page containing the release and already-compiled FIM binary artifacts that you can use for immediate evaluation on the Intel\u00ae FPGA PAC D5005 can be found on the OFS 2023.3 official release drop on GitHub.
                   
                   Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#2-introduction-to-ofs-evaluation-script","title":"2 Introduction to OFS Evaluation Script","text":"
                  By following the setup steps and using the OFS evaluation script you can quickly evaluate many features that the OFS framework provides and also leverage this script for your own development.  
                  "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#21-pre-requisites","title":"2.1 Pre-Requisites","text":"
                  This script uses the following set of software tools which should be installed using the directory structure below. Tool versions can vary.
                   
                     
                  • Intel Quartus\u00ae Prime Pro Software
                    •  
                  • Synopsys\u00ae VCS Simulator
                    •  
                  • Siemens\u00ae Questa\u00ae Simulator
                    •  
                   
                  Figure 2-1 Folder Hierarchy for Software Tools
                   
                   
                     
                  1.  
                    You must create a directory named \"ofs-X.X.X\" where the X represents the current release number, for example ofs-2023.3-2. 
                     
                    2.  
                  2.  
                    You must clone the required OFS repositories as per Figure 2-2 . Please refer to the BKC table for locations., Please go to [OFS Getting Started User Guide] for the instructions for the BKC installation.
                     
                    3.  
                  3.  
                    Once the repositories are cloned, copy the evaluation script (ofs_d5005_eval.sh) which is located at [eval_scripts] beneath the ofs-2023.3-2 directory location as shown in the example below:
                     
                    4.  
                   
                  Figure 2-2 Directory Structure for OFS Project
                   
                  ## ofs-2023.3-2\n##  -> examples-afu\n##  -> linux-dfl\n##  -> ofs-d5005\n##  -> oneapi-asp\n##  -> oneAPI-samples\n##  -> opae-sdk\n##  -> opae-sim\n##  -> ofs_d5005_eval.sh\n
                   
                     
                  1. Open the README file named (README_ofs_D5005_eval.txt) which is located at [eval_scripts] which informs the user which sections to modify in the script prior to building the FIM and running hardware, software and simulation tests. 
                    2.  
                  "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#22-intel-fpga-pac-d5005-evaluation-script-modification","title":"2.2 Intel\u00ae FPGA PAC D5005 Evaluation Script modification","text":"
                  To adapt this script to the user environment please follow the instructions below which explains which line numbers to change in the ofs_d5005_eval.sh script.
                  "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#user-directory-creation","title":"User Directory Creation","text":"
                  The user must create the top-level source directory and then clone the OFS repositories
                   
                  mkdir ofs-2023.3-2\n
                   
                  In the example above we have used ofs-2023.3-2 as the directory name
                  "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#set-up-proxy-server-lines-65-67","title":"Set-Up Proxy Server (lines 65-67)","text":"
                  Please enter the location of your proxy server to allow access to external internet to build software packages.
                   
                  Note: Failing to add proxy server will prevent cloning of repositories and the user will be unable to build the OFS framework.
                   
                  export http_proxy=<user_proxy>\nexport https_proxy=<user_proxy>\nexport no_proxy=<user_proxy>\n
                  "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#license-files-lines-70-72","title":"License Files (lines 70-72)","text":"
                  Please enter the the license file locations for the following tool variables
                   
                  export LM_LICENSE_FILE=<user_license>\nexport DW_LICENSE_FILE=<user_license>\nexport SNPSLMD_LICENSE_FILE=<user_license>\n
                  "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#tools-location-line-85-86-87-88","title":"Tools Location (line 85, 86, 87, 88)","text":"
                  Set Location of Quartus, Synopsys, Questasim and oneAPI Tools
                   
                  export QUARTUS_TOOLS_LOCATION=/home\nexport SYNOPSYS_TOOLS_LOCATION=/home\nexport QUESTASIM_TOOLS_LOCATION=/home\nexport ONEAPI_TOOLS_LOCATION=/opt\n
                  "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#quartus-tools-version-line-93","title":"Quartus Tools Version (line 93)","text":"
                  Set version of Quartus
                   
                  export QUARTUS_VERSION=23.3\n
                   
                  In the example above \"23.3\" is used as the Quartus tools version
                  "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#opae-tools-line-106","title":"OPAE Tools (line 106)","text":"
                  change OPAE SDK VERSION
                   
                  export OPAE_SDK_VERSION=2.10.0-1\n
                   
                  In the example above \"2.10.0-1\" is used as the OPAE SDK tools version
                  "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#pcie-bus-number-lines-231-and-238","title":"PCIe (Bus Number) (lines 231 and 238)","text":"
                  The Bus number must be entered by the user after installing the hardware in the chosen server, in the example below \"b1\" is the Bus Number for a single card as defined in the evaluation script.
                   
                  export ADP_CARD0_BUS_NUMBER=b1\n
                   
                  The evaluation script uses the bus number as an identifier to interrogate the card. The command below will identify the accelerator card plugged into a server. 
                   
                  lspci | grep acc\n\n86:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                   
                  The result identifies the card as being assigned \"86\" as the bus number so the entry in the script changes to
                   
                  export ADP_CARD0_BUS_NUMBER=86\n
                   
                  The user can also run the following command on the ofs_d5005_eval.sh script to automatically change the bus number to 86 in the ofs_d5005_eval.sh script.
                   
                  grep -rli '86' * | xargs -i@ sed -i '86' @
                   
                  if the bus number is 85 for example 
                   
                  85:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)
                   
                  the command to change to 85 in the evaluation script would be
                   
                  grep -rli '86' * | xargs -i@ sed -i '85' @
                   
                  The ofs_d5005_eval.sh script has now been modified to the server set-up and the user can proceed to build, compile and simulate the OFS stack
                   
                  "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#3-using-the-evaluation-script","title":"3 Using the Evaluation Script","text":""},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#31-overview","title":"3.1 Overview","text":"
                  The evaluation script focuses on different evaluation areas. Each of these menu options is described in the next section.  
                   
                  The figure below shows a snapshot of the full evaluation script menu showing all 57 options and each one of 10 sub-menus which focus on different areas of evaluation. Each of these menu options is described in the next section.
                   
                  Figure 3-1 ofs_d5005_eval.sh Evaluation Menu
                   
                  "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#311-tools-menu","title":"3.1.1 TOOLS MENU","text":"
                  By selecting \"List of Documentation for ADP Intel\u00ae FPGA PAC D5005 Project,\" a list of links to the latest OFS documentation appears. Note that these links will take you to documentation for the most recent release which may not correspond to the release version you are evaluating. To find the documentation specific to your release, ensure you clone the intel-ofs-docs tag that corresponds to your OFS version.
                   
                  By selecting \"Check Versions of Operating System and Quartus Premier Design Suite\", the tool verifies correct Operating System, Quartus version, kernel parameters, license files and paths to installed software tools.
                   
                   Menu Option Example Output 1 - List of Documentation for ADP D5005 Project Open FPGA Stack Overview Guides you through the setup and build steps to evaluate the OFS solution https://ofs.github.io 2 - Check versions of Operating System and Quartus Premier Design Suite (QPDS) Checking Linux release Linux version 6.1.41-dfl Checking RedHat release Red Hat Enterprise Linux release RHEL 8.6  Checking Ubuntu release cat: /etc/lsb-release: No such file or directory  Checking Kernel parameters BOOT_IMAGE=(hd0,msdos1)/vmlinuz-6.1.41-dfl-2023.3-1 root=/dev/mapper/rhel-root ro crashkernel=auto resume=/dev/mapper/rhel-swap rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200  Checking Licenses LM_LICENSE_FILE is set to port@socket number:port@socket number DW_LICENSE_FILE is set to port@socket number:port@socket number SNPSLMD_LICENSE_FILE is set to port@socket number:port@socket number  Checking Tool versions QUARTUS_HOME is set to /home/intelFPGA_pro/23.3/quartus QUARTUS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus IMPORT_IP_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../ip QSYS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../qsys/bin  Checking QPDS Patches Quartus Prime Shell Version 23.3"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#312-hardware-menu","title":"3.1.2 HARDWARE MENU","text":"
                  Identifies card by PCIe number, checks power, temperature and current firmware configuration.
                   
                   Menu Option Example Output 3 - Identify Acceleration Development Platform (ADP) D5005 Hardware via PCIe PCIe card detected as 86:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01) Host Server is connected to SINGLE card configuration 4 - Identify the Board Management Controller (BMC) Version and check BMC sensors Intel FPGA Programmable Acceleration Card D5005Board Management Controller, MAX10 NIOS FW version: 2.0.14Board Management Controller, MAX10 Build version: 2.0.8//****** BMC SENSORS ******//Object Id : 0xF000000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00Ports Num : 01Bitstream Id : 288511863935352239Bitstream Version : 4.0.1Pr Interface Id : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98 5 - Identify the FPGA Management Engine (FME) Version Intel FPGA Programmable Acceleration Card D5005Board Management Controller, MAX10 NIOS FW version: 2.0.14Board Management Controller, MAX10 Build version: 2.0.8//****** FME ******//Object Id : 0xF000000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00Ports Num : 01Bitstream Id : 288511863935352239Bitstream Version : 4.0.1Pr Interface Id : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98Boot Page : user 6 - Check Board Power and Temperature Intel FPGA Programmable Acceleration Card D5005Board Management Controller, MAX10 NIOS FW version: 2.0.14Board Management Controller, MAX10 Build version: 2.0.8//****** POWER ******//Object Id : 0xF000000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00Ports Num : 01Bitstream Id : 288511863935352239Bitstream Version : 4.0.1Pr Interface Id : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98( 1) VCCERAM Voltage : 0.90 Voltsetc ......................Intel FPGA Programmable Acceleration Card D5005Board Management Controller, MAX10 NIOS FW version: 2.0.14Board Management Controller, MAX10 Build version: 2.0.8//****** TEMP ******//Object Id : 0xF000000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00Ports Num : 01Bitstream Id : 288511863935352239Bitstream Version : 4.0.1Pr Interface Id : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98( 1) VCCT Temperature : 57.00 Celsiusetc ...................... 7 - Check Accelerator Port status //****** PORT ******// Object Id : 0xEF00000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00 8 - Check MAC and PHY status Intel FPGA Programmable Acceleration Card D5005Board Management Controller, MAX10 NIOS FW version: 2.0.14Board Management Controller, MAX10 Build version: 2.0.8//****** MAC ******//Object Id : 0xF000000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00Ports Num : 01Bitstream Id : 288511863935352239Bitstream Version : 4.0.1Pr Interface Id : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98MAC address : 64:4c:36:f:44:1fIntel FPGA Programmable Acceleration Card D5005Board Management Controller, MAX10 NIOS FW version: 2.0.14Board Management Controller, MAX10 Build version: 2.0.8//****** PHY ******//Object Id : 0xF000000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00Ports Num : 01Bitstream Id : 288511863935352239Bitstream Version : 4.0.1Pr Interface Id : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#313-fimpr-build-menu","title":"3.1.3 FIM/PR BUILD MENU","text":"
                  Builds FIM, Partial Reconfiguration Region and Remote Signal Tap
                   
                   Menu Option Description 9 - Check ADP software versions for ADP Intel\u00ae FPGA PAC D5005 Project OFS_ROOTDIR is set to /home/user_area/ofs-2023.3-2/ofs-d5005OPAE_SDK_REPO_BRANCH is set to release/2.10.0-1 OPAE_SDK_ROOT is set to /home/user_area/ofs-2023.3-2/ofs-d5005/../opae-sdk LD_LIBRARY_PATH is set to /home/user_area/ofs-2023.3-2/ofs-d5005/../opae-sdk/lib64: 10 - Build FIM for Intel\u00ae FPGA PAC D5005 Hardware This option builds the FIM based on the setting for the $ADP_PLATFORM, $FIM_SHELL environment variable. Check these variables in the following file ofs_d5005_eval.sh 11 - Check FIM Identification of FIM for Intel\u00ae FPGA PAC D5005 Hardware The FIM is identified by the following file fme-ifc-id.txt located at $OFS_ROOTDIR/$FIM_WORKDIR/syn/syn_top/ 12 - Build Partial Reconfiguration Tree for Intel\u00ae FPGA PAC D5005 Hardware This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development and also for the OneAPI build flow 13 - Build Base FIM Identification(ID) into PR Build Tree template This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU and OneAPI workloads 14 - Build Partial Reconfiguration Tree for Intel\u00ae FPGA PAC D5005 Hardware with Remote Signal Tap This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development and also for the OneAPI build flow and for the Remote Signal Tap flow 15 - Build Base FIM Identification(ID) into PR Build Tree template with Remote Signal Tap This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree for Remote Signal Tap to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU and OneAPI workloads"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#314-hardware-programmingdiagnostic-menu","title":"3.1.4 HARDWARE PROGRAMMING/DIAGNOSTIC MENU","text":"
                  The following submenu allows you to: * Program and check flash  * Perform a remote system update (RSU) of the FPGA image into the FPGA * Bind virtual functions to VFIO PCIe driver  * Run host exerciser (HE) commands such as loopback to test interfaces VFIO PCI driver binding * Read the control and status registers (CSRs) for bound modules that are part of the OFS reference design.
                   
                   Menu Option Description 16 - Program BMC Image into Intel\u00ae FPGA PAC D5005 Hardware The user must place a new BMC flash file in the following directory $OFS_ROOTDIR/bmc_flash_files. Once the user executes this option a new BMC image will be programmed. A remote system upgrade command is initiated to store the new BMC image 17 - Check Boot Area Flash Image from Intel\u00ae FPGA PAC D5005 Hardware This option checks which location area in FLASH the image will boot from, the default is user1  Boot Page : user1 18 - Program FIM Image into user1 area for Intel\u00ae FPGA PAC D5005 Hardware This option programs the FIM image \"D5005_page1_unsigned.bin\" into user1 area in flash 19 - Initiate Remote System Upgrade (RSU) from user1 Flash Image into Intel\u00ae FPGA PAC D5005 Hardware This option initiates a Remote System Upgrade and soft reboots the server and re-scans the PCIe bus for the new image to be loaded  2022-12-13 07:31:33,244 - [[pci_address(0000:86:00.0), pci_id(0x8086, 0xbcce, 0x8086, 0x138d)]] performing RSU operation 2022-12-13 07:31:33,249 - [[pci_address(0000:85:00.0), pci_id(0x8086, 0x2030, 0x1590, 0x00ea)]] removing device from PCIe bus 2022-12-13 07:31:34,333 - waiting 10.0 seconds for boot 2022-12-13 07:31:44,344 - rescanning PCIe bus: /sys/devices/pci0000:85/pci_bus/0000:85 2022-12-13 07:31:44,377 - RSU operation complete 20 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 21 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 22 - Create Virtual Functions (VF) and bind driver to vfio-pci Intel\u00ae FPGA PAC D5005 Hardware This option creates vfio-pci driver binding for the PF's and VF's Once the VF's have been bound to the driver the user can select menu option 20 to check that the new drivers are bound 23 - Run HE-LB Test This option runs 5 tests  1) checks and generates traffic with the intention of exercising the path from the AFU to the Host at full bandwidth 2) run a loopback throughput test using one cacheline per request 3) run a loopback read test using four cachelines per request 4) run a loopback write test using four cachelines per request 5) run a loopback throughput test using four cachelines per request 24 - Run HE-MEM Test This option runs 2 tests  1) Checking and generating traffic with the intention of exercising the path from FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host 2) run a loopback throughput test using one cacheline per request 25 - Run HE-HSSI Test This option runs 1 test  HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs, and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic  1) Send traffic through the 10G AFU 26 - Read from CSR (Command and Status Registers) for Intel\u00ae FPGA PAC D5005 Hardware This option reads from the following CSR's HE-LB Command and Status Register Default Definitions HE-MEM Command and Status Register Default Definitions HE-HSSI Command and Status Register Default Definitions"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#315-hardware-afu-testing-menu","title":"3.1.5  HARDWARE AFU TESTING MENU","text":"
                  This submenu tests partial reconfiguration by building and loading an memory-mapped I/O example AFU/workload, executes software from host, and tests remote signal tap.
                   
                   Menu Option Description 27 - Build and Compile host_chan_mmio example This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB_EXTERNAL/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS (Green Bit Stream) binary file ready for hardware programming 28 - Execute host_chan_mmio example This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test 29 - Modify host_chan_mmio example to insert Remote Signal Tap This option inserts a pre-defined host_chan_mmio.stp Signal Tap file into the OFS code to allow a user to debug the host_chan_mmio AFU example 30 - Build and Compile host_chan_mmio example with Remote Signal Tap This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB_EXTERNAL/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS (Green Bit Stream) binary file ready for hardware programming with Remote Signal tap enabled 31 - Execute host_chan_mmio example with Remote Signal Tap This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test. The user must open the Signal Tap window when running the host code to see the transactions in the Signal Tap window"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#316-hardware-afu-bbb-testing-menu","title":"3.1.6 HARDWARE AFU BBB TESTING MENU","text":"
                  This submenu tests partial reconfiguration using a hello_world example AFU/workload, executes sw from the host
                   
                   Menu Option Description 32 - Build and Compile hello_world example This option builds the hello_ world example from the following repo $FPGA_BBB_CCI_SRC/tutorial/afu_types/01_pim_ifc/$AFU_BBB_TEST_NAME, where AFU_BBB_NAME=hello_world. This produces a GBS (Green Bit Stream) file ready for hardware programming 33 - Execute hello_world example This option builds the host code for hello_world example and programs the GBS file and then executes the test"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#317-adp-oneapi-project-menu","title":"3.1.7 ADP ONEAPI PROJECT MENU","text":"
                  Builds OneAPI kernel, executes the software from host and runs diagnostic tests
                   
                   Menu Option Result 34 - Check oneAPI software versions for Intel\u00ae FPGA PAC D5005 Project This option checks the setup of the oneAPI software and adds the relevant oneAPI environment variables to the terminal. This option also informs the user to match the oneAPI software version to the oneAPI-samples version 35 - Build and clone shim libraries required by oneAPI host This option builds the oneAPI directory structure 36 - Install OneAPI Host Driver This option Installs the oneAPI Host driver at the following location /opt/Intel/OpenCLFPGA/oneAPI/Boards/, and requires sudo permission 37 - Uninstall One API Host Driver This option Uninstall's the oneAPI Host driver, and requires sudo permissions 38 - Diagnose oneAPI Hardware This option Checks ICD (Intel Client Driver) and FCD (FPGA Client Driver), oneAPI library locations and detects whether oneAPI BSP is loaded into the FPGA 39 - Build oneAPI BSP ofs-d5005 Default Kernel (hello_world) This option Builds the oneAPI BSP using hello_world kernel 40 - Build oneAPI MakeFile Environment This option Builds the oneAPI environment using a Makefile for kernel insertion 41 - Compile oneAPI Sample Application (board_test) for Emulation This option compiles the board_test kernel for Emulation 42 - Run oneAPI Sample Application (board_test) for Emulation This option executes the board_test kernel for Emulation 43 - Generate oneAPI Optimization report for (board_test) This option generates an optimization report for the board_test kernel 44 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 45 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 46 - Create Virtual Function (VF) and bind driver to vfio-pci Intel\u00ae FPGA PAC D5005 Hardware This option creates vfio-pci driver binding for the PF's and VF's Once the VF's have been bound to the driver the user can select menu option 45 to check that the new drivers are bound 47 - Program oneAPI BSP ofs-d5005 Default Kernel (hello_world) This option programs the FPGA with a aocx file based on the hello_world kernel 48 - Compile oneAPI Sample Application (board_test) for Hardware This option compiles the board_test kernel for Hardware 49 - Run oneAPI Sample Application (board_test) for Hardware This option builds the host code for board_test kernel and executes the program running through kernel and host bandwidth tests"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#318-unit-test-project-menu","title":"3.1.8 UNIT TEST PROJECT MENU","text":"
                  Builds, compiles and runs standalone simulation block tests. More unit test examples are found at the following location ofs-d5005/sim/unit_test 
                   
                   Menu Option Result 50 - Generate Simulation files for Unit Test This option builds the simulation file set for running a unit test simulation 51 - Simulate Unit Test dfh_walker and log waveform This option runs the dfh_walker based on the environment variable \"UNIT_TEST_NAME=dfh_walker\" in the evaluation script. A user can change the test being run by modifying this variable"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#319-adp-uvm-project-menu","title":"3.1.9 ADP UVM PROJECT MENU","text":"
                  Builds, compiles and runs full chip simulation tests. The user should execute the options sequentially ie 52, 53, 54 and 55
                   
                   Menu Option Description 52 - Check UVM software versions for Intel\u00ae FPGA PAC D5005 Project DESIGNWARE_HOME is set to /home/synopsys/vip_common/vip_Q-2020.03A UVM_HOME is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm VCS_HOME is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel VERDIR is set to /home/user_area/ofs-2023.3-2/ofs-d5005/verification VIPDIR is set to /home/user_area/ofs-2023.3-2/ofs-d5005/verification 53 - Compile UVM IP This option compiles the UVM IP 54 - Compile UVM RTL and Testbench This option compiles the UVM RTL and Testbench 55 - Simulate UVM ofs_mmio_test and log waveform This option runs the dfh_walking test based on the environment variable \"UVM_TEST_NAME=dfh_walking_test\" in the evaluation script. A user can change the test being run by modifying this variable 56 - Simulate all UVM test cases (Regression Mode) This option runs the Intel\u00ae FPGA PAC D5005 regression mode, cycling through all UVM tests defined in /ofs-d5005/verification/tests/test_pkg.svh file"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#3110-adp-build-all-project-menu","title":"3.1.10 ADP BUILD ALL PROJECT MENU","text":"
                  Builds the complete OFS flow, good for regression testing and overnight builds
                   
                  For this menu, a user can run a sequence of tests (compilation, build and simulation) and executes them sequentially. After the script is successfully executed, a set of binary files is produced which a you can use to evaluate your hardware. Log files are also produced which checks whether the tests passed.
                   
                  A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 57 from the main menu the script will execute 23 tests ie (main menu options 2, 9, 10, 11, 12, 13, 14, 15, 27, 29, 30, 32, 34, 35, 39, 40, 48, 50, 51, 52, 53, 54 and 55. These 23 menu options are chosen to build the complete OFS flow covering build, compile and simulation.
                   
                   Menu Option Result 57 - Build and Simulate Complete Intel\u00ae FPGA PAC D5005  Project Generating Log File with date and timestamp Log file written to /home/user_area/ofs-2023.3-2/log_files/D5005_log_2022_11_10-093649/ofs-d5005_eval.log 
                  Definition of Multi-Test Set-up
                   
                  Menu Option 57 above in the evaluation script can be refined to tailor the number of tests the users runs. The set-up is principally defined by the variable below
                   
                  MULTI_TEST[A,B]=C
                   
                  where
                   
                  A= Total Number of menu options in script B= Can be changed to a number to select the test order C= Menu Option in Script
                   
                  Example 1 MULTI_TEST[57,0]=2
                   
                  A= 57 is the total number of options in the script B= 0 indicates that this is the first test to be run in the script C= Menu option in Script ie 2- List of Documentation for ADP Intel\u00ae FPGA PAC D5005 Project
                   
                  Example 2 MULTI_TEST[57,0]=2 MULTI_TEST[57,1]=9
                   
                  In the example above two tests are run in order ie 0, and 1 and the following menu options are executed ie 2- List of Documentation for ADP Intel\u00ae FPGA PAC D5005 Project and 9 - Check ADP software versions for ADP Intel\u00ae FPGA PAC D5005 Project
                   
                  The user can also modify the build time by de-selecting options they do not wish to use, see below for a couple of use-case scenarios.
                   
                  Default User Case
                   
                  A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 57 from the main menu the script will execute 23 tests ie (main menu options 2, 9, 10, 11, 12, 13, 14, 15, 27, 29, 30, 32, 34, 35, 39, 40, 48, 50, 51, 52, 53, 54 and 55. All other tests with an \"X\" indicates do not run that test
                   
                   
                  User Case for ADP FIM/PR BUILD MENU
                   
                  In the example below when the user selects option 57 from the main menu the script will only run options from the ADP FIM/PR BUILD MENU (7 options, main menu options 9, 10, 11, 12, 13, 14 and 15). All other tests with an \"X\" indicates do not run that test.
                   
                   
                  "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#4-common-test-scenarios","title":"4 Common Test Scenarios","text":"
                  This section will describe the most common compile build scenarios if a user wanted to evaluate an acceleration card on their server. The Pre-requisite column indicates the menu commands that must be run before executing the test eg To run Test 5 then a user needs to have run option 10, 12 and 13 before running options 20, 21, 22, 27 and 28.
                   Test Test Scenario Pre-Requisite Menu Option Menu Option Test 1 FIM Build - 10 Test 2 Partial Reconfiguration Build 10 12 Test 3 Program FIM and perform Remote System Upgrade 10 18, 19 Test 4 Bind PF and VF to vfio-pci drivers - 20, 21, 22 Test 5 Build, compile and test AFU on hardware 10, 12, 13 20, 21, 22, 27, 28 Test 6 Build, compile and test AFU Basic Building Blocks on hardware 10, 12, 13 20, 21, 22, 32, 33 Test 7 Build, compile and test oneAPI on hardware 10, 12, 13 34, 35, 36, 39, 40, 44, 45, 46, 47, 48, 49 Test 8 Build and Simulate Unit Tests - 50, 51 Test 9 Build and Simulate UVM Tests - 52, 53, 54, 55"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                  Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                   
                  OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/","title":"Getting Started Guide: Open FPGA Stack for Intel Stratix 10","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#10-introduction","title":"1.0 Introduction","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#11-about-this-document","title":"1.1 About This Document","text":"
                  This document helps users get started in evaluating Open FPGA Stack (OFS) for Intel\u00ae Stratix 10\u00ae FPGA targeting the Intel\u00ae FPGA PAC D5005. After reviewing the document a user shall be able to:
                   
                     
                  • Set up a development environment with all OFS ingredients
                    •  
                  • Build and install the OFS Linux Kernel drivers
                    •  
                  • Build and install the Open Programmable Acceleration Engine Software Development Kit (OPAE SDK) software on top of the OFS Linux kernel drivers
                    •  
                  • Flash an OFS FIM binary onto the Intel\u00ae FPGA PAC D5005
                    •  
                  • Verify the functionality of OFS on an Intel\u00ae FPGA PAC D5005 board
                    •  
                  • Know where to find additional information on all OFS ingredients
                    •  
                   
                  The following flow charts show a high level overview of the initial bringup process, split into three sequential diagrams.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#diagram-1-installing-the-opae-sdk","title":"Diagram 1: Installing the OPAE SDK","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#diagram-2-installing-the-linux-dfl-drivers","title":"Diagram 2: Installing the Linux DFL Drivers","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#diagram-3-bringing-up-the-intel-d5005","title":"Diagram 3: Bringing up the Intel D5005","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#12-terminology","title":"1.2 Terminology","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#13-introduction-to-ofs","title":"1.3 Introduction to OFS","text":"
                  Each OFS reference FIM targets a specific platform, but the modular hardware, software, simulation and test infrastructure allows you to modify each part of the design and test environment for your own custom acceleration platform card. The current OFS reference FIM for Stratix 10 FPGA targets the Intel\u00ae FPGA PAC D5005 board. This document focuses exclusively on the OFS release targeting the Intel\u00ae FPGA PAC D5005 board.
                   
                  The OFS repositories (in OFS ) on GitHub provide the following components targeting an Intel\u00ae FPGA PAC D5005:
                   
                     
                  • opae-sdk: Contains the Open Programmable Acceleration Software Development Kit source code and build scripts.
                    •  
                  • linux-dfl: Contains Linux kernel-level driver source code and build scripts.
                    •  
                  • intel-ofs-fim: Contains the source code, build scripts and verification suite for FPGA RTL source code
                    •  
                  • ofs-hld-shim: Contains the necessary files to generate Shim/BSP for OFS Cards, using OPAE SDK Interfaces.
                    •  
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#14-intended-audience","title":"1.4 Intended Audience","text":"
                  The information in this document is intended for customers evaluating the Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA on the Intel PAC D5005. This document will cover key topics related to initial setup and development, with links for deeper dives on the topics discussed therein.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#15-reference-documents","title":"1.5 Reference Documents","text":"
                  Please refer to the README on the OFS GitHub for an updated list of collateral on the OFS GitHub page.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#16-component-version-summary","title":"1.6 Component Version Summary","text":"
                  The OFS 2023.3 Release targeting the Intel\u00ae Stratix 10\u00ae FPGA is built upon tightly coupled software and firmware versions. Use this section as a general reference for the versions which comprise this release.
                   
                  The following table highlights the hardware which makes up the Best Known Configuration (BKC) for the OFS 2023.3 release.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#table-1-2-hardware-bkc","title":"Table 1-2: Hardware BKC","text":"Component 1 x Intel\u00ae FPGA PAC D5005 1 x Supported Server Model 1 x Intel FPGA Download Cable II (Optional, only required if loading images via JTAG) 
                  The following table highlights the versions of the software which comprise the OFS stack. The installation of the user-space OPAE SDK on top of the kernel-space linux-dfl drivers is discussed in subsequent sections of this document.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#table-1-3-software-version-summary","title":"Table 1-3: Software Version Summary","text":"Component Version FPGA Platform Intel\u00ae FPGA PAC D5005 OPAE SDK Tag: 2.10.0-1 Kernel Drivers Tag: ofs-2023.3-6.1-2 OFS FIM Source Code Branch: release/ofs-2023.3 Intel Quartus Prime Pro Edition Design Software 23.3 Intel\u00ae Quartus\u00ae Prime Pro Edition Linux Operating System RHEL 8.6 
                  A download page containing the release and already-compiled FIM binary artifacts that you can use for immediate evaluation on the Intel\u00ae FPGA PAC D5005 can be found on the OFS 2023.3 official release drop on GitHub.
                   
                  Note: If you wish to freeze your Red Hat operating system version on the RHEL 8.6, refer to the following solution provided in the Red Hat customer portal.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#20-ofs-stack-architecture-overview-for-reference-platform","title":"2.0 OFS Stack Architecture Overview for Reference Platform","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#21-hardware-components","title":"2.1 Hardware Components","text":"
                  The OFS hardware architecture decomposes all designs into a standard set of modules, interfaces, and capabilities. Although the OFS infrastructure provides a standard set of functionality and capability, the user is responsible for making the customizations to their specific design in compliance with the specifications outlined in the FPGA Interface Manager Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA.
                   
                  OFS is a blanket term which can be used to collectively refer to all ingredients of the OFS reference design, which includes the core hardware components discussed below and software.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#211-fpga-interface-manager","title":"2.1.1 FPGA Interface Manager","text":"
                  The FPGA Interface Manager (FIM) or 'shell' provides platform management functionality, clocks, resets, and interface access to the host and peripheral features on the acceleration platform. The FIM is implemented in a static region of the FPGA device.
                   
                  The primary components of the FIM reference design are:
                   
                     
                  • PCIe Subsystem
                    •  
                  • Transceiver Subsystem
                    •  
                  • Memory Subsystem
                    •  
                  • FPGA Management Engine
                    •  
                  • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                    •  
                  • Board Peripheral Fabric for master to slave CSR accesses from host or AFU
                    •  
                  • Interface to Board Management Controller (BMC)
                    •  
                   
                  The FPGA Management Engine (FME) provides management features for the platform and the loading/unloading of accelerators through partial reconfiguration.
                   
                  For more information on the FIM and its external connections, please refer to the FPGA Interface Manager Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA, and the Intel FPGA Programmable Acceleration Card D5005 Data Sheet. Below is a high-level block diagram of the FIM.
                   
                  Figure 2-1 FIM Overview
                   
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#212-afu","title":"2.1.2 AFU","text":"
                  An AFU is an acceleration workload that interfaces to the FIM. The AFU boundary in this reference design comprises both static and partial reconfiguration (PR) regions. You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region. A port gasket within the design provides all the PR specific modules and logic required for partial reconfiguration. Only one partial reconfiguration region is supported in this design.
                   
                  Similar to the FME, the port gasket exposes its capabilities to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space.
                   
                  You can compile your design in one of the following ways:
                   
                     
                  • Your AFU resides in a partial reconfiguration (PR) region of the FPGA.
                    •  
                  • Your AFU is a part of the static region (SR) and is a compiled flat design.
                    •  
                  • Your AFU contains both static and PR regions.
                    •  
                   
                  The AFU provided in this release is comprised of the following functions:
                   
                     
                  • AFU interface handler to verify transactions coming from the AFU region.
                    •  
                  • PV/VF Mux to route transactions to and from corresponding AFU components, including the ST2MM module, PCIe loopback host exerciser (HE-LB), HSSI host exerciser (HE-HSSI), and Memory Host Exerciser (HE-MEM).
                    •  
                  • AXI4 Streaming to Memory Map (ST2MM) Module that routes MMIO CSR accesses to FME and board peripherals.
                    •  
                  • Host exercisers to test PCIe, memory and HSSI interfaces (these can be removed from the AFU region after your FIM design is complete to provide more resource area for workloads).
                    •  
                  • Port gasket and partial reconfiguration support.
                    •  
                   
                  For more information on the Platform Interface Manager (PIM) and AFU development and testing, please refer to the [OFS AFU Development Guide].
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#22-ofs-software-overview","title":"2.2 OFS Software Overview","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#221-kernel-drivers-for-ofs","title":"2.2.1 Kernel Drivers for OFS","text":"
                  OFS DFL driver software provides the bottom-most API to FPGA platforms. Libraries such as OPAE and frameworks like DPDK are consumers of the APIs provided by OFS. Applications may be built on top of these frameworks and libraries. The OFS software does not cover any out-of-band management interfaces. OFS driver software is designed to be extendable, flexible, and provide for bare-metal and virtualized functionality. An in depth look at the various aspects of the driver architecture such as the API, an explanation of the DFL framework, and instructions on how to port DFL driver patches to other kernel distributions can be found on the DFL Wiki page.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#30-intel-fpga-pac-d5005-card-and-server-requirements","title":"3.0 Intel FPGA PAC D5005 Card and Server Requirements","text":"
                  Currently OFS for Intel\u00ae Stratix 10\u00ae FPGA targets the Intel\u00ae FPGA PAC D5005. Because the Intel\u00ae FPGA PAC D5005 is a production card, you must prepare the card in order to receive a new non-production bitstream. For these instructions, please contact an Intel representative.
                   
                  In addition, refer to sections 2.1-2.3 of the Intel Acceleration Stack Quick Start Guide: Intel FPGA Programmable Acceleration Card D5005 for a complete overview of the physical installation process and ESD precautions for the D5005 platform.
                   
                  Note: Ensure that the system meets all the following requirements   before proceeding to install the Intel\u00ae FPGA PAC D5005 into a server.
                   
                  Table 3-1 Server Requirements for Intel D5005
                   Component Description Server Qualified Servers Main Board PCI Express 3.0 compliant motherboard with at least one dual-width x16 PCIe slot available for card installation Board Power Supply* Auxiliary Power (12V) 
                  * For more information on the required auxiliary power supply, refer to section 2.2.2 of the Intel FPGA Programmable Acceleration Card D5005 Data Sheet.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#31-supported-processors-for-intel-d5005","title":"3.1 Supported Processors for Intel D5005","text":"
                  OFS requires that the deployment machine's Xeon processor must support the following technologies. These options must also be enabled in the BIOS and as kernel parameters. The process to enable these parameters will be discussed in the section on driver installation:
                   
                     
                  • Intel VT-d (Intel Virtualization Technology for IA-32 and Intel 64 Processors)
                    •  
                  • Intel VT-x (Intel Virtualization Technology for Directed I/O)
                    •  
                  • Intel IOMMU
                    •  
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#32-cooling-requirements-for-the-intel-fpga-pac-d5005","title":"3.2 Cooling Requirements for the Intel FPGA PAC D5005","text":"
                  Please refer to sections 8.1 and 8.2 of the Intel FPGA Programmable Acceleration Card D5005 Data Sheet for guidance on cooling specifications that must be met when using the D5005 card. Failure to adhere to these guidelines may result in thermal runaway and/or performance degradation.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#40-ofs-dfl-kernel-drivers","title":"4.0 OFS DFL Kernel Drivers","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#41-ofs-dfl-kernel-driver-environment-setup","title":"4.1 OFS DFL Kernel Driver Environment Setup","text":"
                  All OFS DFL kernel driver code resides in the Linux DFL GitHub repository. This repository is open source and does not require any permissions to access. It includes a snapshot of the latest best-known configuration (BKC) Linux kernel with the OFS driver included in the drivers/fpga/* directory. Downloading, configuration, and compilation will be discussed in this section. Please refer to Table 1-3 for the latest supported OS.
                   
                  The DFL driver suite can be automatically installed using a supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3 Release Page.
                   
                  It is recommended you boot into your operating system's native 4.18.x kernel before attempting to upgrade to the dfl enabled 6.1.41 You may experience issues when moving between two dfl enabled 6.1.41 kernels.
                   
                  This installation process assumes the user has access to an internet connection in order to pull specific GitHub repositories, and to satisfy package dependencies.
                   
                  1. You must make the following changes in order to install all dependencies on the latest BKC Operating System. These are required to both build and install the drivers from source, and to install them from pre-built packages:
                   
                   
                  $ subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms\n$ sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n
                   
                  2. You must satisfy the following package dependencies if building and installing the drivers from source. Double check that all packages have been found and installed:
                   
                   
                  $ sudo dnf install -y python3 python3-pip python3-devel \\\ngdb vim git gcc gcc-c++ make cmake libuuid-devel rpm-build systemd-devel sudo nmap \\\npython3-jsonschema json-c-devel tbb-devel rpmdevtools libcap-devel \\\nspdlog-devel cli11-devel python3-pyyaml hwloc-devel libedit-devel openssl-devel\n\n$ python3 -m pip install --user jsonschema virtualenv pudb pyyaml\n\n$ sudo pip3 uninstall setuptools\n\n$ sudo pip3 install --upgrade setuptools --prefix=/usr\n\n# To Install pybind11 following are the steps\n\n$ curl 'ftp://ftp.pbone.net/mirror/archive.fedoraproject.org/epel/8.2.2020-11-04/Everything/x86_64/Packages/p/python3-pybind11-2.4.3-2.el8.x86_64.rpm' --output ./python3-pybind11-2.4.3-2.el8.x86_64.rpm\n\n$ curl 'ftp://ftp.pbone.net/mirror/archive.fedoraproject.org/epel/8.2.2020-11-04/Everything/x86_64/Packages/p/pybind11-devel-2.4.3-2.el8.x86_64.rpm' --output ./pybind11-devel-2.4.3-2.el8.x86_64.rpm\n\n$ sudo dnf localinstall ./python3-pybind11-2.4.3-2.el8.x86_64.rpm ./pybind11-devel-2.4.3-2.el8.x86_64.rpm\n
                   
                  It is recommended you create an empty top-level directory for their OFS related repositories to keep the working environment clean. All steps in this installation will use a generic top-level directory at /home/user/OFS/. If you have created a different top-level directory, replace this path with your custom path.
                   
                  3. Initialize an empty git repository and clone the LTS tagged DFL driver source code: 
                   
                  $ cd /home/user/OFS/\n$ git init\n$ git clone https://github.com/OPAE/linux-dfl\n$ cd /home/user/OFS/linux-dfl\n$ git checkout tags/ofs-2023.3-6.1-2 -b fpga-ofs-dev-6.1.41\n
                   
                  4. Verify that the correct tag has been checkout out.
                   
                  $ git describe \n$ ofs-2023.3-6.1-2\n
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#42-building-and-installing-the-ofs-dfl-kernel-drivers-from-source","title":"4.2 Building and Installing the OFS DFL Kernel Drivers from Source","text":"
                  1. The following set of instructions walk you through copying an existing kernel configuration file on your machine and changing the minimal required configuration settings:
                   
                  $ cd /home/user/OFS/linux-dfl\n$ cp /boot/config-`uname -r` .config\n$ cat configs/dfl-config >> .config\n$ echo 'CONFIG_LOCALVERSION=\"-dfl\"' >> .config\n$ echo 'CONFIG_LOCALVERSION_AUTO=y' >> .config\n$ sed -i -r 's/CONFIG_SYSTEM_TRUSTED_KEYS=.*/CONFIG_SYSTEM_TRUSTED_KEYS=\"\"/' .config\n$ sed -i '/^CONFIG_DEBUG_INFO_BTF/ s/./#&/' .config\n$ echo 'CONFIG_DEBUG_ATOMIC_SLEEP=y' >> .config\n$ export LOCALVERSION=\n$ make olddefconfig\n
                   
                  (Optional) To use the built-in GUI menu for editing kernel configuration parameters, you can opt to run make menuconfig.
                   
                  2. Linux kernel builds take advantage of multiple processors to parallelize the build process. Display how many processors are available with the nproc command, and then specify how many make threads to utilize with the -j option. Note that number of threads can exceed the number of processors. In this case, the number of threads are set to the number of processors in the system.
                   
                  $ cd /home/user/OFS/linux-dfl\n$ make -j `nproc`\n$ make -j `nproc` modules\n
                   3. The following options are available to o locally build a set of packages:
                   
                     
                  • rpm-pkg: Build both source and binary RPM kernel packages
                    •  
                  • binrpm-pkg: Build only the binary kernel RPM package
                    •  
                  • deb-pkg: Build both source and binary deb kernel packages
                    •  
                  • bindeb-pkg: Build only the binary kernel deb package
                    •  
                   
                  If you are concerned about the size of the resulting package and binaries, you can significantly reduce the size of the package and object files by using the make variable INSTALL_MOD_STRIP. If this is not a concern, feel free to skip this step. The below instructions will build a set of binary RPM packages:
                   
                  $ cd /home/user/OFS/linux-dfl\n$ make INSTALL_MOD_STRIP=1 binrpm-pkg\n
                   
                  4. By default a directory is created in your home directory called rpmbuild. This directory will house all of the kernel packages which have been built. You need to navigate to the newly built kernel packages and install them. The following files were generated using the build command executed in the previous step:
                   
                  $ cd ~/rpmbuild/RPMS/x86_64\n$ ls\n$ kernel-6.1.41_dfl-1.x86_64.rpm  kernel-headers-6.1.41_dfl-1.x86_64.rpm\n$ sudo dnf localinstall kernel*.rpm\n
                   
                  5. The system will need to be rebooted for changes to take effect. After a reboot, select the newly built kernel as the boot target. This can be done pre-boot using the command grub2-reboot, which removes the requirement for user intervention. After boot, verify that the currently running kernel matches expectation.
                   
                  $ uname -r\n$ 6.1.41-dfl\n
                   
                  6. Verify the DFL drivers have been successfully installed. If an Intel\u00ae FPGA PAC D5005 card with the appropriate FIM is on the local system, the kernel driver modules will have been loaded. In the lsmod output the second column corresponds to the size of the kernel module in bytes, the third column displays the number of devices registered to that driver, and the fourth column displays the names of the devices using it. Verify their versions against the below.
                   
                  $ lsmod | grep -e fpga -e dfl\n\nuio_dfl                20480  0\nspi_altera_dfl         20480  0\nuio                    20480  1 uio_dfl\ndfl_emif               16384  0\nspi_altera_core        16384  1 spi_altera_dfl\ndfl_fme_region         20480  0\ndfl_fme_br             16384  0\ndfl_fme_mgr            20480  0\ndfl_afu                36864  0\ndfl_fme                49152  0\ndfl_pci                20480  0\ndfl                    40960  8 dfl_pci,s10hssi,uio_dfl,dfl_fme,dfl_fme_br,dfl_afu,spi_altera_dfl,dfl_emif\nfpga_region            20480  3 dfl_fme_region,dfl_fme,dfl\nfpga_bridge            20480  4 dfl_fme_region,fpga_region,dfl_fme,dfl_fme_br\nfpga_mgr               24576  4 dfl_fme_region,fpga_region,dfl_fme_mgr,dfl_fme\n
                   
                  If an Intel\u00ae FPGA PAC D5005 card is not installed in the system and/or does not have the appropriate FIM configured, the user may read version information of the DFL drivers directly from /lib/modules:
                   
                  $ cd /usr/lib/modules/`uname -r`/kernel/drivers/fpga\n$ modinfo dfl* fpga* | grep ^name\n\nname:           dfl_afu\nname:           dfl_fme_br\nname:           dfl_fme\nname:           dfl_fme_mgr\nname:           dfl_fme_region\nname:           dfl_hssi\nname:           dfl\nname:           dfl_n3000_nios\nname:           dfl_pci\nname:           fpga_bridge\nname:           fpga_mgr\nname:           fpga_regions\n
                   
                  7. Four kernel parameters must be added to the boot command-line for the newly installed kernel. First, open the file grub:
                   
                  $ sudo vim /etc/default/grub\n
                   
                  8. In the variable GRUB_CMDLINE_LINUX add the parameters shown after quiet:
                   
                  GRUB_CMDLINE_LINUX=\"crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\"\n
                   
                  Note: If you wish to instead set hugepages on a per session basis, you can perform the following step. These settings will be lost on reboot.
                   
                  $ mkdir -p /mnt/huge \n$ mount -t hugetlbfs nodev /mnt/huge \n$ echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages \n$ echo 2048 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages \n
                   
                  9. Save your edits, then apply them to the GRUB2 configuration file.
                   
                  $ sudo grub2-mkconfig  -o /boot/efi/EFI/redhat/grub.cfg\n
                   
                  10. Warm reboot. Your kernel parameter changes should have taken affect.
                   
                  $ cat /proc/cmdline\nBOOT_IMAGE=(hd0,gpt2)/vmlinuz-6.1.41-dfl root=/dev/mapper/rhel_bapvedell028-root ro crashkernel=auto resume=/dev/mapper/rhel_bapvedell028-swap rd.lvm.lv=rhel_bapvedell028/root rd.lvm.lv=rhel_bapvedell028/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\n
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#50-opae-software-development-kit","title":"5.0 OPAE Software Development Kit","text":"
                  The OPAE SDK software stack sits in user space on top of the OFS kernel drivers. It is a common software infrastructure layer that simplifies and streamlines integration of programmable accelerators such as FPGAs into software applications and environments. OPAE consists of a set of drivers, user-space libraries, and tools to discover, enumerate, share, query, access, manipulate, and reconfigure programmable accelerators. OPAE is designed to support a layered, common programming model across different platforms and devices.
                   
                  The OPAE SDK source code is contained within a single GitHub repository hosted at the OPAE GitHub. This repository is open source.
                   
                  You may choose to use the supplied Python 3 installation script to handle OPAE SDK installation. This script ships with a README detailing execution instructions on the OFS 2023.3.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#51-opae-sdk-build-environment-setup","title":"5.1 OPAE SDK Build Environment Setup","text":"
                  Ensure the local environment matches the supported Operating System discussed in section Table 1-3: Software Version Summary. This installation process assumes you have access to an internet connection in order to pull specific GitHub repositories, and to satisfy package dependencies.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#511-installing-the-opae-sdk-with-pre-built-packages","title":"5.1.1 Installing the OPAE SDK with Pre-Built Packages","text":"
                  You can skip the entire build process and use a set of pre-built binaries supplied by Intel. Visit the OFS 2023.3 and navigate to the bottom of the page, under the Assets tab you will see a file named opae-2.10.0-1.x86_64-<>_<>.tar.gz. Download this package and extract its contents: 
                  $ dnf install --enablerepo=codeready-builder-for-rhel-8-x86_64-rpms -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel\n\n$ pip3 install --upgrade --prefix=/usr pip setuptools pybind11\n\n$ tar xf opae-2.10.0-1.x86_64-<<date>>_<<build>>.tar.gz\n
                   
                  For a fast installation you can delete the source RPM as it isn't necessary, and install all remaining OPAE RPMs:
                   
                  $ rm opae-*.src.rpm\n$ sudo dnf localinstall opae*.rpm\n
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#512-building-and-installing-the-opae-sdk-from-source","title":"5.1.2 Building and Installing the OPAE SDK from Source","text":"
                  1. Before OPAE SDK installation the user must remove any prior OPAE frameworks.  To remove these packages:
                   
                   
                  $ sudo dnf remove opae*\n
                   
                  2. It is recommended you create an empty top-level directory for their OFS related repositories to keep the working environment clean. All steps in this installation will use a generic top-level directory at /home/user/OFS/. If you have created a different top-level directory, replace this path with your custom path.
                   
                  3. Initialize an empty git repository and clone the tagged OPAE SDK source code:
                   
                   
                  $ cd /home/user/OFS/\n$ git init\n$ git clone https://github.com/OPAE/opae-sdk.git\n$ cd opae-sdk\n$ git checkout tags/2.10.0-1 -b release/2.10.0\n
                   
                  4. Verify that the correct tag has been checkout out:
                   
                  $ git describe --tags\n2.10.0-1\n
                   
                  5. Build the OPAE SDK source code, and pack it into several local RPM packages. Building the code into packages allows for easier installation and removal.
                   
                  $ cd /home/user/OFS\n\n$ podman pull registry.access.redhat.com/ubi8:8.6\n$ podman run -ti -v \"$PWD\":/src:Z -w /src registry.access.redhat.com/ubi8:8.6\n\n# Everything after runs within container:\n\n$ dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n\n$ dnf install --enablerepo=codeready-builder-for-rhel-8-x86_64-rpms -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel\n\n$ pip3 install --upgrade --prefix=/usr pip setuptools pybind11\n\n$ ./opae-sdk/packaging/opae/rpm/create unrestricted\n\n$ exit\n
                   
                  6. After a successful compile there should be 8 packages present:
                   
                  $ ls | grep rpm\n\nopae-2.10.0-1.el8.src.rpm\nopae-2.10.0-1.el8.x86_64.rpm\nopae-debuginfo-2.10.0-1.el8.x86_64.rpm\nopae-debugsource-2.10.0-1.el8.x86_64.rpm\nopae-devel-2.10.0-1.el8.x86_64.rpm\nopae-devel-debuginfo-2.10.0-1.el8.x86_64.rpm\nopae-extra-tools-2.10.0-1.el8.x86_64.rpm\nopae-extra-tools-debuginfo-2.10.0-1.el8.x86_64.rpm\n
                   Remove the opae-2.10.0-1.el8.src.rpm file as it is not used. 
                  $ rm opae-2.10.0-1.el8.src.rpm\n
                   
                  7. Install the user-built OPAE SDK packages:
                   
                  $ sudo dnf clean all\n\n$ sudo dnf localinstall -y opae*.rpm\n
                   
                  8. Check that all packages have been installed:
                   
                  $ rpm -qa | grep opae\n\nopae-extra-tools-2.10.0-1.el8.x86_64\nopae-debugsource-2.10.0-1.el8.x86_64\nopae-2.10.0-1.el8.x86_64\nopae-extra-tools-debuginfo-2.10.0-1.el8.x86_64\nopae-debuginfo-2.10.0-1.el8.x86_64\nopae-devel-2.10.0-1.el8.x86_64\nopae-devel-debuginfo-2.10.0-1.el8.x86_64\n
                   
                  You can query information about each installed package using rpm -qi <package__name>.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#513-fpga-device-access-permissions","title":"5.1.3 FPGA Device Access Permissions","text":"
                  Access to FPGA accelerators and devices is controlled using file access permissions on the Intel\u00ae FPGA device files, /dev/dfl-fme.* and /dev/dfl-port.*, as well as to the files reachable through /sys/class/fpga_region/.
                   
                  In order to allow regular (non-root) users to access accelerators, you need to grant them read and write permissions on /dev/dfl-port.* (with * denoting the respective socket, i.e. 0 or 1). E.g.:
                   
                  sudo chmod a+rw /dev/dfl-port.0\n
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#514-memlock-limit","title":"5.1.4 Memlock limit","text":"
                  Depending on the requirements of your application, you may also want to increase the maximum amount of memory a user process is allowed to lock. The exact way to do this depends on your Linux distribution.
                   
                  You can check the current memlock limit using
                   
                  ulimit -l\n
                   
                  A way to permanently remove the limit for locked memory for a regular user is to add the following lines to your /etc/security/limits.conf:
                   
                  user1    hard   memlock           unlimited\nuser1    soft   memlock           unlimited\n
                   
                  This removes the limit on locked memory for user user1. To remove it for all users, you can replace user1 with *:
                   
                  *    hard   memlock           unlimited\n*    soft   memlock           unlimited\n
                   
                  Note that settings in the /etc/security/limits.conf file don't apply to services. To increase the locked memory limit for a service you need to modify the application's systemd service file and add the line:
                   
                  [Service]\nLimitMEMLOCK=infinity\n
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#52-opae-tools-overview","title":"5.2 OPAE Tools Overview","text":"
                  The OPAE SDK user-space tools sit upon the kernel-space DFL drivers. In order to use OPAE SDK functionality the user needs to complete the steps outlined in the previous section 4.1 OFS DFL Kernel Driver Environment Setup before attempting to run any OPAE commands or flows. You must have at least one D5005 card with the appropriate FIM present in your system. The steps to read and load a new FIM version are discussed in section 6.1 Programming the OFS FIM). After both the DFL kernel-space drivers have been installed and the FIM has been upgraded, you may proceed to test the OPAE commands discussed below.
                   
                  This section covers basic functionality of the commonly used OPAE tools and their expected results. These steps may also be used to verify that all OFS software installation has been completed successfully. A complete overview of the OPAE tools can be found on the OPAE GitHub and in your cloned GitHub repo at <your path>/opae-sdk/doc/src/fpga_tools. More commands are listed than are defined in the list below - most of these are called by other tools and do not need to be called directly themselves.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#521-fpgasupdate","title":"5.2.1 fpgasupdate","text":"
                  The fpgasupdate tool updates the Intel Max10 BMC image and firmware, root entry hash, and FPGA Static Region (SR) and user image (PR). The fpgasupdate will only accept images that have been formatted using PACsign. If a root entry hash has been programmed onto the board, then the image will also need to be signed using the correct keys. Please refer to the [Security User Guide: Intel Open FPGA Stack] for information on created signed images and on programming and managing the root entry hash.
                   
                  The Intel FPGA PAC ships with a factory and user programmed image for both the FIM and BMC FW and RTL on all cards.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#table-5-1-fpgasupdate-overview","title":"Table 5-1: fpgasupdate Overview","text":"
                  Synopsis:
                   
                  fpgasupdate [--log-level=<level>] file [bdf]\n
                   
                  Description: The fpgasupdate command implements a secure firmware update.
                   Command args (optional) Description --log-level  Specifies the log-level which is the level of information output to your command tool. The following seven levels  are available: state, ioctl, debug, info, warning, error, critical. Setting --log-level=state provides the most verbose output. Setting --log-level=ioctl provides the second most information, and so on. The default level is info. file Specifies the secure update firmware file to be programmed. This file may be to program a static region (SR), programmable region (PR), root entry hash, key cancellation, or other device-specific firmware. bdf The PCIe address of the PAC to program. bdf is of the form [ssss:]bb:dd:f, corresponding to PCIe segment, bus, device, function. The segment is optional. If you do not specify a segment, the segment defaults to 0000. If the system has only one PAC you can omit the bdf and let fpgasupdate  determine the address automatically."},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#522-fpgainfo","title":"5.2.2 fpgainfo","text":"
                  Synopsis:
                   
                     fpgainfo [-h] [-S <segment>] [-B <bus>] [-D <device>] [-F <function>] [PCI_ADDR]\n            {errors,power,temp,fme,port,bmc,mac,phy,security}\n
                   
                  Description:  Displays FPGA information derived from sysfs files. The command argument is one of the following: errors, power, temp, port, fme, bmc, phy or mac, security. Some commands may also have other arguments or options that control their behavior.
                   
                  For systems with multiple FPGA devices, you can specify the BDF to limit the output to the FPGA resource with the corresponding PCIe configuration. If not specified, information displays for all resources for the given command.
                   Command args (optional) Description --help, -h Prints help information and exits. --version, -v Prints version information and exits. -S, --segment PCIe segment number of resource. -B, --bus PCIe bus number of resource. -D, --device PCIe device number of resource. -F, --function PCIe function number of resource. errors {fme, port, all} --clear, -c First agument to the errors command specifies the resource type to display in human readable format. The second optional argument clears errors for the given FPGA resource. power Provides total power in watts that the FPGA hardware consumes temp Provides FPGA temperature values in degrees Celsius port Provides information about the port fme Provides information about the FME bmc Provides BMC sensors information mac Provides information about MAC ROM connected to FPGA security Provides information about the security keys, hashes, and flash count, if available. 
                  Note: Your Bitstream ID and PR Interface Id may not match the below examples.
                   
                  The following examples walk through sample outputs generated by fpgainfo.
                   
                  $ sudo fpgainfo fme\n\nOpen FPGA Stack Platform\nBoard Management Controller, MAX10 NIOS FW version: 2.0.14\nBoard Management Controller, MAX10 Build version: 2.0.8\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 288511863935352239\nBitstream Version                : 4.0.1\nPr Interface Id                  : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98\nBoot Page                        : user\n
                   
                  $ sudo fpgainfo bmc\n\nOpen FPGA Stack Platform\nBoard Management Controller, MAX10 NIOS FW version: 2.0.14\nBoard Management Controller, MAX10 Build version: 2.0.8\n//****** BMC SENSORS ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 288511863935352239\nBitstream Version                : 4.0.1\nPr Interface Id                  : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98\n( 1) VCCERAM Voltage                                    : 0.90 Volts\n( 2) VCCT Temperature                                   : 29.00 Celsius\n( 3) 12v Backplane Voltage                              : 12.17 Volts\n( 4) VCCERAM Current                                    : 0.18 Amps\n( 5) FPGA Transceiver Temperature                       : 36.50 Celsius\n( 6) QSFP1 Supply Voltage                               : 0.00 Volts\n( 7) 3.3v Temperature                                   : 29.00 Celsius\n( 8) 12v Backplane Current                              : 2.28 Amps\n( 9) RDIMM3 Temperature                                 : 25.50 Celsius\n(10) VCCR Voltage                                       : 1.12 Volts\n(11) Board Inlet Air Temperature                        : 24.50 Celsius\n(12) 1.8v Temperature                                   : 27.50 Celsius\n(13) 12v AUX Voltage                                    : 12.14 Volts\n(14) VCCR Current                                       : 0.55 Amps\n(15) RDIMM0 Temperature                                 : 24.50 Celsius\n(16) FPGA Core Voltage                                  : 0.88 Volts\n(17) VCCERAM Temperature                                : 27.50 Celsius\n(18) 12v AUX Current                                    : 1.19 Amps\n(19) QSFP0 Temperature                                  : N/A\n(20) VCCT Voltage                                       : 1.12 Volts\n(21) FPGA Core Current                                  : 11.60 Amps\n(22) FPGA Core Temperature                              : 42.50 Celsius\n(23) 12v Backplane Temperature                          : 24.00 Celsius\n(24) VCCT Current                                       : 0.14 Amps\n(25) RDIMM1 Temperature                                 : 24.00 Celsius\n(26) 3.3v Voltage                                       : 3.30 Volts\n(27) VCCR Temperature                                   : 33.50 Celsius\n(28) 1.8v Voltage                                       : 1.80 Volts\n(29) 3.3v Current                                       : 0.32 Amps\n(30) Board Exhaust Air Temperature                      : 26.00 Celsius\n(31) 12v AUX Temperature                                : 25.00 Celsius\n(32) QSFP0 Supply Voltage                               : 0.00 Volts\n(33) QSFP1 Temperature                                  : N/A\n(34) 1.8v Current                                       : 0.54 Amps\n(35) RDIMM2 Temperature                                 : 26.00 Celsius\n
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#523-rsu","title":"5.2.3 rsu","text":"
                  The rsu performs a Remote System Update operation on a device, given its PCIe address. A rsu operation sends an instruction to the device to trigger a power cycle of the card only. This will force reconfiguration from flash for either the BMC or FPGA.
                   
                  The Intel FPGA PAC contains a region of flash the user may store their FIM image. After an image has been programmed with fpgasupdate the user may choose to perform rsu to update the image on the device.
                   
                  Note: The D5005 platform only supports storing and configuring a single user image from flash for the FPGA. It does not include support for the user1/user2 partitions as shown in other OFS related acceleration boards.
                   
                  rsu Overview
                   
                  Synopsis
                   
                  rsu [-h] [-d] {bmc,bmcimg,retimer,sdm,fpgadefault} [PCIE_ADDR]\n
                   
                  rsu bmc --page=(user) [PCIE_ADDR]\nrsu retimer [PCIE_ADDR]\nrsu sdm [PCIE_ADDR]\n
                   
                  Perform RSU (remote system update) operation on PAC device given its PCIe address. An RSU operation sends an instruction to the device to trigger a power cycle of the card only. This will force reconfiguration from flash for either BMC, Retimer, SDM, (on devices that support these) or the FPGA.
                   
                  Note: As a result of using the rsu command, the host rescans the PCI bus and may assign a different Bus/Device/Function (B/D/F) value than the originally assigned value.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#525-bitstreaminfo","title":"5.2.5 bitstreaminfo","text":"
                  Displays authentication information contained with each provided file on the command line. This includes any JSON header strings, authentication header block information, and a small portion  of the payload. The binary is installed by default at /usr/bin/bitstreaminfo.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#526-hssi","title":"5.2.6 hssi","text":"
                  The hssi application provides a means of interacting with the 10G and with the 100G HSSI AFUs. In both 10G and 100G operating modes, the application initializes the AFU, completes the desired transfer as described by the mode-specific options. Only the hssi_10g MODE is currently supported. An example of this command's output can be found in section 5.2.9 Running the Host Exerciser Modules. The binary is installed by default at /usr/bin/hssi.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#527-opaeio","title":"5.2.7 opae.io","text":"
                  Opae.io is a interactive Python environment packaged on top of libopaevfio.so, which provides user space access to PCIe devices via the vfio-pci driver. The main feature of opae.io is its built-in Python command interpreter, along with some Python bindings that provide a means to access Configuration and Status Registers (CSRs) that reside on the PCIe device. opae.io has two operating modes: command line mode and interactive mode. An example of this command's output can be found in section 5.2.9 Running the Host Exerciser Modules. The binary is installed by default at /usr/bin/opae.io.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#528-host_exerciser","title":"5.2.8 host_exerciser","text":"
                  The host exerciser is used to exercise and characterize the various host-FPGA interactions eg. MMIO, Data transfer from host to FPGA , PR, host to FPGA memory etc. An example of this command's output can be found in section 5.2.9 Running the Host Exerciser Modules. The binary is installed by default at /usr/bin/host_exerciser. For more information refer to - Host Exerciser
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#529-running-the-host-exerciser-modules","title":"5.2.9 Running the Host Exerciser Modules","text":"
                  The reference FIM and unchanged compilations contain Host Exerciser Modules (HEMs). These are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. 
                   
                  Note: Before continuing, if huge pages are not set refer to section 4.2, step 7.
                   
                  There are three HEMs present in the OFS FIM - HE-LPBK, HE-HSSI, and HE-MEM. These exercisers are tied to three different VFs that must be enabled before they can be used. The user should enable the VF for each HEM using the below steps:
                   
                  1. Determine the BDF of the Intel\u00ae FPGA PAC D5005 card.
                   
                  The PCIe BDF address is initially determined when the server powers on. The user can determine the addresses of all Intel\u00ae FPGA PAC D5005 boards using lspci:
                   
                  # lspci -d :bcce\n\n3b:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                   
                  Note: Before continuing, if you updated your OFS installation, please also update your PAC FIM to run HEMs.
                   
                  2. Enable three VFs.
                   
                  In this example, the BDF address is 0000:3b:00.0. With this information the user can now enable three VFs with the following:
                   
                  # sudo pci_device 0000:3b:00.0 vf 3\n
                   
                  3. Verify that all three VFs have been created.
                   
                  # lspci -s 3b:00\n\n3b:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n3b:00.1 Processing accelerators: Intel Corporation Device bccf (rev 01)\n3b:00.2 Processing accelerators: Intel Corporation Device bccf (rev 01)\n3b:00.3 Processing accelerators: Intel Corporation Device bccf (rev 01)\n
                   
                  4. Bind the 3 VFs to the vfio-pci driver.
                   
                  $ sudo opae.io init -d 0000:3b:00.1 $USER\n\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:3b:00.1 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:3b:00.1 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:3b:00.1 is 142\nAssigning /dev/vfio/142 to $USER:$USER\nChanging permissions for /dev/vfio/142 to rw-rw----\n\n\n$ sudo opae.io init -d 0000:3b:00.2 $USER\n\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:3b:00.2 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:3b:00.2 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:3b:00.2 is 143\nAssigning /dev/vfio/143 to $USER:$USER\nChanging permissions for /dev/vfio/143 to rw-rw----\n\n\n$ sudo opae.io init -d 0000:3b:00.3 $USER\n\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:3b:00.3 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:3b:00.3 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:3b:00.3 is 144\nAssigning /dev/vfio/144 to $USER:$USER\nChanging permissions for /dev/vfio/144 to rw-rw----\n
                   
                  5. Check that the accelerators are present using fpgainfo. Note your port configuration may differ from the below.
                   
                  $ sudo fpgainfo port\n\n//****** PORT ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0x603B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : 823c334c-98bf-11ea-bb37-0242ac130002\n//****** PORT ******//\nObject Id                        : 0x403B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n//****** PORT ******//\nObject Id                        : 0x203B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#table-5-5-vf-to-hem-mappings","title":"Table 5-5 VF to HEM Mappings","text":"VF BDF HEM BBBB:DD.1 HE-LB BBBB:DD.2 HE-MEM BBBB:DD.3 He-HSSI 
                  HE-MEM / HE-LB
                   
                  HE-LB is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth. HE-MEM is used to exercise the DDR interface; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host. HE-MEM uses external DDR memory (i.e. EMIF) to store data. It has a customized version of the AVMM interface to communicate with the EMIF memory controller. Both exercisers rely on the user-space tool host_exerciser. The following commands are supported by the HE-LB/HE-MEM OPAE driver program. They may need to be run using sudo privileges, depending on your server configuration.
                   
                  Basic operations:
                   
                  $ sudo host_exerciser lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5342\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.067 GB/s\n    Test lpbk(1): PASS\n\n$ sudo host_exerciser --mode lpbk lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5358\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.058 GB/s\n    Test lpbk(1): PASS\n\n$ sudo host_exerciser --mode write lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 0\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 2592\n    Total number of Reads sent: 0\n    Total number of Writes sent: 1024\n    Bandwidth: 6.321 GB/s\n    Test lpbk(1): PASS\n\n$ sudo host_exerciser --mode trput lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 512\n    Host Exerciser numWrites: 513\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 3384\n    Total number of Reads sent: 512\n    Total number of Writes sent: 512\n    Bandwidth: 4.842 GB/s\n    Test lpbk(1): PASS\n
                   
                  Number of cachelines per request 1, 2, and 4. The user may replace --mode lpbk with read, write, trput. The target lpbk can be replaced with mem:
                   
                  $ sudo host_exerciser --mode lpbk --cls cl_1 lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5475\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 2.993 GB/s\n    Test lpbk(1): PASS\n\n\n$ sudo host_exerciser --mode lpbk --cls cl_2 lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5356\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.059 GB/s\n    Test lpbk(1): PASS\n\n\n$ sudo host_exerciser --mode lpbk --cls cl_4 lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 4481\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.656 GB/s\n    Test lpbk(1): PASS\n
                   
                  Interrupt tests (only valid for mode mem):
                   
                  $ sudo host_exerciser --interrupt 0 mem\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\nUsing Interrupts\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1026\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5140\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.188 GB/s\n    Test mem(1): PASS\n\n$ sudo host_exerciser --interrupt 1 mem\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\nUsing Interrupts\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1026\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5079\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.226 GB/s\n    Test mem(1): PASS\n\n\n$ sudo host_exerciser --interrupt 2 mem\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\nUsing Interrupts\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1026\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5525\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.439 GB/s\n    Test mem(1): PASS\n\n\n$ sudo host_exerciser --interrupt 3 mem\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\nUsing Interrupts\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1026\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 4735\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.460 GB/s\n    Test mem(1): PASS\n
                   
                  HE-HSSI
                   
                  HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G ethernet AFU and includes a 10G traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic. Context sensitive information is given by the hssi --help command. Help for the 10G specific test is given by hssi hssi_10g --help Example useage:
                   
                  $ sudo hssi --pci-address 3b:00.3 hssi_10g --eth-ifc s10hssi0 --eth-loopback on --he-loopback=off  --num-packets 100\n\n10G loopback test\n  port: 0\n  eth_loopback: on\n  he_loopback: off\n  num_packets: 100\n  packet_length: 64\n  src_address: 11:22:33:44:55:66\n    (bits):  0x665544332211\n  dest_address: 77:88:99:aa:bb:cc\n    (bits): 0xccbbaa998877\n  random_length: fixed\n  random_payload: incremental\n  rnd_seed0: 5eed0000\n  rnd_seed1: 5eed0001\n  rnd_seed2: 25eed\n  eth: s10hssi0\n
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#60-compiling-ofs-fim","title":"6.0 Compiling OFS FIM","text":"
                  Pre-Compiled FIM binaries are at OFS 2023.3 release page and to compile the OFS FIM for Intel\u00ae FPGA PAC D5005 follow the below steps :
                   
                  1) Compile OFS FIM manually - Steps are provided in the developer guide to compile FIM and generate binaries. Refer to FPGA Interface Manager Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA.
                   
                  2) Compile OFS FIM using evaluation script - The script guides you to the steps required for compilation via selecting options from the menu. Refer to Platform Evaluation Script: Open FPGA Stack for Intel Stratix 10 FPGA.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#70-programming-the-ofs-fim-and-bmc","title":"7.0 Programming the OFS FIM and BMC","text":"
                  Instructions surrounding the compilation and simulation of the OFS FIM have fully moved into the FPGA Interface Manager Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#71-programming-the-ofs-fim","title":"7.1 Programming the OFS FIM","text":"
                  In order to program the OFS FIM, both the OPAE SDK and DFL drivers need to be installed on the host system. Please complete the steps in sections 4.0 OFS DFL Kernel Drivers and 5.0 OPAE Software Development Kit. The OFS FIM version can be identified using the OPAE tool fpgainfo. A sample output of this command is included below.
                   
                  $ sudo fpgainfo fme\n\nIntel FPGA Programmable Acceleration Card D5005\nBoard Management Controller, MAX10 NIOS FW version: 2.0.14\nBoard Management Controller, MAX10 Build version: 2.0.8\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 288511863935352239\nBitstream Version                : 4.0.1\nPr Interface Id                  : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98\nBoot Page                        : user\n
                   
                  Use the value under PR Interface ID to identify that FIM that has been loaded. Refer to the table below for a list of previous FIM releases:
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#table-7-1-previous-fim-releases","title":"Table 7-1 Previous FIM Releases","text":"PR Release PR Interface ID 2023.2 edad864c-99d6-5831-ab67-62bfd81ec654 2022.2 (tag 1.3.0) bf531bcf-a896-5171-ab31-601a4ab754b6 2022.1 Beta (tag: 1.2.0-beta) 2fae83fc-8568-53aa-9157-8f75e9c0ba92 OFS 2.1 Beta (tag: 1.1.0-beta) 99160d37e42a 3f8b586f-c275-594c-92e2-d9f2c23e94d1 OFS 1.0 (tag: ofs-1.0.0) b5f6a71e-daec-59c3-a43a-85567b51fd3f Intel Acceleration Stack for Intel\u00ae FPGA PAC D5005 2.0.1 9346116d-a52d-5ca8-b06a-a9a389ef7c8d 
                  If the user's card does not report a PR Interface ID which matches the above table, then a new FIM will need to be programmed.
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#711-programming-the-fim","title":"7.1.1 Programming the FIM","text":"
                  1. Download the file d5005_page1_unsigned.bin from OFS 2023.3 release page.
                   
                  2. Run PACSign to create an unsigned image with added header for use by fpgasupdate
                   
                  $ PACSign SR -y -v -t UPDATE -s 0 -H openssl_manager -i d5005_page1_unsigned.bin -o d5005_PACsigned_unsigned.bin\n
                   
                  3. Run fpgasupdate to load the image into the user location of the Intel\u00ae FPGA PAC D5005 FPGA flash, NOTE: use \"sudo fpgainfo fme\" command to find the PCIe address for your card.
                   
                  $ sudo fpgasupdate d5005_PACsigned_unsigned.bin 3B:00.0\n
                   
                  4. Run RSU command.
                   
                  $ sudo rsu bmcimg 0000:3B:00.0\n
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#72-programming-the-bmc","title":"7.2 Programming the BMC","text":"
                  1. Download intel-fpga-bmc images(To download OFS Stratix 10 BMC binaries contact Intel Technical Sales Representative)
                   
                  2. The file unsigned_bmc_fw.bin has the newly binary format. This bitstream is programmed with remote system update (RSU) and the bitstream must be signed with PACSign tool to generate.
                   
                  3. Run PACSign to create an unsigned image with added header for use by fpgasupdate
                   
                  $ PACSign BMC -y -v -t UPDATE -s 0 -H openssl_manager -i unsigned_bmc_fw.bin -o PACsigned_unsigned_bmc_fw.bin\n\n2022-04-22 03:07:05,626 - PACSign.log - INFO - OpenSSL version \"OpenSSL 1.1.1k  FIPS 25 Mar 2021\" matches \"1.1.1\"\n2022-04-22 03:07:05,648 - PACSign.log - INFO - Bitstream not previously signed\n2022-04-22 03:07:05,648 - PACSign.log - INFO - platform value is '688128'\n2022-04-22 03:07:05,745 - PACSign.log - INFO - Starting Block 0 creation\n2022-04-22 03:07:05,745 - PACSign.log - INFO - Calculating SHA256\n2022-04-22 03:07:05,747 - PACSign.log - INFO - Calculating SHA384\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Done with Block 0\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Starting Root Entry creation\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Calculating Root Entry SHA\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Starting Code Signing Key Entry creation\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Calculating Code Signing Key Entry SHA\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Code Signing Key Entry done\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Starting Block 0 Entry creation\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Calculating Block 0 Entry SHA\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Block 0 Entry done\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Starting Block 1 creation\n2022-04-22 03:07:05,750 - PACSign.log - INFO - Block 1 done\n2022-04-22 03:07:05,757 - PACSign.log - INFO - Writing blocks to file\n2022-04-22 03:07:05,758 - PACSign.log - INFO - Processing of file 'PACsigned_unsigned_bmc_fw.bin' complete\n
                   
                  4. Run fpgasupdate to perform an upgrade of the BMC.
                   
                  $ sudo fpgasupdate PACsigned_unsigned_bmc_fw.bin 3B:00.0\n\n[2022-04-22 03:08:34.15] [WARNING ] Update starting. Please do not interrupt.\n[2022-04-22 03:08:34.15] [INFO    ] updating from file pacsign_unsigned_bmc_fw.bin with size 819968\n[2022-04-22 03:08:34.15] [INFO    ] waiting for idle\n[2022-04-22 03:08:34.15] [INFO    ] preparing image file\n[2022-04-22 03:09:02.18] [INFO    ] writing image file\n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [819968/819968 bytes][Elapsed Time: 0:00:13.01]\n[2022-04-22 03:09:15.20] [INFO    ] programming image file\n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588][Elapsed Time: 0:00:29.03]\n[2022-04-22 03:09:44.24] [INFO    ] update of 0000:3B:00.0 complete\n[2022-04-22 03:09:44.24] [INFO    ] Secure update OK\n[2022-04-22 03:09:44.24] [INFO    ] Total time: 0:01:10.08\n
                  "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                  Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                   
                  OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/","title":"Simulation User Guide: Open FPGA Stack for Intel Intel\u00ae Stratix 10\u00ae FPGA","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#1-overview","title":"1 Overview","text":""},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#11-about-this-document","title":"1.1 About this Document","text":"
                  This document serves as a set-up and user guide for the UVM simulation tool using OFS. After reviewing the document, you will be able to:
                   
                     
                  • Set-up the UVM verification tool suite
                    •  
                  • Run pre-existing UVM unit tests and also create new UVM tests for your design
                    •  
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#2-introduction-to-uvm","title":"2 Introduction to UVM","text":"
                  OFS (Open FPGA Stack) provides a UVM (Universal Verification Methodology) environment for the FIM (FPGA Interface Manager) with a modular, reusable, and scalable testbench structure via an API framework.
                   
                  The framework consists of a FIM Testbench which is UVM compliant and integrates third party VIPs from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this Testbench. UVM RAL (Register Abstaction Layer) is used for CSR (Command and Status Registers) verification.
                   
                  The qualified verification IPs will help to detect incorrect protocol behavior, help to focus on FIM features and accelerate the verification process.
                   
                  Verification components include:
                   
                     
                  • FIM monitor to detect correct design behavior
                    •  
                  • FIM assertions for signal level integrity testing
                    •  
                  • Arm AMBA Arm\u00ae AMBA\u00ae 4 AXI4 scoreboards to check data integrity
                    •  
                  • FIM coverage to collect functional data
                    •  
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#3-testbench-architecture","title":"3 Testbench Architecture","text":"
                  The testbench connects to the full chip that includes major RTL blocks depicted in Figure 1.
                   
                   
                  Figure 1 Testbench Diagram
                   
                  The major interface is between the Xeon and FPGA where PCIe Verification IP is connected to PCIe Subsystem. Therefore, as a full chip simulation environment, PCIe host VIP is the sole VIP/BFM used. PCIe host VIP connects to PCIe device which resides in FPGA in serial mode. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#4-testbench-infrastructure","title":"4 Testbench Infrastructure","text":""},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#41-traffic-flow","title":"4.1 Traffic Flow","text":"
                  PCIe Host, as the master of FPGA, initiates MMIO read/write requests to FPGA to program registers. The PCIe host also passively receives memory requests from FPGA to read from or write to host memory. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#42-link-up-and-enumeration","title":"4.2 Link Up and Enumeration","text":"
                  With serial mode connection between PCIe host and device, link training and enumeration has to be done before the regular traffic starts. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#421-link-up","title":"4.2.1 Link Up","text":"
                  Linkup sequence(pcie_device_bring_up_link_sequence) is part of configure sequence(ofs_config_seq), which is started in UVM configure phase. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#422-enumeration","title":"4.2.2 Enumeration","text":"
                  PCIe host driver needs to retrieve information from the device hard IP and program necessary configuration space registers, such as PF/VF BAR values. This is done in enumerate_seq, which follows link up sequence in configure sequence(ofs_config_seq). 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#423-pfvf-bar","title":"4.2.3 PF/VF BAR","text":"
                  PF0 BAR0 is set in the base sequence and can be randomized. During enumeration, PF0 BAR0, along with PCIe device hard IP configuration, derives other PF and VF BAR values. These BAR values are stored into base sequence variables and can be used throughout any test sequences that extend the base sequence. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#43-mmio-apis","title":"4.3 MMIO APIs","text":"
                  The base sequence provides APIs for 32-bit and 64-bit MMIO read/write accesses, as well as blocking or non-blocking for MMIO read as described in Table 1. The users can use MMIO APIs without knowing the underlining PCIe sequence items.
                   Name API 32-bit MMIO Write task mmio_write32(input bit [63:0] addr_, input bit [31:0] data_); 64-bit MMIO Write task mmio_write64(input bit [63:0] addr_, input bit [63:0] data_); 32-bit MMIO Read task mmio_read32(input bit [63:0] addr_, output bit [31:0] data_, input blocking_ = 1); 64-bit MMIO Read task mmio_read64(input bit [63:0] addr_, output bit [63:0] data_, input blocking_ = 1); 
                  Table 1 MMIO APIs
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#44-ral","title":"4.4 RAL","text":"
                  UVM RAL is integrated in the testbench providing alternative ways of accessing CSRs in test sequences. RAL is generated from an excel format CSR specification where register name, field, offset, bitmap, attribute, and description are specified.
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#45-vip-dut-connection","title":"4.5 VIP DUT Connection","text":"
                  PCIe host verification IP and DUT connection is achieved by connecting 16 bits lanes. The module for connection from VIP is svt_pcie_device_agent_serdes_x16_x8g_hdl.  
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#5-test-plan","title":"5 Test Plan","text":"
                  The test plan consists of four major categories: MMIO path, HE-LB, HE-MEM, HE-HSSI and interrupt tests. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#51-mmio-path","title":"5.1 MMIO Path","text":"
                  The tests under this category exercise MMIO path including all destination functions or blocks as well as PF/VF mux and different fabrics. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#52-he-lb","title":"5.2 HE-LB","text":"
                  The tests under this category target HE-LB function only. Software which test sequences needs to configure HE-LB CSRs before starting it. These CSRs include SRC_ADDR, DST_ADDR, DSM_ADDR, NUM_LINES, CFG etc.   
                   
                  If HE-LB is configured to have memory read transactions, PCIe host memory has to be initialized before HE-LB is started. This is done by svt_pcie_mem_target_service sequence. In other words, PCIe host VIP programs its internal memory model entries in backdoor way. The same process applies to DSM memory entry. 
                   
                  Once HE-LB is started, HE-LB will function based on what it is programmed to do. When HE-LB is done with all necessary memory transactions, it will perform a final memory write to DSM memory entry. Since the software does not know when hardware is done, software polls DSM memory entry periodically until the DSM status bit is asserted. 
                   
                  For loopback mode, data is compared between source buffer and destination buffer in host memory. 
                   
                  RTL statistic counters are also compared against the corresponding variables inside the test sequence at the end of the simulation. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#53-he-mem","title":"5.3 HE-MEM","text":"
                  HE-MEM tests are duplicates from HE-LB with MMIO to CSRs targeting HE-MEM instead of HE-LB. 
                   
                  The DDR simulation model is inside memory controller IP when being generated.  
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#54-he-hssi","title":"5.4 HE-HSSI","text":"
                  HE-HSSI has indirect registers that are associated with HSSI subsystem, MMIO for indirect registers is different from other functions. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#541-indirect-registers","title":"5.4.1 Indirect Registers","text":"
                  To obtain access to indirect registers, either reading or writing, a MMIO write must be performed.
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#542-tx-loopback","title":"5.4.2 TX Loopback","text":"
                  In TX loopback, HE-HSSI initiates ethernet packets to HSSI subsystem and the packets are looped back to HE-HSSI. The loopback is achieved by hard-wiring HSSI TX and RX lanes. This is done inside RTL and for simulation purposes only. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#55-interrupt-test","title":"5.5 Interrupt Test","text":"
                  The test plan covers the basic interrupt flow for FME error, PORT error and user AFU interrupts. The MSI-X table must be programmed in PF0 BAR4. Corresponding PBA bit is expected to be asserted. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#56-performance-test","title":"5.6 Performance Test","text":"
                  Performance tests are derived from HE-LB tests and they are directed tests. At the end of the simulation, performance number is calculated and printed to terminal and a log file. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#57-csr-test","title":"5.7 CSR Test","text":"
                  CSR consists of two parts. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#571-reset-value-check","title":"5.7.1 Reset Value Check","text":"
                  Front-door MMIO read data is compared against RAL register reset value out of reset. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#572-rw-attribute-csr","title":"5.7.2 RW Attribute CSR","text":"
                  MMIO write-read-compare is performed after reset value check for RW attribute CSRs. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#6-checking-mechanism","title":"6 Checking Mechanism","text":"
                  Since there is only PCIe host verification component in testbench, data checking is done by a self-checking mechanism in the test sequence.  
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#61-protocol-violation","title":"6.1 Protocol Violation","text":"
                  PCIe host VIP has built-in protocol checking on TLP received from FPGA. Abnormal responses are also flagged by the VIP. 
                   
                  Internal AXI Streaming interface has integrated RTL assertion to check AXI Streaming protocol violations. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#62-data-checking","title":"6.2 Data Checking","text":"
                  Data checking is done by self-checking inside a test sequence. MMIO write/read/compare to read-writable CSRs is done inside a sequence. 
                   
                  For memory transactions initiated by functions, backdoor reads from host memory on source buffer and destination buffer is done inside a sequence. Data is compared in case of loopback mode. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#63-counter-checking","title":"6.3 Counter Checking","text":"
                  RTL statistic counters records the number of transactional information that can be read at the end of the simulation and compared against the test expected number. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#64-afu-error-csr","title":"6.4 AFU Error CSR","text":"
                  AFU interface handler provides an error log for illegal transactions that can be read at the end of the simulation. 
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#7-uvm-set-up","title":"7 UVM set-up","text":"
                  To run the tutorial steps in this guide requires the following development environment:
                   Item Version Intel Quartus Prime Pro Intel Quartus Prime Pro 23.3 Simulator (VCS) Synopsys VCS P-2019.06-SP2-5 or newer for UVM simulation of top level FIM Simulator (Questasim) Questasim 2023.2 or newer for UVM simulation of top level FIM"},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#71-uvm-prerequisite","title":"7.1 UVM Prerequisite","text":"
                  Retrieve OFS repositories.
                   
                  The OFS FIM source code is included in the GitHub repository. Create a new directory to use as a clean starting point to store the retrieved files.  The following is a short description of each repository, followed by the git commands for cloning.  The instructions section uses the HTTPS git method for cloning repositories.  Cloning the repo using the HTTPS method requires a personal access token.  Please see this blog post for information about obtaining a personal access token Token authentication requirements for Git operations.
                   
                  Navigate to the location for storage of OFS source, create the top-level source directory and clone OFS repositories.
                   
                  $ mkdir ofs-2023.3-2\n$ cd ofs-2023.3-2\n$ export OFS_BUILD_ROOT=$PWD\n$ git clone --recurse-submodules https://github.com/OFS/ofs-d5005.git\n\nCloning into 'ofs-d5005' ...\nUsername for 'https://github.com': <<Enter your git hub username>>\nPassword for 'https://<<Your username>>': <<Enter your personal access token>>\nremote: Enumerating objects:  ....\n...\n...\nResolving deltas  ..., done.\n\n$ cd ofs-d5005\n$ git checkout tags/release/ofs-2023.3\n
                   
                  Verify that the correct tag/branch have been checked out        
                   
                  $ git describe --tags\n\n$ release/ofs-2023.3\n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#72-license-requirements","title":"7.2 License Requirements","text":"
                  The FIM Testbench is UVM compliant and integrates third party VIPs from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this TB. UVM RAL (Register Abstraction Layer) is used for CSR Verification.
                   
                  The Qualified Verification IPs will help to detect incorrect protocol behavior easily, help to focus on BBS features and accelerate the verification process.
                   
                     
                  • VCS & DVE
                    •  
                  • SNPS-Assertions
                    •  
                  • Verdi
                    •  
                  • VerdiCoverage
                    •  
                  • VerdiSimDB
                    •  
                  • VerdiTransactionDebugUltra
                    •  
                  • VIP-AMBA-AXI-SVT
                    •  
                  • VIP-AMBA-STREAM-SVT
                    •  
                  • VIP-PCIE-SVT
                    •  
                  • VIP-PCIE-TS-SVT
                    •  
                  • VIP-PCIE-G3-OPT-SVT
                    •  
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#73-software-tools-requirements","title":"7.3 Software Tools Requirements","text":"
                  The following tools are required for successful UVM set-up
                   
                     
                  • Python 3.6.8
                    •  
                  • Synopsys PCIE and AMBA AXI UVM VIP Q-2020.03A License
                    •  
                  • Synopsys Verdi R-2020.12-SP2 License         Note: Makefile can be modified to use DVE instead of Verdi
                    •  
                   
                     
                  • VCS R-2020.12-SP2 License 
                    •  
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#74-creating-a-software-tools-script","title":"7.4 Creating a Software Tools Script","text":"
                  The UVM tool set-up is best done by creating a simple set-up script so all applicable tools are sourced before running the tests.
                   
                  The following environment variables can be pasted into a script and used prior to running the UVM verification environment
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#license-files","title":"License Files","text":"
                  export LM_LICENSE_FILE=\nexport SNPSLMD_LICENSE_FILE=\n
                   
                  The license environment variables LM_LICENSE_FILE and SNPSLMD_LICENSE_FILE can point to a server license on your system.
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#general-environment-variables","title":"General Environment Variables","text":"
                  export OFS_BUILD_ROOT=$PWD\nexport OFS_ROOTDIR=<user_path>/ofs-d5005\nexport WORKDIR=$OFS_ROOTDIR\n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#quartus-tools","title":"Quartus Tools","text":"
                  export QUARTUS_HOME=<user_path>/intelFPGA_pro/23.3/quartus\nexport QUARTUS_ROOTDIR=$QUARTUS_HOME\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$PATH\n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-verification-tools","title":"Synopsys Verification Tools","text":"
                  export DESIGNWARE_HOME=<user_path>/synopsys/vip_common/vip_Q-2020.03A\nexport PATH=$DESIGNWARE_HOME/bin:$PATH\nexport UVM_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport PATH=$VCS_HOME/bin:$PATH\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim-verification-tools","title":"QuestaSIM Verification Tools","text":"
                  export MTI_HOME=<user_path>/mentor/questasim/2023.2/linux64\nexport PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\nexport QUESTA_HOME=$MTI_HOME\n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#8-running-a-uvm-simulation-test-and-analysing-results","title":"8 Running a UVM Simulation Test and Analysing Results","text":""},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#81-simulation","title":"8.1 Simulation","text":"
                  The default simulator used in this document is Synopsys VCS-MX but there will be references to Questasim. Users can refer to the options and adopt the options for other simulators. 
                   
                  The script is a makefile that calls vlogan, vcs and simv for compilation, elaboration and simulation, respectively
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#82-file-structure","title":"8.2 File Structure","text":"
                  After cloning the repo, the verification and ofs-common directories contain all UVM verification related files. The directory structure is shown in Figure 2 below.
                   
                   
                  Figure 2 UVM Verification Directory File Structure
                   
                  ofs-d5005/testbench has a testbench, uvm env, virtual sequencer, RAL etc. 
                   
                  ofs-d5005/verification/tests contains all uvm tests and sequences. 
                   
                  Users can run the simulation under \"ofs-d5005/verification/scripts\" directory and the simulation result is outputted to a \"sim\" directory for Synopsys VCS or \"sim_msim\" for Questasim.
                   
                  The simulation result folder is named after the test name with increasing suffix number. If user runs the same test multiple times, the suffix is incremented by 1 each time.
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#83-uvm-test-suite","title":"8.3 UVM Test Suite","text":"
                  The UVM environment contains a variety of tests that have been developed to test out the FIM portion of OFS.
                   
                  The table below lists out the \"Test Name\" which will be used on the command line to execute the test, the \"Test Scenario\" and the \"Checking Criteria\".
                   
                  Tests are located at ofs-d5005/verification/tests
                   Test Name DUT Scope Test Scenario Checking Criteria dfh_walking_test DFH DHF walking offset checking, eol checking flr_reset_test FLR Reset FLR reset to all PFs Reset checking flr_vf0_reset_test FLR Reset FLR reset to VF0 Reset checking flr_vf1_reset_test FLR Reset FLR reset to VF1 Reset checking flr_vf2_reset_test FLR Reset FLR reset to VF2 Reset checking fme_csr_test FME CSR CSR accesses data checking fme_hemem_intr_test Interrupt FME and HE MEM interrupt Interrupts assertion, PBA bits check fme_intr_test Interrupt FME error interrupt Interrupts assertion, PBA bits check he_hssi_csr_test HE-HSSI CSR accesses for HSSI data checking he_hssi_err_test HE-HSSI Error Cases counter checking he_hssi_rx_lpbk_test HE-HSSI RX loopback data checking he_hssi_tx_lpbk_test HE-HSSI TX loopback counter checking he_lpbk_cont_test HE-LPBK Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking he_lpbk_long_rst_test HE-MEM Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_lpbk_long_test HE-MEM Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_lpbk_port_rst_test HE-LPBK Loopback mode. Randomize num_lines, addresses, req_len with port rst data checking he_lpbk_rd_cont_test HE-LPBK Read only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_lpbk_rd_test HE-LPBK Read only mode. Randomize num_lines, addresses, req_len counter checking he_lpbk_reqlen1_test HE-LPBK Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_lpbk_reqlen2_test HE-LPBK Loopback mode. 128 CLs, req_len = 2CL, random addresses. data checking, counter checking he_lpbk_reqlen4_test HE-LPBK Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_lpbk_reqlen8_test HE-LPBK Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_lpbk_test HE-LPBK Loopback mode. Randomize num_lines, addresses, req_len data checking, counter checking he_lpbk_thruput_contmode_test HE-LPBK Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses req_len data checking, counter checking he_lpbk_thruput_test HE-LPBK Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_lpbk_wr_cont_test HE-LPBK Write only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_lpbk_wr_test HE-LPBK Write only mode. Randomize num_lines, addresses, req_len counter checking he_mem_cont_test HE-MEM Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking he_mem_lpbk_long_rst_test HE-LPBK Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_mem_lpbk_long_test HE-LPBK Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_mem_lpbk_reqlen1_test HE-MEM Loopback mode. 128 CLs, req_len = 1CL, random addresses. data checking, counter checking he_mem_lpbk_reqlen2_test HE-MEM Loopback mode. 128 CLs, req_len = 2CL, random addresses. data checking, counter checking he_mem_lpbk_reqlen4_test HE-MEM Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_mem_lpbk_test HE-MEM Loopback mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_rd_cont_test HE-MEM Read only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_rd_test HE-MEM Read only mode. Randomize num_lines, addresses, req_len counter checking he_mem_thruput_contmode_test HE-MEM Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_thruput_test HE-MEM Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_mem_wr_cont_test HE-MEM Write only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_wr_test HE-MEM Write only mode. Randomize num_lines, addresses, req_len counter checking he_random_long_test All HE's Enable all HEs and randomize modes for multiple iterations data checking if in lpbk mode, counter checking he_random_test All HEs Enable all HEs and randomize modes data checking if in lpbk mode, counter checking hehssi_csr_test HE-HSSI CSR accesses for Traffic Control Mail box registers data checking helb_csr_test HE-LPBK CSR accesses data checking helb_rd_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Read only mode data checking, counter checking helb_rd_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Read only mode data checking, counter checking helb_rd_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Read only mode data checking, counter checking helb_thruput_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Thruput mode data checking, counter checking helb_thruput_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Thruput mode data checking, counter checking helb_thruput_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Thruput mode data checking, counter checking helb_wr_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Write only mode data checking, counter checking helb_wr_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Write only mode data checking, counter checking helb_wr_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Write only mode data checking, counter checking hemem_csr_test HE-MEM CSR accesses data checking hemem_intr_test Interrupt HE MEMN Interrupt Interrupts assertion, PBA bits check malformedtlp_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing. maxpayloaderror_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing. MaxTagError_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3.Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing. mini_smoke_test All HEs shorter simpler version of random test for turn-in sanity check data checking if in lpbk mode, counter checking mmio_64b_bar_test PCIe MMIO Path 64-bit bar addess for MMIO data checking mmio_stress_nonblocking_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux with non-blocking MMIO reads data checking mmio_stress_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux data checking mmio_test PCIe MMIO Path MMIO targeting PF0(ST2MM, FME, PMCI, HSSI SS), PF1, PF1.VF1, PF1.VF2 data checking mmio_unimp_test PCIe MMIO Path MMIO acccess to unimplemented addresses MMIO checking MMIODataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing. MMIOInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing. MMIOTimedout_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing. msix_csr_test MSIX CSR CSR accesses data checking pmci_csr_test PMCI CSR CSR accesses data checking port_gasket_csr_test PORT GASKET Port Gasket CSR test port csr checking TxMWrDataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing TxMWrInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing UnexpMMIORspErr_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be retuened on resds. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing 
                  The next section describes how to compile and build the UVM environment prior to running each UVM test and analyzing the results in the log files
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#84-ip-compile","title":"8.4 IP Compile","text":""},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-vcs","title":"Synopsys VCS","text":"
                  To compile all IPs for the Synopsys VCS simulater:
                   
                      cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk cmplib\n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim","title":"Questasim","text":"
                  To compile all IPs for the Questasim simulater:
                   
                      cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk cmplib \n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#85-rtl-test-bench-compile","title":"8.5 RTL & Test Bench Compile","text":"
                  The RTL file list for compilation is located here: verification/scripts/rtl_comb.f
                   
                  The TB file list for compilation is located here: verification/scripts/ver_list.f
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-vcs_1","title":"Synopsys VCS","text":"
                  To compile RTL and Testbench for the Synopsys VCS simulater
                   
                      cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build DUMP=1\n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim_1","title":"Questasim","text":"
                  To compile RTL and Testbench for the Questasim simulater
                   
                      cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build DUMP=1\n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#86-ip-and-rtl-test-bench-compile","title":"8.6 IP and RTL & Test Bench Compile","text":""},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-vcs_2","title":"Synopsys VCS","text":"
                  If the user wants to compile all IPs and RTL Testbench in one command for Synopsys VCS then follow the procedure below
                   
                      cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_all DUMP=1\n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim_2","title":"Questasim","text":"
                  If the user wants to compile all IPs and RTL Testbench in one command for Questasim then follow the procedure below
                   
                      cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_all DUMP=1\n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-vcs_3","title":"Synopsys VCS","text":"
                  To run a simulation for Synopsys VCS:
                   
                      cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk run TESTNAME=mmio_test DUMP=1\n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim_3","title":"Questasim","text":"
                  To run a simulation for Questasim:
                   
                      cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=mmio_test DUMP=1 \n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-vcs_4","title":"Synopsys VCS","text":"
                  To dump the waveform, \"DUMP=1\" must be added into the command line for Synopsys VCS build and simulation. 
                   
                      ofs-d5005/verification/scripts  gmake -f Makefile_VCS.mk build DUMP=1\n\n    ofs-d5005/verification/scripts  gmake -f Makefile_VCS.mk run TESTNAME=<test_case_name> DUMP=1\n
                   Or
                   
                      ofs-d5005/verification/scripts  gmake -f Makefile_VCS.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim_4","title":"Questasim","text":"
                  To dump the waveform, \"DUMP=1\" must be added into the command line for Questasim build and simulation. 
                   
                      ofs-d5005/verification/scripts  gmake -f Makefile_MSIM.mk build DUMP=1\n\n    ofs-d5005/verification/scripts  gmake -f Makefile_MSIM.mk run TESTNAME=<test_case_name> DUMP=1\n
                   Or
                   
                      ofs-d5005/verification/scripts  gmake -f Makefile_MSIM.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                   
                  There are some optimizations in the Table below for convenience if you want to bypass some commands for both Synopsys VCS and Questasim:
                   Command (Synopsys VCS) Command (Questasim) Details gmake -f Makefile_VCS.mk build_all DUMP=1 gmake -f Makefile_MSIM.mk build_all DUMP=1 compile IP + compile RTL gmake -f Makefile_VCS.mk build_run TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk build_run TESTNAME= DUMP=1 compile RTL + run test gmake -f Makefile_VCS.mk do_it_all TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk do_it_all TESTNAME= DUMP=1 compile IP, RTL and run test gmake -f Makefile_VCS.mk rundb TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk rundb TESTNAME= DUMP=1 run test in sim dir + over-writes content"},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#87-uvm-regression-test","title":"8.7 UVM Regression Test","text":"
                  cd $VERDIR/scripts\n\nFor Regression in VCS with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -s vcs -c\n\nResults are created in a sim directory ($VERDIR/sim) with individual testcase log dir\n\nFor Regression in MSIM with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -s msim -c\n
                   
                  Results are created in a sim directory ($VERDIR/sim_msim) with individual testcase log dir
                  "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#88-uvm-waveform-and-transcript-analysis","title":"8.8 UVM Waveform and Transcript Analysis","text":""},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-vcs_5","title":"Synopsys VCS","text":"
                  Running Synopsys VCS UVM tests will generate a ofs-d5005/verification/sim directory
                   
                     
                  • All build time logs are at ofs-d5005/verification/sim
                    •  
                   
                     
                  • Each testcase will have separate directory inside sim ofs-d5005/verification/sim/ 
                    There are two tracker or log files that are available: runsim.log and trans.log. 
                     
                    runsim.log is the simulation log file generated from Synopsys VCS. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 3
                     
                     
                    Figure 3 runsim.log
                     
                    trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 4
                     
                     
                    Figure 4 trans.log
                     
                    The waveform generated is named as \"inter.vpd\". To open the waveform, go to simulation result directory and run 
                     
                    dve -full64 -vpd inter.vpd &\n
                    "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim_5","title":"Questasim","text":"
                    Running Questasim UVM tests will generate a ofs-d5005/verification/sim_msim directory
                     
                       
                    • All build time logs are at ofs-d5005/verification/sim_msim
                      •  
                     
                       
                    • Each testcase will have separate directory inside sim ofs-d5005/verification/sim_msim/ 
                      There are two tracker or log files that are available: runsim.log and trans.log. 
                       
                      runsim.log is the simulation log file generated from Questasim. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 4
                       
                       
                      Figure 4 runsim.log
                       
                      trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 5
                       
                       
                      Figure 5 trans.log
                       
                      The waveform generated is named as \"vsim.wlf\". To open the waveform, go to simulation result directory and run 
                       
                          vsim -view vsim.wlf &    \n
                      "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#9-modifying-uvm-testbench","title":"9 Modifying UVM Testbench","text":"
                      The next section describe what needs to be considered when modifying the UVM, targeting a different device, adding a new interface to the testbench and creating a new UVM test for a customized OFS Accelerator platform.
                      "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#91-modifying-uvm-environment-when-targeting-different-device","title":"9.1 Modifying UVM environment when targeting different device","text":"
                      A new device may have different design feature or flow. The base address must be allocated for the new device. The MMIO targeting the new device must be based on the base address. If it is a new PF or VF, PCIe HIP must be regenerated and enumeration sequence must be updated accordingly. 
                      "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#92-modifying-uvm-environment-when-adding-a-new-interface","title":"9.2 Modifying UVM environment when adding a new interface","text":"
                      Adding a new interface requires signal connections in the testbench. An additional BFM or verification IP is needed to drive the new interface. The main testbench file tb_top.sv is found at the following location
                       
                          $OFS_ROOTDIR/verification/testbench\n
                      "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#93-adding-a-new-uvm-test","title":"9.3 Adding a new UVM test","text":"
                      In the following example we will modify an existing test \"he_lpbk\" and name it \"he_lpbk_new\", and rebuild the test to check it. Please follow the steps below 
                       
                         
                      1.  
                        Create a new test sequence file under ofs-d5005/verification/tests/sequences
                         
                        he_lpbk_seq_new.svh\n
                         
                        2.  
                      2.  
                        Modify ifndef, define and endif statements in new test sequence case i.e he_lpbk_seq_new.svh file
                         
                        `ifndef HE_LPBK_SEQ_NEW_SVH \n`define HE_LPBK_SEQ_NEW_SVH\n`endif // HE_LPBK_SEQ_NEW_SVH\n
                         
                        also replace all occurences of he_lpbk_seq with he_lpbk_seq_new in the he_lpbk_seq_new.svh file
                         
                        3.  
                      3.  
                        Append the new sequence name into ofs-d5005/verification/tests/sequences/seq_lib.svh file
                         
                        `include \"he_lpbk_seq_new.svh\"\n
                         
                        4.  
                      4.  
                        Create a new test under ofs-d5005/verification/tests
                         
                        he_lpbk_test_new.svh\n
                         
                        5.  
                      5.  
                        Modify ifndef, define and endif statements in new test case i.e he_lpbk_test_new.svh file
                         
                        `ifndef HE_LPBK_TEST_NEW_SVH \n`define HE_LPBK_TEST_NEW_SVH\n`endif // HE_LPBK_TEST_NEW_SVH\n
                         
                        also replace all occurences of he_lpbk_test with he_lpbk_test_new in the he_lpbk_test_new.svh file
                         
                        6.  
                      6.  
                        Append the new test name into ofs-d5005/verification/tests/test_pkg.svh file
                         
                        `include \"he_lpbk_test_new.svh\"\n
                         
                        7.  
                      7.  
                        Rebuild UVM test suite for either Synopsys VCS or Questasim simulater
                         
                        cd $VERDIR/scripts\ngmake -f Makefile_VCS.mk build_all\n
                         
                        or
                         
                        cd $VERDIR/scripts\ngmake -f Makefile_MSIM.mk build_all\n
                         
                        8.  
                      8.  
                        Execute new test for either Synopsys VCS or Questasim simulater
                         
                        cd $VERDIR/scripts\ngmake -f Makefile_VCS.mk run TESTNAME=he_lpbk_test_new\n
                         
                        or
                         
                        cd $VERDIR/scripts\ngmake -f Makefile_MSIM.mk run TESTNAME=he_lpbk_test_new\n
                         
                        9.  
                       
                      9) Check new test and log files         cd ofs-d5005/verification/sim/he_lpbk_test_new
                       
                      ```sh\nopen runsim.log\n```\n
                      "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                      Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                       
                      OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                      "},{"location":"hw/doc_modules/Glossary/","title":"Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/doc_modules/Notices_%26_Disclaimers/","title":"Notices & Disclaimers","text":""},{"location":"hw/doc_modules/Notices_%26_Disclaimers/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                      Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                       
                      OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                      "},{"location":"hw/doc_modules/links/","title":"AFU Dev","text":""},{"location":"hw/doc_modules/quartus_installation/","title":"Quartus installation","text":"
                      Intel ${{ env.QUARTUS_PP_VER_L }} is verified to work with the latest OFS release ${{ env.RELEASE }}.  However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.
                       
                      Use ${{ env.HOST_OS_L }} for compatibility with your development flow and also testing your FIM design in your platform. 
                       
                      Prior to installing Quartus:
                       
                         
                      1.  
                        Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.
                         
                           
                        • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                          •  
                        • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.
                          •  
                         
                        2.  
                      2.  
                        Perform the following steps to satisfy the required dependencies.
                         
                        $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                         
                        Apply the following configurations.
                         
                        $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n
                         
                        3.  
                      3.  
                        Create the default installation path: /intelFPGA_pro/, where  is the default path of the Linux workstation, or as set by the system administrator and  is your Quartus version number. 
                        The installation path must satisfy the following requirements:
                         
                           
                        • Contain only alphanumeric characters
                          •  
                        • No special characters or symbols, such as !$%@^&*<>,
                          •  
                        • Only English characters
                          •  
                        • No spaces
                          •  
                         
                      4.  
                        Download your required Quartus Prime Pro Linux version here.
                         
                        5.  
                      5.  
                        Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are ${{ env.QUARTUS_PATCHES }}.
                         
                        6.  
                      6.  
                        After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                         
                        export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                         
                        For example, if the Quartus install directory is /home/intelFPGA_pro/${{ env.QUARTUS_PP_VER_S } then the new line is:
                         
                        export PATH=/home/intelFPGA_pro/${{env.QUARTUS_PP_VER_S }}/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/${ env.QUARTUS_PP_VER_S }}/qsys/bin:$PATH\n
                         
                        7.  
                      7.  
                        Verify, Quartus is discoverable by opening a new shell:
                         
                        $ which quartus\n/home/intelFPGA_pro/${{ env.QUARTUS_PP_VER_S }/quartus/bin/quartus\n
                         
                        8. "},{"location":"hw/doc_modules/wt_clone_fim_repo/","title":"Wt clone fim repo","text":"
                        Perform the following steps to clone the OFS Agilex PCIe Attach FIM Repository:
                         
                           
                        1. Create a new directory to use as a clean starting point to store the retrieved files.     
                          mkdir OFS_BUILD_ROOT\ncd OFS_BUILD_ROOT\nexport OFS_BUILD_ROOT=$PWD\n
                          2.  
                        2. Clone GitHub repository using the HTTPS git method     
                          git clone --recurse-submodules ${{ env.REPO }}.git\n
                          3.  
                        3. Check out the correct tag of the repository     
                          cd ${{ env.FIM_REPO }}\ngit checkout --recurse-submodules tags/${{ env.TAG }}\n
                          4.  
                        "},{"location":"hw/doc_modules/wt_compile_fim_in_prep_for_afu/","title":"Wt compile fim in prep for afu","text":"
                        Perform the following steps to compile a FIM for where the exercisers are removed and replaced with an he_null module while keeping the PF/VF multiplexor connections.
                         
                        Pre-requisites:
                         
                           
                        • This walkthrough requires a development environment. Refer to the [Walkthrough: Set Up Development Environment] Section for instructions on setting up a development environment.
                          •  
                         
                        Steps:
                         
                           
                        1.  
                          Clone the FIM repository (or use an existing cloned repository). Refer to the [Walkthrough: Clone FIM Repository] section for step-by-step instructions.
                           
                          2.  
                        2.  
                          Set development environment variables. Refer to the [Walkthrough: Set Development Environment Variables] section for step-by-step instructions.
                           
                          3.  
                        3.  
                          Compile the FIM with the HE_NULL compile options
                           
                          cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/${{ env.MODEL }}.ofss ${{ env.MODEL }}:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_${{ env.MODEL }}\n
                           
                          4.  
                        "},{"location":"hw/doc_modules/wt_install_git_lfs_rhel/","title":"Wt install git lfs rhel","text":"
                        To install the Git Large File Storage (LFS) extension, execute the following commands:
                         
                           
                        1. Obtain Git LFS package     
                          curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\n
                          2.  
                        2. Install Git LFS package     
                          sudo dnf install git-lfs\n
                          3.  
                        3. Install Git LFS     
                          git lfs install\n
                          4.  
                        "},{"location":"hw/doc_modules/wt_run_individual_unit_level_sim/","title":"Wt run individual unit level sim","text":"
                        Perform the following steps to run an individual unit test.
                         
                        Pre-requisites:
                         
                           
                        • This walkthrough requires a development environment. Refer to the [Walkthrough: Set Up Development Environment] Section for instructions on setting up a development environment.
                          •  
                         
                        Steps:
                         
                           
                        1.  
                          Clone the FIM repository (or use an existing cloned repository). Refer to the [Walkthrough: Clone FIM Repository] section for step-by-step instructions.
                           
                          2.  
                        2.  
                          Set development environment variables. Refer to the [Walkthrough: Set Development Environment Variables] section for step-by-step instructions.
                           
                          3.  
                        3.  
                          Generate the simulation files for the ${{ env.MODEL }}
                           
                          cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/${{ env.MODEL }}.ofss ${{ env.MODEL }}\n
                           
                          4.  
                        4.  
                          Navigate to the common simulation directory   
                          cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n
                           
                          5.  
                        5.  
                          Run the desired unit test using your desired simulator
                           
                             
                          •  
                            Using VCS
                             
                            sh run_sim.sh TEST=<test_name>\n
                             
                            •  
                           
                             
                          •  
                            Using VCSMX
                             
                            sh run_sim.sh TEST=<test_name> VCSMX=1\n
                             
                            •  
                           
                             
                          •  
                            Using QuestaSim
                             
                            sh run_sim.sh TEST=<test_name> MSIM=1\n
                             
                            •  
                           
                             
                          •  
                            For example, to run the DFH walker test using VCSMX:
                             
                            sh run_sim.sh TEST=dfh_walker VCSMX=1\n
                             
                            •  
                           
                          6.  
                        6.  
                          Once the test is complete, check the output for the simulation results. Review the log for detailed test results.
                           
                          Test status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/${{ env.FIM_REPO }}/sim/unit_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time         356233750000\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 356233750000 fs\nCPU Time:     57.730 seconds;       Data structure size:  47.2Mb\nTue Sep  5 09:44:19 2023\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                           
                          7.  
                        "},{"location":"hw/doc_modules/wt_set_fim_dev_env_vars/","title":"Wt set fim dev env vars","text":"
                        Perform the following steps to set the required environment variables. These environment variables must be set prior to simulation or compilation tasks so it is recommended that you create a script to set these variables.
                         
                           
                        1.  
                          Navigate to the top level directory of the cloned OFS FIM repository.
                           
                          cd ${{ env.FIM_REPO }}\n
                           
                          2.  
                        2.  
                          Set project variables   
                          # Set OFS Root Directory - e.g. this is the top level directory of the cloned OFS FIM repository\nexport OFS_ROOTDIR=$PWD\n
                           
                          3.  
                        3.  
                          Set variables based on your development environment   
                          # Set proxies if required for your server\nexport http_proxy=<YOUR_HTTP_PROXY>\nexport https_proxy=<YOUR_HTTPS_PROXY>\nexport ftp_proxy=<YOUR_FTP_PROXY>\nexport socks_proxy=<YOUR_SOCKS_PROXY>\nexport no_proxy=<YOUR_NO_PROXY>\n\n# Set Quartus license path\nexport LM_LICENSE_FILE=<YOUR_LM_LICENSE_FILE>\n\n# Set Synopsys License path (if using Synopsys for simulation)\nexport DW_LICENSE_FILE=<YOUR_DW_LICENSE_FILE>\nexport SNPSLMD_LICENSE_FILE=<YOUR_SNPSLMD_LICENSE_FILE>\n\n# Set Quartus Installation Directory - e.g. $QUARTUS_ROOTDIR/bin contains Quartus executables\nexport QUARTUS_ROOTDIR=<YOUR_QUARTUS_INSTALLATION_DIRECTORY>\n\n# Set the Tools Directory - e.g. $TOOLS_LOCATION contains the 'synopsys' directory if you are using Synopsys. Refer to the $VCS_HOME variable for an example.\nexport TOOLS_LOCATION=<YOUR_TOOLS_LOCATION>\n
                           
                          4.  
                        4.  
                          Set generic environment variables
                           
                          # Set Work directory \nexport WORKDIR=$OFS_ROOTDIR\n\n# Set Quartus Tools variables\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport QUARTUS_VER_AC=$QUARTUS_ROOTDIR\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport IMPORT_IP_ROOTDIR=$IP_ROOTDIR\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\n\n# Set Verification Tools variables (if running simulations)\nexport DESIGNWARE_HOME=$TOOLS_LOCATION/synopsys/vip_common/vip_Q-2020.03A\nexport UVM_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport MTI_HOME=$QUARTUS_ROOTDIR/../questa_fse\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n\n# Set PATH to include compilation and simulation tools\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/../qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/bin:$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$DESIGNWARE_HOME/bin:$VCS_HOME/bin:$PATH\n
                           
                          5.  
                        "},{"location":"hw/doc_modules/wt_set_up_development_environment/","title":"Wt set up development environment","text":"
                        This walkthrough guides you through the process of setting up your development environment in preparation for FIM development. This flow only needs to be done once on your development machine.
                         
                           
                        1.  
                          Ensure that ${{ env.QUARTUS_PP_VER_L }} for Linux with Intel Agilex FPGA device support is installed on your development machine. Refer to the [Walkthrough: Install Quartus Prime Pro Software] section for step-by-step installation instructions.
                           
                             
                          1. Verify version number
                            quartus_sh --version\n
                             
                            Example Output:
                             
                            Quartus Prime Shell\nVersion ${{ env.QUARTUS_PP_VER_S }} Build 94 06/14/2023 SC Pro Edition\nCopyright (C) 2023  Intel Corporation. All rights reserved.\n
                             
                            2.  
                           
                          2.  
                        2.  
                          Ensure that all support tools are installed on your development machine, and that they meet the version requirements.
                           
                             
                          1.  
                            Python ${{ env.PYTHON_VER }} or later
                             
                               
                            1.  
                              Verify version number
                               
                              python --version\n
                               
                              Example Output:
                               
                              Python ${{ env.PYTHON_VER }}\n
                               
                              2.  
                             
                            2.  
                          2.  
                            GCC ${{ env.GCC_VER }} or later
                             
                               
                            1.  
                              Verify version number
                               
                              gcc --version\n
                               
                              Example output:
                               
                              gcc (GCC) ${{ env.GCC_VER }}\n
                               
                              2.  
                             
                            3.  
                          3.  
                            cmake ${{ env.CMAKE_VER }} or later
                             
                               
                            1.  
                              Verify version number
                               
                              cmake --version\n
                               
                              Example output:
                               
                              cmake version ${{ env.CMAKE_VER }}\n
                               
                              2.  
                             
                            4.  
                          4.  
                            git with git-lfs ${{ env.GIT_VER }} or later. Refer to the [Walkthrough: Install Git Large File Storage Extension] section for step-by-step instructions on installing the Git Large File Storage (LFS) extension.
                             
                               
                            1.  
                              Verify version number
                               
                              git --version\n
                               
                              Example output:
                               
                              git version ${{ env.GIT_VER }}\n
                               
                              2.  
                             
                            5.  
                           
                          3.  
                        3.  
                          Clone the ${{ env.FIM_REPO }} repository. Refer to the [Walkthrough: Clone FIM Repository] section for step-by-step instructions.
                           
                          4.  
                        4.  
                          Install UART IP license patch .02.
                           
                             
                          1.  
                            Navigate to the license directory
                             
                            cd $IOFS_BUILD_ROOT/license\n
                             
                            2.  
                          2.  
                            Install Patch 0.02
                             
                            sudo ./quartus-0.0-0.02iofs-linux.run\n
                             
                            3.  
                           
                          5.  
                        5.  
                          Install Quartus Patches ${{ env.QUARTUS_PATCHES }}. All required patches are provided in the Assets of the OFS FIM Release: ${{ env.RELEASE_PATH }}
                           
                          6.  
                        6.  
                          Verify that patches have been installed correctly. They should be listed in the output of the following command.
                           
                          quartus_sh --version\n
                           
                          7.  
                        7.  
                          Set required environment variables. Refer to the [Walkthrough: Set Environment Variables] section for step-by-step instructions.
                           
                          8.  
                         
                        This concludes the walkthrough for setting up your development environment. At this point you are ready to begin FIM development.
                        "},{"location":"hw/f2000x/","title":"Index","text":"
                        Location for SoC Attach Collateral for OFS.
                        "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/","title":"AFU Development Guide: OFS for Intel\u00ae Intel\u00ae Agilex\u00ae 7 FPGA SoC Attach FPGAs","text":"
                        Last updated: February 03, 2024 
                        "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#1-introduction","title":"1. Introduction","text":"
                        This document is a design guide for the creation of an Accelerator Functional Unit (AFU) using Open FPGA Stack (OFS) for Intel\u00ae Agilex\u00ae 7 FPGAs SoC Attach. The AFU concept consists of separating out the FPGA design development process into two parts, the construction of the foundational FPGA Interface Manager (FIM), and the development of the Acceleration Function Unit (AFU), as shown in the diagram below.
                         
                         
                        This diagram shows the separation of FPGA board interface development from the internal FPGA workload creation.  This separation starts with the FPGA Interface Manager (FIM) which consists of the external interfaces and board management functions.  The FIM is the base system layer and is typically provided by board vendors. The FIM interface is specific to a particular physical platform.  The AFU makes use of the external interfaces with user defined logic to perform a specific application.  By separating out the lengthy and complicated process of developing and integrating external interfaces for an FPGA into a board allows the AFU developer to focus on the needs of their workload.  OFS for Intel\u00ae Agilex\u00ae 7 FPGAs SoC Attach provides the following tools for rapid AFU development:
                         
                           
                        • Scripts for both compilation and simulation setup
                          •  
                        • Optional Platform Interface Manager (PIM) which is a set of SystemVerilog shims and scripts for flexible FIM to AFU interfacing
                          •  
                        • Acceleration Simulation Environment (ASE) which is a hardware/software co-simulation environment scripts for compilation and Acceleration
                          •  
                        • Integration with Open Programmable Acceleration Engine (OPAE) SDK for rapid software development for your AFU application
                          •  
                         
                        Please notice in the above block diagram that the AFU region consists of static and partial reconfiguration (PR) regions where the PR region can be dynamically reconfigured while the remaining FPGA design continues to function.  Creating AFU logic for the static region is described in FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae SoC Attach FPGAs. This guide covers logic in the AFU Main region.
                        "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#11-document-organization","title":"1.1. Document Organization","text":"
                        This document is organized as follows:
                         
                           
                        • Description of design flow
                          •  
                        • Interfaces and functionality provided in the Intel\u00ae Agilex\u00ae 7 FPGAs SoC Attach FIM
                          •  
                        • Setup of the AFU Development environment
                          •  
                        • Synthesize the AFU example
                          •  
                        • Testing the AFU example on the Intel IPU Platform F2000X-PL card
                          •  
                        • Hardware/Software co-simulation using ASE
                          •  
                        • Debugging an AFU with Remote Signal Tap
                          •  
                         
                        This guide provides theory followed by tutorial steps to solidify your AFU development knowledge.
                         
                        NOTE: 
                         
                        This guide uses the Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL as the platform for all tutorial steps. Additionally, this guide and the tutorial steps can be used with other Intel\u00ae Agilex\u00ae 7 FPGAs SoC Attach platforms..
                         
                        Some of the document links in this guide are specific to the Intel IPU Platform F2000X-PL .   If using a different platform, please use the associated documentation for your platform as there could be differences in building the FIM and downloading FIM images. 
                         
                        If you have worked with previous Intel Programmable Acceleration products, you will find out that OFS for Intel\u00ae Agilex\u00ae 7 FPGAs SoC Attach is similar. However, there are differences and you are advised to carefully read and follow the tutorial steps to fully understand the design tools and flow.
                        "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#12-prerequisite","title":"1.2. Prerequisite","text":"
                        This guide assumes you have the following FPGA logic design-related knowledge and skills:
                         
                           
                        • FPGA compilation flows including the Intel\u00ae Quartus\u00ae Prime Pro Edition design flow
                          •  
                        • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition software, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
                          •  
                        • RTL and coding practices to create synthesizable logic.
                          •  
                        • Understanding of AXI and Avalon memory mapped and streaming interfaces.
                          •  
                        • Simulation of complex RTL using industry standard simulators (Synopsys\u00ae VCS\u00ae or Siemens\u00ae QuestaSim\u00ae).
                          •  
                        • Signal Tap Logic Analyzer tool in the Intel\u00ae Quartus\u00ae Prime Pro Edition software.
                          •  
                         
                        You are strongly encouraged to review the FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae SoC Attach FPGAs
                        "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#13-acceleration-functional-unit-afu-development-flow","title":"1.3. Acceleration Functional Unit (AFU) Development Flow","text":"
                        The AFU development flow is shown below: 
                        "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#131-understanding-platform-capabilities","title":"1.3.1. Understanding Platform Capabilities","text":"
                        The block diagram of the F2000x Board is shown below:
                         
                         
                        The F2000x FIM provided with this release is shown below:
                         
                         
                        This release F2000x FIM provides the following features:
                         
                           
                        • Host interface
                             
                          • PCIe Gen4 x 16
                            •  
                          • 2 - PFs 
                            •  
                          • MSI-X interrupts
                            •  
                          • Logic to demonstrate simple PCIe loopback
                            •  
                           
                          •  
                        • Network interface
                             
                          • 2 - QSFP28/56 cages
                            •  
                          • 8 X 25 GbE with exerciser logic demonstrating traffic generation/monitoring
                            •  
                           
                          •  
                        • External Memory - DDR4 - 2400
                             
                          • 4 Banks - 4 GB organized as 1 Gb x 32 with 1 Gb x 8 ECC
                            •  
                          • Memory exerciser logic demonstrating external memory operation
                            •  
                           
                          •  
                        • Board Management
                             
                          • SPI interface
                            •  
                          • FPGA management and configuration
                            •  
                          • Example logic showing DFH operation
                            •  
                           
                          •  
                        • Partial reconfiguration control logic
                          •  
                        • SoC - Xeon Icelake-D subsystem with embedded Linux
                             
                          • PCIe Gen4 x 16 interface to FPGA
                            •  
                          • 1 - PF, 3 - VF, AXI-S TLP packets
                            •  
                          • DDR Memory
                               
                            • 2 Banks - 8 GB organized as 1 Gb x 64 with 1 Gb x 8 ECC
                              •  
                            • NVMe SSD - 64 GB
                              •  
                             
                            •  
                           
                          •  
                        "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#132-high-level-data-flow","title":"1.3.2. High Level Data Flow","text":"
                        The OFS high level data flow is shown below:
                         
                        "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#133-considerations-for-pim-usage","title":"1.3.3. Considerations for PIM Usage","text":"
                        When creating an AFU, a designer needs to decide of what type of interfaces the AFU will provide to the platform (FIM).  The AFU can use the native interfaces (i.e. PCIe TLP commands) provided by the FIM or standard memory mapped interfaces (i.e. AXI-MM or AVMM) by using the PIM.  The PIM is an abstraction layer consisting of a collection of SystemVerilog interfaces and shims to enable partial AFU portability across hardware despite variations in hardware topology and native interfaces. The PIM adds a level of logic between the AFU and the FIM converting the native interfaces from the FIM to match the interfaces provided by the AFU.
                         
                         
                        The following resources are available to assist in creating an AFU:
                         
                        PIM Core Concepts provides details on using the PIM and its capabilities. 
                         
                        Connecting an AFU to a Platform using PIM guides you through the steps needed to connect a PIM Based AFU to the FIM. 
                         
                        The AFU Tutorial provides several AFU examples.  These examples can be run with the current OFS FIM package.  There are three AFU types of examples provided (PIM based, hybrid and native).  Each example provides the following:
                         
                           
                        • RTL, which includes the following interfaces: 
                             
                          • Host Channel: 
                               
                            • Host memory, providing a DMA interface.
                              •  
                            • MMIO, providing a CSR interface.  
                              •  
                             
                            •  
                          • Local Memory
                            •  
                           
                          •  
                        • Host software example interfacing to the CSR interface and host memory interface, using the OPAE C API.
                          •  
                        • Accelerator Description File .json file
                          •  
                        • Source file list
                          •  
                        "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#134-afu-interfaces-included-with-intel-ipu-platform-f2000x-pl","title":"1.3.4. AFU Interfaces Included with Intel IPU Platform F2000X-PL","text":"
                        The figure below shows the interfaces available to the AFU in this architecture. It also shows the design hierarchy with module names from the fim (top.sv) to the PR region AFU (afu_main.sv). One of the main differences from the Stratix 10 PAC OFS architecture to this one is the presence of the static port gasket region (port_gasket.sv) that has components to facilitate the AFU and also consists of the PR region (afu_main.sv) via the PR slot. The Port Gasket contains all the PR specific modules and logic, e.g., PR slot reset/freeze control, user clock, remote STP etc. Architecturally, a Port Gasket can have multiple PR slots where user workload can be programmed into. However, only one PR slot is supported for OFS Release for Intel Agilex. Everything in the Port Gasket until the PR slot should be provided by the FIM developer. The task of the AFU developer is to add their desired application in the afu_main.sv module by stripping out unwanted logic and instantiating the target accelerator. As shown in the figure below, here are the interfaces connected to the AFU (highlighted in green) via the SoC Attach FIM:
                         
                           
                        1. AXI Streaming (AXI-S) interface to the Host via PCIe Gen4x16
                          2.  
                        2. AXI Memory Mapped Channels (4) to the DDR4 EMIF interface
                          3.  
                        3. AXI Streaming (AXI-S) interface to the HSSI 25 Gb Ethernet
                          4.  
                         
                        "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#2-set-up-afu-development-environment","title":"2. Set Up AFU Development Environment","text":"
                        This section covers the setup of the AFU development environment.
                        "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#21-prepare-afu-development-environment","title":"2.1. Prepare AFU development environment","text":"
                        A typical development and hardware test environment consists of a development server or workstation with FPGA development tools installed and a separate server with the target OFS compatible FPGA PCIe card installed.  The typical usage and flow of data between these two servers is shown below:
                         
                         
                        Note: both development and hardware testing can be performed on the same server if desired.
                         
                        This guide uses Intel IPU Platform F2000X-PL as the target OFS compatible FPGA PCIe card for demonstration steps.  The Intel IPU Platform F2000X-PL must be fully installed following the Getting Started User Guide: OFS for Intel\u00ae Agilex\u00ae SoC Attach FPGAs. If using a different OFS FPGA PCIe card, contact your supplier for instructions on how to install and operate user developed AFUs.
                         
                        The following is a summary of the steps to set up for AFU development:
                         
                           
                        1. Install Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 for Linux with Agilex device support and required Quartus patches.
                          2.  
                        2. Make sure support tools are installed and meet version requirements.
                          3.  
                        3. Install OPAE SDK.
                          4.  
                        4. Download the Basic Building Blocks repository.
                          5.  
                        5. Build or download the relocatable AFU PR-able build tree based on your Intel\u00ae Agilex FPGA PCIe Attach FIM.
                          6.  
                        6. Download FIM to the Intel\u00ae Agilex FPGA PCIe Attach platform.
                          7.  
                         
                        Building AFUs with OFS for Agilex requires the build machine to have at least 64 GB of RAM. 
                        "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#22-installation-of-quartus-and-required-patches","title":"2.2. Installation of Quartus and required patches","text":"
                        Intel Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 is verified to work with the latest OFS release release/ofs-2023.3.  However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.
                         
                        Use Ubuntu 22.04 for compatibility with your development flow and also testing your FIM design in your platform. 
                         
                        Prior to installing Quartus:
                         
                           
                        1.  
                          Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.
                           
                             
                          • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                            •  
                          • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.
                            •  
                           
                          2.  
                        2.  
                          Perform the following steps to satisfy the required dependencies.
                           
                          $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                           
                          Apply the following configurations.
                           
                          $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n
                           
                          3.  
                        3.  
                          Create the default installation path: /intelFPGA_pro/, where  is the default path of the Linux workstation, or as set by the system administrator and  is your Quartus version number. 
                          The installation path must satisfy the following requirements:
                           
                             
                          • Contain only alphanumeric characters
                            •  
                          • No special characters or symbols, such as !$%@^&*<>,
                            •  
                          • Only English characters
                            •  
                          • No spaces
                            •  
                           
                        4.  
                          Download your required Quartus Prime Pro Linux version here.
                           
                          5.  
                        5.  
                          Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.13 patch (Generic Serial Flash Interface IP), 0.21 (PCIe Subsystem).
                           
                          6.  
                        6.  
                          After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                           
                          export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                           
                          For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:
                           
                          export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n
                           
                          7.  
                        7.  
                          Verify, Quartus is discoverable by opening a new shell:
                           
                          $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                           
                          8. "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#23-installation-of-support-tools","title":"2.3. Installation of Support Tools","text":"
                          Make sure support tools are installed and meet version requirements.
                           
                          The OFS provided Quartus build scripts require the following tools. Verify these are installed in your development environment.
                           Item Version Python 3.6.8 GCC 7.2.0 cmake 3.15 git 1.8.3.1 perl 5.8.8"},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#24-installation-of-opae-sdk","title":"2.4. Installation of OPAE SDK","text":"
                          Follow the instructions in the Getting Started Guide: Open FPGA Stack for Intel IPU Platform F2000X-PL, section 6.2 Installing the OPAE SDK On the Host to build and install the required OPAE SDK for the Intel IPU Platform F2000X-PL.
                           
                          Working with the Intel\u00ae Intel IPU Platform F2000X-PL card requires opae-2.10.0-1. Follow the instructions in the Getting Started Guide: Intel\u00ae Open FPGA Stack for Intel IPU Platform F2000X-PL section 6.2 Installing the OPAE SDK On the Host. Make sure to check out the cloned repository to tag 2.10.0-1 and branch release/2.10.0.
                           
                          $ git checkout tags/2.10.0-1 -b release/2.10.0\n
                           
                          Note: The tutorial steps provided in the next sections assume the OPAE SDK is installed in default system locations, under the directory, /usr. In most system configurations, this will allow the OS and tools to automatically locate the OPAE binaries, scripts, libraries and include files required for the compilation and simulation of the FIM and AFUs.
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#25-download-the-basic-building-blocks-repositories","title":"2.5. Download the Basic Building Blocks repositories","text":"
                          The ofs-platform-afu-bbb repository contains the PIM files as well as example PIM-based AFUs that can be used for testing and demonstration purposes. This guide will use the host_chan_mmio AFU example in the ofs-platform-afu-bbb repository and the hello_world sample in the examples AFU repository to demonstrate how to synthesize, load, simulate, and test a PIM-based AFU using the Intel IPU Platform F2000X-PL card with the SoC Attach FIM.
                           
                          Execute the next commands to clone the BBB repositories.
                           
                            # Create top level directory for AFU development\n$ mkdir OFS_BUILD_ROOT\n$ cd OFS_BUILD_ROOT\n$ export OFS_BUILD_ROOT=$PWD\n\n  # Clone the ofs-platform-afu-bbb repository.\n$ cd $OFS_BUILD_ROOT\n$ git clone https://github.com/OFS/ofs-platform-afu-bbb.git\n\n  # Verify retrieval\n$ cd $OFS_BUILD_ROOT/ofs-platform-afu-bbb\n$ ls\nLICENSE  plat_if_develop  plat_if_release  plat_if_tests  README.md\n
                           
                          The documentation in the ofs-platform-afu-bbb repository further addresses   - The PIM concept.   - The structure of the PIM-based AFU examples.   - How to generate a release and configure the PIM.   - How to connect an AFU to an FIM.
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#26-build-or-download-the-relocatable-pr-build-tree","title":"2.6. Build or download the relocatable PR build tree","text":"
                          A relocatable PR build tree is needed to build the AFU partial reconfiguration area for the intended FIM. The tree is relocatable and may be copied to a new location. It does not depend on files in the original FIM build.
                           
                          You can use the Intel IPU Platform F2000X-PL release package and download the PR build tree and FIM images, to develop your AFU.  These are located at OFS-F2000X-PL release 
                           
                          Or you can build your own FIM and generate the PR build tree during the process.
                           
                          To download and untar the pr_build_template:
                           
                          $ wget https://github.com/OFS/ofs-f2000x-pl/releases/download/ofs-2023.3-1/f2000x-images_ofs-2023-3-2.tar.gz\n$ tar -zxvf f2000x-images_ofs-2023-3-2.tar.gz\n$ cd f2000x-images_ofs-2023-3-2/\n$ mkdir pr_build_template\n$ tar -zxvf pr_template-f2000x.tar.gz -C ./pr_build_template\n$ cd pr_build_template\n$ export OPAE_PLATFORM_ROOT=$PWD\n
                           
                          To build your own FIM and generate the PR build tree for the Intel IPU Platform F2000X-PL platform, refer the FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae SoC Attach FPGAs and follow the Out-of-Tree PR FIM build flow.  If you are using a different platform, refer to the FPGA Interface Manager Developer Guide for your platform and follow the Out-of-Tree PR FIM build flow.
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#27-download-fim-to-fpga","title":"2.7. Download FIM to FPGA","text":"
                          The AFU requires that the FIM from which the AFU is derived be loaded onto the FPGA.   
                           
                          If you are using the Intel IPU Platform F2000X-PL release package downloaded in the previous section, copy the associated FIM files to the SoC: 
                          # On Development Host\n$ cd $OFS_BUILD_ROOT/f2000x-images_ofs-2023-3-2/\n$ scp ofs_top_page1_unsigned_user1.bin <user>@<SoC IP address>:</remote/directory>\n$ scp ofs_top_page2_unsigned_user2.bin <user>@<SoC IP address>:</remote/directory>\n
                           
                          If you are generating your own FIM, use the unsigned FPGA binary images from your FIM build and copy over to the SoC. 
                           
                          To downlaod the FIM to the Intel IPU Platform F2000X-PL platform: 
                          # On SoC\n$ sudo fpgasupdate ofs_top_page1_unsigned_user1.bin \n$ sudo fpgasupdate ofs_top_page2_unsigned_user2.bin \n$ sudo rsu fpga --page=user1 \n
                           
                          If you are using a different platform, refer to the documentation for your platform to download the FIM images onto your Intel\u00ae Agilex\u00ae SoC Attach Platform.
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#28-set-up-required-environment-variables","title":"2.8. Set up required Environment Variables","text":"
                          Set the required environment variables as shown below. These environment variables must be set prior to simulation or compilation tasks. You can create a simple script to set these variables and save time going forward.
                           
                          # If not already done, export OFS_BUILD_ROOT to the top level directory for AFU development\n$ export OFS_BUILD_ROOT=<path to ofs build directory>\n\n# If not already done, export OPAE_PLATFORM_ROOT to the PR build tree directory\n$ export OPAE_PLATFORM_ROOT=<path to pr build tree>\n\n# Quartus Tools\n# Note, QUARTUS_HOME is your Quartus installation directory, e.g. $QUARTUS_HOME/bin contains Quartus executable.\n$ export QUARTUS_HOME=<user_path>/intelFPGA_pro/23.3/quartus\n$ export QUARTUS_ROOTDIR=$QUARTUS_HOME\n$ export QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\n$ export QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\n$ export IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\n$ export IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\n$ export QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys\n$ export PATH=$QUARTUS_HOME/bin:$QSYS_ROOTDIR/bin:$QUARTUS_HOME/sopc_builder/bin/:$PATH\n\n# OPAE SDK release\n$ export OPAE_SDK_REPO_BRANCH=release/2.10.0\n\n# The following environment variables are required for compiling the AFU examples. \n\n# Location to clone the ofs-platform-afu-bbb repository which contains PIM files and AFU examples.\n$ export OFS_PLATFORM_AFU_BBB=$OFS_BUILD_ROOT/ofs-platform-afu-bbb \n\n# OPAE and MPF libraries must either be on the default linker search paths or on both LIBRARY_PATH and LD_LIBRARY_PATH.  \n$ export OPAE_LOC=/usr\n$ export LIBRARY_PATH=$OPAE_LOC/lib:$LIBRARY_PATH\n$ export LD_LIBRARY_PATH=$OPAE_LOC/lib64:$LD_LIBRARY_PATH\n
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#3-compiling-an-afu","title":"3. Compiling an AFU","text":"
                          In this section, you will use the relocatable PR build tree created in the previous steps from the FIM to compile an example PIM-based AFU. This section will be developed around the host_chan_mmio and hello_world AFU examples to showcase the synthesis of a PIM-based AFU.
                           
                          The build steps presented below demonstrate the ease in building and running an actual AFU on the board. To successfully execute the instructions in this section, you must have set up your development environment and have a relocateable PR Build tree as instructed in section 2 of this document.
                           
                          The build steps presented below demonstrate the ease in building and running an actual AFU on the board. To successfully execute the instructions in this section, you must have set up your development environment and have a relocateable PR Build tree as instructed in section 2 of this document.
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#31-creating-the-afu-synthesis-environment","title":"3.1. Creating the AFU Synthesis Environment","text":"
                          The PIM flow provides the script afu_synth_setup to create the synthesis environment to build the AFU examples. See how to use it below.
                           
                          usage: afu_synth_setup [-h] -s SOURCES [-p PLATFORM] [-l LIB] [-f] dst\n\nGenerate a Quartus build environment for an AFU. A build environment is\ninstantiated from a release and then configured for the specified AFU. AFU\nsource files are specified in a text file that is parsed by rtl_src_config,\nwhich is part of the OPAE base environment.\n\npositional arguments:\n  dst                   Target directory path (directory must not exist).\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -s SOURCES, --sources SOURCES\n                        AFU source specification file that will be passed to\n                        rtl_src_config. See \"rtl_src_config --help\" for the\n                        file's syntax. rtl_src_config translates the source\n                        list into either Quartus or RTL simulator syntax.\n  -p PLATFORM, --platform PLATFORM\n                        FPGA platform name.\n  -l LIB, --lib LIB     FPGA platform release hw/lib directory. If not\n                        specified, the environment variables OPAE_FPGA_HW_LIB\n                        and then BBS_LIB_PATH are checked.\n  -f, --force           Overwrite target directory if it exists.\n
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#32-building-and-running-host_chan_mmio-example-afu","title":"3.2. Building and Running host_chan_mmio example AFU","text":"
                          The $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio is a simple example demonstrating both hardware and software access to an AFU. The host_chan_mmio example AFU consists of the following files.  The hw directory contains the RTL to implement the hardware functionality using Avalon and AXI interfaces. However, this guide will use the AXI version of the host_chan_mmio AFU to go through the compilation steps. The sw directory of the AFU contains the source code of the host application that communicates with the actual AFU hardware.
                           
                          host_chan_mmio\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_avalon.sv\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_axi.sv\n\u2502       \u251c\u2500\u2500 host_chan_mmio.json\n\u2502       \u251c\u2500\u2500 test_mmio_avalon0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon1.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon2_512rw.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi1.txt\n\u2502       \u2514\u2500\u2500 test_mmio_axi2_512rw.txt\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 main.c\n    \u251c\u2500\u2500 Makefile\n
                           
                          Execute afu_synth_setup as follows to create the synthesis environment for a host_chan_mmio AFU that fits the SoC Attach FIM previously constructed.
                           
                          $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio\n$ afu_synth_setup -s ./hw/rtl/test_mmio_axi1.txt afu_dev\n
                           
                          Now, move into the synthesis environment afu_dev directory just created. From there, execute the afu_synth command. The successful completion of the command will produce the host_chan_mmio.gbs file under the synthesis environment directory, $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_dev.
                           
                          $ cd afu_dev\n$ $OPAE_PLATFORM_ROOT/bin/afu_synth\nCompiling ofs_top ofs_pr_afu\nGenerating host_chan_mmio.gbs\n==================================\n...\n...\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n Design meets timing\n===========================================================================\n
                           
                          The previous output indicates the successful compilation of the AFU and the compliance with the timing requirements. Analyze the reports generated in case the design does not meet timing. The timing reports are stored in the directory, $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_dev/build/syn/syn_top/output_files/timing_report.
                           
                          Once the compilation finishes successfully, load the new host_chan_mmio.gbs bitstream file into the partial reconfiguration region of the target Intel IPU Platform F2000X-PL board. Keep in mind, that the loaded image is dynamic - this image is not stored in flash and if the card is power cycled, then the PR region is re-loaded with the default AFU.
                           
                          To load the image, perform the following steps:
                           
                          # On Development Host\n$ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_dev\n# Copy FIM files to SoC\n$ scp host_chan_mmio.gbs <user>@<SoC IP address>:</remote/directory>\n
                           
                          # On SoC\n$ cd </remote/directory>\n$ fpgasupdate host_chan_mmio.gbs \n[2022-04-15 20:22:18.85] [WARNING ] Update starting. Please do not interrupt.\n[2022-04-15 20:22:19.75] [INFO    ] \nPartial Reconfiguration OK\n[2022-04-15 20:22:19.75] [INFO    ] Total time: 0:00:00.90\n
                           
                          Set up your board to work with the newly loaded AFU.
                           
                          # On SoC\n\n# Verify PCIe b.d.f\n# For the following example, the F2000x SKU2 PCIe b:d.f is 15:00.0,\n# however this may be different in your system\n$  fpgainfo fme\nIntel IPU Platform F2000X-PL\nBoard Management Controller NIOS FW version: 1.2.4\nBoard Management Controller Build version: 1.2.4\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\n...\n\n# Create the Virtual Functions (VFs) provided by the FIM, the default FIM has 3 VFs\n$ pci_device 15:00.0 vf 3\n\n# Verify the VFs have been added (device id: bccf)\n$ lspci -s 15:00\n15:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n15:00.1 Processing accelerators: Intel Corporation Device bccf\n15:00.2 Processing accelerators: Intel Corporation Device bccf\n15:00.3 Processing accelerators: Intel Corporation Device bccf\n\n\n# Bind VFs to VFIO driver. \n\n$  opae.io init -d 0000:15:00.1 \nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:15:00.1 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:15:00.1 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:15:00.1 is 52\n\n$ opae.io init -d 0000:15:00.2\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:15:00.2 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:15:00.2 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:15:00.2 is 53\n\n$  opae.io init -d 0000:15:00.3\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:15:00.3 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:15:00.3 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:15:00.3 is 54\n\n# Verify the new AFU is loaded.  The host_chan_mmio AFU GUID is \"76d7ae9c-f66b-461f-816a-5428bcebdbc5\".\n\n$ fpgainfo port\n//****** PORT ******//\nObject Id                        : 0xF100000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0x6015000000000000\nPCIe s:b:d.f                     : 0000:15:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nAccelerator GUID                 : d15ab1ed-0000-0000-0210-000000000000\n//****** PORT ******//\nObject Id                        : 0x4015000000000000\nPCIe s:b:d.f                     : 0000:15:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nAccelerator GUID                 : d15ab1ed-0000-0000-0110-000000000000\n//****** PORT ******//\nObject Id                        : 0x2015000000000000\nPCIe s:b:d.f                     : 0000:15:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nAccelerator GUID                 : 76d7ae9c-f66b-461f-816a-5428bcebdbc5\n
                           
                          Now, navigate to the directory of the host_chan_mmio AFU containing the host application's source code, $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw. Once there, compile the host_chan_mmio host application and execute it on the host server to excercise the functionality of the AFU.
                           
                          Note: If OPAE SDK libraries were not installed in the default systems directories under /usr, you need to set the OPAE_LOC, LIBRARY_PATH, and LD_LIBRARY_PATH environment variables to the custom locations where the OPAE SDK libraries were installed.
                           
                          # On Development Host, move to the sw directory of the the host_chan_mmio AFU. This directory holds the source for the host application.\n$ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw\n$ export OPAE_LOC=/usr\n$ export LIBRARY_PATH=$OPAE_LOC/lib:$LIBRARY_PATH\n$ export LD_LIBRARY_PATH=$OPAE_LOC/lib64:$LD_LIBRARY_PATH\n$ make\n# Copy application to SoC\n$ scp host_chan_mmio <user>@<SoC IP address>:</remote/directory>\n
                           
                          # On SoC, Run the application\n$ cd </remote/directory>\n$  ./host_chan_mmio\nAFU ID:  76d7ae9cf66b461f 816a5428bcebdbc5\nAFU MMIO interface: AXI Lite\nAFU MMIO read bus width: 64 bits\n512 bit MMIO write supported: yes\nAFU pClk frequency: 470 MHz\n\nTesting 32 bit MMIO reads:\n  PASS - 4 tests\n\nTesting 32 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 64 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 512 bit MMIO writes:\n  PASS\n
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#33-building-and-running-the-hello_world-example-afu","title":"3.3. Building and running the hello_world example AFU","text":"
                          The platform-independent examples AFU repository also provides some interesting example AFUs. In this section, you will compile and execute the PIM based hello_world AFU. The RTL of the hello_world AFU receives from the host application an address via memory mapped I/O (MMIO) write and generates a DMA write to the memory line at that address. The content written to memory is the string \"Hello world!\". The host application spins, waiting for the memory line to be updated. Once available, the software prints out the string.
                           
                          The hello_world example AFU consists of the following files. The hw directory contains the RTL to implement the hardware functionality using CCIP, Avalon, and AXI interfaces. However, this guide will use the AXI version of the AFU to go through the compilation steps. The sw directory of the AFU contains the source code of the host application that communicates with the AFU hardware.
                           
                          hello_world\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u251c\u2500\u2500 ccip\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_ccip.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u2514\u2500\u2500 hello_world.json\n\u251c\u2500\u2500 README.md\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 hello_world\n    \u251c\u2500\u2500 hello_world.c\n    \u251c\u2500\u2500 Makefile\n    \u2514\u2500\u2500 obj\n        \u251c\u2500\u2500 afu_json_info.h\n        \u2514\u2500\u2500 hello_world.o\n
                           
                          The following instructions can be used to compile other AFU samples accompanying this repository.
                           
                             
                          1. If not done already, download and clone the examples AFU repository.
                            2.  
                           
                          $ cd $OFS_BUILD_ROOT \n$ git clone https://github.com/OFS/examples-afu.git\n
                           
                             
                          1.  
                            Compile the hello_word sample AFU.  
                            $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world\n$ afu_synth_setup --source hw/rtl/axi/sources.txt afu_dev\n$ cd afu_dev\n$ ${OPAE_PLATFORM_ROOT}/bin/afu_synth\nCompiling ofs_top ofs_pr_afu\nGenerating hello_world.gbs\n==================================\n.\n.\n.\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'hello_world.gbs'\n Design meets timing\n===========================================================================\n
                             
                            2.  
                          2.  
                            To test the AFU in actual hardware, load the hello_world.gbs to the Intel IPU Platform F2000X-PL card. For this step to be successful, the SoC Attach FIM must have already been loaded to the Intel IPU Platform F2000X-PL card following the steps described in Section 2 of this document.
                             
                            3.  
                           
                          $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/afu_dev/\n# Copy FIM files to SoC\n$ scp hello_world.gbs <user>@<SoC IP address>:</remote/directory>\n
                           
                          # On SoC\n$ cd </remote/directory>\n$ fpgasupdate hello_world.gbs \n[2022-04-15 20:22:18.85] [WARNING ] Update starting. Please do not interrupt.\n[2022-04-15 20:22:19.75] [INFO    ] \nPartial Reconfiguration OK\n[2022-04-15 20:22:19.75] [INFO    ] Total time: 0:00:00.90\n
                           
                          Set up your Intel IPU Platform F2000X-PL board to work with the newly loaded hello_world.gbs file.
                           
                          # On SoC\n\n# Verify PCIe b.d.f\n# For the following example, the F2000x SKU2 PCIe b:d.f is 15:00.0,\n# however this may be different in your system\n$  fpgainfo fme\nIntel IPU Platform F2000X-PL\nBoard Management Controller NIOS FW version: 1.2.4\nBoard Management Controller Build version: 1.2.4\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\n...\n\n# Create the Virtual Functions (VFs) provided by the FIM, the default FIM has 3 VFs\n$ pci_device 15:00.0 vf 3\n\n# Verify the VFs have been added (device id: bccf)\n$ lspci -s 15:00\n15:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n15:00.1 Processing accelerators: Intel Corporation Device bccf\n15:00.2 Processing accelerators: Intel Corporation Device bccf\n15:00.3 Processing accelerators: Intel Corporation Device bccf\n\n\n# Bind VFs to VFIO driver.\n\n$ opae.io init -d 0000:15:00.1\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:15:00.1 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:15:00.1 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:15:00.1 is 52\n\n$ opae.io init -d 0000:15:00.2\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:15:00.2 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:15:00.2 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:15:00.2 is 53\n\n$ opae.io init -d 0000:15:00.3\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:15:00.3 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:15:00.3 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:15:00.3 is 54\n\n# Verify the new AFU is loaded.  The hello_world AFU GUID is \"c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\".\n\n$ fpgainfo port\n//****** PORT ******//\nObject Id                        : 0xF100000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0x6015000000000000\nPCIe s:b:d.f                     : 0000:15:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nAccelerator GUID                 : d15ab1ed-0000-0000-0210-000000000000\n//****** PORT ******//\nObject Id                        : 0x4015000000000000\nPCIe s:b:d.f                     : 0000:15:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nAccelerator GUID                 : d15ab1ed-0000-0000-0110-000000000000\n//****** PORT ******//\nObject Id                        : 0x2015000000000000\nPCIe s:b:d.f                     : 0000:15:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nAccelerator GUID                 : c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\n
                           
                             
                          1. Compile and execute the host application of the hello_world AFU. You should see the application outputs the \"Hello world!\" message in the terminal.
                            2.  
                           
                          # On Development Host, move to the sw directory of the hello_world AFU and build application\n$ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/sw\n$ make\n# Copy application to SoC\n$ scp hello_world <user>@<SoC IP address>:</remote/directory>\n
                           
                          # On SoC, Run the application\n$ cd </remote/directory>\n$ ./hello_world\nHello world!\n
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#34-modify-the-afu-user-clocks-frequency","title":"3.4. Modify the AFU user clocks frequency","text":"
                          An OPAE compliant AFU specifies the frequency of the uclk_usr and uclk_usr_div2 clocks through the JSON file for AFU configuration located under the <afu_example>/hw/rtl directory of an AFU design. For instance, the AFU configuration file of the host_chan_mmio example is $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/host_chan_mmio.json.
                           
                          The AFU specifies the frequency for uClk_usr in its platform configuration file using the following key:value pairs:
                           
                            \"clock-frequency-high\": [<float-value>|\u201dauto\u201d|\u201dauto-<float-value>\u201d]\n  \"clock-frequency-low\": [<float-value>|\u201dauto\u201d|\u201dauto-<float-value>\u201d]\n
                           
                          These key:value tuples are used to configure the PLL of the target platform that provides the user clocks through the AFU clocks interface. In addition, the specified frequency affects the timing closure process on the user clocks during AFU compilation. 
                           
                          Setting the value field to a float number (e.g., 315.0 to specify 315 MHz) drives the AFU generation process to close timing within the bounds set by the low and high values and sets the AFU's JSON metadata to specify the user clock PLL  frequency values.
                           
                          The following example shows the JSON file of the host_chan_mmio to set the AFU uClk to 500 MHz and uClk_div2 to 250 MHz.
                           
                          {\n   \"version\": 1,\n   \"afu-image\": {\n      \"power\": 0,\n      \"clock-frequency-high\": 500,\n      \"clock-frequency-low\": 250,\n      \"afu-top-interface\":\n         {\n            \"class\": \"ofs_plat_afu\"\n         },\n      \"accelerator-clusters\":\n         [\n            {\n               \"name\": \"host_chan_mmio\",\n               \"total-contexts\": 1,\n               \"accelerator-type-uuid\": \"76d7ae9c-f66b-461f-816a-5428bcebdbc5\"\n            }\n         ]\n   }\n}\n
                           
                          Save the changes to host_chan_mmio.json file, then execute the afu_synth_setup script to create a new copy of the AFU files with the modified user clock settigns.
                           
                          $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/\n$ afu_synth_setup -s $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/test_mmio_axi1.txt afu_clks\n\nCopying build from /home/<user_area>/ofs-f2000x-pl/work_pr/pr_build_template/hw/lib/build...\nConfiguring Quartus build directory: build_F2000x_afu_clks/build\nLoading platform database: /home/<user_area>/ofs-f2000x-pl/work_pr/pr_build_template/hw/lib/platform/platform_db/ofs_agilex_adp.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting platform/platform_afu_top_config.vh\nWriting platform/platform_if_addenda.qsf\nWriting ../hw/afu_json_info.vh\n
                           Compile the host_chan_mmio AFU with the new frequency values.
                           
                          $ cd afu_clks\n$ $OPAE_PLATFORM_ROOT/bin/afu_synth\n
                           
                          During the compilation phase, you will observe the Timing Analyzer uses the specified user clock frequency values as the target to close timing.
                           
                           
                          AFU developers must ensure the AFU hardware design meets timing. The compilation of an AFU that fails timing shows a message similar to the following.
                           
                          .\n.\n.\n\nWrote host_chan_mmio.gbs\n\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n\n  *** Design does not meet timing\n  *** See build/syn/syn_top/output_files/timing_report\n\n===========================================================================\n
                           
                          The previous output indicates the location of the timing reports for the AFU designer to identify the failing paths and perform the necessary design changes. Next, is a listing of the timing report files from a host_chan_mmio AFU that fails to meet timing after modifying the user clock frequency values.
                           
                          $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_clks\n$ ls build/syn/syn_top/output_files/timing_report\n\nclocks.rpt  clocks.sta.fail.summary  clocks.sta.pass.summary\n
                           
                          Warning: AFU developers must inform software developers of the maximum operating frequency (Fmax) of the user clocks to avoid any unexpected behavior of the accelerator and potentially of the overall system.
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#4-simulating-an-afu-using-ase","title":"4. Simulating an AFU using ASE","text":"
                          The Application Simulation Environment (ASE) is a hardware/software co-simulation environment for your AFU. See diagram below illustrating ASE operation:
                           
                           
                          ASE uses the simulator Direct Programming Interface (DPI) to provide HW/SW connectivity.  The PCIe connection to the AFU under testing is emulated with a transactional model.
                           
                          The following list describes ASE operation:
                           
                             
                          • Attempts to replicate the transactions that will be seen in real system.
                            •  
                          • Provides a memory model to AFU, so illegal memory accesses can be identified early.
                            •  
                          • Not a cache simulator.
                            •  
                          • Does not guarantee synthesizability or timing closure.
                            •  
                          • Does not model system latency.
                            •  
                          • No administrator privileges are needed to run ASE.  All code is user level.
                            •  
                           
                          The remainder of this section is a tutorial providing the steps on how to run ASE with either Synopsys\u00ae VCS\u00ae or Siemens\u00ae QuestaSim\u00ae using an example AFU and the AFU build tree previously created in this guide.
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#41-set-up-steps-to-run-ase","title":"4.1. Set Up Steps to Run ASE","text":"
                          In this section you will set up your server to support ASE by independently downloading and installing OPAE SDK and ASE. Then, set up the required environment variables.
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#411-install-opae-sdk","title":"4.1.1. Install OPAE SDK","text":"
                          Follow the instructions documented in the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 FPGAs Targeting the Intel IPU Platform F2000X-PL, section 6.2 Installing the OPAE SDK On the Host to build and install the required OPAE SDK for the Intel IPU Platform F2000X-PL card.
                           
                          The F2000x SKU2 card requires 2.10.0-1. Follow the instructions provided in the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 FPGAs Targeting the Intel IPU Platform F2000X-PL, section 6.2 Installing the OPAE SDK On the Host. However, just make sure to check out the cloned repository to tag 2.10.0-1 and branch release/2.10.0.
                           
                          $ git checkout tags/2.10.0-1 -b release/2.10.0\n
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#412-install-ase-tools","title":"4.1.2 Install ASE Tools","text":"
                          ASE is an RTL simulator for OPAE-based AFUs. The simulator emulates both the OPAE SDK software user space API and the AFU RTL interface. The majority of the FIM as well as devices such as PCIe and local memory are emulated with simple functional models.
                           
                          ASE must be installed separatedly from the OPAE SDK. However, the recommendation is to install it in the same target directory as OPAE SDK.
                           
                             
                          1.  
                            If not done already, set the environment variables as described in section, Set Up AFU Development Environment.
                             
                            2.  
                          2.  
                            Clone the opae-sim repository. 
                            $ cd $OFS_BUILD_ROOT\n$ git clone https://github.com/OFS/opae-sim.git\n$ cd opae-sim\n$ git checkout tags/2.10.0-1 -b release/2.10.0 \n
                             
                            3.  
                          3.  
                            Create a build directory and build ASE to be installed under the default system directories along with OPAE SDK. 
                            $ mkdir build\n$ cd build\n$ cmake  -DCMAKE_INSTALL_PREFIX=/usr ..\n$ make\n
                             
                            4.  
                           
                          Optionally, if the desire is to install ASE binaries in a different location to the system's default, provide the path to CMAKE through the CMAKE_INSTALL_PREFIX switch, as follows. 
                          $ cmake -DCMAKE_INSTALL_PREFIX=<</some/arbitrary/path>> ..  \n
                           
                             
                          1. Install ASE binaries and libraries under the system directory /usr. 
                            $ sudo make install  \n
                            2.  
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#413-setup-required-ase-environment-variables","title":"4.1.3. Setup Required ASE Environment Variables","text":"
                          The values set to the following environment variables assume the OPAE SDK and ASE were installed in the default system directories below /usr. Setup these variables in the shell where ASE will be executed. You may wish to add these variables to the script you created to facilitate configuring your environment.
                           
                          $ export OPAE_PLATFORM_ROOT=<path to PR build tree>\n$ cd /usr/bin\n$ export PATH=$PWD:$PATH\n$ cd /usr/lib/python*/site-packages\n$ export PYTHONPATH=$PWD\n$ cd /usr/lib\n$ export LIBRARY_PATH=$PWD\n$ cd /usr/lib64\n$ export LD_LIBRARY_PATH=$PWD\n$ cd $OFS_BUILD_ROOT/ofs-platform-afu-bbb\n$ export OFS_PLATFORM_AFU_BBB=$PWD\n\n  ## For VCS, set the following:\n$ export VCS_HOME=<Set the path to VCS installation directory>\n$ export PATH=$VCS_HOME/bin:$PATH\n\n  ## For QuestaSIM, set the following:\n$ export MTI_HOME=<path to Modelsim installation directory>\n$ export PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\n
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#42-simulating-the-host_chan_mmio-afu","title":"4.2. Simulating the host_chan_mmio AFU","text":"
                          The $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio is a simple example demonstrating both hardware and software access to an AFU. The host_chan_mmio example AFU consists of the following files:
                           
                          host_chan_mmio\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_avalon.sv\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_axi.sv\n\u2502       \u251c\u2500\u2500 host_chan_mmio.json\n\u2502       \u251c\u2500\u2500 test_mmio_avalon0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon1.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon2_512rw.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi1.txt\n\u2502       \u2514\u2500\u2500 test_mmio_axi2_512rw.txt\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 main.c\n    \u251c\u2500\u2500 Makefile\n
                           
                          This example AFU contains examples using both Avalon and AXI interface buses. This guide will use the AXI version of the host_chan_mmio AFU.
                           
                          ASE uses client-server application architecture to deliver hardware/software co-simulation.  You require one shell for the hardware based simulation and another shell where the software application is running. The hardware is started first with a simulation compilation and simulator startup script, once the simulator has loaded the design, it will wait until the software process starts. Once the software process starts, the simulator proceeds.  Transaction logging and waveform capture is performed.
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#421-set-up-and-run-the-hw-simulation-process","title":"4.2.1 Set Up and Run the HW Simulation Process","text":"
                          You will run the afu_sim_setup script to create the scripts for running the ASE environment.  The afu_sim_setup script has the following usage:
                           
                          usage: afu_sim_setup [-h] -s SOURCES [-p PLATFORM] [-t {VCS,QUESTA,MODELSIM}]\n                     [-f] [--ase-mode ASE_MODE] [--ase-verbose]\n                     dst\n\nGenerate an ASE simulation environment for an AFU. An ASE environment is\ninstantiated from the OPAE installation and then configured for the specified\nAFU. AFU source files are specified in a text file that is parsed by\nrtl_src_config, which is also part of the OPAE base environment.\n\npositional arguments:\n  dst                   Target directory path (directory must not exist).\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -s SOURCES, --sources SOURCES\n                        AFU source specification file that will be passed to\n                        rtl_src_config. See \"rtl_src_config --help\" for the\n                        file's syntax. rtl_src_config translates the source\n                        list into either Quartus or RTL simulator syntax.\n  -p PLATFORM, --platform PLATFORM\n                        FPGA Platform to simulate.\n  -t {VCS,QUESTA,MODELSIM}, --tool {VCS,QUESTA,MODELSIM}\n                        Default simulator.\n  -f, --force           Overwrite target directory if it exists.\n  --ase-mode ASE_MODE   ASE execution mode (default, mode 3, exits on\n                        completion). See ase.cfg in the target directory.\n  --ase-verbose         When set, ASE prints each CCI-P transaction to the\n                        command line. Transactions are always logged to\n                        work/ccip_transactions.tsv, even when not set. This\n                        switch sets ENABLE_CL_VIEW in ase.cfg.\n
                           
                          Run afu_sim_setup to create the ASE simulation environment for the host_chan_mmio example AFU. The '-t VCS' option indicates to prepare the ASE simulation environment for Synopsys\u00ae VCS\u00ae.
                           
                          $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio\n$ afu_sim_setup -s ./hw/rtl/test_mmio_axi1.txt -t VCS afu_sim\n\nCopying ASE from /opae-sdk/install-opae-sdk/share/opae/ase...\n#################################################################\n#                                                               #\n#             OPAE Intel(R) Xeon(R) + FPGA Library              #\n#               AFU Simulation Environment (ASE)                #\n#                                                               #\n#################################################################\n\nTool Brand: VCS\nLoading platform database: /ofs-f2000x-pl/work_pr/build_tree/hw/lib/platform/platform_db/ofs_agilex_adp.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting rtl/platform_afu_top_config.vh\nWriting rtl/platform_if_addenda.txt\nWriting rtl/platform_if_includes.txt\nWriting rtl/ase_platform_name.txt\nWriting rtl/ase_platform_config.mk and rtl/ase_platform_config.cmake\nASE Platform: discrete (FPGA_PLATFORM_DISCRETE)\n
                           
                          The afu_sim_setup creates the ASE scripts in the directory host_chan_mmio_sim where the afu_sim_setup script was run.  Start the simulator as shown below:
                           
                          $ cd afu_sim\n$ make\n$ make sim\n
                           
                          This process launches the AFU hardware simulator. Before moving to the next section, pay attention to the simulator output highlighted in the image below.
                           
                           
                          The simulation artifacts are stored in host_chan_mmio/work and consist of:
                           
                          log_ase_events.tsv\nlog_ofs_plat_host_chan.tsv \nlog_ofs_plat_local_mem.tsv \nlog_pf_vf_mux_A.tsv \nlog_pf_vf_mux_B.tsv \n
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#422-set-up-and-run-the-sw-process","title":"4.2.2 Set Up and Run the SW Process","text":"
                          Open an additional shell to build and run the host application that communicates with the actual AFU hardware. Set up the same environment variable you have set up in the shell you have been working on until this point. 
                           
                          Additionally, as indicated by the hardware simulator output that is currently executing in the \"simulator shell\", copy and paste the line \"export ASE_WORKDIR=...\", into the new \"software shell\". See the last image of the previous section.
                           
                          $ export ASE_WORKDIR= $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_sim/work\n
                           Then, go to the sw directory of the host_chan_mmio AFU example to compile the host application.
                           
                          $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw  \n$ make\n\nafu_json_mgr json-info --afu-json=../hw/rtl/host_chan_mmio.json --c-hdr=obj/afu_json_info.h\nWriting obj/afu_json_info.h\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c main.c -o obj/main.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c test_host_chan_mmio.c -o obj/test_host_chan_mmio.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/connect.c -o obj/connect.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/csr_mgr.c -o obj/csr_mgr.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/hash32.c -o obj/hash32.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/test_data.c -o obj/test_data.o\ncc -o host_chan_mmio obj/main.o obj/test_host_chan_mmio.o obj/connect.o obj/csr_mgr.o obj/hash32.o obj/test_data.o  -z noexecstack -z relro -z now -pie -luuid -lopae-c-ase\n
                           
                          Now, launch the host application to exercise the AFU hardware running on the simulator shell. The next image shows the AFU hardware simulation process on the left side shell. The right hand shell shows the host application's output of a successful simulation.
                           
                          $ with_ase ./host_chan_mmio\n  [APP]  Initializing simulation session ...\nRunning in ASE mode\nAFU ID:  76d7ae9cf66b461f 816a5428bcebdbc5\nAFU MMIO interface: AXI Lite\nAFU MMIO read bus width: 64 bits\n512 bit MMIO write supported: yes\nAFU pClk frequency: 470 MHz\n\nTesting 32 bit MMIO reads:\n  PASS - 4 tests\n\nTesting 32 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 64 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 512 bit MMIO writes:\n  PASS\n  [APP]  Deinitializing simulation session\n  [APP]         Took 1,003,771,568 nsec\n  [APP]  Session ended\n
                           
                           
                           
                          Finally, on the hardware simulation shell, you can view the wave forms by invoking the following command.
                           
                          $ make wave\n
                           
                          This brings up the Synopsys\u00ae VCS\u00ae simulator GUI and loads the simulation waveform files. Use the Hierarchy window to navigate to the afu instance located under, ase_top | ase_top_plat | ase_afu_main_pcie_ss | ase_afu_main_emul | afu_main | port_afu_instances | ofs_plat_afu | afu , as shown below.
                           
                           
                          Right click on the afu (afu) entry to display the drop-down menu. Then, click on Add to Waves | New Wave View to display the following waveforms window.
                           
                           
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#43-simulating-the-hello_world-afu","title":"4.3 Simulating the hello_world AFU","text":"
                          In this section you will quickly simulate the PIM-based hello_world sample AFU accompanying the examples-afu repository.
                           
                             
                          1.  
                            Set the environment variables as described in section 4.1. Set Up Steps to Run ASE.
                             
                            2.  
                          2.  
                            Prepare an RTL simulation environment for the AXI version of the hello_world AFU.
                             
                            3.  
                           
                          Simulation with ASE requires two software processes, one to simulate the AFU RTL and the other to run the host software that excercises the AFU. To construct an RTL simulation environment under the directory simulation, execute the following.
                           
                          $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world\n$ afu_sim_setup -s ./hw/rtl/axi/sources.txt -t VCS afu_sim\n\n\nCopying ASE from /usr/local/share/opae/ase...\n#################################################################\n#                                                               #\n#             OPAE Intel(R) Xeon(R) + FPGA Library              #\n#               AFU Simulation Environment (ASE)                #\n#                                                               #\n#################################################################\n\nTool Brand: VCS\nLoading platform database: /home/<user_area>/ofs-f2000x-pl/work_pr/pr_build_template/hw/lib/platform/platform_db/ofs_agilex_adp.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting rtl/platform_afu_top_config.vh\nWriting rtl/platform_if_addenda.txt\nWriting rtl/platform_if_includes.txt\nWriting rtl/ase_platform_name.txt\nWriting rtl/ase_platform_config.mk and rtl/ase_platform_config.cmake\nASE Platform: discrete (FPGA_PLATFORM_DISCRETE)\n
                           
                          The afu_sim_setup script constructs an ASE environment in the hello_world_sim subdirectory. If the command fails, confirm that the path to the afu_sim_setup is on your PATH environment variable (in the OPAE SDK bin directory) and that your Python version is at least 2.7.
                           
                             
                          1. Build and execute the AFU RTL simulator.
                            2.  
                           
                          $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/afu_sim\n$ make\n$ make sim  \n
                           
                          The previous commands will build and run the Synopsys\u00ae VCS\u00ae RTL simulator, which prints a message saying it is ready for simulation. The simulation process also prints a message instructing you to set the ASE_WORKDIR environment variable in a second shell.
                           
                             
                          1.  
                            Open a second shell where you will build and execute the host software. In this new \"software shell\", set up the environment variables you have set up so far in the \"hardware simulation\" shell.
                             
                            2.  
                          2.  
                            Also, set the ASE_WORKDIR environment variable following the instructions given in the \"hardware simulation\" shell. 
                            $ export ASE_WORKDIR=$OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/afu_sim/work\n
                             
                            3.  
                          3.  
                            Then, move to the sw directory of the hello_world AFU sample to build the host software. 
                            $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/sw\n$ make      \n
                             
                            4.  
                          4.  
                            Run the hello_world host application to resume the work of the RTL simulation. The host software process and the RTL simulation execute in lockstep. If successful, you should see the Hello world! output.
                             
                            5.  
                           
                          $ with_ase ./hello_world\n\n  [APP]  Initializing simulation session ...\nHello world!\n  [APP]  Deinitializing simulation session\n  [APP]         Took 43,978,424 nsec\n  [APP]  Session ended\n
                           
                          The image below shows the simulation of the AFU hardware and the execution of the host application side-by-side.
                           
                           
                           
                           
                             
                          1. Finally, on the hardware simulation shell, you can view the wave forms by invoking the following command. 
                            make wave\n
                            2.  
                           
                          This brings up the DVE GUI and loads the simulation waveform files. Use the Hierarchy window to navigate to the afu instance located under, ase_top | ase_top_plat | ase_afu_main_pcie_ss | ase_afu_main_emul | afu_main | port_afu_instances | ofs_plat_afu | hello_afu, as shown below.
                           
                           
                          Right click on the hello_afu entry to display the drop-down menu. Then, click on Add to Waves | New Wave View to display the following waveforms window.
                           
                           
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#5-adding-remote-signal-tap-logic-analyzer-to-debug-the-afu","title":"5. Adding Remote Signal Tap Logic Analyzer to debug the AFU","text":"
                          Remote Signal Tap is currently not supported in F2000x base FIM configuration.
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#6-how-to-modify-the-pfvf-mux-configuration","title":"6. How to modify the PF/VF MUX configuration","text":"
                          For information on how to modify the PF/VF mapping for your own design, refer to the FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae SoC Attach FPGAs.
                          "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                          Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                           
                          OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/","title":"Intel\u00ae FPGA Interface Manager Developer Guide: Intel Agilex\u00ae 7 SoC Attach: Open FPGA Stack","text":"
                          Last updated: February 03, 2024 
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#1-introduction","title":"1 Introduction","text":""},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#11-about-this-document","title":"1.1 About This Document","text":"
                          This document serves as a design guide for FPGA developers, system architects, and hardware developers using Open FPGA Stack (OFS) as a starting point for the creation of an FPGA Interface Manager (FIM) for a custom FPGA acceleration board.   
                           
                          OFS addresses the demand for FPGA acceleration boards and workloads by providing a powerful methodology for the rapid development of FPGA Acceleration systems.  This methodology addresses the challenges and responsibilities of the board, platform, and workload developers by providing a complete FPGA project consisting of RTL and simulation code, build scripts, and software.  This provided FPGA project can be rapidly customized to meet new market requirements.
                           
                          OFS separates the FPGA design into two areas: FPGA Interface Manager (FIM) and workload (or Acceleration Function Unit) as shown in the figure below:
                           
                           
                          As can be seen in this diagram, the OFS FPGA structure has a natural separation into two distinct areas: 
                           
                             
                          • FPGA Interface Manager (FIM or sometimes called the \"the shell\") containing:
                               
                            • FPGA external interfaces and IP cores (e.g. Ethernet, DDR-4, PCIe, etc)
                              •  
                            • PLLs/resets
                              •  
                            • FPGA - Board management infrastructure
                              •  
                            • Interface to Acceleration Function Unit (AFU)
                              •  
                             
                            •  
                          • Acceleration Function Unit (\"the workload\")
                               
                            • Uses the FIM interfaces to perform useful work inside the FPGA
                              •  
                            • Contains logic supporting partial reconfiguration
                              •  
                            • Remote Signal Tap core for remote debugging of SoC AFU PR region
                              •  
                             
                            •  
                           
                          This guide is organized as follows: 
                           
                             
                          • Introduction
                            •  
                          • Top Level Block Diagram description
                               
                            • Control and data flow
                              •  
                             
                            •  
                          • Description of Sub-systems
                               
                            • Command/status registers (CSR) and software interface
                              •  
                            • Clocking, resets, and interfaces
                              •  
                            • High-Speed Serial Interface Sub-System (HSSI-SS) - also known as the Ethernet Sub-System
                              •  
                            • External Memory Interface Sub-System (MEM-SS)
                              •  
                             
                            •  
                          • High-level development flow description
                               
                            • Installation of OFS RTL and development packages
                              •  
                            • Compiling FIM
                              •  
                            • Simulation  
                              •  
                             
                            •  
                          • Design customization walkthroughs
                            •  
                           
                          This document uses the  Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL as the platform to illustrate key points and demonstrate how to extend the capabilities provided in OFS.  The demonstration steps serve as a tutorial for the development of your OFS knowledge.  
                           
                          This document covers OFS architecture lightly. For more details on the OFS architecture, please see Open FPGA Stack Technical Reference Manual.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#111-glossary","title":"1.1.1 Glossary","text":"
                          The following table describes several terms that are used in this document.
                           Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#12-release-capabilities","title":"1.2 Release Capabilities","text":"
                          Intel Agilex 7 SoC Attach OFS supports the following features.
                           FIM BASE Intel IPU Platform F2000X-PL f2000x PCIe Configuration Host: PCIe Gen4x16SoC: PCIe Gen4x16 SR-IOV support Host: 2 PFs, No VFsSoC:  1 PFs, 3 VFs AXI ST datapath 512b @ 470MHz Transceiver Subsystem Configuration 2x4x25G 
                          The FIM also integrates:
                           
                             
                          • SoC AFU and Host AFU
                            •  
                          • Exercisers demonstrating PCIe, external memory, and Ethernet interfaces
                            •  
                          • FME CSR
                            •  
                          • Remote Signal Tap
                            •  
                          • Partial Reconfiguration of the SoC AFU
                            •  
                           
                          The Host exercisers are provided for the quick evaluation of the FIM and can be leveraged for the verification of the platform's functionality and capabilities.  The host exercisers can be removed by the designer to release FPGA real estate to accommodate new workload functions. To compile the FIM without host exercisers go to How to compile the FIM in preparation for designing your AFU.
                           
                          OFS is extensible to meet the needs of a broad set of customer applications.  The general use cases listed below are examples where the OFS base design is easily extended to build a custom FIM:
                           
                             
                          1. Use OFS design example as-is
                               
                            • Porting the code to another platform that is identical to OFS reference platform changing targeted FPGA device and pinout
                              •  
                            • Change I/O assignments without changing design
                              •  
                             
                            2.  
                          2. Update the configuration of peripheral IP in OFS design example, not affecting FIM architecture
                               
                            • External memory settings
                              •  
                            • Ethernet Subsystem analog settings
                              •  
                             
                            3.  
                          3. Remove/update peripheral feature in OFS design example, not affecting FIM architecture
                               
                            • External memory speed/width change
                              •  
                            • Change number of VFs supported
                              •  
                             
                            4.  
                          4. Add new features as an extension to OFS design example, not affecting the FIM architecture
                               
                            • Add/remove external memory interface to the design
                              •  
                            • Add/remove user clocks for the AFU
                              •  
                            • Add/remove IP to the design with connection to the AFU
                              •  
                             
                            5.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#13-knowledge-prerequisites","title":"1.3 Knowledge Prerequisites","text":"
                          OFS is an advanced application of FPGA technology. This guide assumes you have the following FPGA logic design-related knowledge and skills:
                           
                             
                          • FPGA compilation flows using Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3.
                            •  
                          • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
                            •  
                          • RTL (System Verilog) and coding practices to create synthesized logic.
                            •  
                          • RTL simulation tools.
                            •  
                          • Intel\u00ae Quartus\u00ae Prime Pro Edition Signal Tap Logic Analyzer tool software.
                            •  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#2-top-level-description","title":"2 Top Level Description","text":"
                          The FIM targets operation in the  Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL the block diagram is shown below.  
                           
                           
                             
                          • Host interface 
                               
                            • PCIe Gen4 x 16
                              •  
                             
                            •  
                          • SoC Interface
                               
                            • PCIe Gen4 x 16
                              •  
                             
                            •  
                          • Network interface
                               
                            • 2 - QSFP28/56 cages
                              •  
                            • Eight Arm\u00ae AMBA\u00ae 4 AXI4-Stream channels of 25G Ethernet interfacing to an E-tile Ethernet Subsystem.
                              •  
                             
                            •  
                          • External Memory - DDR4
                               
                            • Four Fabric DDR4-2400 banks - 4 GB organized as 1Gb x 32 with 1 Gb x 8 ECC (ECC login not implemented in default FIM) 
                              •  
                             
                            •  
                          • Board Management
                               
                            • SPI interface
                              •  
                            • FPGA configuration
                              •  
                             
                            •  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#21-top-level-fpga","title":"2.1 Top Level FPGA","text":"
                          The internal FPGA architecture is shown below:
                           
                           
                          The following Platform Designer IP subsystems are used to implement the following:
                           
                             
                          • P-tile PCIe Subsystem
                            •  
                          • E-Tile Ethernet Subsystem
                            •  
                          • Memory Subsystem
                            •  
                           
                          Documentation on the above Platform Designer IP subsystems is available by request to your Intel support team.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#22-fim-fpga-resource-usage","title":"2.2 FIM FPGA Resource Usage","text":"
                          The provided design includes both required board management and control functions as well as optional interface exerciser logic that both creates transactions and validates operations.  These exerciser modules include:
                           
                             
                          • HE_MEM - this module creates external memory transactions to the DDR4 memory and then verifies the responses.
                            •  
                          • HE_MEM-TG -The memory traffic generator (TG) AFU provides a way for users to characterize local memory channel bandwidth with a variety of traffic configuration features including request burst size, read/write interleave count, address offset, address strobe, and data pattern.
                            •  
                          • HE_HSSI - this module creates ethernet transactions to the HSSI Subsystem and then verifies the responses.
                            •  
                           
                          The FIM uses a small portion of the available FPGA resources.  The table below shows resource usage for a base FIM built with 2 channels of external memory, a small AFU instantiated that has host CSR read/write, external memory test and Ethernet test functionality.
                           
                          Note: The host exerciser modules allow you to evaluate the FIM in hardware and are removed when you begin development. 
                           
                          The AGFC023R25A2E2VR0  FPGA has the following resources available for base FIM design :
                           Resource needed / total on device (%) Logic utilization (ALMs) 229,622 / 782,400 ( 29 % ) M20K blocks 1,241 / 10,464 (12 %) Pins 518 / 742 ( 70 % ) IOPLLs 10 / 15 ( 67 % ) 
                          The resource usage for the FIM base:
                           Entity Name ALMs needed ALM Utilization % M20Ks M20K Utilization % top 229,646.10 29.35 1241 11.86 soc_afu 87,364.80 11.17 273 2.61 soc_pcie_wrapper 37,160.80 4.75 195 1.86 pcie_wrapper 36,233.40 4.63 187 1.79 host_afu 26,462.20 3.38 140 1.34 hssi_wrapper 20,066.30 2.56 173 1.65 pmci_wrapper 8,449.90 1.08 186 1.78 mem_ss_top 7,907.10 1.01 60 0.57 auto_fab_0 2,708.90 0.35 13 0.12 soc_bpf 1,210.20 0.15 0 0.00 qsfp_1 635.50 0.08 4 0.04 qsfp_0 628.70 0.08 4 0.04 fme_top 628.60 0.08 6 0.06 host_soc_rst_bridge 151.40 0.02 0 0.00 rst_ctrl 16.80 0.00 0 0.00 soc_rst_ctrl 16.50 0.00 0 0.00 sys_pll 0.50 0.00 0 0.00 
                          The following example without the he_lb,he_hssi,he_mem,he_mem_tg:
                           Entity Name ALMs needed ALM Utilization % M20Ks M20K Utilization % top 162,010.20 20.71 992 9.48 pcie_wrapper 36,771.70 4.70 195 1.86 soc_afu_top 34,851.30 4.45 85 0.81 pcie_wrapper 33,358.90 4.26 175 1.67 hssi_wrapper 20,109.90 2.57 173 1.65 afu_top 14,084.20 1.80 91 0.87 pmci_wrapper 8,447.90 1.08 186 1.78 mem_ss_top 8,379.70 1.07 60 0.57 alt_sld_fab_0 2,725.10 0.35 13 0.12 bpf_top 1,213.00 0.16 0 0.00 fme_top 638.30 0.08 6 0.06 qsfp_top 626.70 0.08 4 0.04 qsfp_top 619.20 0.08 4 0.04 axi_lite_rst_bridge 147.40 0.02 0 0.00 rst_ctrl 17.40 0.00 0 0.00 rst_ctrl 15.90 0.00 0 0.00 sys_pll 0.50 0.00 0 0.00 
                          The following example without the Ethernet Subsystem (no_hssi):
                           Entity Name ALMs needed ALM Utilization % M20Ks M20K Utilization % top 189,827.00 24.26 980 9.37 soc_afu_top 67,751.40 8.66 197 1.88 pcie_wrapper 36,909.30 4.72 195 1.86 pcie_wrapper 36,077.70 4.61 187 1.79 afu_top 26,549.40 3.39 140 1.34 pmci_wrapper 8,688.10 1.11 186 1.78 mem_ss_top 8,079.00 1.03 60 0.57 alt_sld_fab_0 1,751.90 0.22 9 0.09 bpf_top 1,186.00 0.15 0 0.00 dummy_csr 664.70 0.08 0 0.00 dummy_csr 662.80 0.08 0 0.00 dummy_csr 661.20 0.08 0 0.00 fme_top 649.40 0.08 6 0.06 axi_lite_rst_bridge 161.70 0.02 0 0.00 rst_ctrl 16.30 0.00 0 0.00 rst_ctrl 16.00 0.00 0 0.00 sys_pll 0.50 0.00 0 0.00 
                          The following example without the Ethernet Subsystem (no_hssi) + no host exercisers (he_lb, he_hssi, he_mem, he_mem_tg):
                           Entity Name ALMs needed ALM Utilization % M20Ks M20K Utilization % top 139,105.70 17.78 807 7.71 pcie_wrapper 36,518.80 4.67 195 1.86 pcie_wrapper 33,234.50 4.25 175 1.67 soc_afu_top 32,700.00 4.18 85 0.81 afu_top 14,178.20 1.81 91 0.87 pmci_wrapper 8,693.20 1.11 186 1.78 mem_ss_top 7,999.00 1.02 60 0.57 alt_sld_fab_0 1,758.40 0.22 9 0.09 bpf_top 1,183.50 0.15 0 0.00 dummy_csr 667.20 0.09 0 0.00 dummy_csr 666.30 0.09 0 0.00 dummy_csr 663.10 0.08 0 0.00 fme_top 652.80 0.08 6 0.06 axi_lite_rst_bridge 153.80 0.02 0 0.00 rst_ctrl 18.20 0.00 0 0.00 rst_ctrl 16.50 0.00 0 0.00 sys_pll 0.50 0.00 0 0.00"},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#3-description-of-sub-systems","title":"3 Description of Sub-Systems","text":""},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#31-host-control-and-data-flow","title":"3.1 Host Control and Data Flow","text":"
                          The host control and data flow is shown in the diagram below:
                           
                           
                          The control and data paths are composed of the following:
                           
                             
                          • Host Interface Adapter (PCIe)
                            •  
                          • SoC Interface Adapter (PCIe)
                            •  
                          • Low Performance Peripherals
                               
                            • Slow speed peripherals (JTAG, I2C, Smbus, etc)
                              •  
                            • Management peripherals (FME)
                              •  
                             
                            •  
                          • High Performance Peripherals
                               
                            • Memory peripherals
                              •  
                            • Acceleration Function peripherals (eg. AFUs)
                              •  
                            • HPS Peripheral
                              •  
                             
                            •  
                          • Fabrics
                               
                            • Peripheral Fabric (multi drop)
                              •  
                            • AFU Streaming fabric (point to point)
                              •  
                             
                            •  
                           
                          Peripherals are connected to one another using AXI, either:
                           
                             
                          • Via the peripheral fabric (AXI4-Lite, multi drop)
                            •  
                          • Via the AFU streaming fabric (AXI-S, point to point)
                            •  
                           
                          Peripherals are presented to software as:
                           
                             
                          • OFS managed peripherals that implement DFH CSR structure.  
                            •  
                          • Native driver managed peripherals (i.e. Exposed via an independent PF, VF)
                            •  
                           
                          The peripherals connected to the peripheral fabric are primarily Intel OPAE managed resources, whereas the peripherals connected to the AFU are \u201cprimarily\u201d managed by native OS drivers. The word \u201cprimarily\u201d is used since the AFU is not mandated to expose all its peripherals to Intel OPAE. 
                           
                          OFS uses a defined set of CSRs to expose the functionality of the FPGA to the host software.  These registers are described in Open FPGA Stack Reference Manual - MMIO Regions section.
                           
                          If you make changes to the FIM that affect the software operation, then OFS provides a mechanism to communicate that information to the proper software driver that works with your new hardware.  The Device Feature Header (DFH) structure is followed to provide compatibility with OPAE software.  Please see FPGA Device Feature List (DFL) Framework Overview for a description of DFL operation from the driver perspective.
                           
                          In the default design, the SoC and Host AFUs are isolated from each other. You must develop mechanisms for Host - SoC communication if desired.
                           
                          Note: The default configuration of the Board Peripheral Fabric, there is a connection from the Host Interface to the PMCI-SS, however the PMCI-SS is not in the Host DFL, and is not discovered by Host SW by default. If you want to guarantee that the Host can not access the PMCI-SS, and by extension the Board BMC, you must implement a filtering mechanism, for example, in the Host ST2MM module to prevent access to the PMCI-SS address space.
                           
                          Refer to the following documents for more information on sub-systems:
                           
                             
                          • Intel FPGA PCI Express Subsystem IP User Guide
                            •  
                          • Intel FPGA Memory Subsystem IP User Guide
                            •  
                          • Intel FPGA Ethernet Subsystem IP User Guide
                            •  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#4-high-level-development-flow","title":"4 High Level Development Flow","text":"
                          OFS provides a framework of FPGA synthesizable code, simulation environment, and synthesis/simulation scripts.  FIM designers can use the provided code as-is, modify the provided code, or add new code to meet your specific product requirements. The instructions provided after this point are for you to either evaluate the existing design (the current section) or to modify and generate your own design (described in the Custom FIM Development Flow section). 
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#41-development-pre-requisites","title":"4.1 Development Pre-requisites","text":"
                          The following pre-requisites must be satisfied to go through the development flow.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#411-tutorial-pre-requisites","title":"4.1.1 Tutorial Pre-requisites","text":"
                          To run the FPGA compilation steps covered in this guide, requires the following:
                           
                             
                          1. Workstation or server with a Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 installed on a Intel Quartus Prime Pro-supported Linux distribution.  See Operating System Support.  The Linux distribution known to work with this version of RedHatEnterprise Linux\u00ae (RHEL) 8.6 . Note, Windows is not supported.
                            2.  
                          2. Compilation targeting Intel Agilex 7 devices requires a minimum of 64 GB of RAM.
                            3.  
                          3. Simulation of lower level functionality (not chip level) is supported by Synopsys\u00ae VCS and Mentor Graphics\u00ae QuestaSim SystemVerilog simulators.
                            4.  
                          4. Simulation of chip level requires Synopsys VCS and VIP
                            5.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#412-development-environment","title":"4.1.2 Development Environment","text":"
                          To run the tutorial steps in this guide requires this development environment:
                           Item Version Intel Quartus Prime Pro 23.3 OPAE SDK Branch Tag: release/2.10.0 Simulator Synopsys VCS P-2019.06-SP2-5 or newer for UVM simulation of top level FIM Python 3.6.8 GCC 7.2.0 cmake 3.15 git with git-lfs 1.8.3.1 PERL 5.8.8 
                          Note: Steps to install Intel Quartus Prime Pro are provided in the Installation of OFS section.
                           
                          To install the Git Large File Storage (LFS) extension run the following commands:
                           
                          curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\nsudo dnf install git-lfs\ngit lfs install\n
                           
                          To test FPGA image files on hardware, this version of OFS only targets  Intel IPU Platform F2000X-PL. You may modify the build scripts and pin files to target different boards with Intel Agilex 7 FPGA devices.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#42-installation-of-ofs","title":"4.2 Installation of OFS","text":"
                          In this section you set up a development machine for compiling the OFS FIM. These steps are separate from the setup for a deployment machine where the FPGA acceleration card is installed.  Typically, FPGA development and deployment work is performed on separate machines, however, both development and deployment can be performed on the same server if desired.  Please see the Getting Started Guide: Intel\u00ae Open FPGA Stack for Intel FPGA for instructions on installing software for deployment of your FPGA FIM, AFU and software application on a server.  
                           
                          Building the OFS FIM requires the build machine to have at least 64 GB of RAM.  
                           
                          The following is a summary of the steps to set up for FIM development:
                           
                             
                          1. Install Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 Linux with Agilex device support
                            2.  
                          2. Make sure support tools are installed and meet version requirements
                            3.  
                          3. Clone the  repository
                            4.  
                          4. Install required Intel Quartus Prime Pro patches which are included in the cloned ofs-f2000x repository
                            5.  
                          5. Review the files provided in the repo
                            6.  
                          6. Test installation by building the FIM
                            7.  
                           
                          Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 is the currently verified version of Intel Quartus Prime Pro used for building the FIM and AFU images.  Porting to newer versions of Intel Quartus Prime Pro may be performed by developers, however, you will need to verify operation.
                           
                          The recommended Best Known Configuration (BKC) for development of the OFS FIM is RedHatEnterprise Linux\u00ae (RHEL) 8.6, which is the assumed operating system for this developer guide. 
                           
                             
                          1.  
                            Prior to installing Intel Quartus Prime Pro, perform the following steps to satisfy the required dependencies.
                             
                            sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                             
                            2.  
                          2.  
                            Apply the following configurations.
                             
                            sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \nsudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \nsudo ln -s /usr/bin/python3 /usr/bin/python\n
                             
                            3.  
                          3.  
                            Download Intel\u00ae Quartus\u00ae Prime Pro Edition Linux.
                             
                            4.  
                          4.  
                            After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                             
                            export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                             
                            For example, if the Intel Quartus Prime Pro install directory is /home/intelFPGA_pro/23.3 then the new line is:
                             
                            export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n
                             
                            5.  
                          5.  
                            Verify, Intel Quartus Prime Pro is discoverable by opening a new shell:
                             
                            which quartus\n
                             
                            Example output:   
                            /home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                             
                            6.  
                          6.  
                            Install Quartus two patches 0.11 and 0.19.  The patch files are included as assets in the OFS-F2000X-PL release.  Scroll to the bottom of the page and download the patches to your Quartus development computer and then install the patches by running the patch script and following the installation items.
                             
                            sudo ./quartus-23.2-0.11-linux-internal-donot-distribute.run\nsudo ./quartus-23.2-0.19-linux-internal-donot-distribute.run\n
                             
                            7.  
                          7.  
                            Verify Quartus the version of Quartus.
                             
                            quartus_sh --version\nVersion 23.2.0 Build 94 06/14/2023 Patches 0.11,0.19 SC Pro Edition\nCopyright (C) 2023  Intel Corporation. All rights reserved.\n
                             
                            8.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#421-clone-the-ofs-git-repo","title":"4.2.1 Clone the OFS Git Repo","text":"
                          Retrieve the OFS FIM source code from the OFS f2000x FIM Github Branch repository. Create a new directory to use as a clean starting point to store the retrieved files.  The following is a short description of each repository, followed by the git commands for cloning.  The instructions section uses the HTTPS git method for cloning repositories.
                           
                             
                          1.  
                            Navigate to the location you want to clone the OFS source files, and create the top-level source directory.
                             
                            mkdir IOFS_BUILD_ROOT\n
                             
                            2.  
                          2.  
                            Clone OFS repositories.
                             
                            cd IOFS_BUILD_ROOT\ngit clone --recurse-submodules https://github.com/OFS/ofs-f2000x-pl\n
                             
                            3.  
                          3.  
                            Checkout the proper tag
                             
                            cd ofs-f2000x-pl\ngit checkout --recurse-submodules tags/ofs-2023.3-1\n
                             
                            4.  
                          4.  
                            Ensure that ofs-common has been cloned as well
                             
                            git submodule status\n
                             
                            Example output:
                             
                            ea585a4f48d50faf3ae7ecfbec82525a8d22c730 ofs-common (ofs-2023.3-1)\n
                             
                            5.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#43-directory-structure-of-ofs","title":"4.3 Directory Structure of OFS","text":"
                          List the directory contents of the cloned repo to verify that the following directories and files are present in $IOFS_BUILD_ROOT/ofs-f2000x directory.  
                           
                          ls -1\n
                           Expected output: 
                          ipss\nLICENSE.txt\nofs-common\nREADME.md  \nSECURITY.md\nsim\nsrc\nsyn\ntools\nverification\n
                           
                          Use the following command to show how the directories are arranged:
                           
                          find . -mindepth 1 -maxdepth 2 -type d -not -path '*/\\.*' -print | sed -e 's/[^-][^\\/]*\\//--/g' -e 's/--/|  /g' -e 's/|-/|   /g'\n
                           Expected output:
                           
                          |  ipss\n|  |  hssi\n|  |  mem\n|  |  pcie\n|  |  pmci\n|  |  qsfp\n|  ofs-common\n|  |  scripts\n|  |  src\n|  |  tools\n|  |  verification\n|  sim\n|  |  bfm\n|  |  common\n|  |  scripts\n|  |  unit_test\n|  src\n|  |  afu_top\n|  |  includes\n|  |  pd_qsys\n|  |  top\n|  syn\n|  |  scripts\n|  |  setup\n|  |  syn_top\n|  tools\n|  |  pfvf_config_tool\n|  verification\n|  |  scripts\n|  |  testbench\n|  |  tests\n|  |  unit_tb\n|  |  verifplan\n
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#44-compiling-the-ofs-fim","title":"4.4 Compiling the OFS FIM","text":"
                          OFS provides a build script with the following FPGA image creation options:
                           
                             
                          • Flat compile which combines the FIM and AFU into one FPGA image that is loaded into the entire FPGA device
                            •  
                          • A PR compile which creates a FPGA image consisting of the FIM that is loaded into the static region of the FPGA and a default AFU that is loaded into dynamic region. The AFU image may be loaded into the dynamic region using partial reconfiguration.  
                            •  
                           
                          The build scripts included with OFS are verified to run in a bash shell. Other shells have not been tested. The full build script typically takes around 3 hours to complete.
                           
                          The build script flow is the primary flow described in this user guide. For instructions on compiling using the Intel Quartus Prime Pro GUI, refer to the Compiling the OFS FIM Using Quartus GUI section.
                           
                          The following sections describe how to set up the environment and build the provided FIM and AFU. Follow these steps as a tutorial to learn the build flow.  You will use this environment and build scripts for the creation of your specialized FIM.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#441-setting-up-required-environment-variables","title":"4.4.1. Setting Up Required Environment Variables","text":"
                          Set required environment variables as shown below.  These environment variables must be set prior to simulation or compilation tasks so creating a simple script to set these variables saves time. 
                           
                          Set the following environment variables based on your environment: 
                          export QUARTUS_MAINPATH=/<YOUR_QUARTUS_DIRECTORY>/23.3 \nexport TOOLS_LOCATION=<YOUR_TOOLS_DIRECTORY> \nexport LM_LICENSE_FILE=<YOUR_LM_LICENSE_PATH>\nexport DW_LICENSE_FILE=<YOUR_DW_LICENSE_PATH>\nexport SNPSLMD_LICENSE_FILE=<YOUR_SNPSLMD_LICENSE_PATH>\n
                           
                          Note: The TOOLS_LOCATION directory is where the Synopsys tools reside. Refer to the UVM_HOME variable below for an example.
                           
                          Then set the remaining environment variables: 
                          export QUARTUS_ROOTDIR=$QUARTUS_MAINPATH/quartus \nexport QUARTUS_HOME=$QUARTUS_ROOTDIR \nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR \nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR \nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip \nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip \nexport INTELFPGAOCLSDKROOT=$QUARTUS_MAINPATH/hld \nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin \nexport IOFS_BUILD_ROOT=$PWD\nexport OFS_ROOTDIR=$IOFS_BUILD_ROOT/ofs-f2000x-pl\nexport WORKDIR=$OFS_ROOTDIR \nexport VERDIR=$OFS_ROOTDIR/verification/ofs-f2000x/common:$OFS_ROOTDIR/verification \nexport OFS_PLATFORM_AFU_BBB=$IOFS_BUILD_ROOT/ofs-platform-afu-bbb \nexport OPAE_SDK_REPO_BRANCH=release/$OPAE_SDK_VERSION\nexport OPAE_PLATFORM_ROOT=$OFS_ROOTDIR/work_dir/build_tree    \nexport LIBRARY_PATH=$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/lib \nexport LD_LIBRARY_PATH=$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/lib64 \nexport OPAE_LOC=/install-opae-sdk \nexport QUARTUS_NUM_PARALLEL_PROCESSORS=8 \nexport DESIGNWARE_HOME=$TOOLS_LOCATION/synopsys/vip_common/vip_Q-2020.03A\nexport UVM_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm \nexport VCS_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel \nexport MTI_HOME=$TOOLS_LOCATION/intelFPGA_pro/questa_fse \nexport PATH=$PATH:$QUARTUS_HOME/bin:$QUARTUS_HOME/qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/bin:$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$DESIGNWARE_HOME/bin:$VCS_HOME/bin\n
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#442-compiling-the-fim","title":"4.4.2 Compiling the FIM","text":"
                          The f2000x  FIM build flow uses the bash script $OFS_ROOTDIR/ofs-common/scripts/common/syn/build_top.sh.  There are several setup files that must be put in place before compilation, which is handled by the build script. If you wish to compile the f2000x FIM using the Intel Quartus Prime Pro GUI, you must at least run the setup portion of the build_top.sh script before compiling with the GUI. For instructions on compiling the FIM using the Quartus GUI, refer to the Compiling the OFS FIM Using Quartus GUI section.
                           
                          The usage of the compile build script is shown below: 
                          ofs-common/scripts/common/syn/build_top.sh/build_top.sh [-p] f2000x[:OPTIONS]  work_dir \n
                           
                          Note: Refer to the $OFS_ROOTDIR/ofs-common/scripts/common/syn/README for detailed information on using this script
                           
                             
                          • To build the provided design using a flat, non-PR build flow use the following commands:
                            cd $OFS_ROOTDIR\n./ofs-common/scripts/common/syn/build_top.sh f2000x:flat  work_dir\n
                             
                            •  
                           
                             
                          • To build the provided design with in-tree PR enabled use the following commands:
                            cd $OFS_ROOTDIR\n./ofs-common/scripts/common/syn/build_top.sh f2000x work_dir\n
                             
                            •  
                           
                             
                          • To build the provided design with a relocatable (out-of-tree) PR directory use the following commands:
                            cd $OFS_ROOTDIR\n./ofs-common/scripts/common/syn/build_top.sh -p f2000x work_dir\n
                             
                            Refer to the Create a Relocatable PR Directory Tree section for more information on out-of-tree PR builds.
                             
                            •  
                           
                          The build takes ~3 hours to complete. A successful build will report the following:
                           
                          Compile work directory:     <$IOFS_BUILD_ROOT>/ofs-f2000x/work_f2000x /syn/syn_top\nCompile artifact directory: <$IOFS_BUILD_ROOT>/ofs-f2000x/work_f2000x /syn/syn_top/output_files\n\n***********************************\n***\n***        OFS_PROJECT: f2000x \n***        OFS_FIM: base\n***        OFS_BOARD: adp\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 0\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                           
                          The build script copies the ipss, ofs-common, sim, src,syn and tools directories to the specified work directory and then these copied files are used in the Intel Quartus Prime Pro compilation process.
                           
                          Some key output directories are described in the following table:
                           Directory Description $OFS_ROOTDIR/<WORK_DIR>/syn/syn_top Intel Quartus Prime Pro project (ofs_top.qpf) and other Intel Quartus Prime Pro specific files $OFS_ROOTDIR/<WORK_DIR>/syn/syn_top/output_files Directory with build reports and FPGA programming files 
                          The build script will run PACSign (if installed) and create an unsigned FPGA programming files for both user1 and user2 locations of the f2000x  FPGA flash.  Please note, if the f2000x  has the root entry hash key loaded, then PACsign must be run to add the proper key to the FPGA binary file.
                           
                          The following table provides a detailed description of the generated *.bin files.
                           File Description ofs_top.bin This is an intermediate, raw binary file. This intermediate raw binary file is produced by taking the Intel Quartus Prime Pro generated *.sof file, and converting it to *.pof using quartus_pfg, then converting the *.pof to *.hexout using quartus_cpf, and finally converting the *.hexout to *.bin using objcopy. ofs_top.bin - Raw binary image of the FPGA. ofs_top_page0_unsigned_factory.bin This is the unsigned PACSign output generated for the Factory Image. Unsigned means that the image has been signed with an empty header. ofs_top_page1_user1.bin This is an input file to PACSign to generate ofs_top_page1_unsigned_user1.bin. This file is created by taking the ofs_top.bin file and assigning the User1 or appending factory block information. Unsigned means that the image has been signed with an empty header. ofs_top_page1_unsigned_user1.bin This is the unsigned FPGA binary image generated by the PACSign utility for the User1 Image. This file is used to load the FPGA flash User1 Image using the fpgasupdate tool. Unsigned means that the image has been signed with an empty header. ofs_top_page2_user2.bin This is an input file to PACSign to generate ofs_top_page2_unsigned_user2.bin. This file is created by taking the ofs_top.bin file and assigning the User2 or appending factory block information. ofs_top_page2_unsigned_user2.bin This is the unsigned FPGA binary image generated by the PACSign utility for the User2 Image. This file is used to load the FPGA flash User2 Image using the fpgasupdate tool. Unsigned means that the image has been signed with an empty header. ofs_top.sof This image is used to generate ofs_top.bin, and can also be used to program the Intel Agilex 7 device directly through JTAG 
                          Note: The build/output_files/timing_report directory contains clocks report, failing paths and passing margin reports. 
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#443-create-a-relocatable-pr-directory-tree-from-the-base_x16-fim","title":"4.4.3 Create a Relocatable PR Directory Tree from the base_x16 FIM","text":"
                          If you are developing a FIM to be used by another team developing AFU workload(s), scripts are provided that create a relocatable PR directory tree. ODM and board developers will make use of this capability to enable a broad set of AFUs to be loaded on a board using PR.
                           
                          You can create this relocatable PR directory tree by either:
                           
                             
                          • Build FIM and AFU using ofs-common/scripts/common/syn/build_top.sh followed by running /ofs-common/scripts/common/syn/generate_pr_release.sh
                            •  
                          • Build FIM and AFU using ofs-common/scripts/common/syn/build_top.sh with optional -p switch included
                            •  
                           
                          The generate_pr_release.sh has the following command structure:
                           
                          ofs-common/scripts/common/syn/generate_pr_release.sh -t <work_directory>/build_tree <target_board> <work_directory>\n
                           Where:
                           
                             
                          • <work_directory>/build_tree is the location for your relocatable PR directory tree in the work directory
                            •  
                          • <target_board> is the name of the board target/FIM e.g. f2000x 
                            •  
                          • <work_directory> is the work directory
                            •  
                           
                          Here is an example of running the generate_pr_release.sh script:
                           
                          cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/generate_pr_release.sh -t work_f2000x/build_tree f2000x   work_f2000x \n
                           
                          The resulting relocatable build tree has the following structure:
                           
                          .\n\u251c\u2500\u2500 bin\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 afu_synth\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 build_env_config\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 run.sh -> afu_synth\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 update_pim\n\u251c\u2500\u2500 hw\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 blue_bits\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u251c\u2500\u2500 ofs_top_page0_unsigned_factory.bin\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u251c\u2500\u2500 ofs_top_page1_unsigned_user1.bin\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u251c\u2500\u2500 ofs_top_page2_unsigned_user2.bin\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u2514\u2500\u2500 ofs_top.sof -> ../lib/build/syn/syn_top/output_files/ofs_top.sof\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 lib\n\u2502\u00a0\u00a0     \u251c\u2500\u2500 build\n\u2502\u00a0\u00a0     \u251c\u2500\u2500 fme-ifc-id.txt\n\u2502\u00a0\u00a0     \u251c\u2500\u2500 fme-platform-class.txt\n\u2502\u00a0\u00a0     \u2514\u2500\u2500 platform\n\u2514\u2500\u2500 README\n
                           This build tree can be moved to a different location and used for AFU development of PR-able AFU to be used with this board.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#444-compiling-the-ofs-fim-using-quartus-gui","title":"4.4.4 Compiling the OFS FIM Using Quartus GUI","text":"
                          Perform the following steps to compile the OFS FIM using the Quartus GUI:
                           
                             
                          1.  
                            Set the environment variables as described in the Setting Up Required Environment Variables section.
                             
                            2.  
                          2.  
                            Run the setup portion of the build script. This takes a few seconds to complete.
                             
                            ./ofs-common/scripts/common/syn/build_top.sh --stage setup f2000x work_dir\n
                             
                            3.  
                          3.  
                            Open the OFS FIM project using the Intel Quartus Prime Pro GUI. The project is located at $OFS_ROOTDIR/work_dir/syn/syn_top/ofs_top.qpf.
                             
                            4.  
                          4.  
                            Ensure the checkbox next to Assembler (Generate programming files) is marked.
                             
                            5.  
                          5.  
                            Click the arrow next to Compile Design in the Compilation Flow window to start full compilation.
                             
                            6.  
                          6.  
                            After compilation is complete, check the $OFS_ROOTDIR/work_dir/syn/syn_top/output_files directory. This directory will contain the output SOF image generated by Quartus, named ofs_top.sof.
                             
                            7.  
                          7.  
                            Run the finish portion of the build script. This will generate the images that can be programmed to f2000x FPGA flash. Use the optional -p argument to create an out-of-tree PR build.
                             
                            ./ofs-common/scripts/common/syn/build_top.sh --stage finish f2000x work_dir\n
                             
                            8.  
                          8.  
                            Check the $OFS_ROOTDIR/work_dir/syn/syn_top/output_files directory again to see that all output files have been generated.
                             
                            9.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#45-unit-level-simulation","title":"4.5 Unit Level Simulation","text":"
                          Unit level simulation of key components in the FIM is provided. These simulations provide verification of the following areas:
                           
                             
                          • Ethernet
                            •  
                          • PCIe
                            •  
                          • External Memory
                            •  
                          • Core FIM
                            •  
                           
                          The Unit Level simulations work with Synopsys VCS-MX or Mentor Graphics Questasim simulators. Readme files are provided explaining how to run the simulation of each component. 
                           
                          The scripts to run the unit level simulations are located in $OFS_ROOTDIR/sim/unit_test, which contains subdirectories soc_tests and host_tests. The table below lists the tests that are provided.
                           Test Type Test Name SoC Tests csr_test dfh_walker flr he_lb_test he_mem_test hssi_kpi_test hssi_test mem_ss_csr_test mem_ss_rst_test mem_tg_test pf_vf_access_test qsfp_test regress_run.py Host Tests csr_test he_lb_test pcie_csr_test pf_vf_access_test pmci_csr_test pmci_mailbox_test pmci_rd_default_value_test pmci_ro_mailbox_test pmci_vdm_b2b_drop_err_scenario_test pmci_vdm_len_err_scenario_test pmci_vdm_mctp_mmio_b2b_test pmci_vdm_multipkt_error_scenario_test pmci_vdm_multipkt_tlp_err_test pmci_vdm_tlp_error_scenario_test pmci_vdm_tx_rx_all_random_lpbk_test pmci_vdm_tx_rx_all_toggle_test pmci_vdm_tx_rx_lpbk_test regress_run.py"},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#451-run-comprehensive-unit-tests","title":"4.5.1 Run Comprehensive Unit Tests","text":"
                          The regress_run.py script is provided to automatically run all unit tests for either the SoC or the Host. Note that running all tests will take around three hours for the SoC tests and around 2.5 hours for the Host tests to complete.
                           
                          Perform the following steps to run comprehensive tests:
                           
                             
                          1. Set the environment variables as described in the Setting Up Required Environment Variables section.
                            2.  
                          2.  
                            Navigate to the test directory you wish to run from.
                             
                               
                            • SoC Tests
                              cd $OFS_ROOTDIR/sim/unit_test/soc_tests\n
                               
                              •  
                             
                               
                            • Host Tests
                              cd $OFS_ROOTDIR/sim/unit_test/host_tests\n
                               
                              •  
                             
                            3.  
                          3.  
                            Run the tests with the regress_run.py. Use the -h argument to display the help menu.
                             
                            For example, to run all tests locally, using VCS, with 8 processors:
                             
                            python regress_run.py -l -n 8 -k all -s vcs\n
                             
                            4.  
                          4.  
                            Once all tests have completed, the comprehensive test summary will be shown. The following is an example test summary after running the SoC tests:
                             
                            2023-05-14 19:10:55,404: Passing Unit Tests:12/12 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-05-14 19:10:55,404:    csr_test:......... PASS -- Time Elapsed:0:03:57.713154\n2023-05-14 19:10:55,404:    dfh_walker:....... PASS -- Time Elapsed:0:02:46.025067\n2023-05-14 19:10:55,404:    flr:.............. PASS -- Time Elapsed:0:03:26.289900\n2023-05-14 19:10:55,404:    he_lb_test:....... PASS -- Time Elapsed:0:06:41.142643\n2023-05-14 19:10:55,404:    he_mem_test:...... PASS -- Time Elapsed:1:39:01.824096\n2023-05-14 19:10:55,404:    hssi_kpi_test:.... PASS -- Time Elapsed:2:21:33.007328\n2023-05-14 19:10:55,404:    hssi_test:........ PASS -- Time Elapsed:2:16:36.821034\n2023-05-14 19:10:55,404:    mem_ss_csr_test:.. PASS -- Time Elapsed:0:38:45.836540\n2023-05-14 19:10:55,404:    mem_ss_rst_test:.. PASS -- Time Elapsed:0:40:51.065354\n2023-05-14 19:10:55,404:    mem_tg_test:...... PASS -- Time Elapsed:0:54:00.210146\n2023-05-14 19:10:55,404:    pf_vf_access_test: PASS -- Time Elapsed:0:03:25.561919\n2023-05-14 19:10:55,404:    qsfp_test:........ PASS -- Time Elapsed:0:39:29.192304\n2023-05-14 19:10:55,404: Failing Unit Tests: 0/12 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-05-14 19:10:55,404: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-05-14 19:10:55,404:       Number of Unit test results captured: 12\n2023-05-14 19:10:55,404:       Number of Unit test results passing.: 12\n2023-05-14 19:10:55,404:       Number of Unit test results failing.:  0\n2023-05-14 19:10:55,404:     End Unit regression running at date/time................: 2023-05-14 19:10:55.404641\n2023-05-14 19:10:55,404:     Elapsed time for Unit regression run....................: 2:22:39.310455\n
                             
                            5.  
                          5.  
                            Output logs for each individual test are saved in the respective test's directory
                             
                            $OFS_ROOTDIR/sim/unit_test/<TEST_TYPE>/<TEST_NAME>/<SIMULATOR>/transcript\n
                             
                            For example, the log for the SoC DFH Walker test using VCS can be found in:
                             
                            $OFS_ROOTDIR/sim/unit_test/soc_tests/dfh_walker/sim_vcs/transcript\n
                             
                            6.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#452-run-individual-unit-tests","title":"4.5.2 Run Individual Unit Tests","text":"
                          The run_sim.sh scripts are provided to run individual unit tests for either the SoC or the Host. Before you can run any unit tests, you must generate the IP simulation files. Note that the regress_run.py script used for comprehensive testing does this step automatically. Perform the following steps to generate the IP simulation files:
                           
                             
                          1. Set the environment variables as described in the Setting Up Required Environment Variables section.
                            2.  
                          2. Generate the IP simulation files for all unit tests:
                            cd `$OFS_ROOTDIR/ofs-common/scripts/common/sim`\nsh gen_sim_files.sh f2000x \n
                             
                            3.  
                           
                          Next, perform the following steps to run individual tests:
                           
                             
                          1.  
                            Navigate to the directory of the test you wish to run
                             
                            cd $OFS_ROOTDIR/sim/unit_test/<TEST_TYPE>/<TEST_NAME>\n
                             
                            For example, to run the DFH walker test for the SoC:
                             
                            cd $OFS_ROOTDIR/sim/unit_test/soc_tests/dfh_walker\n
                             
                            2.  
                          2.  
                            Run the test with your desired simulator:
                             
                               
                            •  
                              Using VCS
                               
                              sh run_sim.sh\n
                               
                              •  
                             
                               
                            •  
                              Using Questasim
                               
                              sh run_sim.sh MSIM=1\n
                               
                              •  
                             
                               
                            •  
                              Using VCS-MX
                               
                              sh run_sim.sh VCSMX=1\n
                               
                              •  
                             
                            3.  
                          3.  
                            Once the test has completed, the test summary will be shown. The following is an example test summary after running the SoC DFH Walker Test:
                             
                            Test status: OK\n\n********************\nTest summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/applications.fpga.ofs.fim-f2000x-pl/sim/unit\n_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time            355393750\n        V C S   S i m u l a t i o n   R e p o r t\nTime: 355393750 ps\nCPU Time:     59.870 seconds;       Data structure size:  91.2Mb\nSun May 14 16:54:20 2023\n
                             
                            4.  
                          4.  
                            The log for the test is stored in a transcript file in the simulation directory of the test that was run.
                             
                            $OFS_ROOTDIR/sim/unit_test/<TEST_TYPE>/<TEST_NAME>/<SIMULATOR>/transcript\n
                             
                            For example, the log for the SoC DFH Walker test using VCS can be found in:
                             
                            $OFS_ROOTDIR/sim/unit_test/soc_tests/dfh_walker/sim_vcs/transcript\n
                             
                            The simulation waveform database is saved as vcdplus.vpd for post simulation review. You are encouraged to run the additional simulation examples to learn about each key area of the OFS shell.
                             
                            5.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5-custom-fim-development-flow","title":"5 Custom FIM Development Flow","text":"
                          FIM development for a new acceleration card consists of the following steps:
                           
                             
                          1. Installation of OFS and familiarization with scripts and source code
                            2.  
                          2. Development of high level block diagram with your specific functionality
                               
                            1. Determination of requirements and key performance metrics
                              2.  
                            2. Selection of IP cores
                              3.  
                            3. Selection of FPGA device
                              4.  
                            4. Software memory map
                              5.  
                             
                            3.  
                          3. Selection and implementation of FIM Physical interfaces including:
                               
                            1. External clock sources and creation of internal PLL clocks
                              2.  
                            2. General I/O
                              3.  
                            3. Ethernet modules
                              4.  
                            4. External memories
                              5.  
                            5. FPGA programming methodology
                              6.  
                             
                            4.  
                          4. Device physical implementation
                               
                            1. FPGA device pin assignment
                              2.  
                            2. Inclusion of logic lock regions
                              3.  
                            3. Creation of timing constraints
                              4.  
                            4. Create Intel Quartus Prime Pro FIM test project and validate:
                                 
                              1. Placement
                                2.  
                              2. Timing constraints
                                3.  
                              3. Build script process
                                4.  
                              4. Review test FIM FPGA resource usage
                                5.  
                               
                              5.  
                             
                            5.  
                          5. Select FIM to AFU interfaces and development of PIM
                            6.  
                          6. FIM design implementation
                               
                            1. RTL coding
                              2.  
                            2. IP instantiation
                              3.  
                            3. Development of test AFU to validate FIM
                              4.  
                            4. Unit and device level simulation
                              5.  
                            5. Timing constraints and build scripts
                              6.  
                            6. Timing closure and build validation
                              7.  
                             
                            7.  
                          7. Creation of FIM documentation to support AFU development and synthesis
                            8.  
                          8. Software Device Feature discovery
                            9.  
                          9. Hardware/software integration, validation and debugging
                            10.  
                          10. High volume production preparation
                            11.  
                           
                          The FIM developer works closely with the hardware design of the target board, software development and system validation.
                           
                          This section describes how to perform specific customizations of areas of the FIM. Each section can be done independently. The following walkthroughs are provided:
                           Section Walkthrough 5.1 How to add a new module to the FIM 5.2 How to debug the FIM with Signal Tap 5.3 How to compile the FIM in preparation for designing your AFU 5.4 How to resize the Partial Reconfiguration Region 5.5 How to modify the Memory Subsystem 5.6 How to compile the FIM with no HSSI 5.7 How to change the PCIe Device ID and Vendor ID 5.8 How to migrate to a different Intel Agilex 7 device number 5.9 How to change the Ethernet interface from 8x25 GbE to 8x10 GbE 5.10 How to change the Ethernet interface from 8x25 GbE to 2x100 GbE 5.11 How to add more transceiver channels to the FIM 5.12 How to modify the PF/VF MUX configuration 5.13 How to create a minimal FIM 
                          In each section, it is assumed that:
                           
                             
                          1. You have a clean, unmodified clone of the OFS repo.  See the Clone the OFS Git Repo section.
                            2.  
                          2. After cloning, you must set various environment variables. See the Setting Up Required Environment Variables section.
                            3.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#51-how-to-add-a-new-module-to-the-fim","title":"5.1 How to add a new module to the FIM","text":"
                          This section provides a walkthrough for adding a custom module to the FIM, simulating the new design, compiling the new design, implementing and testing the new design on hardware, and debugging the new design on hardware.
                           
                          If you intend to add a new module to the FIM area, then you will need to inform the host software of the new module. The FIM exposes its functionalities to host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). This set of CSR registers and their operation is described in FIM MMIO Regions.
                           
                          See FPGA Device Feature List (DFL) Framework Overview for a description of the software process to read and process the linked list of Device Feature Header (DFH) CSRs within a FPGA.
                           
                          The Hello FIM example adds a simple DFH register and 64bit scratchpad register connected to the Board Peripheral Fabric (BPF) that can be accessed by the SoC. You can use this example as the basis for adding a new feature to your FIM. 
                           
                          The Hello FIM design can be verified by Unit Level simulation, Universal Verification Methodology (UVM) simulation, and running in hardware on the f2000x  card. The process for these are described in this section. 
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#511-board-peripheral-fabric-bpf","title":"5.1.1 Board Peripheral Fabric (BPF)","text":"
                          The Hello FIM module will be connected to the Board Peripheral Fabric (BPF), and will be connected such that it can only be mastered by the SoC. The BPF is an interconnect generated by Platform Designer. The figure below shows the APF/BPF Master/Slave interactions, as well as the added Hello FIM module.
                           
                           
                          You can create, modify, and/or generate the BPF manually in Platform Designer or more conveniently by executing a provided script.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#512-soc-mmio-region","title":"5.1.2 SoC MMIO Region","text":"
                          We will add the Hello FIM module to an un-used address space in the SoC MMIO region. The table below shows the MMIO region for the SoC with the Hello FIM module added at base address 0x16000.
                           Offset Feature CSR set 0x00000 FME AFU 0x01000 Thermal Management 0x03000 Global Performance 0x04000 Global Error 0x10000 PCIe Interface 0x12000 QSFP Controller 0 0x13000 QSFP Controller 1 0x14000 HSSI Interface 0x15000 EMIF Interface 0x16000 Hello FIM 0x80000 PMCI Controller 0x100000 ST2MM (Streaming to Memory-Mapped) 0x130000 PR Control & Status (Port Gasket) 0x131000 Port CSRs (Port Gasket) 0x132000 User Clock (Port Gasket) 0x133000 Remote SignalTap (Port Gasket) 0x140000 AFU Errors (AFU Interface Handler) 
                          Refer to the FIM Technical Reference Manual: Interconnect Fabric for more information on the default MMIO region.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#513-hello-fim-csr","title":"5.1.3 Hello FIM CSR","text":"
                          The Hello FIM CSR will consist of the three registers shown in the table below. The DFH and Hello FIM ID registers are read-only. The Scratchpad register supports read and write accesses.
                           Offset Attribute Description Default Value 0x016000 RO DFH(Device Feature Headers) register 0x30000006a0000100 0x016030 RW Scrachpad register 0x0 0x016038 RO Hello FIM ID register 0x6626070150000034"},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#514-files-to-edit-to-support-hello-fim","title":"5.1.4 Files to Edit to Support Hello FIM","text":"
                          The table below shows all files in $OFS_ROOTDIR that will be modified or created in order to implement the Hello FIM module. The build_top.sh script copies files from $OFS_ROOTDIR into the target work directory and then the copied files are used in the Quartus build process. Details on editing these files is given in subsequent sections.
                           Category Status Path File Description Source Modify src/top top.sv Top RTL Modify src/pd_qsys bpf_top.sv BPF top RTL New src/hello_fim hello_fim_top.sv Hello FIM top RTL New src/hello_fim hello_fim_com.sv Hello FIM common RTL Design Files Modify syn/syn_top ofs_top.qsf Quartus setting file Modify syn/syn_top ofs_top_sources.tcl Define sources for top level design New syn/setup hello_fim_design_files.tcl Define RTLs of Hello FIM Modify syn/setup fabric_design_files.tcl Define IPs for fabric Platform Designer Modify src/pd_qsys/fabric bpf.txt Text definition of BPF interconnect Modify src/pd_qsys/fabric bpf.qsys BPF Qsys file Simulation Modify sim/unit_test/soc_tests/dfh_walker test_csr_defs.sv Define CSRs for simulation Verification Modify /ofs-common/verification/fpga_family/agilex/tests/sequences mmio_seq.svh MMIO testbench Modify /ofs-common/verification/fpga_family/agilex/tests/sequences dfh_walking_seq.svh DFH Walking testbench Modify ofs-common/verification/fpga_family/agilex/scripts Makefile_VCS.mk Makefile for VCS"},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#515-pre-requisites-for-adding-hello-fim","title":"5.1.5 Pre-Requisites for Adding Hello FIM","text":"
                          The following pre-requisites must be satisfied before adding the Hello FIM module.
                           
                             
                          1. Clone the design repositories. See the Clone the OFS Git Repo section.
                            2.  
                          2. Set the environment variables. See the Setting Up Required Environment Variables section.
                            3.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#516-file-modification","title":"5.1.6 File Modification","text":"
                          This section describes the steps to add the Hello FIM module to the FIM.  The steps in this simple example are the basis for modifying the FIM for more complex functions.
                           
                             
                          1.  
                            Modify syn/syn_top/ofs_top.qsf
                             
                               
                            1.  
                              Define the INCLUDE_HELLO_FIM Verilog macro to the Verilog Macros section. This will enable instantiation of the Hello FIM module. If this is not set, a dummy register will be instantiated instead.
                               
                              ######################################################\n# Verilog Macros\n######################################################\n.....\nset_global_assignment -name VERILOG_MACRO \"INCLUDE_HELLO_FIM\"     # Includes Hello FIM\n
                               
                              2.  
                             
                            2.  
                          2.  
                            Modify syn/syn_top/ofs_top_sources.tcl
                             
                               
                            1.  
                              Add hello_fim_design_files.tcl to the list of subsystems in the Design Files section.
                               
                              ############################################\n# Design Files\n############################################\n...\n# Subsystems\n...\nset_global_assignment -name SOURCE_TCL_SCRIPT_FILE ../setup/hello_fim_design_files.tcl\n
                               
                              2.  
                             
                            3.  
                          3.  
                            Create syn/setup/hello_fim_design_files.tcl
                             
                               
                            1.  
                              Create hello_fim_design_files.tcl with the following contents:
                               
                              # Copyright 2023 Intel Corporation.\n#\n# THIS SOFTWARE MAY CONTAIN PREPRODUCTION CODE AND IS PROVIDED BY THE\n# COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED\n# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\n# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR\n# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,\n# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE\n# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,\n# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# Hello FIM Files\n#--------------------\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_com.sv\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_top.sv\n
                               
                              2.  
                             
                            4.  
                          4.  
                            Modify /src/pd_qsys/fabric/fabric_design_files.tcl
                             
                               
                            1.  
                              Add bpf_hello_fim_slv.ip to the list of files in the BPF section.
                               
                              #--------------------\n# BPF\n#--------------------\n...\nset_global_assignment -name IP_FILE ../ip_lib/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip\n
                               
                              2.  
                             
                            5.  
                          5.  
                            Modify src/top/top.sv
                             
                               
                            1.  
                              Add bpf_hello_fim_slv_if to AXI4-Lite Interfaces:
                               
                              // AXI4-lite interfaces\nofs_fim_axi_lite_if #(.AWADDR_WIDTH(12), .ARADDR_WIDTH(12)) bpf_hello_fim_slv_if();\n
                               
                              2.  
                            2.  
                              Modify the value of NEXT_DFH_OFFSET of mem_ss_top from 24'h6B000 to 24'h1000
                               
                              //*******************************\n// Memory Subsystem\n//*******************************\n`ifdef INCLUDE_DDR4\n   mem_ss_top #(\n      .FEAT_ID          (12'h009),\n      .FEAT_VER         (4'h1),\n      .NEXT_DFH_OFFSET  (24'h1000),\n      .END_OF_LIST      (1'b0)\n   ) mem_ss_top (\n
                               
                              3.  
                            3.  
                              Modify the value of NEXT_DFH_OFFSET of the Memory Subsystem dummy_csr from 24'h6B000 to 24'h1000
                               
                              // Placeholder logic if no mem_ss\ndummy_csr #(\n   .FEAT_ID          (12'h009),\n   .FEAT_VER         (4'h1),\n   .NEXT_DFH_OFFSET  (24'h1000),\n   .END_OF_LIST      (1'b0)\n) emif_dummy_csr (\n
                               
                              4.  
                            4.  
                              Add Hello FIM instance and dummy CSR after the Memory Subsystem. Set the NEXT_DFH_OFFSET to 24'h6A000 for both
                               
                              //*******************************\n// Hello FIM Subsystem\n//*******************************\n\n`ifdef INCLUDE_HELLO_FIM\nhello_fim_top #(\n   .ADDR_WIDTH       (12),\n   .DATA_WIDTH       (64),\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (24'h6A000),\n   .END_OF_LIST      (1'b0)\n) hello_fim_top_inst (\n    .clk (clk_csr),\n    .reset(~rst_n_csr),\n    .csr_lite_if    (bpf_hello_fim_slv_if)         \n);\n`else\ndummy_csr #(   \n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (24'h6A000),\n   .END_OF_LIST      (1'b0)\n) hello_fim_dummy (\n   .clk         (clk_csr),\n   .rst_n       (rst_n_csr),\n   .csr_lite_if (bpf_hello_fim_slv_if)\n);\n\n`endif \n
                               
                              5.  
                             
                            6.  
                          6.  
                            Modify /src/pd_qsys/bpf_top.sv
                             
                               
                            1.  
                              Add bpf_hello_fim_slv_if to the interface descriptions
                               
                              ...\nmodule bpf_top (\n...\n//BPF funtions\n...\nofs_fim_axi_lite_if.master bpf_hello_fim_slv_if\n
                               
                              2.  
                            2.  
                              Add bpf_hello_fim_slv_if to the module
                               
                              module bpf_top (\n...\n);\n...\n.bpf_hello_fim_slv_awaddr    (bpf_hello_fim_slv_if.awaddr     ),\n.bpf_hello_fim_slv_awprot    (bpf_hello_fim_slv_if.awprot     ),\n.bpf_hello_fim_slv_awvalid   (bpf_hello_fim_slv_if.awvalid    ),\n.bpf_hello_fim_slv_awready   (bpf_hello_fim_slv_if.awready    ),\n.bpf_hello_fim_slv_wdata     (bpf_hello_fim_slv_if.wdata      ),\n.bpf_hello_fim_slv_wstrb     (bpf_hello_fim_slv_if.wstrb      ),\n.bpf_hello_fim_slv_wvalid    (bpf_hello_fim_slv_if.wvalid     ),\n.bpf_hello_fim_slv_wready    (bpf_hello_fim_slv_if.wready     ),\n.bpf_hello_fim_slv_bresp     (bpf_hello_fim_slv_if.bresp      ),\n.bpf_hello_fim_slv_bvalid    (bpf_hello_fim_slv_if.bvalid     ),\n.bpf_hello_fim_slv_bready    (bpf_hello_fim_slv_if.bready     ),\n.bpf_hello_fim_slv_araddr    (bpf_hello_fim_slv_if.araddr     ),\n.bpf_hello_fim_slv_arprot    (bpf_hello_fim_slv_if.arprot     ),\n.bpf_hello_fim_slv_arvalid   (bpf_hello_fim_slv_if.arvalid    ),\n.bpf_hello_fim_slv_arready   (bpf_hello_fim_slv_if.arready    ),\n.bpf_hello_fim_slv_rdata     (bpf_hello_fim_slv_if.rdata      ),\n.bpf_hello_fim_slv_rresp     (bpf_hello_fim_slv_if.rresp      ),\n.bpf_hello_fim_slv_rvalid    (bpf_hello_fim_slv_if.rvalid     ),\n.bpf_hello_fim_slv_rready    (bpf_hello_fim_slv_if.rready     ),\n...\nendmodule\n
                               
                              3.  
                             
                            7.  
                          7.  
                            Create src/hello_fim
                             
                               
                            1.  
                              Create src/hello_fim directory
                               
                              mkdir $OFS_ROOTDIR/src/hello_fim\n
                               
                              2.  
                             
                            8.  
                          8.  
                            Create src/hello_fim/hello_fim_top.sv
                             
                               
                            1.  
                              Create hello_fim_top.sv with the following contents:
                               
                              // ***************************************************************************\n//                               INTEL CONFIDENTIAL\n//\n//        Copyright (C) 2023 Intel Corporation All Rights Reserved.\n//\n// The source code contained or described herein and all  documents related to\n// the  source  code  (\"Material\")  are  owned  by  Intel  Corporation  or its\n// suppliers  or  licensors.    Title  to  the  Material  remains  with  Intel\n// Corporation or  its suppliers  and licensors.  The Material  contains trade\n// secrets  and  proprietary  and  confidential  information  of  Intel or its\n// suppliers and licensors.  The Material is protected  by worldwide copyright\n// and trade secret laws and treaty provisions. No part of the Material may be\n// used,   copied,   reproduced,   modified,   published,   uploaded,  posted,\n// transmitted,  distributed,  or  disclosed  in any way without Intel's prior\n// express written permission.\n//\n// No license under any patent,  copyright, trade secret or other intellectual\n// property  right  is  granted  to  or  conferred  upon  you by disclosure or\n// delivery  of  the  Materials, either expressly, by implication, inducement,\n// estoppel or otherwise.  Any license under such intellectual property rights\n// must be express and approved by Intel in writing.\n//\n// You will not, and will not allow any third party to modify, adapt, enhance, \n// disassemble, decompile, reverse engineer, change or create derivative works \n// from the Software except and only to the extent as specifically required by \n// mandatory applicable laws or any applicable third party license terms \n// accompanying the Software.\n//\n// -----------------------------------------------------------------------------\n// Engineer     : \n// Create Date  : Nov 2021\n// Module Name  : hello_fim_top.sv\n// Project      : IOFS\n// -----------------------------------------------------------------------------\n//\n// Description: \n// This is a simple module that implements DFH registers and \n// AVMM address decoding logic.\n\nmodule hello_fim_top  #(\n   parameter ADDR_WIDTH  = 12, \n   parameter DATA_WIDTH = 64, \n   parameter bit [11:0] FEAT_ID = 12'h001,\n   parameter bit [3:0]  FEAT_VER = 4'h1,\n   parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n   parameter bit END_OF_LIST = 1'b0\n)(\n   input  logic    clk,\n   input  logic    reset,\n// -----------------------------------------------------------\n//  AXI4LITE Interface\n// -----------------------------------------------------------\n   ofs_fim_axi_lite_if.slave   csr_lite_if\n);\n\nimport ofs_fim_cfg_pkg::*;\nimport ofs_csr_pkg::*;\n\n//-------------------------------------\n// Signals\n//-------------------------------------\n   logic [ADDR_WIDTH-1:0]              csr_waddr;\n   logic [DATA_WIDTH-1:0]              csr_wdata;\n   logic [DATA_WIDTH/8-1:0]            csr_wstrb;\n   logic                               csr_write;\n   logic                               csr_slv_wready;\n   csr_access_type_t                   csr_write_type;\n\n   logic [ADDR_WIDTH-1:0]              csr_raddr;\n   logic                               csr_read;\n   logic                               csr_read_32b;\n   logic [DATA_WIDTH-1:0]              csr_readdata;\n   logic                               csr_readdata_valid;\n   logic [ADDR_WIDTH-1:0]              csr_addr;\n\n   logic [63:0]                        com_csr_writedata;\n   logic                               com_csr_read;\n   logic                               com_csr_write;\n   logic [63:0]                        com_csr_readdata;\n   logic                               com_csr_readdatavalid;\n   logic [5:0]                         com_csr_address;\n\n// AXI-M CSR interfaces\nofs_fim_axi_mmio_if #(\n   .AWID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .AWADDR_WIDTH (ADDR_WIDTH),\n   .WDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH),\n   .ARID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .ARADDR_WIDTH (ADDR_WIDTH),\n   .RDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH)\n) csr_if();\n\n// AXI4-lite to AXI-M adapter\naxi_lite2mmio axi_lite2mmio (\n   .clk       (clk),\n   .rst_n     (~reset),\n   .lite_if   (csr_lite_if),\n   .mmio_if   (csr_if)\n);\n\n//---------------------------------\n// Map AXI write/read request to CSR write/read,\n// and send the write/read response back\n//---------------------------------\nofs_fim_axi_csr_slave #(\n   .ADDR_WIDTH (ADDR_WIDTH),\n   .USE_SLV_READY (1'b1)\n\n   ) csr_slave (\n   .csr_if             (csr_if),\n\n   .csr_write          (csr_write),\n   .csr_waddr          (csr_waddr),\n   .csr_write_type     (csr_write_type),\n   .csr_wdata          (csr_wdata),\n   .csr_wstrb          (csr_wstrb),\n   .csr_slv_wready     (csr_slv_wready),\n   .csr_read           (csr_read),\n   .csr_raddr          (csr_raddr),\n   .csr_read_32b       (csr_read_32b),\n   .csr_readdata       (csr_readdata),\n   .csr_readdata_valid (csr_readdata_valid)\n);\n\n// Address mapping\nassign csr_addr             = csr_write ? csr_waddr : csr_raddr;\nassign com_csr_address      = csr_addr[5:0];  // byte address\nassign csr_slv_wready       = 1'b1 ;\n// Write data mapping\nassign com_csr_writedata    = csr_wdata;\n\n// Read-Write mapping\nalways_comb\nbegin\n   com_csr_read             = 1'b0;\n   com_csr_write            = 1'b0;\n   casez (csr_addr[11:6])\n      6'h00 : begin // Common CSR\n         com_csr_read       = csr_read;\n         com_csr_write      = csr_write;\n      end   \n      default: begin\n         com_csr_read       = 1'b0;\n         com_csr_write      = 1'b0;\n      end\n   endcase\nend\n\n// Read data mapping\nalways_comb begin\n   if (com_csr_readdatavalid) begin\n      csr_readdata       = com_csr_readdata;\n      csr_readdata_valid = 1'b1;\n   end\n   else begin\n      csr_readdata       = '0;\n      csr_readdata_valid = 1'b0;\n   end\nend\n\nhello_fim_com  #(\n   .FEAT_ID          (FEAT_ID),\n   .FEAT_VER         (FEAT_VER),\n   .NEXT_DFH_OFFSET  (NEXT_DFH_OFFSET),\n   .END_OF_LIST      (END_OF_LIST)\n) hello_fim_com_inst (\n   .clk                   (clk                     ),\n   .reset                 (reset                   ),\n   .writedata             (com_csr_writedata       ),\n   .read                  (com_csr_read            ),\n   .write                 (com_csr_write           ),\n   .byteenable            (4'hF                    ),\n   .readdata              (com_csr_readdata        ),\n   .readdatavalid         (com_csr_readdatavalid   ),\n   .address               (com_csr_address         )\n   );\nendmodule\n
                               
                              2.  
                             
                            9.  
                          9.  
                            Create src/hello_fim/hello_fim_com.sv
                             
                               
                            1.  
                              Create hello_fim_com.sv with the following contents:
                               
                              module hello_fim_com #(\n   parameter bit [11:0] FEAT_ID = 12'h001,\n   parameter bit [3:0]  FEAT_VER = 4'h1,\n   parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n   parameter bit END_OF_LIST = 1'b0\n)(\ninput clk,\ninput reset,\ninput [63:0] writedata,\ninput read,\ninput write,\ninput [3:0] byteenable,\noutput reg [63:0] readdata,\noutput reg readdatavalid,\ninput [5:0] address\n);\n\nwire reset_n = !reset;  \nreg [63:0] rdata_comb;\nreg [63:0] scratch_reg;\n\nalways @(negedge reset_n ,posedge clk)  \n   if (!reset_n) readdata[63:0] <= 64'h0; else readdata[63:0] <= rdata_comb[63:0];\n\nalways @(negedge reset_n , posedge clk)\n   if (!reset_n) readdatavalid <= 1'b0; else readdatavalid <= read;\n\nwire wr = write;\nwire re = read;\nwire [5:0] addr = address[5:0];\nwire [63:0] din  = writedata [63:0];\nwire wr_scratch_reg = wr & (addr[5:0]  == 6'h30)? byteenable[0]:1'b0;\n\n// 64 bit scratch register\nalways @( negedge  reset_n,  posedge clk)\n   if (!reset_n)  begin\n      scratch_reg <= 64'h0;\n   end\n   else begin\n   if (wr_scratch_reg) begin \n      scratch_reg <=  din;  \n   end\nend\n\nalways @ (*)\nbegin\nrdata_comb = 64'h0000000000000000;\n   if(re) begin\n      case (addr)  \n        6'h00 : begin\n                rdata_comb [11:0]   = FEAT_ID ;  // dfh_feature_id  is reserved or a constant value, a read access gives the reset value\n                rdata_comb [15:12]  = FEAT_VER ;  // dfh_feature_rev    is reserved or a constant value, a read access gives the reset value\n                rdata_comb [39:16]  = NEXT_DFH_OFFSET ;  // dfh_dfh_ofst is reserved or a constant value, a read access gives the reset value\n                rdata_comb [40]     = END_OF_LIST ;        //dfh_end_of_list\n                rdata_comb [59:40]  = 20'h00000 ;  // dfh_rsvd1     is reserved or a constant value, a read access gives the reset value\n                rdata_comb [63:60]  = 4'h3 ;  // dfh_feat_type  is reserved or a constant value, a read access gives the reset value\n        end\n        6'h30 : begin\n                rdata_comb [63:0]   = scratch_reg; \n        end\n        6'h38 : begin\n                rdata_comb [63:0]       = 64'h6626_0701_5000_0034;\n        end\n        default : begin\n                rdata_comb = 64'h0000000000000000;\n        end\n      endcase\n   end\nend\n\nendmodule\n
                               
                              2.  
                             
                            10.  
                          10.  
                            Modify src/pd_qsys/fabric/bpf.txt
                             
                               
                            1.  
                              Add hello_fim as a slave in the BPF, and enable the SoC as a master for it.
                               
                              #### - '#' means comment\n# NAME   TYPE      BASEADDRESS    ADDRESS_WIDTH    SLAVES\napf         mst     n/a             21             fme,soc_pcie,hssi,qsfp0,qsfp1,emif,pmci,hello_fim\n...\nhello_fim   slv     0x16000         12             n/a\n
                               
                              2.  
                             
                            11.  
                          11.  
                            Execute the helper script to re-generate the BPF design files
                             
                            cd $OFS_ROOTDIR/src/pd_qsys/fabric/\nsh gen_fabrics.sh\n
                             
                            12.  
                          12.  
                            After the shell script finishes, you can find the generated bpf_hello_fim_slv.ip file in $OFS_ROOTDIR/src/pd_qsys/fabric/ip/bpf/. This is the ip variant of the axi4lite shim that bridges the Hello FIM module with the BPF. The updated bpf.qsys file is located in $OFS_ROOTDIR/src/pd_qsys/fabric. You can view the updated bpf file in Platform designer as follows.
                             
                            cd $OFS_ROOTDIR/src/pd_qsys/fabric\nqsys-edit bpf.qsys --quartus-project=$OFS_ROOTDIR/syn/syn_top/ofs_top.qpf\n
                             
                            The image below shows the BPF that integrates the bpf_hello_fim_slv axi4lite shim at address 0x16000, generated through the helper script gen_fabrics.sh.
                             
                             
                            13.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#517-unit-level-simulation-of-hello-fim-design","title":"5.1.7 Unit Level Simulation of Hello FIM Design","text":"
                          The following section describes the file modifications that need to be made to perform unit level simulations of the Hello FIM design, followed by instructions for running the unit level simulations.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5171-unit-level-simulation-file-modification","title":"5.1.7.1 Unit Level Simulation File Modification","text":"
                          Perform the following steps to modify the Unit Level simulation files to support the Hello FIM design.
                           
                             
                          1.  
                            Modify $OFS_ROOTDIR/sim/unit_test/soc_tests/dfh_walker/test_csr_defs.sv
                             
                               
                            1.  
                              Add a HELLO_FIM_IDX entry to the t_dfh_idx enumeration:
                               
                              typedef enum {\n   FME_DFH_IDX,\n   THERM_MNGM_DFH_IDX,\n   GLBL_PERF_DFH_IDX,\n   GLBL_ERROR_DFH_IDX,\n   QSFP0_DFH_IDX,\n   QSFP1_DFH_IDX,\n   HSSI_DFH_IDX,\n   EMIF_DFH_IDX,\n   HELLO_FIM_DFH_IDX,\n   PMCI_DFH_IDX,\n   ST2MM_DFH_IDX,\n   PG_PR_DFH_IDX,\n   PG_PORT_DFH_IDX,\n   PG_USER_CLK_DFH_IDX,\n   PG_REMOTE_STP_DFH_IDX,\n   AFU_ERR_DFH_IDX,\n   MAX_DFH_IDX\n} t_dfh_idx;\n
                               
                              2.  
                            2.  
                              Add an entry for HELLO_FIM_IDX into the get_dfh_names() function:
                               
                              function automatic dfh_name[MAX_DFH_IDX-1:0] get_dfh_names();\n   dfh_name[MAX_DFH_IDX-1:0] dfh_names;\n\n   dfh_names[FME_DFH_IDX]         = \"FME_DFH\";\n   dfh_names[THERM_MNGM_DFH_IDX]  = \"THERM_MNGM_DFH\";\n   dfh_names[GLBL_PERF_DFH_IDX]   = \"GLBL_PERF_DFH\";\n   dfh_names[GLBL_ERROR_DFH_IDX]  = \"GLBL_ERROR_DFH\";\n   dfh_names[QSFP0_DFH_IDX]       = \"QSFP0_DFH\";\n   dfh_names[QSFP1_DFH_IDX]       = \"QSFP1_DFH\";\n   dfh_names[HSSI_DFH_IDX]        = \"HSSI_DFH\";\n   dfh_names[EMIF_DFH_IDX]        = \"EMIF_DFH\";\n   dfh_names[HELLO_FIM_DFH_IDX]   = \"HELLO_FIM_DFH\";\n   dfh_names[PMCI_DFH_IDX]        = \"PMCI_DFH\";\n   dfh_names[ST2MM_DFH_IDX]       = \"ST2MM_DFH\";\n   dfh_names[PG_PR_DFH_IDX]       = \"PG_PR_DFH\";\n   dfh_names[PG_PORT_DFH_IDX]     = \"PG_PORT_DFH\";\n   dfh_names[PG_USER_CLK_DFH_IDX] = \"PG_USER_CLK_DFH\";\n   dfh_names[PG_REMOTE_STP_DFH_IDX] = \"PG_REMOTE_STP_DFH\";\n   dfh_names[AFU_ERR_DFH_IDX] = \"AFU_ERR_DFH\";\n\n   return dfh_names;\nendfunction\n
                               
                              3.  
                            3.  
                              Modify the expected DFH value of the EMIF from 64'h3_00000_06B000_1009 to 64'h3_00000_001000_1009 and add the expected value for HELLO_FIM as 64'h3_00000_06A000_0100:
                               
                              function automatic [MAX_DFH_IDX-1:0][63:0] get_dfh_values();\n   logic[MAX_DFH_IDX-1:0][63:0] dfh_values;\n\n   dfh_values[FME_DFH_IDX]        = 64'h4000_0000_1000_0000;\n   dfh_values[THERM_MNGM_DFH_IDX] = 64'h3_00000_002000_0001;\n   dfh_values[GLBL_PERF_DFH_IDX]  = 64'h3_00000_001000_0007;\n   dfh_values[GLBL_ERROR_DFH_IDX] = 64'h3_00000_00e000_1004;\n   dfh_values[QSFP0_DFH_IDX]      = 64'h3_00000_001000_0013;\n   dfh_values[QSFP1_DFH_IDX]      = 64'h3_00000_001000_0013;\n   dfh_values[HSSI_DFH_IDX]       = 64'h3_00000_001000_100f;\n   dfh_values[EMIF_DFH_IDX]       = 64'h3_00000_001000_1009;\n   dfh_values[HELLO_FIM_DFH_IDX]  = 64'h3_00000_06A000_0100;\n   dfh_values[PMCI_DFH_IDX]       = 64'h3_00000_080000_1012;\n   dfh_values[ST2MM_DFH_IDX]      = 64'h3_00000_030000_0014;\n   dfh_values[PG_PR_DFH_IDX]      = 64'h3_00000_001000_1005;\n   dfh_values[PG_PORT_DFH_IDX]     = 64'h4_00000_001000_1001;\n   dfh_values[PG_USER_CLK_DFH_IDX] = 64'h3_00000_001000_1014;\n   dfh_values[PG_REMOTE_STP_DFH_IDX] = 64'h3_00000_00d000_2013;\n   dfh_values[AFU_ERR_DFH_IDX] = 64'h3_00001_000000_2010;\n\n   return dfh_values;\nendfunction\n
                               
                              4.  
                             
                            2.  
                          2.  
                            Regenerate the simulation files
                             
                            cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\nsh gen_sim_files.sh f2000x \n
                             
                            3.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5172-run-dfh-walker-simulation","title":"5.1.7.2 Run DFH Walker Simulation","text":"
                          After the simulation files have been re-generated, run the DFH Walker test to ensure the Hello FIM module can be accessed by the SoC through the BPF.
                           
                             
                          1.  
                            Run the DFH Walker test
                             
                            cd $OFS_ROOTDIR/sim/unit_test/soc_tests/dfh_walker\nsh run_sim.sh\n
                             
                            2.  
                          2.  
                            Check the output for the presence of the HELLO_FiM module at address 0x16000:
                             
                            ********************************************\nRunning TEST(0) : test_dfh_walking\n********************************************\n\n...\n\nREAD64: address=0x00015000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x3000000010001009\n\nEMIF_DFH\n   Address   (0x15000)\n   DFH value (0x3000000010001009)\n\nREAD64: address=0x00016000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x30000006a0000100\n\nHELLO_FIM_DFH\n   Address   (0x16000)\n   DFH value (0x30000006a0000100)\n\nREAD64: address=0x00080000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x3000000800001012\n\nPMCI_DFH\n   Address   (0x80000)\n   DFH value (0x3000000800001012)\n\n...\n\nTest status: OK\n\n********************\nTest summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n
                             
                            3.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#518-uvm-verfication-of-the-hellofim","title":"5.1.8 UVM Verfication of the HelloFIM","text":"
                          The following section describes the file modifications that need to be made to perform UVM verifaction of the Hello FIM design, followed by instructions for running the UVM simulations.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5181-uvm-simulation-file-modification","title":"5.1.8.1 UVM Simulation File Modification","text":"
                          Perform the following steps to modify the UVM simulation files to support the Hello FIM design.
                           
                             
                          1.  
                            Modify $OFS_ROOTDIR/verification/tests/sequences/dfh_walking_seq.svh
                             
                               
                            1.  
                              Modify the dfh_offset_array to insert the Hello FIM.
                               
                              dfh_offset_array = new[16];\ndfh_offset_array[ 0] = tb_cfg0.PF0_BAR0;                    // FME_DFH                0x8000_0000\ndfh_offset_array[ 1] = dfh_offset_array[ 0] + 64'h0_1000;   // THERM_MNGM_DFH         0x8000_1000\ndfh_offset_array[ 2] = dfh_offset_array[ 1] + 64'h0_2000;   // GLBL_PERF_DFH          0x8000_3000\ndfh_offset_array[ 3] = dfh_offset_array[ 2] + 64'h0_1000;   // GLBL_ERROR_DFH         0x8000_4000\ndfh_offset_array[ 4] = dfh_offset_array[ 3] + 64'h0_E000;   // QSFP0_DFH              0x8001_2000\ndfh_offset_array[ 5] = dfh_offset_array[ 4] + 64'h0_1000;   // QSFP1_DFH              0x8001_3000\ndfh_offset_array[ 6] = dfh_offset_array[ 5] + 64'h0_1000;   // HSSI_DFH               0x8001_4000\ndfh_offset_array[ 7] = dfh_offset_array[ 6] + 64'h0_1000;   // EMIF_DFH               0x8001_5000\ndfh_offset_array[ 8] = dfh_offset_array[ 7] + 64'h0_1000;   // HELLO_FIM_DFH          0x8001_6000\ndfh_offset_array[ 9] = dfh_offset_array[ 8] + 64'h6_a000;   // PMCI_DFH               0x8008_0000\ndfh_offset_array[ 10] = dfh_offset_array[ 9] + 64'h8_0000;  // ST2MM_DFH              0x8010_0000\ndfh_offset_array[ 11] = dfh_offset_array[10] + 64'h3_0000;  // PG_PR_DFH_IDX          0x8013_0000\ndfh_offset_array[ 12] = dfh_offset_array[11] + 64'h0_1000;  // PG_PORT_DFH_IDX        0x8013_1000\ndfh_offset_array[ 13] = dfh_offset_array[12] + 64'h0_1000;  // PG_USER_CLK_DFH_IDX    0x8013_2000\ndfh_offset_array[ 14] = dfh_offset_array[13] + 64'h0_1000;  // PG_REMOTE_STP_DFH_IDX  0x8013_3000\ndfh_offset_array[ 15] = dfh_offset_array[14] + 64'h0_D000;  // PG_AFU_ERR_DFH_IDX     0x8014_0000\n
                               
                              2.  
                             
                            2.  
                          2.  
                            Modify $OFS_ROOTDIR/verification/tests/sequences/mmio_seq.svh
                             
                               
                            1.  
                              Add test code related to the Hello FIM. This code will verify the scratchpad register at 0x16030 and read only the register at 0x16038.
                               
                              // HELLO_FIM_Scratchpad 64 bit access\n`uvm_info(get_name(), $psprintf(\"////Accessing PF0 HELLO_FIM_Scratchpad Register %0h+'h16030////\", tb_cfg0.PF0_BAR0), UVM_LOW)\n\nassert(std::randomize(wdata));\naddr = tb_cfg0.PF0_BAR0+'h1_6000+'h30;\n\nmmio_write64(.addr_(addr), .data_(wdata));\nmmio_read64 (.addr_(addr), .data_(rdata));\n\nif(wdata !== rdata)\n    `uvm_error(get_name(), $psprintf(\"Data mismatch 64! Addr = %0h, Exp = %0h, Act = %0h\", addr, wdata, rdata))\nelse\n    `uvm_info(get_name(), $psprintf(\"Data match 64! addr = %0h, data = %0h\", addr, rdata), UVM_LOW)\n\naddr = tb_cfg0.PF0_BAR0+'h1_6000+'h38;\nwdata = 64'h6626_0701_5000_0034;\nmmio_read64 (.addr_(addr), .data_(rdata));\nif(wdata !== rdata)\n    `uvm_error(get_name(), $psprintf(\"Data mismatch 64! Addr = %0h, Exp = %0h, Act = %0h\", addr, wdata, rdata))\nelse\n    `uvm_info(get_name(), $psprintf(\"Data match 64! addr = %0h, data = %0h\", addr, rdata), UVM_LOW)\n
                               
                              Note: uvm_info and uvm_error statements will put a message into log file.
                               
                              2.  
                             
                            3.  
                          3.  
                            Modify $OFS_ROOTDIR/verification/scripts/Makefile_VCS.mk
                             
                               
                            1.  
                              Add INCLUDE_HELLO_FIM define option to enable Hello FIM on UVM
                               
                              VLOG_OPT += +define+INCLUDE_HELLO_FIM\n
                               
                              2.  
                             
                            4.  
                          4.  
                            Re-generate the UVM files
                             
                               
                            1.  
                              Navigate to the verification scripts directory
                               
                              cd $VERDIR/scripts\n
                               
                              2.  
                            2.  
                              Clean the output of previous builds
                               
                              gmake -f Makefile_VCS.mk clean\n
                               
                              3.  
                            3.  
                              Compile the IP files
                               
                              gmake -f Makefile_VCS.mk cmplib_adp\n
                               
                              4.  
                            4.  
                              Build the RTL and Test Benches
                               
                              gmake -f Makefile_VCS.mk build_adp DUMP=1 DEBUG=1 \n
                               
                              Note: Using the DEBUG option will provide more detail in the log file for the simulation.
                               
                              5.  
                             
                            5.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5182-run-uvm-dfh-walker-simulation","title":"5.1.8.2 Run UVM DFH Walker Simulation","text":"
                          Perform the following steps to run the UVM DFH Walker Simulation.
                           
                             
                          1.  
                            Run the DFH Walker simulation
                             
                            cd $VERDIR/scripts\ngmake -f Makefile_VCS.mk run TESTNAME=dfh_walking_test DUMP=1 DEBUG=1\n
                             
                            Note: Using the DEBUG option will provide more detail in the log file for the simulation.
                             
                            2.  
                          2.  
                            The output logs are stored in the $VERDIR/sim/dfh_walking_test directory. The main files to note are described in the table below:
                             File Name Description runsim.log A log file of UVM trans.log A log file of transactions on PCIe bus inter.vpd A waveform for VCS 
                            3.  
                          3.  
                            Run the following command to quickly verify- that the Hello FIM module was successfully accessed. In the example below, the message DFH offset Match! Exp = 80016000 Act = 80016000 shows that the Hello FIM module was successfully accessed.
                             
                            cd $VERDIR/sim/dfh_walking_test\ncat runsim.log | grep \"DFH offset\"\n
                             
                            Expected output:
                             
                            UVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 111950000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp = 80000000 Act = 80000000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 112586000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80001000 Act = 80001000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 113222000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80003000 Act = 80003000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 113858000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80004000 Act = 80004000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 114494000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80012000 Act = 80012000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 115147000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80013000 Act = 80013000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 115801000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80014000 Act = 80014000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 116628000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80015000 Act = 80015000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 117283000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80016000 Act = 80016000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 117928000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80080000 Act = 80080000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 118594000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80100000 Act = 80100000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 119248000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80130000 Act = 80130000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 119854000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80131000 Act = 80131000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 120460000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80132000 Act = 80132000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 121065000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80133000 Act = 80133000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 121672000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80140000 Act = 80140000\n
                             
                            4.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5183-run-uvm-mmio-simulation","title":"5.1.8.3 Run UVM MMIO Simulation","text":"
                          Perform the following steps to run the UVM MMIO Simulation.
                           
                             
                          1.  
                            Run the MMIO test
                             
                            cd $VERDIR/scripts\ngmake -f Makefile_VCS.mk run TESTNAME=mmio_test DUMP=1\n
                             
                            2.  
                          2.  
                            Run the following commands to show the result of the scratchpad register and Hello FIM ID register. You can see the \"Data match\" message indicating that the registers are successfuly verified.
                             
                            cd $VERDIR/sim/mmio_test\ncat runsim.log | grep \"Data\" | grep 1603\n
                             
                            Expected output:
                             
                            UVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/mmio_seq.svh(68) @ 115466000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] Data match 64! addr = 80016030, data = 880312f9558c00e1\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/mmio_seq.svh(76) @ 116112000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] Data match 64! addr = 80016038, data = 6626070150000034\n
                             
                            3.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#519-compile-the-f2000x-design-with-hello-fim","title":"5.1.9 Compile the f2000x Design with Hello FIM","text":"
                          Perform the following to compile the Hello FIM design.
                           
                             
                          1. Ensure the pre-requesites described in the Pre-Requisites for Adding Hello FIM section are satisfied.
                            2.  
                          2. Ensure that Intel Quartus Prime Pro is in your $PATH
                            3.  
                          3.  
                            Compile the design
                             
                            $OFS_ROOTDIR/ofs-common/scripts/common/syn/build_top.sh -p f2000x work_hello_fim\n
                             
                            4.  
                          4.  
                            Once compilation is complete, the output files can be found in the $OFS_ROOTDIR/work_hello_fim/syn/syn_top/output_files directory.
                             
                            5.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5110-program-the-f2000x-with-the-hellofim-design","title":"5.1.10 Program the f2000x with the HelloFIM Design","text":"
                          Perform the following steps to program the f2000x with the HelloFIM design generated in the previous section. This flow uses the RSU feature, which requires that an OPAE enabled design is already loaded on the FPGA. All OPAE commands are run from the SoC, and the new image will be updated from the SoC.
                           
                             
                          1.  
                            Use the fpgainfo fme command to obtain the PCIe s:b:d.f associated with your board. In this example, the PCIe s:b:d.f is 0000:15:00.0.
                             
                            fpgainfo fme\n
                             
                            Example output:
                             
                            Intel IPU Platform f2000x\nBoard Management Controller NIOS FW version: 1.2.4 \nBoard Management Controller Build version: 1.2.4 \n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010302A97C08A3\nBitstream Version                : 5.0.1\nPr Interface Id                  : fb25ff1d-c31a-55d8-81d8-cbcedcfcea17\nBoot Page                        : user1\nUser1 Image Info                 : a566ceacaed810d43c60b0b8a7145591\nUser2 Image Info                 : a566ceacaed810d43c60b0b8a7145591\nFactory Image Info               : None\n
                             
                            2.  
                          2.  
                            Navigate to the output directory in the Hello FIM work directory that contains the output files from compilation. 
                             
                            cd $OFS_ROOTDIR/work_hello_fim/syn/syn_top/output_files\n
                             
                            3.  
                          3.  
                            Initiate the User Image 1 update by running fpgasupdate from a shell in the SoC. This will update User Image 1 stored in FPGA Flash. Remember to use the PCIe BDF associated with your board.
                             
                            sudo fpgasupdate ofs_top_page1_unsigned_user1.bin 0000:15:00.0\n
                             
                            4.  
                          4.  
                            Run the rsu command to re-configure the FPGA with User Image 1 from FPGA Flash.
                             
                            sudo rsu fpga --page=user1 0000:15:00.0\n
                             
                            5.  
                          5.  
                            Run the fpgainfo fme command again to verify the User1 Image Info has been updated.
                             
                            Example Output:
                             
                            Intel IPU Platform f2000x\nBoard Management Controller NIOS FW version: 1.2.4 \nBoard Management Controller Build version: 1.2.4 \n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010302A97C08A3\nBitstream Version                : 5.0.1\nPr Interface Id                  : fb25ff1d-c31a-55d8-81d8-cbcedcfcea17\nBoot Page                        : user1\nUser1 Image Info                 : 81d8cbcedcfcea17fb25ff1dc31a55d8\nUser2 Image Info                 : a566ceacaed810d43c60b0b8a7145591\nFactory Image Info               : None\n
                             
                            6.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5111-verify-the-hello-fim-design-on-the-f2000x-using-opae","title":"5.1.11 Verify the Hello FIM Design on the f2000x Using OPAE","text":"
                          This section will describe how to access the Hello FIM registers using the opae.io tool.
                           
                             
                          1.  
                            Confirm the driver software on 0000:15:00.0
                             
                            opae.io ls\n
                             
                            Example output:
                             
                            [0000:15:00.0] (0x8086:0xbcce 0x8086:0x17d4) Intel IPU Platform f2000x (Driver: dfl-pci)\n
                             
                            2.  
                          2.  
                            Initialize the opae.io tool
                             
                            opae.io init -d 15:00.0\n
                             
                            3.  
                          3.  
                            Confirm the driver software on 0000:15:00.0 has been updated
                             
                            opae.io ls\n
                             
                            Example Output:
                             
                            [0000:15:00.0] (0x8086:0xbcce 0x8086:0x17d4) Intel IPU Platform f2000x (Driver: vfio-pci)\n
                             
                            4.  
                          4.  
                            Run the DFH Walker test to verify there is a module at offset 0x16000
                             
                            opae.io walk -d 15:00.0\n
                             
                            Example output:
                             
                            offset: 0x0000, value: 0x4000000010000000\n   dfh: id = 0x0, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x4\noffset: 0x1000, value: 0x3000000020000001\n    dfh: id = 0x1, rev = 0x0, next = 0x2000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x3000, value: 0x3000000010000007\n    dfh: id = 0x7, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x4000, value: 0x30000000e0001004\n    dfh: id = 0x4, rev = 0x1, next = 0xe000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x12000, value: 0x3000000010000013\n    dfh: id = 0x13, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x13000, value: 0x3000000010000013\n    dfh: id = 0x13, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x14000, value: 0x3000000010002015\n    dfh: id = 0x15, rev = 0x2, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x15000, value: 0x3000000010001009\n    dfh: id = 0x9, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x16000, value: 0x30000006a0000100\n    dfh: id = 0x100, rev = 0x0, next = 0x6a000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x80000, value: 0x3000000800002012\n    dfh: id = 0x12, rev = 0x2, next = 0x80000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x100000, value: 0x3000000300000014\n    dfh: id = 0x14, rev = 0x0, next = 0x30000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x130000, value: 0x3000000010001005\n    dfh: id = 0x5, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x131000, value: 0x4000000010001001\n    dfh: id = 0x1, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x4\noffset: 0x132000, value: 0x3000000010001014\n    dfh: id = 0x14, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x133000, value: 0x30000000d0002013\n    dfh: id = 0x13, rev = 0x2, next = 0xd000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x140000, value: 0x3000010000002010\n    dfh: id = 0x10, rev = 0x2, next = 0x0, eol = 0x1, reserved = 0x0, feature_type = 0x3\n
                             
                            5.  
                          5.  
                            Read all of the registers in the Hello FIM module
                             
                               
                            1.  
                              Read the DFH Register
                               
                              opae.io -d 15:00.0 -r 0 peek 0x16000\n
                               
                              Example Output:
                               
                              0x30000006a0000100\n
                               
                              2.  
                            2.  
                              Read the Scratchpad Register
                               
                              opae.io -d 15:00.0 -r 0 peek 0x16030\n
                               
                              Example Output:
                               
                              0x0\n
                               
                              3.  
                            3.  
                              Read the ID Register
                               
                              opae.io -d 15:00.0 -r 0 peek 0x16038\n
                               
                              Example Output:
                               
                              0x6626070150000034\n
                               
                              4.  
                             
                            6.  
                          6.  
                            Verify the scratchpad register at 0x16030 by writing and reading back from it.
                             
                               
                            1.  
                              Write to Scratchpad register
                               
                              opae.io -d 0000:15:00.0 -r 0 poke 0x16030 0x123456789abcdef\n
                               
                              2.  
                            2.  
                              Read from Scratchpad register
                               
                              opae.io -d 15:00.0 -r 0 peek 0x16030\n
                               
                              Expected output:
                               
                              0x123456789abcdef\n
                               
                              3.  
                            3.  
                              Write to Scratchpad register
                               
                              opae.io -d 15:00.0 -r 0 poke 0x16030 0xfedcba9876543210\n
                               
                              4.  
                            4.  
                              Read from Scratchpad register
                               
                              opae.io -d 15:00.0 -r 0 peek 0x16030\n
                               
                              Expected output:
                               
                              0xfedcba9876543210\n
                               
                              5.  
                             
                            7.  
                          7.  
                            Release the opae.io tool
                             
                            opae.io release -d 15:00.0\n
                             
                            8.  
                          8.  
                            Confirm the driver has been set back to dfl-pci
                             
                            opae.io ls\n
                             
                            Example output:
                             
                            [0000:15:00.0] (0x8086:0xbcce 0x8086:0x17d4) Intel IPU Platform f2000x (Driver: dfl-pci)\n
                             
                            9.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#52-how-to-debug-the-fim-with-signal-tap","title":"5.2 How to Debug the FIM with Signal Tap","text":"
                          For debugging issues within the FIM, Signal Tap can be used to gain internal visibility into your design. This section describes the process of adding a Signal Tap instance to the Hello FIM design example described in the How to add a new module to the FIM section, however the process can be used for any design.
                           
                          For more detailed information on Signal Tap please see refer to Quartus Prime Pro Edition User Guide: Debug Tools (RDC Document ID 683819).
                           
                          Signal Tap uses the Intel FPGA Download Cable II USB device to provide access.  Please see Intel FPGA Download Cable II for more information. This device is widely available via distributors for purchase.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#521-adding-signal-tap-to-the-hello-fim-example","title":"5.2.1 Adding Signal Tap to the Hello FIM example","text":"
                          The following steps guide you through the process of adding a Signal Tap instance to your design. The added Signal Tap instance provides hardware to capture the desired internal signals and connect the stored trace information via JTAG. Please be aware that the added Signal Tap hardware will consume FPGA resources and may require additional floorplanning steps to accommodate these resources. Some areas of the FIM use logic lock regions and these regions may need to be re-sized. These steps assume the use of the  Intel IPU Platform F2000X-PL.  
                           
                          The steps below use the hello_fim example to add Signal Tap, however the general process can be used for any design.
                           
                             
                          1.  
                            The design must be synthesized before adding Signal Tap.
                             
                               
                            •  
                              If you are using the previously built Hello FIM design, copy the work directory and rename it so that we have a work directory dedicated to the Hello FIM Signal Tap design.
                               
                              cp -r $OFS_ROOTDIR/work_hello_fim $OFS_ROOTDIR/work_hello_fim_with_stp\n
                               
                              •  
                             
                               
                            •  
                              If you are adding signal tap to a new design that has not yet been synthesized, perform the following steps to synthesize the design.
                               
                                 
                              1. Set the environment variables as described in the Setting Up Required Environment Variables section.
                                2.  
                              2.  
                                Run the setup portion of the build script to create a working directory based on the original source files.
                                 
                                ./ofs-common/scripts/common/syn/build_top.sh --stage setup f2000x <YOUR_STP_WORK_DIR>\n
                                 
                                3.  
                              3.  
                                Open the project in the Intel Quartus Prime Pro GUI. The Intel Quartus Prime Pro project is named ofs_top.qpf and is located in the work directory $OFS_ROOTDIR/<YOUR_STP_WORK_DIR>/syn/syn_top/ofs_top.qpf.
                                 
                                4.  
                              4. In the Compilation Flow window, run Analysis & Synthesis.
                                5.  
                              5. Once Synthesis has completed, you may skip to Step 3.
                                6.  
                               
                              •  
                             
                            2.  
                          2.  
                            Open the Hello FIM Signal Tap project in the Intel Quartus Prime Pro GUI. The Intel Quartus Prime Pro project is named ofs_top.qpf and is located in the work directory $OFS_ROOTDIR/work_hello_fim_with_stp/syn/syn_top/ofs_top.qpf.
                             
                            3.  
                          3.  
                            Select Tools > Signal Tap Logic Analyzer to open the Signal Tap GUI.
                             
                             
                            4.  
                          4.  
                            Accept the \"Default\" selection and click \"Create\".
                             
                             
                            5.  
                          5.  
                            This opens the Signal Tap Logic Analyzer window.
                             
                             
                            6.  
                          6.  
                            Set up the clock for the STP instance. This example instruments the hello_fim_top module previously intetegrated into the FIM. If unfamiliar with code, it is helpful to use the Intel Quartus Prime Pro Project Navigator to find the block of interest and open the design instance for review. For example, see the image below using Project Navigator to open the top module where hello_fim_top_inst is instantiated.
                             
                             
                            7.  
                          7.  
                            Assign the clock for sampling the Signal Tap instrumented signals of interest. Note, that the clock selected should be associated with the signals you want to view for best trace fidelity. Different clocks can be used, however, there maybe issues with trace inaccuracy due to sampling time differences. Ensure that all signals that are to be sampled are on the same clock domain as the clock you select here. In the middle right of the Signal Tap window, under Signal Configuration, Clock:, select \"\u2026\" as shown below:
                             
                             
                            8.  
                          8.  
                            In the Node Finder tool that popped up, input \"hello_fim_top_inst|clk\" into the \"Named:\" textbox and click \"Search\". Select \"clk\" in the Matching Nodes list and click the \">\" button to select this clock as shown below. Click \"OK\" to close the Node Finder dialog.
                             
                             
                            9.  
                          9.  
                            Update the sample depth and other Signal Tap settings as needed for your debugging criteria.
                             
                             
                            10.  
                          10.  
                            In the Signal Tap GUI add the nodes to be instrumented by double-clicking on the \"Double-click to add nodes\" legend.
                             
                             
                            11.  
                          11.  
                            This brings up the Node Finder to add the signals to be traced. In this example we will monitor the memory mapped interface to the Hello FIM. Select the signals that appear from the  search patterns hello_fim_top_inst|reset and hello_fim_top_inst|csr_lite_if*. Click Insert and close the Node Finder dialog.
                             
                             
                            12.  
                          12.  
                            To provide a unique name for your Signal Tap instance, select \"auto signaltap_0\", right-click, and select Rename Instance (F2). Provide a descriptive name for your instance, for example, \"STP_For_Hello_FIM\".
                             
                             
                            13.  
                          13.  
                            Save the newly created Signal Tap file, and give it the same name as the instance. Ensure that the Add file to current project checkbox is ticked.
                             
                             
                            14.  
                          14.  
                            In the dialog that pops up, click \"Yes\" to add the newly created Signal Tap file to the project settings files.
                             
                             
                            This will aurtomatically add the following lines to $OFS_ROOTDIR/work_hello_fim_with_stp/syn/syn_top/ofs_top.qsf:
                             
                            set_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE STP_For_Hello_FIM.stp\nset_global_assignment -name SIGNALTAP_FILE STP_For_Hello_FIM.stp\n
                             
                            15.  
                          15.  
                            Close all Quartus GUIs.
                             
                            16.  
                          16.  
                            Compile the project with the Signal Tap file added to the project. Use the -k switch to perform the compilation using the files in a specified working directory and not the original ones from the cloned repository. 
                             
                            ofs-common/scripts/common/syn/build_top.sh -p -k f2000x work_hello_fim_with_stp\n
                             
                            Alternatively, you can copy the ofs_top.qsf and STP_For_Hello_FIM.stp files from the Hello FIM with STP work directory to replace the original files in the cloned OFS repository. In this scenario, all further FIM compilation projects will include the Signal Tap instance integrated into the design. Execute the following commands for this alternative flow:
                             
                            Copy the modified file \"work_hello_fim_with_stp/syn/syn_top/ofs_top.qsf\" over to the source OFS repository, into \"syn/syn_top/\".
                             
                            cd $OFS_ROOTDIR/work_hello_fim_with_stp/syn/syn_top\ncp ofs_top.qsf $OFS_ROOTDIR/syn/syn_top\ncp STP_For_Hello_FIM.stp $OFS_ROOTDIR/syn/syn_top\n
                             
                            Compile the FIM using the files from the OFS repository to create a new work directory.
                             
                            cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x work_hello_fim_with_stp_from_src_repo\n
                             
                            17.  
                          17.  
                            Ensure that the compile completes successfully and meets timing:
                             
                            ***********************************\n***\n***        OFS_PROJECT: f2000x\n***        OFS_FIM: base\n***        OFS_BOARD: adp\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 0\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                             
                            18.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#522-configuring-the-fpga-with-a-sof-image-via-jtag","title":"5.2.2 Configuring the FPGA with a SOF Image via JTAG","text":"
                          Every successful run of build_top.sh script creates the file $OFS_ROOTDIR/<WORK_DIRECTORY>/syn/syn_top/output_files/ofs_top.sof which can be used with the Intel FPGA Download Cable II to load the image into the FPGA using the f2000x  JTAG access connector. 
                           
                          Perform the steps described in the following sections to load the ofs_fim.sof image created in the previous section into the Intel Agilex 7 FPGA using the Intel FPGA Download Cable II. You will also use the Intel FPGA Download Cable II to access the Signal Tap instance via JTAG.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5221-connecting-to-intel-fpga-download-cable","title":"5.2.2.1 Connecting to Intel FPGA Download Cable","text":"
                          The f2000x  has a 10 pin JTAG header on the top side of the board.  This JTAG header provides access to either the Intel Agilex 7 FPGA or Cyclone\u00ae 10 BMC device.  Perform the followign steps to connect the Itel FPGA Download Cable II and target the Intel Agilex 7 device:
                           
                             
                          1.  
                            Locate SW2 and SW3 on the f2000x board as shown in the following figure.
                             
                             
                            2.  
                          2.  
                            Set the switches described in the following table:
                             Switch Position SW2 ON SW3.3 ON 
                            3.  
                          3.  
                            Connect the Intel FPGA Download Cable to the JTAG header of the f2000x as shown in the figure below.
                             
                             
                            4.  
                          4.  
                            Depending on your server, install the card in a slot that allows room for the JTAG cable. The figure below shows the f2000x installed in a Supermicro server slot.
                             
                             
                            5.  
                          5.  
                            There are two JTAG modes that exist. Short-chain mode is when only the Cyclone 10 device is on the JTAG chain. Long-chain mode is when both the Cyclone 10 and the Intel Agilex 7 FPGA are on the JTAG chain. Check which JTAG mode the f2000x board is in by running the following command.
                             
                            $QUARTUS_ROOTDIR/bin/jtagconfig\n
                             
                               
                            •  
                              Example output when in short-chain mode (only Cyclone 10 detected):
                               
                              1) USB-BlasterII [3-4]\n020F60DD    10CL080(Y|Z)/EP3C80/EP4CE75\n
                               
                              •  
                             
                               
                            •  
                              Example output when in long-chain mode (both Cyclone 10 and Intel Agilex 7 detected):
                               
                              1) USB-BlasterII [3-4]\n020F60DD   10CL080(Y|Z)/EP3C80/EP4CE75\n234150DD   AGFC023R25A(.|AE|R0)\n
                                 If the Intel Agilex 7 device does not appear on the chain, ensure that the switches described in Step 1 have been set correctly and power cycle the board. Also ensure that the JTAG Longchain bit is set to 0 in BMC Register 0x378. The BMC registers are accessed through SPI control registers at addresses 0x8040C and 0x80400 in the PMCI. Use the following commands to clear the JTAG Longchain bit in BMC register 0x378. 
                               
                              •  
                             
                            Note: These commands must be executed as root user from the SoC.
                             
                            Note: You may find the PCIe BDF of your card by running fpgainfo fme.
                             
                            opae.io init -d <BDF>\nopae.io -d <BDF> -r 0 poke 0x8040c 0x000000000\nopae.io -d <BDF> -r 0 poke 0x80400 0x37800000002\nopae.io release -d <BDF>\n
                             For example, for a board with PCIe BDF 15:00.0: 
                            opae.io init -d 15:00.0\nopae.io -d 15:00.0 -r 0 poke 0x8040c 0x000000000\nopae.io -d 15:00.0 -r 0 poke 0x80400 0x37800000002\nopae.io release -d 15:00.0\n
                             
                            6.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5222-programming-the-intel-agilex-7-fpga-via-jtag","title":"5.2.2.2 Programming the Intel Agilex 7 FPGA via JTAG","text":"
                          Perform the following steps to program the Intel Agilex 7 FPGA via JTAG.
                           
                             
                          1.  
                            The generated SOF file is located in the work directory $OFS_ROOTDIR/work_hello_fim_with_stp/syn/syn_top/output_files/ofs_top.sof. If the target FPGA is on a different server, then transfer ofs_top.sof and STP_For_Hello_FIM.stp files to the server with the target FPGA.
                             
                            2.  
                          2.  
                            You can use a Full Intel Quartus Prime Pro Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the f2000x  is installed or on a separte machine such as a laptop.
                             
                            Note: You can download the Quartus Prime Programmer and Tools by clicking on the \"Additional Software\" tab on the FPGA download page. The Quartus Prime Programmer and Tools come with Quartus programmer as well as System Console which are needed to program the flash devices.
                             
                            Note: If using the Intel FGPA download Cable on Linux, add the udev rule as described in Intel FPGA Download Cable (formerly USB-Blaster) Driver for Linux.
                             
                            3.  
                          3.  
                            Temporarily disable the PCIe AER feature and remove the PCIe port for the board you are going to update.. This is required because when you program the FPGA using JTAG, the f2000x PCIe link goes down for a moment causing a server surprise link down event. To prevent this server event, temporarily disable PCIe AER and remove the PCIe port using the following commands:
                             
                            Note: enter the following commands as root.
                             
                               
                            1.  
                              Find the PCIe BDF and Device ID of your board from the SoC. You may use the OPAE command fpaginfo fme on the SoC to display this information. Run this command on the SoC.
                               
                              fpgainfo fme\n
                               
                              Example output:
                               
                              Intel IPU Platform F2000X-PL\nBoard Management Controller NIOS FW version: 1.2.3\nBoard Management Controller Build version: 1.2.3\n//****** FME ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x50103024BF5B5B1\nBitstream Version                : 5.0.1\nPr Interface Id                  : e7926956-9b1b-5ea1-b02c-307f1cb33446\nBoot Page                        : user1\nUser1 Image Info                 : b02c307f1cb33446e79269569b1b5ea1\nUser2 Image Info                 : b02c307f1cb33446e79269569b1b5ea1\nFactory Image Info               : b02c307f1cb33446e79269569b1b5ea1\n
                               
                              In this case, the PCIe BDF for the board on the SoC is 15:00.0, and the Device ID is 0xBCCE.
                               
                              2.  
                            2.  
                              From the SoC, use the pci_device OPAE command to \"unplug\" the PCIe port for your board using the PCIe BDF found in Step 3.a. Run this command on the SoC.
                               
                              pci_device <B:D.F> unplug\n
                               
                              For example:
                               
                              pci_device 15:00.0 unplug\n
                               
                              3.  
                            3.  
                              Find the PCIe BDF of your board from the Host. Use lspci and grep for the device ID found in Step 3.a to get the PCIe BDF. Run this command on the Host.
                               
                              For example:
                               
                              lspci | grep bcce\n
                               
                              Example output:
                               
                              31:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n31:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                               
                              In this case, the board has PCIe BDF 31:00.0 from the Host.
                               
                              4.  
                            4.  
                              From the Host, use the pci_device OPAE command to \"unplug\" the PCIe port for your board using the PCIe BDF found in Step 3.c. Run this command on the Host.
                               
                              pci_device <B:D.F> unplug\n
                               
                              For example:
                               
                              pci_device 31:00.0 unplug\n
                               
                              5.  
                             
                            4.  
                          4.  
                            Launch \"Quartus Prime Programmer\" software from the device which the Intel FPGA Programmer is connected.
                             
                            $QUARTUS_ROOTDIR/bin/quartus_pgmw\n
                             
                            Click on Hardware Setup, select USB-BlasterII in the Current Selected Hardware list, and ensure the JTAG Hardware Frequency is set to 16Mhz (The default is 24MHz).
                             
                             
                            Alternatively, use the following command from the command line to change the clock frequency:
                             
                            jtagconfig \u2013setparam \u201cUSB-BlasterII\u201d JtagClock 16M\n
                             
                            5.  
                          5.  
                            Click Auto Detect and make sure the Intel Agilex 7 Device is shown in the JTAG chain. Select the Cyclone 10 and Intel Agilex 7 devices if prompted.
                             
                             
                            6.  
                          6.  
                            Right-click on the cell in the File column for the Intel Agilex 7 device and click on Change file
                             
                             
                            7.  
                          7.  
                            Select the generated ofs_top.sof file for the Intel Agilex 7 FPGA with the Signal Tap instrumented Hello FIM example. Remember that the output files are located under work_hello_fim_with_stp/syn/syn_top/output_files/.
                             
                            8.  
                          8.  
                            Tick the checkbox below \"Program/Configure\" column and click on Start to program this .sof file.
                             
                             
                            9.  
                          9.  
                            After successful programming, you can close the \"Quartus Prime Programmer\" software. You can answer 'No' if a dialog pops up asking to save the 'Chain.cdf' file. This completes the Intel Agilex 7 .sof programming.
                             
                            10.  
                          10.  
                            Re-plug the PCIe ports on both the SoC and Host.
                             
                            Note: enter the following commands as root.
                             
                               
                            1.  
                              From the SoC, use the pci_device OPAE command to \"plug\" the PCIe port for your board using the PCIe BDF found in Step 3.a. Run this command on the SoC.
                               
                              pci_device <B:D.F> plug\n
                               
                              For example:
                               
                              pci_device 15:00.0 plug\n
                               
                              2.  
                            2.  
                              From the Host, use the pci_device OPAE command to \"plug\" the PCIe port for your board using the PCIe BDF found in Step 3.c. Run this command on the Host.
                               
                              pci_device <B:D.F> plug\n
                               
                              For example:
                               
                              pci_device 31:00.0 plug\n
                               
                              3.  
                             
                            11.  
                          11.  
                            Verify the f2000x is present by checking expected bitstream ID using fpaginfo fme on the SoC:
                             
                            fpgainfo fme\n
                             
                            Example output: 
                            Intel IPU Platform f2000x-PL\nBoard Management Controller NIOS FW version: 1.1.9\nBoard Management Controller Build version: 1.1.9\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010302A97C08A3\nBitstream Version                : 5.0.1\nPr Interface Id                  : cf00eed4-a82b-5f07-be25-0528baec3711\nBoot Page                        : user1\nUser1 Image Info                 : 9e3db8b6a4d25a4e3e46f2088b495899\nUser2 Image Info                 : 9e3db8b6a4d25a4e3e46f2088b495899\nFactory Image Info               : None\n
                             
                            Note: The Image Info fields will not change, because these represent the images stored in flash. In this example, we are programming the Intel Agilex 7 FPGA directly, thus only the Bitstream ID should change.
                             
                            12.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#523-signal-tap-trace-acquisition-of-hello-fim-signals","title":"5.2.3 Signal Tap trace acquisition of Hello FIM signals","text":"
                             
                          1.  
                            Once the instrumented HelloFIM SOF file is downloaded into the Intel Agilex 7 FPGA, start the Quartus Signal Tap GUI.
                             
                            $QUARTUS_ROOTDIR/bin/quartus_stpw\n
                             
                            2.  
                          2.  
                            In the Signal Tap GUI, open your STP file. Your STP file settings will load. In this example we used STP_For_Hello_FIM.stp.
                             
                             
                            3.  
                          3.  
                            In the right pane of the Signal Tap GUI, in the Hardware: selection box select the cable USB-BlasterII. In the Device: selection box select the Intel Agilex 7 device.
                             
                             
                            4.  
                          4.  
                            If the Intel Agilex 7 Device is not displayed in the Device: list, click the 'Scan Chain' button to re-scan the JTAG device chain.
                             
                            5.  
                          5.  
                            If not already set, you can create the trigger conditions. In this example, we will capture data on a rising edge of the Read Address Valid signal.
                             
                             
                            6.  
                          6.  
                            Start analysis by selecting the 'STP_For_Hello_FIM' instance and pressing 'F5' or clicking the Run Analysis icon in the toolbar. You should see a green message indicating the Acquisition is in progress. Then, move to the Data Tab to observe the signals captured.
                             
                             
                            7.  
                          7.  
                            To generate traffic in the csr_lite_if signals of the Hello FIM module, go back to the terminal and walk the DFH list or peek/poke the Hello FIM registers as was done during the creation of the Hello FIM design example.
                             
                            opae.io init -d 0000:15:00.0\nopae.io walk -d 0000:15:00.0\nopae.io release -d 0000:15:00.0\n
                             
                            The signals should be captured on the rising edge of arvalid in this example. Zoom in to get a better view of the signals.
                             
                             
                            8.  
                          8.  
                            The PCIe AER feature is automatically re-enabled by rebooting the server. 
                             
                            9.  
                           
                          This concludes the example on how to instrument an OFS FIM with the Quartus Prime Signal Tap Logic Analyzer.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#53-how-to-compile-the-fim-in-preparation-for-designing-your-afu","title":"5.3 How to compile the FIM in preparation for designing your AFU","text":"
                          To save area, the default Host Excercisers in the FIM can be replaced by a \"he_null\" block during compile-time. There are a few things to note:
                           
                             
                          • \"he_null\" is a minimal block with registers that responds to PCIe MMIO request. MMIO responses are required to keep PCIe alive (end points enabled in PCIe-SS needs service downstream requests).
                            •  
                          • If an exerciser with other I/O connections such as \"he_mem\" or \"he_hssi\" is replaced, then then those I/O ports are simply tied off.
                            •  
                          • The options supported are null_he_lb, null_he_hssi, null_he_mem and null_he_mem_tg. Any combination, order or all can be enabled at the same time. 
                            •  
                          • Finer grain control is provided for you to, for example, turn off only the exercisers in the Static Region in order to save area.
                            •  
                           
                          To compile a FIM for where the exercisers are removed and replaced with a he_null module and keeping the PF/VF multiplexor connections, execute the following command.
                           
                          cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x:null_he,null_he_hssi,null_he_mem,null_he_mem_tg work_null_he\n
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#54-how-to-resize-the-partial-reconfiguration-region","title":"5.4 How to Resize the Partial Reconfiguration Region","text":"
                          To take advantage of the available resources in the Intel Agilex 7 FPGA for an AFU design, you can adjust the size of the AFU PR partition. An example reason for the changing the size of PR region is if you add more logic to the FIM region, then you may need to reduce the size of the PR region to fit the additional logic into the static region.  Similiarly, if you reduce logic in the FIM region, then you can increase the size of the PR region to provide more logic resources for the AFU.
                           
                          After the compilation of the FIM, the resources usage broken down by partitions is reported in the Logic Lock Region Usage Summary sections of following two files:
                           
                             
                          • $OFS_ROOTDIR/<YOUR_WORK_DIRECTORY>/syn/syn_top/output_files/ofs_top.fit.place.rpt
                            •  
                          •  
                            $OFS_ROOTDIR/<YOUR_WORK_DIRECTORY>/syn/syn_top/output_files/ofs_top.fit.rpt
                             
                             
                            •  
                           
                          In this case, the default size for the afu_top|port_gasket|pr_slot|afu_main PR partition is large enough to comfortably hold the logic of the default AFU, which is mainly occupied by the Host Exercisers. However, larger designs might require additional resources.
                           
                          Perform the following steps to customize the resources allocated to the AFU in the PR regions:
                           
                             
                          1.  
                            The $OFS_ROOTDIR/syn/setup/pr_assignments.tcl TCL file defines the Logic Lock Regions in the design, including the PR partition where the AFU is allocated.
                             
                            The default design uses the the following Logic Lock Regions:
                             
                            set TOP_MEM_REGION    \"X115 Y310 X219 Y344\"\nset BOTTOM_MEM_REGION \"X0 Y0 X294 Y20\"\nset SUBSYSTEM_REGION  \"X0 Y0 X60 Y279; X0 Y0 X300 Y39; X261 Y0 X300 Y129;\"\n\nset AFU_PLACE_REGION  \"X61 Y40 X260 Y309; X220 Y130 X294 Y329; X12 Y280 X114 Y329;\"\nset AFU_ROUTE_REGION  \"X0 Y0 X294 Y329\"\n
                             
                            Each region is made up of rectangles defined by the origin (X0,Y0) and the top right corner (X1,Y1).
                             
                            2.  
                          2.  
                            Use Quartus Chip Planner to identify the locations of the resources available within the Intel Agilex 7 device for placement and routing your AFU. The image below shows the default floorplan for the f2000x Intel Agilex 7 device.
                             
                             
                            3.  
                          3.  
                            Make changes to the $OFS_ROOTDIR/syn/setup/pr_assignments.tcl file based on your findings in Quartus Chip Planner. You can modify the size and location of existing Logic Lock Regions or add new ones and assign them to the AFU PR partition. You will need to modify the coordinates of other regions assigned to the FIM accordingly to prevent overlap.
                             
                            4.  
                          4.  
                            Recompile your FIM and create the PR relocatable build tree using the following commands:
                             
                            cd $OFS_ROOTDIR    \nofs-common/scripts/common/syn/build_top.sh -p f2000x  <YOUR_WORK_DIRECTORY>\n
                             
                            5.  
                          5.  
                            Analyze the resource utilization report per partition produced after recompiling the design.
                             
                            6.  
                          6.  
                            Make further modifications to the PR regions until the results are satisfactory. Make sure timing constraints are met.
                             
                            7.  
                           
                          Refer to the following documentation for more information on how to optimize the floor plan of your Partial Reconfiguration design:
                           
                             
                          • Analyzing and Optimizing the Design Floorplan
                            •  
                          • Partial Reconfiguration Design Flow - Step 3: Floorplan the Design
                            •  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#55-how-to-modify-the-memory-subsystem","title":"5.5 How to modify the Memory Subsystem","text":"
                          In this example we will modify the Memory Subsystem to enable ECC on all of the existing memory interfaces. You may make different modifications to meet your own design requirements. Perform the following steps to make this change.
                           
                             
                          1.  
                            Clone the design repositories or use an existing design. See the Clone the OFS Git Repo section.
                             
                            2.  
                          2.  
                            Set the environment variables as described in the Setting Up Required Environment Variables section.
                             
                            3.  
                          3.  
                            Navigate to the directory containing the Memory Subsystem IP file mem_ss_fm.ip.
                             
                            cd $OFS_ROOTDIR/ipss/mem/qip/mem_ss/ \n
                             
                            4.  
                          4.  
                            Open the Memory Subsystem IP file in Platform Designer to perform the required edits. 
                             
                            qsys-edit mem_ss_fm.ip\n
                             
                            The IP Parameter Editor GUI opens as shown below.
                             
                             
                            5.  
                          5.  
                            Select the \"Memory Interfaces\" tab to view the current configuration of the Memory Interfaces. You may make edits to the configuration of any of the interfaces as needed. The figure below shows the default configuration of Interface 3.
                             
                             
                            6.  
                          6.  
                            In this example we will enable ECC for all four interfaces. In Interface tabs 0 through 3, change the drop-down selection for Memory DQ width from 32 (no ECC) to 40 (with ECC). The figure below shows this change for Interface 3.
                             
                             
                            7.  
                          7.  
                            Generate the HDL code by clicking the Generate HDL... button at the bottom right corner of the Platform Designer window. In the dialog box that appears next, review the HDL generation options and click the Generate button at the bottom right corner of the dialog box. Save the system if prompted to do so. Once the generation process is finished, close the Platform designer windows. 
                             
                            8.  
                          8.  
                            Edit the $OFS_ROOTDIR/ipss/mem/rtl/mem_ss_pkg.sv file to change the DDR4_DQ_WIDTH from 32 to 40.
                             
                            // DDR PARAMS\n...\nlocalparam DDR4_DQ_WIDTH      = 40;\n
                             
                            9.  
                          9.  
                            Edit the $OFS_ROOTDIR/syn/setup/emif_loc.tcl file to assign the pins required for ECC enabled interfaces.
                             
                               
                            1.  
                              Uncomment the DQS4 pin assignments for all memory interfaces
                               
                              #---------------------------------------------------------\n# EMIF CH0\n#---------------------------------------------------------\n...\n# # CH0 DQS4 (ECC)\nset_location_assignment PIN_A39 -to ddr4_mem[0].dq[32]\nset_location_assignment PIN_J35 -to ddr4_mem[0].dq[33]\nset_location_assignment PIN_C38 -to ddr4_mem[0].dq[34]\nset_location_assignment PIN_G34 -to ddr4_mem[0].dq[35]\nset_location_assignment PIN_G38 -to ddr4_mem[0].dq[36]\nset_location_assignment PIN_C34 -to ddr4_mem[0].dq[37]\nset_location_assignment PIN_J39 -to ddr4_mem[0].dq[38]\nset_location_assignment PIN_A35 -to ddr4_mem[0].dq[39]\nset_location_assignment PIN_C36 -to ddr4_mem[0].dqs[4]\nset_location_assignment PIN_A37 -to ddr4_mem[0].dqs_n[4]\nset_location_assignment PIN_G36 -to ddr4_mem[0].dbi_n[4]\n\n#---------------------------------------------------------\n# EMIF CH1\n#---------------------------------------------------------\n...\n# # CH1 DQS4 (ECC)\nset_location_assignment PIN_N7  -to ddr4_mem[1].dq[32]\nset_location_assignment PIN_L12 -to ddr4_mem[1].dq[33]\nset_location_assignment PIN_L6  -to ddr4_mem[1].dq[34]\nset_location_assignment PIN_U14 -to ddr4_mem[1].dq[35]\nset_location_assignment PIN_U7  -to ddr4_mem[1].dq[36]\nset_location_assignment PIN_W12 -to ddr4_mem[1].dq[37]\nset_location_assignment PIN_W6  -to ddr4_mem[1].dq[38]\nset_location_assignment PIN_N14 -to ddr4_mem[1].dq[39]\nset_location_assignment PIN_L9  -to ddr4_mem[1].dqs[4]\nset_location_assignment PIN_N10 -to ddr4_mem[1].dqs_n[4]\nset_location_assignment PIN_W9  -to ddr4_mem[1].dbi_n[4]\n\n#---------------------------------------------------------\n# EMIF CH2\n#---------------------------------------------------------\n...\n# CH2 DQS4 (ECC)\nset_location_assignment PIN_GC37 -to ddr4_mem[2].dq[32]\nset_location_assignment PIN_GC41 -to ddr4_mem[2].dq[33]\nset_location_assignment PIN_FY37 -to ddr4_mem[2].dq[34]\nset_location_assignment PIN_GE40 -to ddr4_mem[2].dq[35]\nset_location_assignment PIN_FV36 -to ddr4_mem[2].dq[36]\nset_location_assignment PIN_FY41 -to ddr4_mem[2].dq[37]\nset_location_assignment PIN_GE36 -to ddr4_mem[2].dq[38]\nset_location_assignment PIN_FV40 -to ddr4_mem[2].dq[39]\nset_location_assignment PIN_GE38 -to ddr4_mem[2].dqs[4]\nset_location_assignment PIN_GC39 -to ddr4_mem[2].dqs_n[4]\nset_location_assignment PIN_FV38 -to ddr4_mem[2].dbi_n[4]\n\n#---------------------------------------------------------\n# EMIF CH3\n#---------------------------------------------------------\n...\n# # CH3 DQS4 (ECC)\nset_location_assignment PIN_FP46 -to ddr4_mem[3].dq[32]\nset_location_assignment PIN_FT43 -to ddr4_mem[3].dq[33]\nset_location_assignment PIN_FH47 -to ddr4_mem[3].dq[34]\nset_location_assignment PIN_FP42 -to ddr4_mem[3].dq[35]\nset_location_assignment PIN_FT47 -to ddr4_mem[3].dq[36]\nset_location_assignment PIN_FH43 -to ddr4_mem[3].dq[37]\nset_location_assignment PIN_FK46 -to ddr4_mem[3].dq[38]\nset_location_assignment PIN_FK42 -to ddr4_mem[3].dq[39]\nset_location_assignment PIN_FP44 -to ddr4_mem[3].dqs[4]\nset_location_assignment PIN_FT45 -to ddr4_mem[3].dqs_n[4]\nset_location_assignment PIN_FK44 -to ddr4_mem[3].dbi_n[4]\n
                               
                              2.  
                            2.  
                              Change the pin assignment for ddr4_mem[1].dbi_n[4] from PIN_G12 to PIN_W9
                               
                              set_location_assignment PIN_W9  -to ddr4_mem[1].dbi_n[4]\n
                               
                              3.  
                             
                            10.  
                          10.  
                            Compile the design.
                             
                            cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x <YOUR_WORK_DIRECTORY>\n
                             
                            11.  
                          11.  
                            You may need to adjust the floorplan of the design in order to meet timing after a design change such as this. Refer to the How to Resize the Partial Reconfiguration Region section for information regarding modifications to the floorplan.
                             
                            12.  
                           
                          The configuration edits described here were made to the original source files of the cloned OFS repository. Therefore, these modifications will present in subsequent FIM compilations. This is because the FIM compilation process links and copies source files from the cloned OFS repository to the FIM compilation work directory.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#56-how-to-compile-the-fim-with-no-hssi","title":"5.6 How to compile the FIM with No HSSI","text":"
                          In this example we will compile f2000x with the Ethernet subsystem removed. To perform the flat compile of the FIM with no Ethernet subsystem, pass the no_hssi and flat options to the build_top.sh script:
                           
                          cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh f2000x :flat,no_hssi <YOUR_WORK_DIRECTORY>\n
                           
                          If you wish to build a PR enabled design, you may adjust the Logic Lock regions to allocate more resources to the PR region since the Ethernet subsystem has been removed from the FIM. Refer to the How to Resize the Partial Reconfiguration Region section for information regarding modifications to the floorplan.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#57-how-to-change-the-pcie-device-id-and-vendor-id","title":"5.7 How to change the PCIe device ID and Vendor ID","text":"
                          The PCIe configuration registers contains the Vendor, Device and Subsystem Vendor ID registers which are used in PCIe add-in cards to uniquely identify the card for assignment to software drivers.  OFS has these registers set with Intel values for out of the box usage.  If you are using OFS for a PCIe add in card that your company manufactures, then update the PCIe Subsytem Subsystem ID and Vendor ID registers as described below and change OPAE provided software code to properly operate with your company's register values.
                           
                          The Vendor ID is assigned to your company by the PCI-SIG (Special Interest Group). The PCI-SIG is the only body officially licensed to give out IDs. You must be a member of PCI-SIG to request your own ID. Information about joining PCI-SIG is available here: PCI-SIG. You select the Subsystem Device ID for your PCIe add in card.
                           
                          Follow the instructions below to customize the PCIe device ID and Vendor ID of the f2000x  PLPlatform.
                           
                          You can display the current settings using the command lspci -nvmms <PCIe B.D.f> as shown below:
                           
                          lspci -nvmms 98:00\n
                           Example Output:
                           
                          Slot:   98:00.0\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nPhySlot:        1\nRev:    01\nNUMANode:       1\n\nSlot:   98:00.1\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nPhySlot:        1\nNUMANode:       1\n\nSlot:   98:00.2\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nPhySlot:        1\nNUMANode:       1\n\nSlot:   98:00.3\nClass:  1200\nVendor: 1af4\nDevice: 1000\nSVendor:        1af4\nSDevice:        1771\nPhySlot:        1\nNUMANode:       1\n\nSlot:   98:00.4\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nPhySlot:        1\nNUMANode:       1\n
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#571-changing-the-pcie-subsystem-device-id-and-vendor-id","title":"5.7.1 Changing the PCIe Subsystem Device ID and Vendor ID","text":"
                          You will use IP Parameter Editor to modify the PCIe configuration registers.
                           
                             
                          1.  
                            Navigate to the PCIe Subsystem IP file and bring up IP Parameter Editor to change values.
                             
                            Note: Both the Host and SoC PCIe subsystems use the same IP module, so the following changes to Device ID and Vendor ID will be reflected in both the Host and SoC PCIe Subsystems.
                             
                            cd $OFS_ROOTDIR/ipss/pcie/qip/ss\nqsys-edit pcie_ss.ip\n
                             The IP Parameter Editor GUI will open.  Close any tool pop-ups.
                             
                             
                            2.  
                          2.  
                            Scroll down through the PCIe subsystem settings tabs to the PCIe Interfaces 0 Ports Settings tab as shown below:
                             
                             
                            Select the PCIe0 Device Identification Registers tab. You can edit the values of Vendor ID, Device ID, Subsystem Vendor ID and Subsystem Device ID for each PF/VF in use.
                             
                            3.  
                          3.  
                            Once you have made changes, click Generate HDL and save. 
                             
                            4.  
                          4. Make sure the environment variables are set as described in the Setting Up Required Environment Variables section.
                            5.  
                          5.  
                            Build your new FPGA image with build_top.sh script
                             
                            cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x work_pcie_vid\n
                             
                            6.  
                           
                          Be aware that OPAE FPGA management commands require recognition of the FPGA PCIe Device ID for control.  If there is a problem between OPAE management recognition of FPGA PCIe values, then control of the card will be lost.  For this reason, you are strongly encouraged to initially confiugre the FPGA via JTAG to load the test FPGA image. Instructions for thes process are given in the Configuring the FPGA with a SOF Image via JTAG section. If there is a problem with the SOF image working with your host software that is updated for the new PCIe settings, then you can load a known good SOF file to recover.  Once you sure that both the software and FPGA work properly, you can load the FPGA into FPGA flash using the OPAE command fpgasupdate.
                           
                          The changes to software required to work with new PCIe settings are described in [Software Reference Manual: Open FPGA Stack] 
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#58-how-to-migrate-to-a-different-intel-agilex-7-device-number","title":"5.8 How to migrate to a different Intel Agilex 7 device number","text":"
                          The following instructions enable you to change the Intel Agilex 7 FPGA device part number of the f2000x, for example, to migrate to a device with a larger density. Be aware that this release works with Intel Agilex 7 devices that have P-tile for PCIe and E-tile for Ethernet.
                           
                          The default device for the Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL is AGFC023R25A2E2VR0
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#581-migrating-to-a-larger-device-with-the-same-package","title":"5.8.1 Migrating To a Larger Device With the Same Package","text":"
                          Perform the following steps to change the device to a larger density with the same package.
                           
                             
                          1.  
                            Clone the design repository. See the Clone the OFS Git Repo section.
                             
                            2.  
                          2.  
                            Set the environment variables as described in the Setting Up Required Environment Variables section.
                             
                            3.  
                          3.  
                            Navigate to the OFS Root Directory
                             
                            cd $OFS_ROOTDIR\n
                             
                            4.  
                          4.  
                            Use the following command to change the device part number throughout the OFS Root directory heirarchy, replacing <DEFAULT_OPN> and <NEW_OPN> with the part numbers specific to your update:
                             
                            grep -rli '<DEFAULT_OPN>' * | xargs -i@ sed -i 's/<DEFAULT_OPN>/<NEW_OPN>/g' @\n
                             
                            For example, use the following command to change from part AGFC023R25A2E2VR0 to part AGFA027R25A2E2VR0:
                             
                            grep -rli 'AGFC023R25A2E2VR0'* | xargs -i@ sed -i 's/AGFC023R25A2E2VR0/AGFA027R25A2E2VR0/g' @\n
                             
                            This changes all occurrences of the default device (AGFC023R25A2E2VR0) in the $OFS_ROOTDIR directory to the new device number (AGFA027R25A2E2VR0).
                             
                            5.  
                          5.  
                            Compile the flat (non-PR) design to verify the compilation is successful with the new part. The flat design is compiled without any Logic Lock constraints.
                             
                            cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh f2000x:flat  <YOUR_WORK_DIRECTORY>\n
                             
                            6.  
                          6.  
                            To enable the PR region, use Quartus Chip Planner to analyze the compiled flat design and adjust the Logic Lock constraints defined in $OFS_ROOTDIR/syn/setup/pr_assignments.tcl for the new device layout. Refer to the How to Resize the Partial Reconfiguration Region section for instructions. Re-compile the design with the out-of-tree PR region enabled.
                             
                            cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x  <YOUR_WORK_DIRECTORY>\n
                             
                            7.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#59-how-to-change-ethernet-interface-from-8x25-gbe-to-8x10-gbe","title":"5.9 How to change Ethernet interface from 8x25 GbE to 8x10 GbE","text":"
                          This section describes steps to change the Ethernet interface from 8x25 GbE to 8x10 GbE. 
                           
                             
                          1.  
                            Edit the HSSI IP Subsystem $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss_8x25g.ip to be 8x10 GbE using IP Platform Editor.
                             
                            cd $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss\nqsys-edit hssi_ss_8x25g.ip\n
                             
                            2.  
                          2.  
                            The IP Prameter Editer comes up - expect 2-5 minutes for this process to complete.  When the pop-up indicates Open System Completed, click Close.  When the General Configuration window comes up, scroll down and switch ports 0 through 7 from 25GbE to 10GbE as shown below:
                             
                             
                            3.  
                          3.  
                            Click the IP Configuration tab and note the default settings of OFF for AN/LT and SYNCE.  You may optionally change these settings based on your application needs. The settings for P0 IP cover ports 0 to 3.  The settings for P4 cover ports 4 to 7.
                             
                             
                            4.  
                          4.  
                            Click \"P0 Configuration\" tab and note the default settings for maximum frame size.  You may optionally change these settings based on your application needs.  Set \"P4 Configuration\" as needed. 
                             
                             
                            5.  
                          5.  
                            Leave other settings at default values.
                             
                            6.  
                          6. Click File and Save As hssi_ss_8x10g.ip  Click Generate HDL in the bottom right hand corner of IP Editor and enable simulation support.
                            7.  
                          7.  
                            Edit $OFS_ROOTDIR/ipss/hssi/eth_design_files.tcl to comment out 8x25g and add in 8x10g.ip
                             
                            #-----------------\n# HSSI SS IP\n#-----------------\n#set_global_assignment -name IP_FILE               ../ip_lib/ipss/hssi/qip/hssi_ss/hssi_ss_8x25g.ip\nset_global_assignment -name IP_FILE               ../ip_lib/ipss/hssi/qip/hssi_ss/hssi_ss_8x10g.ip\nset_global_assignment -name IP_FILE               ../ip_lib/ipss/hssi/qip/ptp_iopll/ptp_sample_clk_pll.ip\n
                             
                            8.  
                          8.  
                            Edit $OFS_ROOTDIR/syn/syn_top/ofs_top.qsf and $OFS_ROOTDIR/syn/syn_top/ofs_pr_afu.qsf to add new macro definition:
                             
                            set_global_assignment -name VERILOG_MACRO \"ETH_10G\"    # Change Ethernet from 8x25 to 8x10 GbE\n
                             
                            9.  
                          9.  
                            Build new 8x10G FIM
                             
                            cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x work_8x10gbe\n
                             
                            10.  
                          10.  
                            You may need to adjust the floorplan of the design in order to meet timing after a design change such as this. Refer to the How to Resize the Partial Reconfiguration Region section for information regarding modifications to the floorplan.
                             
                            11.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#510-how-to-change-ethernet-interface-from-8-x-25-gbe-to-2-x-100-gbe","title":"5.10 How to change Ethernet interface from 8 X 25 GbE to 2 X 100 GbE","text":"
                          This section describes steps to change the Ethernet interface from 8 X 25 GbE to 2 x 100 GbE. 
                           
                             
                          1.  
                            Edit HSSI IP Subsystem $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss/hssi_ss_8x25g.ip to be 2 X 100 GbE using IP Platform Editor.
                             
                            cd $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss\nqsys-edit hssi_ss_8x25g.ip\n
                             
                            2.  
                          2.  
                            The IP Prameter Editer comes up - expect 2-5 minutes for this process to complete.  When pop-up indicates Open System Completed, click 'Close'.  The General Configuration window comes up, change ports to 2 and set \"PORT0_PROFILE\" and \"PORT4_PROFILE\" to \"100GCAUI-4\" as shown below:
                             
                             
                            3.  
                          3.  
                            Click the IP Configuration tab and note the default settings of OFF for AN/LT and SYNCE.  You may optionally change these settings based on your application needs.
                             
                             
                            4.  
                          4.  
                            Click \"P0 Configuration\" tab and note the default settings for maximum frame size.  You may optionally change these settings based on your application needs.  Set \"P4 Configuration\" as needed. 
                             
                             
                            5.  
                          5.  
                            Leave other settings at default values.
                             
                            6.  
                          6.  
                            Click File and Save As hssi_ss_2x100g.  Click Generate HDL in the bottom right hand corner of IP Editor and enable simulation support.
                             
                            7.  
                          7.  
                            Edit $OFS_ROOTDIR/ipss/hssi/eth_design_files.tcl to comment out 8x25g and add in 2x100g.ip
                             
                            #-----------------\n# HSSI SS IP\n#-----------------\n#set_global_assignment -name IP_FILE               ../ip_lib/ipss/hssi/qip/hssi_ss/hssi_ss_8x25g.ip\nset_global_assignment -name IP_FILE               ../ip_lib/ipss/hssi/qip/hssi_ss/hssi_ss_2x100g.ip\nset_global_assignment -name IP_FILE               ../ip_lib/ipss/hssi/qip/ptp_iopll/ptp_sample_clk_pll.ip\n
                             
                            8.  
                          8.  
                            Edit $OFS_ROOTDIR/syn/syn_top/ofs_top.qsf and $OFS_ROOTDIR/syn/syn_top/ofs_pr_afu.qsf to add new macro definition:
                             
                            set_global_assignment -name VERILOG_MACRO \"ETH_100G\"      # Change Ethernet from 8x25 to 2x100 GbE\n
                             
                            9.  
                          9.  
                            Update $OFS_ROOTDIR/syn/setup/eth_top.sdc:
                             
                            #Timing for 100G\nset_false_path -from [get_clocks {sys_pll|iopll_0_clk_100m}] -to [get_clocks {hssi_wrapper|hssi_ss|hssi_ss_0|U_hssi_ss_ip_wrapper|U_hssi_ss_ip_top_p*|alt_ehipc3_fm_0|alt_ehipc3_fm_top_p*|alt_ehipc3_fm_hard_inst|E100GX4_FEC.altera_xcvr_native_inst|xcvr_native_s10_etile_0_example_design_4ln_ptp|tx_clkout|ch0}]; \n\nset_false_path -from [get_clocks {hssi_wrapper|hssi_ss|hssi_ss_0|U_hssi_ss_ip_wrapper|U_hssi_ss_ip_top_p*|alt_ehipc3_fm_0|alt_ehipc3_fm_top_p*|alt_ehipc3_fm_hard_inst|E100GX4_FEC.altera_xcvr_native_inst|xcvr_native_s10_etile_0_example_design_4ln_ptp|tx_clkout|ch0}] -to [get_clocks {sys_pll|iopll_0_clk_100m}];   \n
                             
                            10.  
                          10.  
                            Build new 2x100G FIM.
                             
                            cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x work_2x100gbe\n
                             
                            11.  
                          11.  
                            You may need to adjust the floorplan of the design in order to meet timing after a design change such as this. Refer to  for information regarding modifications to the floorplan.
                             
                            12.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#511-how-to-add-more-transceiver-channels-the-ethernet-subsystem","title":"5.11 How to add more Transceiver channels the Ethernet Subsystem","text":"
                          This section describes how to add 4 extra Ethernet channels to the existing f2000x FIM design which uses the 8x25G (2x4x25G) as the default ethernet configuration.
                           
                          In this exercise we will add 4 extra channels to make a total of 12 channels. This configuration will be called 12x25G.
                           
                             
                          1. Clone the design repository or use an existing design. See the Clone the OFS Git Repo section.
                            2.  
                          2. Set the environment variables. See the Setting Up Required Environment Variables section.
                            3.  
                          3.  
                            Navigate to the directory containing the existing Ethernet Subsystem IP hssi_ss_8x25g.ip.
                             
                            cd $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss\n
                             
                            4.  
                          4.  
                            Create a copy of the existing 8x25G IP and name it hssi_ss_12x25g.ip.
                             
                            cp hssi_ss_8x25g.ip hssi_ss_12x25g.ip\n
                             
                            5.  
                          5.  
                            Open the newly created hssi_ss_12x25g.ip file in Platform Designer.
                             
                            qsys-edit hssi_ss_12x25g.ip\n
                             
                            6.  
                          6.  
                            In the HSSI Subsystem > Device 0 Configuration > Main Configuration tab, change the NUM_ENABLED_PORTS value from 8 to 12. 
                             
                             
                            7.  
                          7.  
                            In the HSSI Subsystem > Device 0 Configuration > Main Configuration tab, enable Ports 8 through 11, using the same configuration as the original 8 transceiver ports.
                             
                             
                             
                            8.  
                          8.  
                            Click Generate HDL. Close Platform Designer once generation is complete with no errors.
                             
                            9.  
                          9.  
                            Edit $OFS_ROOTDIR/ipss/hssi/eth_design_files.tcl
                             
                               
                            1.  
                              Comment out the old 8x25G IP and and the new 12x25G IP in the HSSI SS IP section
                               
                              #set_global_assignment -name IP_FILE ../ip_lib/ipss/hssi/qip/hssi_ss/hssi_ss_8x25g.ip\nset_global_assignment -name IP_FILE ../ip_lib/ipss/hssi/qip/hssi_ss/hssi_ss_12x25g.ip\n
                               
                              2.  
                             
                            10.  
                          10.  
                            Edit $OFS_ROOTDIR/ipss/hssi/rtl/hssi_wrapper.sv
                             
                               
                            1.  
                              In the HSSI SS Instantiation section, replace the existing 8x25G IP instantiation with the new 12x25G IP instantiation.
                               
                              //hssi_ss_8x25g\nhssi_ss_12x25g\n
                               
                              2.  
                            2.  
                              In the Serial signal mapping to QSFP section, after the else statement, add the 4 new Ethernet ports:
                               
                              `ifdef INCLUDE_HSSI_PORT_8\nassign serial_rx_p[PORT_8] = qsfp_serial[2].rx_p[0];\nassign serial_rx_n[PORT_8] = 1'b0;\nassign qsfp_serial[2].tx_p[0] = serial_tx_p[PORT_8];\n`endif\n`ifdef INCLUDE_HSSI_PORT_9\nassign serial_rx_p[PORT_9] = qsfp_serial[2].rx_p[1];\nassign serial_rx_n[PORT_9] = 1'b0;\nassign qsfp_serial[2].tx_p[1] = serial_tx_p[PORT_9];\n`endif\n`ifdef INCLUDE_HSSI_PORT_10\nassign serial_rx_p[PORT_10] = qsfp_serial[2].rx_p[2];\nassign serial_rx_n[PORT_10] = 1'b0;\nassign qsfp_serial[2].tx_p[2] = serial_tx_p[PORT_10];\n`endif\n`ifdef INCLUDE_HSSI_PORT_11\nassign serial_rx_p[PORT_11] = qsfp_serial[2].rx_p[3];\nassign serial_rx_n[PORT_11] = 1'b0;\nassign qsfp_serial[2].tx_p[3] = serial_tx_p[PORT_11];\n`endif\n
                               
                              3.  
                             
                            11.  
                          11.  
                            Edit $OFS_ROOTDIR/ipss/hssi/rtl/inc/ofs_fim_eth_plat_defines.svh
                             
                               
                            1.  
                              Define the new port macros in the section configuring 25G with no CVL
                               
                              `define INCLUDE_HSSI_PORT_8\n`define INCLUDE_HSSI_PORT_9\n`define INCLUDE_HSSI_PORT_10\n`define INCLUDE_HSSI_PORT_11\n
                               
                              2.  
                             
                            12.  
                          12.  
                            Edit $OFS_ROOTDIR/ipss/hssi/rtl/inc/ofs_fim_eth_plat_if_pkg.sv
                             
                               
                            1.  
                              Change the parameter defining the number of QSFP ports from 2 to 3
                               
                              localparam NUM_QSFP_PORTS = 3; // QSFP cage on board\n
                               
                              2.  
                            2.  
                              Change the number of ethernet channels parameter for the 25G with no CVL configuration from 8 to 12
                               
                              ```verilog localparam NUM_ETH_CHANNELS = 12; // Ethernet Ports ````
                               
                              3.  
                             
                            13.  
                          13.  
                            Edit $OFS_ROOTDIR/syn/setup/eth_loc.tcl
                             
                               
                            1.  
                              Edit the pinout file to assign pins for the new QSFP. In this example we are using Channels 8-11 in the E-tile.
                               
                              set_location_assignment PIN_DL1  -to qsfp_serial[2].tx_p[0]\nset_location_assignment PIN_DN4  -to qsfp_serial[2].tx_p[1]\nset_location_assignment PIN_DY1  -to qsfp_serial[2].tx_p[2]\nset_location_assignment PIN_EB4  -to qsfp_serial[2].tx_p[3]\n\nset_location_assignment PIN_DL8  -to qsfp_serial[2].rx_p[0]\nset_location_assignment PIN_DN13 -to qsfp_serial[2].rx_p[1]\nset_location_assignment PIN_DY8  -to qsfp_serial[2].rx_p[2]\nset_location_assignment PIN_EB13 -to qsfp_serial[2].rx_p[3]\n
                               
                              2.  
                             
                            14.  
                          14.  
                            Compile the design
                             
                            cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x <YOUR_WORK_DIRECTORY>\n
                             
                            15.  
                          15.  
                            You may need to adjust the floorplan of the design in order to meet timing after a design change such as this. Refer to the How to Resize the Partial Reconfiguration Region section for information regarding modifications to the floorplan.
                             
                            16.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#512-how-to-modify-the-pfvf-mux-configuration","title":"5.12 How to modify the PF/VF MUX configuration","text":"
                          The PF/VF Configuration Tool allows you to easily reconfigure the default number of PFs and VFs on both the SoC and Host side of your design. To modify the PF/VF configuration, you must:
                           
                             
                          1.  
                            Decide which PCIe PF/VFs require modification.  If you are modifying host side PF/VF configuration, you must edit file pcie_host.ofss file found in $OFS_ROOTDIR/tools/pfvf_config_tool.  If you want to modify SoC-side PF/VF configuration, edit the pcie_soc.ofss file found in the same location.  
                             
                            The code given below show the default Host *.ofss file:
                             
                            [ProjectSettings]\nplatform = f2000x \nfamily = Agilex\nfim = base_x16\nPart = AGFC023R25A2E2VR0\nIpDeployFile = pcie_ss.sh\nIpFile = pcie_ss.ip\nOutputName = pcie_ss\nComponentName = pcie_ss\nis_host = True\n\n[pf0]\n\n[pf1]\n
                             
                            This default configuration is made up of two physical functions (PF), and neither of them has any virtual functions (VF). 
                             
                            2.  
                          2.  
                            Modify the OFSS file with the new PF/VF configuration.
                             
                            An example modification to the OFSS file is shown below.  In this example we have changed the configuration to: 6 PFs in total, 4 VFs in PF0, 1 VF in PF2, and 2 VFs on PF3.  You can add up to 8 PFs and could conceivably add up to the number of VFs supported by the PCIe IP. Note that more PFs/VFs will use more FPGA resources, which may cause fitter challenges.
                             
                            [ProjectSettings]\nplatform = f2000x \nfamily = Agilex\nfim = base_x16\nPart = AGFC023R25A2E2VR0\nIpDeployFile = pcie_ss.sh\nIpFile = pcie_ss.ip\nOutputName = pcie_ss\nComponentName = pcie_ss\nis_host = True\n\n[pf0]\nnum_vfs = 4\n\n[pf1]\n\n[pf2]\nnum_vfs = 1\n\n[pf3]\nnum_vfs = 2\n\n[pf4]\n\n[pf5]\n
                             
                            3.  
                          3.  
                            Run the gen_ofs_settings.py script found in $OFS_ROOTDIR/ofs-fim-common/tools/pfvf_config_tool.
                             
                            ./gen_ofs_settings.py\u00a0 --ini $OFS_ROOTDIR/tools/pfvf_config_tool/<PCIE_SOURCE>.ofss --platform <PLATFORM>\n
                             
                            For example, execute the following command to generate Host settings in an f2000x design:
                             
                            ./gen_ofs_settings.py\u00a0 --ini $OFS_ROOTDIR/tools/pfvf_config_tool/pcie_host.ofss --platform f2000x\n
                             
                            This script reconfigures the FIM by:
                             
                               
                            1. Updating the PF/VF Mux Package APIs:
                                 
                              • $OFS_ROOTDIR/src/afu_top/mux/top_cfg_pkg.sv
                                •  
                               
                              2.  
                            2. Adding/removing AFU endpoints
                                 
                              • PF0 VFs - afu_main.port_afu_instances
                                •  
                              • All other functions: afu_top.fim_afu_instances
                                •  
                              • New AFUs will be instantiated as HE-NULL (he_null.sv) AFUs
                                •  
                               
                              3.  
                            3. Updating the pcie_ss.sh \"ip-deploy\" file
                                 
                              • Generating the new pcie_ss.ip file ($OFS_ROOTDOR/ipss/pcie/ss/pcie_ss.ip)
                                •  
                              • Adding scratch register reads/writes to sim/csr_test for added functions
                                •  
                              • Updating the simulation filelists ($OFS_ROOTDIR/sim/common/pfvf_sim_pkg.sv)
                                •  
                               
                              4.  
                             
                            If the port gasket is enabled in the OFSS file, all functions in the PR region must be a virtual function (VF) on physical function 0 (PF0) and are routed to Port AFU Instances (port_afu_instances.sv) in the port gasket.  You can enable the port gasket in the ini (*.ofss file) by adding pg_enable = True under the num_vfs in PF0. In the 2023.3 OFS SoC Attach Release for Intel IPU Platform F2000X-PL the PR region is in the SoC AFU, so the SoC OFSS file has the port gasket enabled by default:
                             
                            [ProjectSettings]\nplatform = f2000x \nfamily = Agilex\nfim = base_x16\nPart = AGFC023R25A2E2VR0\nIpDeployFile = pcie_ss.sh\nIpFile = pcie_ss.ip\nOutputName = pcie_ss\nComponentName = pcie_ss\nis_host = False\n\n[pf0]\npg_enable = True\nnum_vfs = 3\n
                             
                            If the port gasket is disabled, the virtual functions on PF0 are routed to FIM AFU Instances (fim_afu_instances.sv) in the static region.  All physical functions and virtual functions not on PF0 are routed to the FIM AFU Instances module (fim_afu_instances.sv) in afu_top.
                             
                            After you run the gen_ofs_settings.py script you should see a final success report:
                             
                            2022.07.28.21:46:38 Info: Regenerate these scripts whenever you make any change to any Quartus-generated IP in your project.\n2022.07.28.21:46:38 Info: Finished: Create simulation script\nsh: /home/applications.fpga.ofs.rtl/env_not_shipped/f2000x/update_sim.sh: No such file or directory\nSuccess!  Thank you for using the IP-Deploy Tool\n
                             
                            4.  
                          4.  
                            Recompile the FIM using the build_top.sh script described in the Compiling the FIM section of this guide. 
                             
                            5.  
                          5.  
                            Verify the correct functionality of new the PF/VF Mux configuration.
                             
                            New PF/VF are seamlessly connected to their own CSR stub, which can be read at DFH Offset 0x0. You can bind to the function and perform opae.io peek commands to read from the stub CSR. Similarly, perform opae.io poke commands to write into the stub CSRs. Use this mechanism to verify that the new PF/VF Mux configuration allows to write and read back values from the stub CSRs. 
                             
                            The GUID for every new PF/VF CSR stub is the same.   
                             
                               
                            • NULL_GUID_L           = 64'haa31f54a3e403501
                              •  
                            • NULL_GUID_H           = 64'h3e7b60a0df2d4850
                              •  
                             
                            Limitations: Setting 0 virtual functions on SoC PF0 is not supported. This is because the PR region cannot be left unconnected. A loopback may need to be instantiated in this special case. 
                             
                            Load the newly compiled FIM to the card to test the functionality of the new PF/VF functions. Use the following commands to verify the number of PFs/VFs created:
                             
                            sudo lspci -vvv -s b1:00.0 | grep VF\n
                             Example output: 
                            Initial VFs: 4, Total VFs: 4, Number of VFs: 0, Function Dependency Link: 00\nVF offset: 6, stride: 1, Device ID: bccf               \n
                             
                            Note: The PCIe B:D.F associated with your board may be different. Use the fpgainfo fme command to see the PCIe B:D:F for your board.
                             
                            6.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#513-how-to-create-a-minimal-fim","title":"5.13 How to Create a Minimal FIM","text":"
                          In this example, the exercisers and Ethernet subsystem are removed and a new AFU PR area is used to make use of the added area from the removed components.  This minimal FIM is useful for HDL applications.
                           
                          To create this minimal FIM, perform the following steps:
                           
                             
                          1.  
                            Change the PF/VF configuration to support only 1 PF with 1 VF. Edit the following files in the $OFS_ROOTDIR/tools/pfvf_config_tool/ directory as shown below:
                             
                               
                            1.  
                              $OFS_ROOTDIR/tools/pfvf_config_tool/pcie_host.ofss
                               
                              [ProjectSettings]\nplatform = f2000x \nfamily = Agilex\nfim = base_x16\nPart = AGFC023R25A2E2VR0\nIpDeployFile = pcie_ss.sh\nIpFile = pcie_ss.ip\nOutputName = pcie_ss\nComponentName = pcie_ss\nis_host = True\n\n[pf0]\n
                               
                              2.  
                            2.  
                              $OFS_ROOTDIR/tools/pfvf_config_tool/pcie_soc.ofss 
                               
                              [ProjectSettings]\nplatform = f2000x \nfamily = Agilex\nfim = base_x16\nPart = AGFC023R25A2E2VR0\nIpDeployFile = pcie_ss.sh\nIpFile = pcie_ss.ip\nOutputName = soc_pcie_ss\nComponentName = pcie_ss\nis_host = False\n\n[pf0]\nnum_vfs = 1\npg_enable = True\n
                               
                              3.  
                             
                            2.  
                          2.  
                            Save the modified ofs_dev.ofss file and build a new configuration.
                             
                            python3 gen_ofs_settings.py --ini $OFS_ROOTDIR/tools/pfvf_config_tool/pcie_host.ofss --platform f2000x\n\npython3 gen_ofs_settings.py --ini $OFS_ROOTDIR/tools/pfvf_config_tool/pcie_soc.ofss --platform f2000x\n
                             
                            3.  
                          3.  
                            Compile the new FIM with exercisers removed.
                             
                            cd $OFS_ROOTDIR\n./ofs-common/scripts/common/syn/build_top.sh -p f2000x:null_he,null_he_lp,null_he_hssi,null_he_mem,null_he_mem_tg,no_hssi work_null_he_no_hssi\n
                             
                            4.  
                          4.  
                            The build will complete with reduced resources as compared to the base version. You may review the floorplan in Quartus Chip Planner and modify the Logic Lock regions to allocate more resources to the PR region if desired. Refer to the How to Resize the Partial Reconfiguration Region section for information regarding modifications to the floorplan. 
                             
                            5.  
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#6-single-event-upset-reporting","title":"6 Single Event Upset Reporting","text":"
                          A Single Event Upset (SEU) is the change in state of a storage element inside a device or system. They are caused by ionizing radiation strikes that discharge the charge in storage elements, such as configuration memory cells, user memory and registers.
                           
                          Error Detection CRC (EDCRC) circuitry in the Card BMC is used to detect SEU errors. The CRC function is enabled in Intel Quartus Prime Pro to enable CRC status to be reported to the FM61 via the dedicated CRC_ERROR pin.
                           
                          With the EDCRC there is no method to determine the severity of an SEU error i.e. there is not way to distinguish between non-critical and catastrophic errors. Hence once the SEU error is detected, the Host system must initiate the Card BMC reset procedure.
                           
                          SEU errors can be read from either the Card BMC SEU Status Register or the PMCI Subsystem SEU Error Indication Register. The processes to read these registers are described in greater detail in the BMC User Guide. Contact your Intel representative for access to the BMC User Guide.
                           
                          Additionally, refer to the [Intel Agilex SEU Mitigation User Guide] for more information on SEU detection and mitigation.
                          "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                          Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                           
                          OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                          "},{"location":"hw/f2000x/doc_modules/Glossary/","title":"Glossary","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to user space."},{"location":"hw/f2000x/doc_modules/Notices_%26_Disclaimers/","title":"Notices & Disclaimers","text":""},{"location":"hw/f2000x/doc_modules/Notices_%26_Disclaimers/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                          Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Performance varies by use, configuration and other factors. Your costs and results may vary. You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for the safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/","title":"FPGA Interface Manager Technical Reference Manual for Intel Agilex 7 SoC Attach: Open FPGA Stack","text":"
                          Last updated: February 03, 2024 
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1-overview","title":"1 Overview","text":""},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#11-about-this-document","title":"1.1 About this Document","text":"
                          This document describes the hardware architecture for the SoC Attach reference FIM of the Open FPGA Stack (OFS) targeting the Intel\u00ae Agilex 7 FPGA.  After reviewing this document you should understand the features and functions of the components that comprise the FPGA Interface Manager (FIM), also known as the \"shell.\"
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#12-glossary","title":"1.2 Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#13-introduction-to-open-fpga-stack","title":"1.3 Introduction to Open FPGA Stack","text":"
                          The Open FPGA Stack (OFS) is a modular infrastructure of hardware platform components, open source unstreamed software, and broad ecosystem support that enables an efficient path to develop a custom FPGA platform.  OFS Provides a framework of FPGA synthesizable code, simulation environment and synthesis/simulation scripts. 
                           
                          The key components of OFS include:
                           
                             
                          • Target development platforms such as Intel-branded  Acceleration Development Platforms (ADPs) and third-party platforms.
                            •  
                           
                             
                          • Board Management Controller RTL and firmware that supports telemetry monitoring and capability for remote configuration updates.
                            •  
                           
                             
                          • Source accessible, modular FPGA Interface  manager (FIM) RTL with a UVM infrastructure unit tests that can be leveraged for your own custom FIM design. The FIM can be thought of as the FPGA shell that provides the I/O ring and timing closed management components for the FPGA.
                            •  
                           
                             
                          • Basic building blocks for interconnect and PF/VF translation and arbitration; Platform Interface Manager (PIM) which provides Avalon\u00ae and Arm\u00ae AMBA\u00ae 4 AXI4 bus compliant interfaces.
                            •  
                           
                             
                          • AFU examples both in the git repository and workload examples provided by 3rd party vendors.
                            •  
                           
                             
                          • Unit level simulation test suite
                            •  
                           
                             
                          • System level simulation through a unified verification methodology (UVM)
                            •  
                           
                             
                          • OPAE software development kit (APIs, up-streamed Linux drivers and software tools)
                            •  
                           
                             
                          • Support for other frameworks to be built on top of the OPAE such as DPDK 
                            •  
                           
                          These components are available in a two GitHub locations:
                           
                             
                          • OFS hardware GitHub site
                            •  
                          • OPAE software GitHub site
                            •  
                           
                          The OFS hardware repository supports hardware development and simulation.  Repositories for OFS high-level design support and board management controller RTL and firmware source code are also provided.  These repositories can be found in the Intel Opensource Technology GitHub location, which requires entitlement access.  To request access, please contact your local Intel sales representative.
                           
                          Table 1-2 OFS Hardware Repositories
                           Repository Contains OFS f2000x FIM Github Branch Contains FIM or shell RTL, automated compilation scripts, and unit tests and UVM framework. 
                          The OPAE software GitHub site is fully opensource and contains resources for both software and workload developers.
                           
                          Table 1-3  OFS Software Repositories
                           OPAE Git Repository Folder Contains OPAE SDK Repo Contains the files for building and installing OPAE SDK from source. Linux DFL Contains OFS Linux drivers that are being upstreamed to the Linux kernel. ofs-platform-afu-bbb Contains the files and scripts to build the platform interface manager. opae-sim Contains the files for an AFU developer to build the Accelerator Functional Unit Simulation Environment (ASE) for workload development. 
                          Providing the hardware and software source code and supporting test frameworks in a GitHub repository allows you to customize your designs with the latest versions easily.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14-ofs-features","title":"1.4 OFS Features","text":"
                          The OFS architecture within the FPGA comprises two partitions:
                           
                             
                          • FPGA Interface Manager (FIM)
                            •  
                           
                             
                          • Accelerator Functional Unit (AFU)     -   AFU SoC Dynamic Region and Static Region      -   AFU Host Static Region 
                            •  
                           
                          The FIM or shell provides platform management functionality, clocks, resets, and interface access to the host and peripheral features of the acceleration platform. 
                           
                          The FIM architecture along with the supporting OPAE software supports features such as partial reconfiguration and virtualization.
                           
                          The FIM provides a standard Arm\u00ae AMBA\u00ae 4 AXI4 datapath interface. The FIM resides in the static region of the FPGA.
                           
                          The AFU partition is provided for custom acceleration workloads and may contain both static and partial reconfiguration regions.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#141-fpga-interface-manager-fim","title":"1.4.1 FPGA Interface Manager (FIM)","text":"
                          The updated OFS architecture for Intel\u00ae Agilex\u00ae 7 FPGA devices improves upon the modularity, configurability and scalability of the first release of the OFS architecture while maintaining compatibility with the original design.  The primary components of the FPGA Interface Manager or shell of this reference design are: 
                           
                             
                          • P-tile PCIe Subsystem
                            •  
                          • E-Tile Ethernet Subsystem
                            •  
                          • Memory Subsystem
                            •  
                          • Reset Controller
                            •  
                          • FPGA Management Engine (FME)
                            •  
                          • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                            •  
                          • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                            •  
                          • Platform Management Controller Interface (PMCI) to the board management controller
                            •  
                           
                          The AFU Region provides design space for custom workloads and contains both static and partial reconfiguration regions.  Partial reconfiguration allows you to update your specific logic blocks or entire workload while the rest of your static design is still in operation.
                           
                          Note that as discussed previously, the BMC RTL and firmware, the OFS OPAE software stack and support for building your own customer board support package are also provided in separate OFS repositories.
                           
                          Figure 1-1 OFS for OFS FIM Platform Block Diagram
                           
                           
                          The table provides an overview of the OFS features targeting the Intel\u00ae Agilex\u00ae 7 FPGA.  This reference FIM (shell) is a starting point for your custom FPGA design.  With this initial starting point, you can add or subtract interfaces or ports to different Intel Agilex 7 devices.
                           
                          Table 1-4 OFS FIM for Intel\u00ae Agilex\u00ae 7 FPGA Features
                           Key Feature Description PCIe P-tile PCIe* Gen4x16 to the HostP-tile PCIe* Gen4x16 to the SoC (IceLake-D) Virtualization Host: 2 physical functions SoC:   1 physical function and 3 Virtual functions Memory Four Fabric DDR4 banks, x40 (optional ECC, be configured as x32 and ECC x8 ), 1200 MHz, 4GB Ethernet Eight Arm\u00ae AMBA\u00ae 4 AXI4-Stream channels of 25G Ethernet interfacing to an E-tile Ethernet Subsystem. Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration) * Platform Controller Management Interface (PMCI) Module contained within the Intel Agilex 7 FPGA that interfaces through Avalon-Streaming x8 QuadSPI and SPI to a Board Management Controller Partial Reconfiguration Partial Reconfiguration region supported in hardware and software Software Support * Linux DFL drivers targeting OFS FIMs * OPAE Software Development Kit * OPAE Tools"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#subsystem-interfaces","title":"Subsystem Interfaces","text":"
                          The PCIe, Memory and Ethernet interfaces in this design use a new flexible subsystem design that provides a standard Arm\u00ae AMBA\u00ae 4 AXI4 interface.  To access these Intel FPGA IP Subsystem documents. Please go here and search for the following ID numbers: * 690604: PCIe Subsystem IP User Guide (Note: you must login to myIntel and request entitled access) * 686148: Memory Subsystem IP User Guide (Note: you must login to myIntel and request entitled access) * 773413: [Ethernet Subsystem Intel FPGA IP] (public document)
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#fpga-management-engine-fme","title":"FPGA Management Engine (FME)","text":"
                          The FIM contains only one FME, regardless of the number of host interfaces to the FIM. The FME provides management features for the platform and controls reset and loading of the AFU into the partial reconfiguration region of the FPGA.
                           
                          Any feature, such as a memory interface or global error control that you want to control through FME, must expose its capability to host software drivers.  New features are exposed to the FME by adding a device feature header (DFH) register at the beginning of the feature's control status register (CSR) space. The FME CSR maps to physical function 0 (PF0) Base address register 0 (BAR0) so that software can access it through a single PCIe link.  For more information about DFHs, refer to the FPGA Device Feature List Framework Overview
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#streaming-datapath","title":"Streaming Datapath","text":"
                          The FIM implements an Arm\u00ae AMBA\u00ae 4 AXI4-Stream bus protocol for data transfer in the FIM. Arm\u00ae AMBA\u00ae 4 AXI4-Stream channels send data packets to and from the host channel IP without data abstraction. Memory-mapped I/O (MMIO) CSR accesses are routed to the ST2MM module, which converts the Arm\u00ae AMBA\u00ae 4 AXI4-Stream to an Arm\u00ae AMBA\u00ae 4 AXI4 memory-mapped protocol.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#virtualization","title":"Virtualization","text":"
                          This design supports virtualization by making use of the virtualization functionality in the PCIe Hard IP and mapping packets to the appropriate physical or virtual function through a PF/VF multiplexer.  
                           
                          This reference FIM example supports 2 PFs for the host and 1PF, 3VFs for the SoC; however, you may extend your configuration to whatever the PCIe Hard IP can support or your application requires.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#142-afu","title":"1.4.2 AFU","text":"
                          An AFU is an acceleration workload that interfaces with the FIM. The AFU boundary in this design comprises both static and partial reconfiguration (PR) regions.  You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region.  A port gasket within the design provides all the PR-specific modules and logic required for partial reconfiguration. Only one partial reconfiguration region is supported in this design for SoC AFU.
                           
                          Similar to the FME, the port gasket exposes its capability to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space.  
                           
                          You can compile your design in one of the following ways:  * Your AFU (workload) for Host (Static Area) and SoC resides in the partial reconfiguration area.  * Your AFU is part of the static region and is compiled from a flat design. 
                           
                          In this design, the AFU region is comprised of: * AFU Interface handler to verify transactions coming from AFU region. * PF/VF Mux to route transactions to and from corresponding AFU components:    * Host:     * ST2MM module.     * PCIe loopback host exerciser (HE-LPBK) .
                           
                          * SoC:     * ST2MM module.     * Ethernet Subsystem (previous HSSSI) host exerciser (HE-HSSI).     * Memory Host Exerciser (HE-MEM).     * Traffic Generator to memory (HE-MEM-TG).     * Port Gasket (PRG).
                           
                             
                          • Arm\u00ae AMBA\u00ae 4 AXI4 Streaming to Memory Map (ST2MM) Module that routes MMIO CSR accesses to FME and board peripherals.
                            •  
                          • Host exercisers to test PCIe, memory and Ethernet Interfaces (these can be removed from the AFU region after your FIM design is complete to provide more resource area for workloads)
                            •  
                          • Port gasket and partial reconfiguration support.
                            •  
                          • Component for handling PLDM over MCTP over PCIe Vendor Defined Messages (VDM)
                            •  
                           
                          For this design the PF/VF Mux provides the following mappings (found in /src/afu_top/mux/top_cfg_pkg.sv and /src/afu_top/mux/soc_top_cfg_pkg.sv):
                           
                          Table 1-5 PF/VF Mapping
                           
                          SoC (IceLake-D)
                           PF VF Feature BAR Bar Size PF0 OFS managed peripherals BAR 0 2MB AFU peripherals  Board peripherals BAR 0 256KB  768KB PF0 VF0 HE-MEM BAR 0 2MB VF1 HE-HSSI BAR 0 2MB VF2 HE-MEM TG BAR 0 2MB 
                          Xeon Host
                           PF VF Feature BAR Bar Size PF0 OFS managed peripherals BAR 0 1MB AFU peripherals  Board peripherals 256KB  768KB PF1 HE-LPBK BAR 0 16KB 
                          Figure 1-2 AFU Diagram 
                           
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#143-platform-interface-manager","title":"1.4.3 Platform Interface Manager","text":"
                          The PIM provides a way to abstract the Arm\u00ae AMBA\u00ae 4 AXI4-Stream interface to the AFU by providing a library of shims that convert the host channel native packet into other protocols such as Arm\u00ae AMBA\u00ae 4 AXI4 memory-mapped, Avalon\u00ae streaming (Avalon-ST) or Avalon\u00ae memory-mapped (Avalon-MM). 
                           
                          The FPGA or AFU developer implements these interface abstractions in the AFU regions (afu_top and soc_afu_top) of the design.  
                           
                          For more information, refer to the AFU Developer's Guide.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#144-platform-feature-discovery","title":"1.4.4 Platform Feature Discovery","text":"
                          This reference design comes with specific Intel FPGA drivers that are upstreamed to linux-dfl.  These drivers abstract the hardware and operating system specific details of the platform to the host. 
                           
                          The FIM implements a device feature header (DFH) at the beginning of each host-discoverable feature in a linked list format that allows an FPGA platform driver running on the host to discover FME, port, and AFU features. 
                           
                          You must implement a 64-bit DFH Device Feature Header register at the beginning (first 8B aligned address) of the feature CSR space for a new feature to be discovered or enumerated by a driver.  
                           
                          During host discovery, the driver traverses the DFH of the first feature from the first address on PF0 BAR0. Based on the information in the DFH, a driver can determine the CSR address range of the feature and other associated details. The end of the DFH contains a \"next DFH offset\" field that points the driver to the DFH of the next feature. 
                           
                          The software must continue traversing the linked list until it sees the EOL (End-Of-List) bit set to 1 in the \"next DFH offset\" field it inspects.  A 1 indicates this is the last feature in the feature set. The figure below gives a simple illustration of the feature discovery by traversing the DFH registers. This model is similar to how PCIe enumeration occurs.
                           
                          Figure 1-3 Device Feature Header Linked List Traversal
                           
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#145-ofs-reference-design","title":"1.4.5 OFS Reference Design","text":"
                          OFS provides FIM designs you can use as a starting point for your own custom design. These designs target a specific programmable acceleration card or development kit and exercise key FPGA device interfaces. 
                           
                          The Intel Agilex\u00ae 7 code line for OFS targets the Intel IPU Platform F2000X-PL.  FIM designs are released to for evaluation and use. 
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#146-fim-simulation","title":"1.4.6 FIM Simulation","text":"
                          OFS provides unit tests and a UVM environment for the FIM and a framework for new feature verification. UVM provides a modular, reusable, and scalable testbench structure by providing an API framework that can be deployed across multiple projects. 
                           
                          The FIM testbench is UVM compliant and integrates third-party verification IPs from Synopsys that require a license. 
                           
                          Verification components include:
                           
                             
                          • FIM monitor to detect correct design behavior
                            •  
                           
                             
                          • FIM assertions for signal level integrity testing
                            •  
                           
                             
                          • Arm\u00ae AMBA\u00ae 4 AXI4 scoreboards to check data integrity
                            •  
                           
                             
                          • FIM coverage to collect functional data
                            •  
                           
                          The verification infrastructure can be found here for evaluation and use. Please refer to the Simulation User Guide for more information.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#2-ofs-high-level-architecture","title":"2 OFS High Level Architecture","text":"
                          OFS provides distinct data paths that simplify the design and integration process for adding or removing interface modules:
                           
                             
                          • High Bandwidth data path for AFU-attached high-performance peripherals (Ethernet Subsystem, Memory,  workload).
                            •  
                          • Low Bandwidth data path for OFS management and slow peripheral components (JTAG, I2C, SMBus).
                            •  
                          • AFU Peripheral Fabric (APF) to Board Peripheral Fabric (BPF) path to communicate with interface control and status registers (CSRs) and board components.
                            •  
                          • Peer-to-peer datapath between AFU components.
                            •  
                          • Peer-to-peer datapath between BPF components.
                            •  
                           
                          Depending on your design goals, you can present peripherals to software as:
                           
                             
                          • OFS managed peripherals with a device feature header that is part of a device feature list.
                            •  
                          • Native driver managed peripherals that are exposed through an independent physical function or virtual function.
                            •  
                           
                          Figure 2-1 OFS Datapath Structure 
                           
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#3-pcie-subsystem","title":"3 PCIe Subsystem","text":"
                          The FIM's PCIe Subsystem is a hierarchical design that targets the P-tile PCIe* hard IP and is configured to support Gen4 speeds and Arm\u00ae AMBA\u00ae 4 AXI4-Stream Data Mover functional mode.  The IP supports SR-IOV and is configured to provide  2 PFs for the host and 1PF, 3VFs for the SoC.  Native PCIe TLP packets are sent through the PCIe usingArm\u00ae AMBA\u00ae 4 AXI4 Stream Protocol.  Before they reach the AFU, the packets go through an adapter in the subsystem that converts any headers to a data mover format.  Tag allocation and management for packets sent from the application to the host are done by the PF/VF Mux which is part of the AFU region.
                           
                          Figure 3-1 OFS FIM RX-TX Datapath
                           
                           
                          Some key features of the PCIe interface are:
                           Feature OFS for Intel Intel Agilex 7 FPGA SoC Attach Subsystem Configuration Mode Host: PCIe Gen4x16SoC: PCIe Gen4x16 Tile P-Tile Port Mode Native Endpoint SR-IOV Host: 2 PFs, No VFsSoC:  1 PFs, 3 VFs MSI-X Support Yes Functional Mode Data Mover Profile Basic TLP Bypass No Header Packing Scheme Simple Data Width 512-bit (64-byte) Arm\u00ae AMBA\u00ae 4 AXI4-ST Clock Frequency 500 MHz Tags Supported 128 Reordering Enabled with buffer 64 KB Maximum Payload Size 512 Bytes Memory Requests Supported 1CL, 2CL, 4CL MMIO transaction Size 4B, 8B Control Shadow Interface Enabled Completion Timeout Interface Enabled 
                          The PCIe PF, VF and Base Address Register (BAR) configuration can be modified in the PCIe Subsystem Platform Designer GUI interface.  The current implementation for the OFS FIM for Intel IPU Platform F2000X-PL is as follows:
                           
                          Table 3-1 Function and BAR Table for OFS for Intel IPU Platform F2000X-PL
                           
                          SoC (IceLake-D)
                           PF VF Feature BAR Bar Size PF0 OFS managed peripherals BAR 0 2MB AFU peripherals  Board peripherals BAR 0 256KB  768KB PF0 VF0 HE-MEM BAR 0 2MB VF1 HE-HSSI BAR 0 2MB VF2 HE-MEM TG BAR 0 2MB 
                          Xeon Host
                           PF VF Feature BAR Bar Size PF0 OFS managed peripherals BAR 0 1MB AFU peripherals  Board peripherals 256KB  768KB PF1 HE-LPBK BAR 0 16KB"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#31-pcie-subsystem-header-format","title":"3.1 PCIe Subsystem Header Format","text":"
                          The first 32 bytes of the TLP from the PCIe subsystem denotes the PCIe header. There are two types of header format supported \u2013 Power User Mode header and Data Mover mode header. The tuser_vendor[0] bit on the Arm\u00ae AMBA\u00ae 4 AXI4-Stream channel indicates the header format of the TLP; tuser_vendor[0] =0 indicates Power User Mode header and tuser_vendor[0] =1 indicates Data Mover Mode header.
                           
                          The OFS FIM for Intel Intel Agilex 7 FPGA implements the Data Mover Functional Mode.  With this implementation, the application has the flexibility to use either mode for PCIe transaction, as shown in the following table. For more detailed information about the PCIe Subsystem, see the PCIe Subsystem Intel FPGA User Guide. 
                           
                          Table 3-2 PCIe Subsystem Header Format Support for OFS for Intel Agilex 7 FPGA
                           Direction Type Power User Data Mover Host to Endpoint  MWr, MRd Yes No CPL/CPLd Yes Yes Msg Yes No Endpoint to Host MWr, MRd Yes Yes Intr Yes (MWr)  Yes CPL/CPLd Yes Yes Msg Yes Yes"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#311-power-user-header-format","title":"3.1.1 Power User Header Format","text":"
                          The Power User Format provides user complete control over PCIe Hard IP. The user can implement functionality of interest with finer control over PCIe Transaction Layer Packet (TLP), credit handling and various mode provided by HIP. 
                           
                          The lower 16 bytes of the Power User Format are standard PCIe header as defined by PCIe specification, and the upper 16 bytes are specific to the PCIe Subsystem Intel FPGA IP. 
                           
                          Table 3-3 Power User Header Format
                           
                           
                          The mapping of the PCIe defined header to the lower 16 bytes of the Arm\u00ae AMBA\u00ae 4 AXI4-Stream data channel is shown in the figure below. Each double word (DW) or 4 bytes in the PCIe header is mapped from big-endian to little-endian on Arm\u00ae AMBA\u00ae 4 AXI4-S data channel.  
                           
                          Figure 3-2 Power User Header Mapping to Arm\u00ae AMBA\u00ae 4 AXI4 Channel
                           
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#312-data-mover-header-format","title":"3.1.2 Data Mover Header Format","text":"
                          The data mover mode allows high bandwidth data transfers to and from Host memory. It hides the complexity of handling PCIe TLPs. This format provides a simple interface for reading and writing to Host Memory. The data mover checks link partner credits before transmitting packets. It also provides MSI-X interrupt generation capability. 
                           
                          In Data Mover Mode, the lower 16 bytes are data mover specific and do not follow a PCIe standard header.  
                           
                          Table 3-4 Data Mover Header Format
                           
                           
                           
                          The mapping of the data mover header to the lower 16 bytes of the Arm\u00ae AMBA\u00ae 4 AXI4-S data channel is shown below. Each byte in the data mover header is mapped directly to the Arm\u00ae AMBA\u00ae 4 AXI4-S data channel.  
                           
                          Figure 3-3 Data Mover Header Mapping to Arm\u00ae AMBA\u00ae 4 AXI4 Channel
                           
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#32-pcie-subsystem-interface-module","title":"3.2 PCIe Subsystem Interface Module","text":"
                          The PCIe Subsystem Interface module (/ipss/pcie/rtl/pcie_ss_if.sv), provides the supporting interface between software and the PCIe subsystem.  
                           
                          The interface module provides the following:
                           
                             
                          • Device Feature Header Registers
                            •  
                          • Control and Status Registers
                            •  
                          • Indirect access to PCIe subsystem CSR registers through a CSR mailbox in the PCIe Subsystem Interface.
                            •  
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#33-data-mover-request-cycles","title":"3.3 Data Mover Request Cycles","text":"
                          For Host read request cycles using the OFS FIM for Intel Agilex 7 FPGA: * All requests in the RX direction will be MMIO.  * Requester ID from the request does get sent to the AFU. It is the AFU's responsibility to send back a completion to the host with the correct completer ID.  * Prefix is not supported.  * Memory Mapped (MM) Mode is not supported. * Slot Number is 0. * Base address is not sent to the AFU. * Local Address field is not used.
                           
                          For AFU/application request cycles using the OFS FIM for Intel Agilex 7 FPGA: * All requests in the TX direction will be Memory Read/Write.  * The tag must be generated by the AFU/application. * Prefix is not supported.  * MM Mode is not supported. * Slot Number is 0 (non-0 only for switch) * VF Active, VF number and PF number are obtained from Data Mover Header Packet.
                           
                          Figure 3-4 Data Mover Request Cycles
                           
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#34-data-mover-completion-cycles","title":"3.4 Data Mover Completion Cycles","text":"
                          For Host completion cycles using the OFS FIM for Intel Agilex 7 FPGA: * All completions in the RX direction will be Data Completions.  * Prefix is not supported. * MM Mode is not supported. * Slot Number is 0. * Data packet responses (for Memory Read requests from AFU) from the PCIe SS may come out of order when the size is >64B.
                           
                          For AFU/application completion cycles using the OFS FIM for Intel Agilex 7 FPGA: * All requests in the TX direction will be Memory Read/Write.  * Requester ID is generated within the FIM. * That tag must be generated by the AFU/application. * Prefix is not supported.  * MM Mode is not supported. * Slot Number is 0. * VF Active, VF Number and PF number are obtained from the Data Mover Header Packet. 
                           
                          Figure 3-5 Data Mover Completion Cycles
                           
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#4-platform-interface-manager","title":"4 Platform Interface Manager","text":"
                          The FIM interfaces to an application in the AFU region through Arm\u00ae AMBA\u00ae 4 AXI4-Stream channels.  This format allows the AFU to access the host channel's raw interface without any translation. 
                           
                          As a FIM developer, you have the option to provide the raw data format associated with the host interface channel to the workload or AFU developer or you can provide an intermediate protocol using Platform Interface Manager Components or your own custom interface.
                           
                          If you expose the raw Arm\u00ae AMBA\u00ae 4 AXI4-Stream interface of the FIM, workload developers also have the option to convert to a desired protocol using the PIM resources as well.  
                           
                          Refer to the AFU Developer Guide and the FPGA Interface Manager Developer Guide for more information on using the PIM in your design.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#5-interconnect-fabric","title":"5 Interconnect Fabric","text":"
                          There are three types of interconnect fabric in the OFS FIM design:  * Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF mux/demux fabric  * AFU Peripheral Fabric (APF)  * Board Peripheral Fabric (BPF)
                           
                          Figure 5-1 Interconnect Fabric Diagram
                           
                           
                          TLP packets sent from upstream PCIe Subsystem on Arm\u00ae AMBA\u00ae 4 AXI4-Stream channel are demultiplexed in the Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF mux/demux fabric and routed to the respective PF/VF function based on the PF/VF information in the TLP header, such as vf_active or the PF/VF number. In the opposite direction, TLP packets from downstream PF/VF function are muxed in the fabric and sent to PCIe subsystem over Arm\u00ae AMBA\u00ae 4 AXI4-Stream channel.
                           
                          All host MMIO requests targeting PF0 BAR0 are routed to the ST2MM module. The ST2MM converts MMIO TLP packets into Arm\u00ae AMBA\u00ae 4 AXI4-Lite memory requests and places the requests onto AFU Peripheral Fabric (APF). AFU peripherals, such as OFS managed AFU features and ST2MM, and Board Peripheral Fabric (BPF) are interconnected by APF. The BPF is the interconnect fabric one hierarchy below APF which connects all the board peripherals. Both APF and BPF allow multiple Arm\u00ae AMBA\u00ae 4 AXI4-Lite master and slave interconnect topology.
                           
                          If you are modifying the APF or BPF connections, you must use Platform Designer to generate the fabrics directly.  Please refer to the FPGA Interface Manager Developer Guide for directions on what files must be modified and how to generate the interconnect.
                           
                          For modifying the PF/VF mux you must update parameters in these files: * src/includes/top_cfg_pkg.sv * src/common/pf_vf_mux.sv Then make the corresponding update to AFU top level instantiation and connections: * src/afu_top/soc_afu_top.sv(SoC_AFU) and src/afu_top/afu_top.sv (HOST_AFU)
                           
                          For details on these modifications, please refer to the FIM Interface Manager Developer Guide.  
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#51-afu-peripheral-fabric-apf","title":"5.1 AFU Peripheral Fabric (APF)","text":"
                          The AFU Peripheral Fabric (APF) is a 64-bit Arm\u00ae AMBA\u00ae 4 AXI4-lite compliant interconnect fabric that connects AFU peripheral modules to board peripheral modules through the Board Peripheral Fabric (BPF). 
                           
                          The fabric is clocked by clk_csr and has a read allowance and write allowance of 1, i.e. only 1 active write/read is allowed in the fabric at any single time. 
                           
                          The address mapping for components interconnected by the APF is listed below. All components are mapped to PF0 BAR0 and implement Arm\u00ae AMBA\u00ae 4 AXI4-lite slave interface. Note that none of the features in the APF mapping are designed to act as a master.
                           
                          Table 5-1 APF Address Mapping
                           
                          SoC PF0 BAR 0
                           Address Size (Byte) Feature 0x00000\u20130xFFFFF 1024K Board Peripherals (See BPF address mapping) AFU Peripherals 0x100000 \u2013 0x10FFFF 64K ST2MM 0x130000 \u2013 0x13FFFF 64K PR Gasket: 4K= PR Gasket DFH, control and status 4K= Port DFH 4K=User Clock 52K=Remote STP 0x140000 \u2013 0x14FFFF 64K AFU Error Reporting (Protocol checker) 
                          Host PF0 BAR 0
                           Address Size (Byte) Feature 0x00000\u20130xFFFFF 1024K Board Peripherals (See BPF address mapping) AFU Peripherals 0x100000 \u2013 0x10FFFF 64K ST2MM 0x140000 \u2013 0x14FFFF 64K AFU Error Reporting (Protocol checker)"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#52-board-peripheral-fabric-bpf","title":"5.2 Board Peripheral Fabric (BPF)","text":"
                          The Board Peripheral Fabric is the 64-bit Arm\u00ae AMBA\u00ae 4 AXI4-Lite compliant interconnect fabric that connects board peripheral modules to APF. The fabric is clocked by clk_csr and has a read allowance and write allowance of 1, i.e. only 1 active write/read is allowed in the fabric at any single time.
                           
                          The address mapping for components interconnected by BPF is listed below. All components are mapped to PF0 BAR0 and implement Arm\u00ae AMBA\u00ae 4 AXI4-lite slave interface. The Master column indicates if a component also implements Arm\u00ae AMBA\u00ae 4 AXI4-lite master interface which can send requests to the BPF.
                           
                          Table 5-2 BPF Address Mapping
                           
                          SoC PF0 BAR 0
                           Address Size (Byte) Feature Master 0x00000 \u2013 0x0FFFF 64K FME (FME, Error, etc) No 0x10000 \u2013 0x10FFF 4K SoC PCIe Interface No 0x11000 \u2013 0x11FFF 4K Reserved - 0x12000 \u2013 0x12FFF 4K QSFP Controller 0 No 0x13000 \u2013 0x13FFF 4K QSFP Controller 1 No 0x14000 \u2013 0x14FFF 4K Ethernet Subsystem - 0x15000 \u2013 0x15FFF 4K Memory Subsystem 0x80000 \u2013 0xFFFFF 512K PMCI Controller Yes 
                          Host PF0 BAR 0
                           Address Size (Byte) Feature Master 0x00000 \u2013 0x00FFF 4K Host PCIe Interface No 0x80000 \u2013 0xFFFFF 512K PMCI Controller Yes"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#53-arm-amba-4-axi4-stream-pfvf-muxdemux","title":"5.3 Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF Mux/Demux","text":"
                          The Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF Mux/Demux routes the PCIe TLP packets from the PCIe subsystem Arm\u00ae AMBA\u00ae 4 AXI4-Stream RX channel to downstream PF/VF based on the pf_num and vf_num information in the PCIe TLP header.
                           
                          The Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF mux arbitrates PCIe TLP packets from downstream PF/VF to the PCIe SS Arm\u00ae AMBA\u00ae 4 AXI4-S TX channel. The PF/VF Mux/Demux is an M X N switch that allows any M port to target any N port, and any N port to target any M port, where M is the number of host/upstream ports, and N is the numbers functions/downstream ports.   
                           
                          The fpga top package file, /src/afu_top/mux/top_cfg_pkg.sv and soc_top_cfg_pkg.sv and, contains the PF/VF parameters and mappings.
                           
                          Figure 5-2 PF/VF Mux
                           
                           
                          The PF/VF mux integration is part of afu_top (src/afu_top/afu_top.sv and /soc_afu_top).  There are two independent TX PF/VF MUX trees, labeled \"A\" and \"B\".
                           
                          Both an A and a B port are passed to each AFU component with a unique PF/VF. You can design your AFU components to send all requests to the primary A port or partition requests across both A and B ports. A typical high-performance AFU sends read requests to the B port and everything else to the A port, giving the arbiter freedom to keep both the host TX and RX channels busy.
                           
                          In the reference FIM provided for Intel Agilex 7 OFS, the A and B TX trees have been multiplexed down to a single channel for A and another for B. The A/B multiplexer merges them into a single TX stream that will be passed to the tag remapper.
                           
                          The tag remapper provides unique tags as required by the PCIe specification.  Tags are not provided by the PCIe Subsystem FPGA IP.  When creating your own AFU you can leverage this module to generate unique tags.
                           
                          Note that the primary PF/VF Mux A supports RX and TX ports.  For the secondary PF/VF Mux B only TX ports are supported and the RX input to the Mux is tied off.  
                           
                          The default mapping is shown below:
                           
                          Table 5-3 PF/VF Mapping
                           
                          SoC (IceLake-D)
                           PF VF Feature BAR Bar Size PF0 OFS managed peripherals BAR 0 2MB AFU peripherals  Board peripherals BAR 0 256KB  768KB PF0 VF0 HE-MEM BAR 0 2MB VF1 HE-HSSI BAR 0 2MB VF2 HE-MEM TG BAR 0 2MB 
                          Xeon Host
                           PF VF Feature BAR Bar Size PF0 OFS managed peripherals BAR 0 1MB AFU peripherals  Board peripherals 256KB  768KB PF1 HE-LPBK BAR 0 16KB 
                          For information on how to modify the PF/VF mapping for your own design, refer to the OFS FIM Developer User Guide.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#54-afu-interface-handler","title":"5.4 AFU Interface Handler","text":"
                          The AFU Interface Handler resides inline between the PCIe Arm\u00ae AMBA\u00ae 4 AXI4-Stream Adapter and the Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF Demux/Mux logic. Its main function is to provide: * Unique PCIe tags \u2013 Each PCIe transaction shares the 512 tags across all VFs in the AFU region * AFU error logging for all VFs in the AFU region
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#541-unified-tag-remapping","title":"5.4.1 Unified Tag Remapping","text":"
                          When a FPGA function sends out a read cycle, it allocates a unique tag which is subsequently used to identify the read completion.  The tag is considered busy; it cannot be assigned to another read cycle until read completion.  While a tag may be unique within a unit, two different units could unknowingly send out two read cycles of the same tag.  The PCIe subsystem requires unique tags for all read cycles irrespective of their origins.  Therefore, a mechanism is needed to uniquify tag globally across different units.
                           
                          OFS contains a tag remapper (tag_remap.sv) that intercepts the read cycle, finds a globally unique tag, and replaces the original tag value.  It also restores the original tag value when returning completion to the read requester.  tag_remap is placed between the Arm\u00ae AMBA\u00ae 4 AXI4-Stream interface of the PCIE subsystem and the PF/VF Mux/Demux.
                           
                          The logic is described as follows:
                           
                             
                          1. A sub-module (ofs_fim_tag_pool) maintains a pool of available tags.
                            2.  
                          2. TX read requests are held until a tag is available from the pool by setting tvalid=0 to the host, and tready=0 to the PF/VF Mux/Demux.
                            3.  
                          3. When a TX read is dispatched, the tag is marked busy in the pool.
                            4.  
                          4. The original tag is stored in tag_reg, so it can be recovered when returning a completion to the unit/function.
                            5.  
                          5. Because completion to a read request can split into multiple smaller transfer sizes, responses are monitored and the final completion is detected using PCIe TLP rules.
                            6.  
                          6. Tags are released in the pool only when all requested data are transferred.
                            7.  
                          7. When the completion returns, the original tag is restored from tag_reg.
                            8.  
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#542-afu-error-handling","title":"5.4.2 AFU Error Handling","text":"
                          In this OFS design, the AFU Interface Handler handles error logging for all VFs in the AFU. Errors handled are as follows
                           
                          Table 5-4 AFU Error Descriptions
                           Checker Field Description AFU protocol checker (PCIe TLP) Malformed TLP AFU PCIe TLP contains unsupported format type MaxPayloadError AFU memory write payload size exceeds max_payload_length limit MaxReadReqSizeError AFU memory read payload size exceeds max_read_request_size limit MaxTagError AFU memory read request tag value exceeds the maximum supported tag count UnalignedAddrErr The address field in AFU memory write/read request TLP is not DW-aligned. UnexpMMIOResp AFU is sending a MMIO read response with no matching MMIO read request. MMIOTimedOutAFU is not responding to a MMIO read request within the pre-defined response timeout period.  MMIODataPayloadOverrunThe number of data payload sent by AFU for a MMIO response (cplD) is more than the data length specified in the response. MMIOInsufficientDataThe number of data payload sent by AFU for a MMIO response (cplD) is less than the data length specified in the response. TxMWrDataPayloadOverrun    The number of data payload sent by AFU for a memory write request is more than the data length specified in the request. TxMWrInsufficientData  The number of data payload sent by AFU for a memory write request is less than the data length specified in the request. AFU Protocol Checker (Arm\u00ae AMBA\u00ae 4 AXI4-Stream)TxValidViolationThree checkers are implemented in the FIM to catch errors and protocol violations."},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#55-tlp-to-arm-amba-4-axi4-lite-memory-mapped-bridge-st2mm","title":"5.5 TLP to Arm\u00ae AMBA\u00ae 4 AXI4-Lite Memory Mapped Bridge (ST2MM)","text":"
                          ST2MM implements the following key features: * Host MMIO bridge     * Maps MMIO TLP packets received from the PCIe Subsystem over streaming interface to Arm\u00ae AMBA\u00ae 4 AXI4-Lite memory-mapped request. The memory-mapped request is sent to AFU or Board peripherals over APF and BPF.     * Maps Arm\u00ae AMBA\u00ae 4 AXI4-lite MM response received from AFU or Board peripherals to TLP packets and send the packets over ST streaming channel to host HIA subsystem. * Sends MMIO response of all 0\u2019s for MMIO read to unused BAR region. * Interrupt     * Sends interrupt packets to the PCIe subsystem when interrupt requests are received from the peripherals. Interrupts can be requested by a peripheral through a memory write to interrupt CSR registers in the ST2MM.
                           
                          Figure 5-3 ST2MM Module
                           
                           
                          ST2MM implements both Arm\u00ae AMBA\u00ae 4 AXI4-lite master and slave interfaces that are connected to the designated slave and master port on APF. Host memory requests are sent on the ST2MM master interface to AFP where the requests are routed to the targeted peripherals. 
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#6-mmio-regions","title":"6 MMIO Regions","text":"
                          The FIM and AFU expose their functionalities to the host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). An MMIO region is an address space within a base address register (BAR) region to which features are memory mapped.  
                           
                          For example, when a feature is mapped to an MMIO region, the CSR registers of that feature are located within the address range of that region. There can be multiple MMIO regions within a BAR region. 
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#61-feature-region","title":"6.1 Feature Region","text":"
                          A group of related CSRs can be categorized as a feature region. For example, a DMA engine has queue management function and quality of service (QoS) function; these are two different features of the DMA engine. A feature region is contained within a single PCIe BAR and cannot span across two BAR region boundaries. 
                           
                          A Device Feature Header (DFH) register marks the start of the feature region and sub-feature region, and you must place it at the first address of the region. Each DFH starts at 4KB boundary. A DFH register contains information that OPAE software requires to enumerate the feature. It also has an offset field that points to the next DFH in a feature list. OPAE software traverses the linked list of DFHs in each BAR region to discover all the features implemented on the platform. 
                           
                          The EOL field in a DFH marks the end of a DFH list and is only set in the DFH of the last feature in the feature list. The feature type field in the DFH is used to differentiate between the different types of feature region. Basic building blocks (BBB) and private features are always a child of an AFU or FPGA Interface Unit (FIU) and must be contained within an AFU or FIU, respectively.  
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#611-device-feature-header-dfh-structure","title":"6.1.1 Device Feature Header (DFH) Structure","text":"
                          All DFHs must follow the following structure to be compatible with OPAE software.  Note that only features you want to be exposed to the OPAE software must have a DFH.
                           
                          Table 6-1: DFH Register Structure
                           Bitfield Name Range Access Description FeatureType 63:60 RO 4\u2019b0000 \u2013 Reserved 4\u2019b0001 \u2013 AFU4\u2019b0010 \u2013 BBB4\u2019b0011 \u2013 Private Feature4'b0100 \u2013 FIM Reserved 59:41 Rsvd Reserved EOL 40 RO End of DFH List1'b0=No other feature header beyond this one1'b1=This is the last feature header NextDFHByteOffset 39:16 RO Next DFH byte offsetNext DFH Address= Current DFH address + Next DFH byte offset.  You can also use this value as an indication of the maximum size of the MMIO region occupied by this feature. FeatureRev 15:12 RO For AFU Feature type= AFU major version number that is user defined.All other feature types= Feature revision number FeatureID 11:0 RO For AFU feature type= CoreFIM version numberFor BBB feature type= Intel defined ID for BBBFor private feature type= User-defined ID to identify within an AFU For FIU type=ID for FIU unit (ex. 0 for FME, 1 for Port) 
                          You must increment a feature revision number if a feature changes.  This change requires a corresponding change in the software to detect the new version and report mismatches between the hardware and software revision number. 
                           
                          When indicating \"FeatureType\" in your DFH Structure, use the following guidance: * FIM= Any static region feature you want discovered through software.  FIM features must implement mandatory FIM registers, including a FIM feature ID. * AFU= User Application region; may be PR region.  AFU feature must implement mandatory AFU register including an AFU ID. * Private Feature = Linked list of features within the FIM or AFU which are enumerated and managed by separate software and do not require an ID. * Basic Building Blocks (BBB) = Special features within the AFU or FIM that are meant to be reusable basic building blocks.  They are enumerated and called by separate software services.  
                           
                          Table 6-2: Mandatory FIM and AFU Registers
                           
                          The following table details the registers required for a FIM feature or AFU to be discovered by OPAE software.  The ID_H and ID_L fields allow you to create a unique feature or AFU identifier that remains unchanged while the FeatureRev in the DFH register increments.
                           Byte Address offset with respect to DFH Register Name 0x0000 DFH with corresponding FeatureType field implemented 0x0008 ID_L: Lower 64 bits of unique ID number (GUID) 0x0010 ID_H: Upper 64 bits of unique ID number (GUID) 0x0018 Next AFU 
                          Table 6-3: Next AFU Register Definition
                           
                          The following table details the \"Next AFU Register Definition.\"  This register is used if you have multiple AFUs that can be used with your FIM.  
                           Bit Attribute Description 63:24 Reserved Reserved 23:0 RO Next AFU DFH Byte Offset Next AFU DFH address =current address + offset; where a value of 0 implies this is the last AFU in the list. Example: AFU0 at address 0x0: Next AFU offset= 0x100  AFU1@ address 0x100: Next AFU offset = 0x100 AFU2 at address 0x200: Next AFU offset = 0x0 (End of AFU List) 
                          6.2 Control and Status Registers
                           
                          All the Control and Status Registers (CSRs) in the FIM are 64-bit registers with the following MMIO write, and MMIO read support.
                           
                          Table 6-4: CSR MMIO Read and Write Support
                           Request Memory Attribute Payload size Memory Ordering MMIO Write UC 4B or 8B Strongly ordered MMIO Read UC 4B or 8B Strongly ordered 
                          The FIM does not reorder the MMIO requests or responses. For MMIO writes, there is no reordering of requests in FIM, and UC ordering rules are followed. Similarly, for MMIO reads, there is no re-ordering of requests or responses in the FIM. An AFU may opt to re-order the MMIO read responses but the FIM does not enforce read response ordering. 
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#621-software-access-to-registers","title":"6.2.1 Software Access to Registers","text":"
                             
                          • Software accesses 64-bit registers as aligned quadwords. For example, to modify a field (bit or byte) in a 64-bit register, the entire quadword is read, the appropriate field(s) are modified, and the entire quadword is written back.
                            •  
                          • When updating registers through multiple accesses (whether in software or due to hardware disassembly), certain registers may have specific requirements on how the accesses must be ordered for proper behavior. These are documented as part of the respective register descriptions.
                            •  
                          • For compatibility with future extensions or enhancements, software must assign the last read value to all \u201cReserved and Preserved\u201d (RsvdP) fields when written. In other words, any updates to a register must be read so that the appropriate merge between the RsvdP and updated fields occurs. Also, software must assign a value of zero for \u201cReserved and Zero\u201d (RsvdZ) fields when written.
                            •  
                          • PCIe locked operations to FPGA hardware registers are not supported. Software must not issue locked operations to access FPGA hardware registers.
                            •  
                           
                          In the following two cases, the FIM terminates MMIO Read requests by sending a completion with the data (CplD) specified below: * MMIO Timeout:  This occurs when the AFU does not respond within a set timeout. The timeout value is currently configured to 512 pclks (clk_2x). In this case, the FIM returns all 1s.
                           
                             
                          • Illegal MMIO Accesses: This occurs when the read is accessing undefined registers in the FIM or if an AFU access violation.  An example of an access violation is when a PF attempts to access the AFU when it is set to VF mode, or when a VF attempts to access the AFU when it is set to PF mode. In this case, the FME will returns all 0s.
                            •  
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#62-register-attribute-definition","title":"6.2 Register Attribute Definition","text":"
                          Table 6-5: OFS Register Attribute Definitions
                           Attribute Expansion Description RW Read/Write This bit can be read or written by software. RO Read Only The bit is set by hardware only. Software can only read this bit. Writes do not have any effect. RW1C Read/ Write 1 to Clear Software can read or clear this bit. The software must write 1 to clear this bit. Writing zero to RW1C bit has no effect. Note that a multi-bit RW1C field may exist. In this case, all bits in the field are cleared if a 1 is written to any of the bits. RW1S Read/ Write 1 to Set Software can read this bit. Writing a 1 to the bit sets it to 1. Writing a 0 has no effect. It is not possible for software to set this bit to 0. The 1 to 0 transition can only be performed by HW. RW1CS Read/Write 1 to Clear Sticky Software can read and clear this bit. Writing a 1 to a bit clears it, while writing a 0 to a bit has no effect. This bit is only reinitialized to its default value by a power-on reset. RWD Read/Write Sticky across Hard Reset The bit can be read or written by SW. This bit is sticky or unchanged by any reset type, including Hard Reset. The bit gets cleared only with power on. *S Sticky across Soft Reset The bit will be sticky or unchanged by soft reset. These bits are only re-initialized to their default value by a power-on reset. *D Sticky across Hard Reset The bit is sticky or unchanged by or unchanged by any reset type, including hard reset. The bit gets cleared only with power on. Rsvd Reserved Reserved for future definitions. Currently don\u2019t care bits. RsvdP Reserved and Protected Reserved for future RW implementations. The software must preserve the value of this bit by read modify write. RsvdZ Reserved and Zero Reserved for future RW1C implementations. The software must write zero to this bit."},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#623-csr-offset-in-bars","title":"6.2.3 CSR Offset in BARs","text":"
                          The table below captures the FIM and AFU features in the supported BAR regions. The highlighted offset indicates the first DFH in the DFH list of a BAR region where device driver starts the DFH traversal.
                           
                          Table 6-6: PF0 BAR0 Features SoC AFU
                           Offset Feature CSR set 0x0000 FME AFU 0x10000 PCIe Interface 0x12000 QSFP Controller 0 0x13000 QSFP Controller 1 0x14000 Ethernet Interface 0x15000 Memory Interface 0x80000 PMCI 0x100000 St2MM 0x130000 PR Control &  Status (Port Gasket) 0x131000 Port CSRs (Port Gasket) 0x132000 User Clock (Port  Gasket) 0x133000 Remote SignalTap (Port Gasket) 0x140000 AFU Errors (Protocol Checker) 
                          Table 6-7: PF0 BAR0 Features Host AFU
                           Offset Feature CSR  set 0x0000 PCIe Interface"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#7-clocks","title":"7 Clocks","text":"
                          The following table provides external clock sources which correspond to pins on the FPGA that drive blocks of internal logic or are used as a reference clock for internal PLLs.  The names in the table are used in the top.sv or are the logical clock names used by their respective functional blocks.  
                           
                          Table 7-1: External Clock Source 
                           Clock Frequency Description SYS_REFCLK 100MHz Reference clock to system IOPLL (sys_pll) which provides FIM system clocks. PCIE_REFCLK0 100MHz PCIe  Refclk 0 PCIE_REFCLK1 100MHz PCIe Refclk 1 SOC_PCIE_REFCLK0 100MHz PCIe Refclk 0 on the SOC side SOC_PCIE_REFCLK1 100MHz PCIe Refclk 1 on the SOC side qsfp_ref_clk 156.25MHz Ethernet  Refclk ddr4[0].ref_clk 150MHz Refclk  for EMIF channel 0 ddr4[1].ref_clk 150MHz Refclk for EMIF channel 1 ddr4[2].ref_clk 150MHz Refclk  for EMIF channel 2 ddr4[3].ref_clk 150MHz Refclk for EMIF channel 3 sdm_config_clk 125MHz SDM  Config Clock altera_reserved_tck 10MHz Default  JTAG Clock 
                          Table 7-2: Internal Clocks
                           Clock Frequency Description clk_sys 400MHz System clock. Primary clock for PCIe datapath. clk_100m 100MHz 100MHz clock. For RemoteSTP and user  clock, or any logic that requires a 100MHz clock. clk_csr 100MHz Arm\u00ae AMBA\u00ae 4 AXI4-lite  interconnect fabric and CSR clock. Driven by clk_100m. clk_2x 400MHz 2x clock to AFU in PR slot. Driven by clk_sys. clk_1x 171.43MHz 1x clock to AFU in PR slot. uclk_usr 312.50  MHz Configurable  \u201cH\u201d clk for AFU. Default 312.50 MHz uclk_usr_div2 156.25 MHz Configurable  \u201cL\u201d clk for AFU. Default 156.25 MHz"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#8-reset","title":"8 Reset","text":""},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#81-reset-signals","title":"8.1 Reset Signals","text":"
                          The FIM system reset signals are driven according to a their respective requirements derived from their use cases.  The behavioral code defining the reset operation is located in the file rst_ctrl.sv. The FIM System Reset Drivers table below provides a list of the input and feedback signals that compose the various resets.
                           
                          Table 8-1: FIM System Reset Drivers
                           Reset Description nPERST pin Active low PCIe reset pin from the PCIe card edge that may be set by the host machine for cold or warm resets. nCONFIG pin Active low input to the FPGA that causes the FPGA to lose its configuration data, enter a reset state, and tri-state all I/O pins. Host software must reload the FPGA FIM after nCONFIG is activated. ninit_done Active low signal indicating FPGA core configuration is complete and FPGA has entered usermode.  This signal is provided by the configuration monitor block in rst_ctrl.sv. pcie_reset_status Active high reset status from PCIe hard IP. When driven high, this signal indicates that the PCIe IP core is not ready for usermode. locked Active high SYS IOPLL locked signal pcie_cold_rst_ack_n Reset Acknowledge signal from the PCIe subsystem that lets the Reset Controller know that the Cold Reset sequence has been completed. pcie_warm_rst_ack_n Reset Acknowledge signal from the PCIe subsystem that lets the Reset Controller know that the Warm Reset sequence has been completed. 
                          Upon power-on, the reset module in the FIM holds the FIM in reset until all the reset conditions are de-activated:
                           
                             
                          • nPERST signal is asserted.
                            •  
                          • The INIT_DONE pin is driven high to indicate core configuration is complete.
                            •  
                          • The SYS IOPLL is locked.
                            •  
                          • The reset status from PCIe hard IP is de-asserted indicating the IP is ready for transfer. 
                            •  
                           
                          The reset module places the FIM back into reset if any of these conditions becomes active again. The only way to invoke a system reset to the FIM after power-up is to deassert the nPERST pin either by performing a warm reboot or through PCIe driver intervention. There are soft reset signals set aside to allow software to reset the Port, AFU and partial reconfiguration IP. 
                           
                          The following table lists the reset outputs from the rst_ctrl.sv block.
                           
                          \u200b                                                      Table 8-2: FIM System Reset Outputs
                           Reset Description rst_n_sys pin System general reset synchronous to clk_sys.  This is a warm reset of the FPGA logic.  Sticky bits in the FME and other CSR registers will not be reset by this signal. rst_n_100m pin System general reset synchronous to clk_100m. ninit_done Active low signal indicating FPGA core configuration is complete and FPGA has entered usermode.  This signal is provided by the configuration monitor block in rst_ctrl.sv. pwr_good_n Hardware reset conditioned by ninit_done and the pll_locked signal.  The signal is generally set when power has been cycled or a physical hardware fault has occurred, requiring a reset of the FPGA logic.   This signal is synchronous to clk_sys. pcie_cold_rst_ack_n Hardware reset conditioned by the pcie_reset_status which is a summary reset driven by the PCIe Hard IP logic tile on the FPGA die.  This signal is synchronous to clk_sys. pcie_warm_rst_ack_n Soft reset conditioned by nPERST when the pcie_reset_status is not asserted, meaning a warm reset is desired.  Cold reset sticky bits in the PCIe subsystem will not be reset by this signal."},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#9-interrupts","title":"9 Interrupts","text":"
                          The OFS platform supports interrupts through MSI-X feature implemented in the PCIe SS. The MSI-X Interrupt feature handles FME and AFU interrupts. 
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#91-msi-x","title":"9.1 MSI-X","text":"
                          FME interrupts are primarily used to notify the host of error events happened in the FIM. When any of the bits in the FME error status registers is set, an interrupt request is sent to the MSI-X module. There are FME error status registers for various FPGA features such as RAS Error, Temperature monitoring etc.\u202fIf any of those error registers log an error, we trigger an FME interrupt.
                           
                          An AFU sends an interrupt to the MSI-X module in the PCIE SS on the AXI interrupt request channel. 
                           
                          The MSI-X table entries and PBA vectors are implemented in the PCIE SS. The PCIE SS supports upto 4096 vectors in Static MSI-X mode
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#92-msi-x-entry-table","title":"9.2 MSI-X Entry Table","text":"
                          Each entry in the MSI-X table stores the message address, message data, and vector control for one interrupt vector. The address, data, and vector control information are populated by software (usually done by the PCIe SRIOV driver) after reading the PCIe endpoint configuration space.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#93-msi-x-pba-table","title":"9.3 MSI-X PBA Table","text":"
                          The PBA table contains a corresponding bit for each MSI-X interrupt vector. This PBA bit is set if that interrupt is triggered but is masked. When the interrupt is unmasked, the PBA bit is unset, and an interrupt is sent to the Host. When the Application generates an interrupt from an IRQ source, it sets the corresponding PBA bit. 
                           
                          When the interrupt is triggered by an IRQ Source, the IRQ Processor checks the masking of that interrupt to determine if the PBA bit should be set. If unmasked, the IRQ Processor looks up MSI-X address and data for the interrupt vector and generates the interrupt request over the PCIe EP.
                           
                          The FIM implements the pending bit array as per the MSI-X specification.  When interrupts are enabled the FIM immediately generates an interrupt in response to a suitable event.  When interrupt vectors are masked, the pending bit array entry for the corresponding interrupt vector is set without issuing an interrupt.  When interrupt vectors are re-enabled, any pending interrupt entries are issued to the PCIe EP
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#94-interrupts-supported","title":"9.4 Interrupts Supported","text":"
                          Table 9-1: OFS for Intel Agilex 7 FPGA Interrupts Supported
                           PF/VF Vector Assignment PF0 0-5 Reserved 6 FME Error Interrupt PF0.VF00-3User AFU Interrupt"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#95-msi-x-table-locations","title":"9.5 MSI-X Table Locations","text":"
                          The MSI-X vector tables are at BAR4, address 0x2000. The MSI-X PBA tables are at BAR4, address 0x3000.
                           
                          Table 9-2: PF0 BAR4 Features
                           Offset Feature CSR set 0x02000 MSI-X Vector Tables 0x03000 MSI-X PBA Tables"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#10-external-memory-interface-emif","title":"10 External Memory Interface (EMIF)","text":"
                          There are 4 EMIF channels (4 DDR4 banks) on the f2000x platform which is targeted OFS FIM  for Intel Agilex 7 FPGA. The HE-MEM exerciser module in AFU. ECC is not implemented in this design. Both memory subsystem and HE-MEM implement Arm\u00ae AMBA\u00ae 4 AXI4-MM interface.
                           
                          **Table 10-1 Memory Subsystem Configuration on the Intel IPU Platform F2000X-PL platform **
                           EMIF  Channel # Width ECC Size (GB) Speed  (MT/s) DDR4 Device FPGA Bank 0 x32 x8 4 2400 Three 1Gx16 3D 1 x32 x8 4 2400 Three 1Gx16 3A 2 x32 x8 4 2400 Three 1Gx16 2F 3 x32 x8 4 2400 Three 1Gx16 2E 
                          Figure 10-1: EMIF Interfaces
                           
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#101-emif-csr","title":"10.1 EMIF CSR","text":"
                          TheMIF is implemented as an external FME feature in OFS EA and the CSR for the EMIF feature is memory mapped to the FME BAR region. The following table captures the EMIF CSR registers.
                           
                          Table 10-2: EMIF CSR Registers
                           EMIF_DFH 0x000 0x300000001000100F EMIF Management DFH FIELD NAME RANGE ACCESS DEFAULT DESCRIPTION FeatureType [63:60] RO 0x3 Feature Type = Private Feature Reserved40 [59:40] RsvdZ 0x0 Reserved NextDfhByteOffset [39:16] RO 0x1000 Next DFH Byte  offset FeatureRev [15:12] RO 0x1 Feature Revision FeatureID [11:0] RO 0x00F Feature Id 
                          \u200b               
                           EMIF_STAT 0x008 0x0000000000000000 EMIF Calibration Status Register FIELD NAME RANGE ACCESS DEFAULT DESCRIPTION Reserved [63:8] RsvdZ 0x0 Reserved CalFaliure [7:4] RO 0x0 EMIF PHY Calibration Failure (1 bit per interface) CalSuccess [3:0] RO 0x0 EMIF PHY  Calibration Successful (1 bit per  interface) EMIF_CAPABILI TY 0x010 0x000000000000000F EMIF Capabliity Register FIELD NAME RANGE ACCESS DEFAULT DESCRIPTION Reserved [63:4] RsvdZ 0x0 Reserved EMIFCap [3:0] RO 0xF Attached Memory  Capability (1 bit per  interface)"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#11-ethernet-interface-subsystem","title":"11 Ethernet Interface Subsystem","text":""},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#111-ethernet-interface-overview","title":"11.1 Ethernet Interface Overview","text":"
                          The Ethernet Subsystem provides portability to different Ethernet configurations across platforms and generations and reusability of the hardware framework and software stack. This reference FIM implements an E-tile Ethernet Subsystem IP in a 2x4x25GbE configuration and provides a Linux driver that can be leveraged for customization. Each group of 4x25GbE routes to a QSFP.
                           
                          For more information about how to reconfigure the Ethernet Subsystem please refer to the [Ethernet Subsystem Intel FPGA IP].
                           
                          Table 11-1: Ethernet Configurations for example OFS FIMs for Intel Agilex 7 FPGAs
                           Configuration/Mode Base Fim IP file name hssi_ss_8x25g Number of ports enabled 8 Enabled ports Port 0-7 Port{x} profile 25GbE Port{x} subprofile MAC+PCS Port{x} RSFEC True Port{x} PTP False Enable AN/LT False (for simulation efficiency) Enable NPDME True Enable JTAG to Avalon master bridge False Enable Tx packet classifier NA PTP accuracy mode NA Enable iCAL and pCAL recipe at power True TX/RX maximum frame size 1518 Enforce maximum frame size False Link fault generation mode Bidirectional Stop TX traffic when link partner sends PAUSE Yes Bytes to remove from RX frames Remove CRC bytes Forward RX PAUSE requests False Use source address insertion False Enable TX/RX VLAN detection True Enable asynchronous adapter clocks False Enable preamble Passthrough False Enable asynchronous adapter clocks False Enable strict preamble check False Enable strict SFD check False Average IPG 12 Enable adaptation load soft IP True Additional IPG removed as per AM period 0 PMA adaptation select NRZ_28Gbps_VSR 
                          To determine which Transceiver Subsystem port maps to the QSFP A and B lanes, please refer to the following table:
                           
                          Table 11-2: Transceiver Subsystem Port Mapping
                           Port Number base 2x4x25G 0 QSFP-A Lane-0 1 QSFP-A Lane-1 2 QSFP-A Lane-2 3 QSFP-A Lane-3 4 QSFP-B Lane-0 5 QSFP-B Lane-1 6 QSFP-B Lane-2 7 QSFP-B Lane-3 8 NA 9 NA 10 NA 11 NA 12 NA 13 NA 14 NA 15 NA 
                          Figure 11-1: Transceiver Subsystem Block Diagram
                           
                           
                          A host exerciser, named he-hssi, is provided in the pr_slot of the AFU partition. The Transceiver susbystem interface to the AFU has an Arm\u00ae AMBA\u00ae 4 AXI4-Stream data and sideband interface.  
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#112-ofs-ethernet-subsystem-interface-module","title":"11.2 OFS  Ethernet Subsystem Interface Module","text":"
                          A wrapper around the  Ethernet Subsystem integrates the following features:
                           
                             
                          • DFH registers 
                            •  
                          • Control & status registers 
                            •  
                          • Indirect access to Transceiver SS CSR registers via CSR mailbox in the Ethernet Subsystem interface 
                            •  
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1121-ethernet-subsystem-control-and-status-register-csr-map","title":"11.2.1 Ethernet Subsystem Control and Status Register (CSR) Map","text":"
                          The Ethernet Subsystem connects to the APF which is memory mapped to PF0 BAR0. The Ethernet Subsystem CSR space in the FIM consists of two parts:
                           
                             
                          • Ethernet Subsystem CSRs assigned from offset 0x000 to 0x7FF
                            •  
                          • Additional CSRs are added to FIM at offset 0x800 to 0xFFF
                            •  
                           
                          The PCIe subsystem uses Arm\u00ae AMBA\u00ae 4 AXI4 Memory Mapped accesses to read and write Ethernet Subsystem Control and Status Registers in the FIM. The Ethernet Subsystem CSR Map structure is designed to scale according to IP capabilities.
                           
                          The Ethernet Subsystem CSR Map can be found ipss/hssi/HSSI_SS_CSR.xls.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#114-ethernet-subsytem-software","title":"11.4 Ethernet Subsytem Software","text":"
                          There are two pieces of software related to running the Ethernet Subsystem: The Linux* dfl network driver and the user space OPAE Tools.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1141-ethernet-subsystem-linux-driver","title":"11.4.1 Ethernet Subsystem Linux Driver","text":"
                          The Ethernet subystem is exposed as a feature in the PCIe PF BAR0 region. It has a Device Feature Header (DFH) specifying the interface.
                           
                          The primary functionality of the driver is to interact with the ethernet MAC and PHY through an indirect register access mailbox implemented by the HSSI_RCFG_CMD0, HSSI_RCFG_DATA0 registers described above. To aid in RTL bringup, the driver provides a debugfs interface directly to the indirect register access mailbox. For each HSSI interface in the system there would be a directory with the following form containing two files, regaddr and regval: /sys/kernel/debug/dfl-fme.X.Y
                           
                          To read a register offset in the MAC/PHY write the offset to the regaddr file as a C hex string (e.g. 0x04) and then read the value as string out of regval file. To write a register offset in the MAC/PHY write the offset to the regaddr file as a C hex string (e.g. 0x04) and then write the value as a C hex string to regval file.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1142-ethernet-subsystem-opae-user-space-tool","title":"11.4.2 Ethernet Subsystem OPAE User Space Tool","text":"
                          User space OPAE Tools that are part of OPAE SDK provide support for the Ethernet Subsystem. You can use the --help option to print more information about each of these commands:
                           
                             
                          • hssi: Provides a means of interacting with the 10G and 100G HSSI AFUs through the host exerciser.
                            •  
                          • hssiloopback: Enables and disables Ethernet loopback.
                            •  
                          • hssistats: Provides MAC statistics.
                            •  
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#12-partial-reconfiguration","title":"12 Partial Reconfiguration","text":"
                          Partial Reconfiguration (PR) is an Intel FPGA technology that allows users to reconfigure parts of the FPGA device dynamically, while the remainder of the device continues to operate. In a non-partial reconfiguration flow, any change to the design requires full reprogramming of the entire configuration RAM (CRAM) arrays in the device. With partial reconfiguration, you can dynamically reprogram one or more CRAM frames. A partial reconfiguration design has a static region, and a PR regions, which can be modified to implement new logic. The portion of the CRAM on the chip to be reconfigured is contained within a PR region. For the PR flow, your design must be partitioned into static region and reconfigurable region. The static region is the area of your FPGA that is not reconfigured without reprogramming the entire FPGA. An area of the chip that you plan to partially reconfigure is a PR region.
                           
                          The Port Gasket contains all the PR specific modules and logic, such as PR slot reset/freeze control, user clock, remote STP etc. For this reference example only one PR slot is supported.
                           
                          The Figure below shows the fundamental concepts required to support PR in OFS platform. There are 4 OFS management DFH \u2013 PR, Port, User Clock and Remote STP in Port Gasket that is exposed to OFS software. These platform capabilities are generally used in conjunction to Partial Reconfiguration. The Figure below also demonstrates the concepts of adding a new interface to the PR region.
                           
                          Figure 12-1 Partial Reconfiguration Gasket
                           
                           
                          The isolation logic is provided on the output signals of the PR region to ensure they don\u2019t glitch and affect the interfacing logic in the Static Region (SR). The isolation logic is controlled by the PR Freeze logic during PR operation.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#121-user-clock-support","title":"12.1 User Clock Support","text":"
                          The reference platform provides two user configurable clock (uclk_usr, uclk_usr_div2) for use by the AFU. These clocks are generated by a reconfigurable IOPLL. The control and status of these clocks is expose through two pairs of command and status registers (USR_CLK_FREQ_CMD0 / USR_CLK_FREQ_STS0 & USR_CLK_FREQ_CMD1 / USR_CLK_FREQ_STS1). The first pair controls the fPLL reconfiguration functionality. The second pair controls a clock frequency measurement block.
                           
                          The following are the default values of the userclk.
                           
                          uclk_usr: 312.5 MHz
                           
                          uclk_usr_div2: 156.2 MHz
                           
                          Table 12-1 usr_clk_* Acceptable Programming Range
                           Fabric Frequency Range uclk_usr (H) uclk_usr_div2 (L) Details 0-400 MHz 0-800 MHz 0-400 MHz Clocks set on 2x relationship for L<400 MHz 400-800 MHz 400-800 MHz 400-800 MHz Clks are equal for L>400 MHz 
                          PLL Reconfiguration
                           
                          The blue bit stream hardware exposes the low level IOPLL reconfiguration interfaces directly to software control. Through the USR_CLK_FREQ_CMD0 register software can select the reference clock, assert the PLL power down pin and issue reads and writes on the IOPLL Avalon-mm reconfiguration bus. Through the USR_CLK_FREQ_STS0 register software can query the IOPLL active reference clock, locked status and readdata returned from the IOPLL AVMM interface for read requests.
                           
                          Clock Frequency Counter
                           
                          The user clocks, generated by the reconfigurable IOPLL, are connected to a frequency measurement circuit. Software selects which of the two clocks to measure by setting the clock select bit in the USER_CLK_FREQ_CMD1 register. After 10ms software can read the frequency, in 10 KHz units, from the USER_CLK_FREQ_STS1 register. Reading this register before 10ms has passed will return undefined data but otherwise has no effect.
                           
                          Configuring User Clocks
                           
                          To program the user clock to the desired frequency, user will set the clock-frequency-low and clock-frequency-high fields in the PR AFU GBS .json file to the desired frequency value. During PR, SW will try to provide the closest possible frequency to the value specified in the .json file.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#13-host-exercisers","title":"13 Host Exercisers","text":"
                          The Host Exerciser (HE) modules are responsible for generating traffic with the intent of exercising the specific interface they are designed to connect to. They are intended to test the interface in simulation and hardware, enable measurement of bandwidth and other performance KPIs and, in some cases, provide an example of data movement between interfaces (PCIe to DDR for e.g.) for adoption into a customer application.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#131-he-lb-and-he-mem-host-exerciser","title":"13.1 HE-LB and HE-MEM Host Exerciser","text":"
                          The Host Exerciser Loopback module is a traffic generator that can be attached both to host memory, over PCIe (HE-LB), and to local memory, such as board-attached DDR (HE-MEM). The Host Exerciser (HE) is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth as well as demonstrating data path correctness. The FIM picks up the HE-LB module behind the PIM (Platform Interface Manager). The PIM converts the Arm\u00ae AMBA\u00ae 4 AXI4 with TLP out of the PCIe SS to standard Arm\u00ae AMBA\u00ae 4 AXI4 (MM for completions and Lite for CSR) interfaces. The figure below shows how the HE-LB is instantiated in the FIM.
                           
                          Figure 13-1 HE-LB Dataflow Diagram 
                           
                           
                          The exerciser has two primary modes: loopback, in which read data is fed back as writes, and read/write mode, in which reads and writes are generated independently. Furthermore, the host_exerciser software provided with OPAE that drives the RTL has two major modes: \"lpbk\" and \"mem\". These modes only select the UUID of the HE AFU, with lpbk selecting a version configured with no local memory (56e203e9-864f-49a7-b94b-12284c31e02b) and mem seeking a version with local memory (8568ab4e-6ba5-4616-bb65-2a578330a8eb). The behavior of each engine with and without local memory is described below.
                           
                          Figure 13-2 HE Engine Heirarchy 
                           
                          flowchart TB\n    classDef gr fill:green,color:white;\n    classDef ol fill:olive,color:white;\n    classDef rd fill:maroon,color:white;\n\n    he_lb(\"he_lb_top\"):::gr --- he_lb_main\n    he_mem(\"he_mem_top\"):::gr --- he_lb_main\n    he_lb_main:::gr ---- he_lb_engines\n    he_lb_engines:::gr ---- mode_rdwr:::rd\n    he_lb_engines:::gr ---- mode_lpbk:::rd\n    he_lb_engines:::gr ---- mode_atomic:::rd\n    he_lb_main --- he_lb_csr:::ol
                           
                          For more details of HE-LB and HE-MEM IP, please refer to ofs-fim-common/src/common/he_lb/README.md
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#132-memory-traffic-generator-he-mem-tg","title":"13.2 Memory Traffic Generator  (HE-MEM-TG)","text":"
                          The memory traffic generator (TG) AFU provides a way for users to characterize local memory channel bandwidth with a variety of traffic configuration features including request burst size, read/write interleave count, address offset, address strobe, and data pattern.
                           
                          The AFU consists of a series of configurable Arm\u00ae AMBA\u00ae 4 AXI4-MM traffic generators connected to the available local memory channels not attached to HE-MEM. It has a separate CSR block for AFU feature information and high-level status, including a TG_CAPABILITY register that describes the available traffic generators with a 1 bit active high mask and a TG_STAT register that provides the 4-bit, one-hot status of each TG in adjacent fields. The status is decoded left to right as follows: pass, fail, timeout, active. For additional detail, refer to the MEM_TG CSR table ofs-fim-common/src/common/mem_tg/MEM_TG_CSR.xls
                           
                          Each traffic generator is configured through a separate Avalon-MM interface at incremental offsets of 0x1000 from the AFU DFH. For details on the TG2 configuration space, refer to the MEM_TG_CSR.xls. The default configuration for each TG performs a single-word write followed by a read at address 0. Triggering the start of the test on a TG will initiate a counter to measure the duration of the test which is recorded in the AFU CSR block and used to report memory channel bandwidth.
                           
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#132-hssi-host-exerciser-he-hssi","title":"13.2 HSSI Host Exerciser (HE-HSSI)","text":"
                          HE-HSSI is an Ethernet AFU that handles client side ethernet traffic. The reference HE-HSSI has following features:
                           
                             
                          • HE-HSSI provides an E-tile compatible interface with the Transceiver Subsystem.
                            •  
                          • Includes a traffic generator and checker or monitor
                            •  
                          • Provides pause signals to the Transceiver Subsystem for XON and XOFF generation
                            •  
                          • Generates traffic or incoming traffic that can be looped back into transmit path by enabling loopback mode, which will bypass traffic generator
                            •  
                          • At the HE-HSSI interface boundary the Ethernet data interface is Arm\u00ae AMBA\u00ae 4 AXI4-Stream with 64-bit data at eth_clk clock
                            •  
                          • The Traffic generator and checker modules have a 64-bit data interface at eth_clk clock.
                            •  
                          • The traffic generator supports the following modes:
                               
                            • Fixed length or Random Length
                              •  
                            • Incremental pattern or Random pattern
                              •  
                             
                            •  
                          • Traffic checker does a 32-bit crc check in 10/25G. In the 100G configuration, there is no data integrity check, only a packet counter.
                            •  
                          • The CSR of this AFU is accessible through Arm\u00ae AMBA\u00ae 4 AXI4-Stream PCIe TLP interface
                            •  
                          • The PCIe TLP to CSR Interface Conversion module converts PCIe TLPs into simple CSR interface
                            •  
                          • The CSR space of the traffic generator and checker modules are accessed in an indirect way using mailbox registers
                            •  
                          • If used for more than one channel, each channel has a separate traffic generator and traffic checker with separate CSR space.
                            •  
                          • Reads and Writes to individual traffic controller CSR spaces can be done by selecting that particular channel using channel select register.
                            •  
                           
                          The HE-HSSI Ethernet block diagram is below.
                           
                          Figure 13-3: HE-HSSI Block Diagram Block Diagram
                           
                           
                          The diagram below shows the path followed depending on if you enable loopback mode in HE-HSSI or not. In loopback mode, traffic is looped back into the transmit path which will bypass traffic. Alternatively, the traffic generates traffic.
                           
                          Figure 13-4: HE-HSSI Operation Mode Traffic Patterns 
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1322-he-hssi-csr-map","title":"13.2.2 HE-HSSI CSR Map","text":"
                          The reference HSSI AFU contains the following registers and a similar arrangement of register space can be implemented for other use case-specific HSSI AFUs.
                           
                             
                          • AFU DFH Register: Device feature header for the AFU (AFU_DFH)
                            •  
                          • AFU ID Registers: 128-bit UUID for the AFU which occupies two 64-bit registers (AFU_ID_L, AFU_ID_H)
                            •  
                          • Mailbox Registers: Command and Data register for traffic controller register access. It follows the standard access method defined for OFS. Access method and implementation is same as Reconfiguration Interface defined for the HSSI FIM. (TRAFFIC_CTRL_CMD, TRAFFIC_CTRL_DATA)
                            •  
                          • Channel Select Register: Channel select register for traffic controller mailbox access. It is used in cases where more than one channel is in the AFU, else it defaults to zero, meaning channel-0 is selected. (TRAFFIC_CTRL_PORT_SEL)
                            •  
                          • Scratchpad Register: Scratchpad register for CSR access checking. (AFU_SCRATCHPAD)
                            •  
                           
                          The CSR excel for HE-HSSI module can be found at ofs-fim-common/src/common/he_hssi/HE_HSSI_CSR.xls.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#133-he-null-overview","title":"13.3 HE-Null Overview","text":"
                          This module is a simple stub that is used to replace various HE and other blocks in the FIM whenever they are bypassed using the qsf compiler directive such as null_he_lb, null_he_hssi, null_he_mem and null_he_mem_tg. To find out more about these compiler directives, refer to the [Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs].
                           
                          Table 13-1 HE-Null DFH
                           REGISTER NAME ADDRESS ACCESS DEFAULT DESCRIPTION DFH 0x0000 RO 0x0000_0000_1000_0000 DFH register GUID_L 0x0008 RO 0xaa31f54a3e403501 Lower 64b of GUID GUID_H 0x0010 RO 0x3e7b60a0df2d4850 Upper 64b of GUID SCRATCHPAD 0x0018 RW 0x0 Scratchpad 
                          Figure 13-5: HE-Null Block Diagram
                           
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14-reliability-accessibility-serviceability-ras-and-error-handling","title":"14 Reliability, Accessibility, Serviceability (RAS) and Error Handling","text":"
                             
                          1. Downstream AFU checker: Identifies AFU violations.  For example, these checker flags violations of the interface specification. 
                            2.  
                          2. Upstream software or PCIe link checker: Identifies invalid traffic from PCIe that violates FIM or PCIe specifications. For example, this checker flags an application sending traffic if it violates the FIM specification or creates a PCIe link issue by causing completion timeout or malformed TLP. 
                            3.  
                          3. FIM - Checks for bugs in the FIM fabric.
                            4.  
                           
                          Errors reported by the checker are logged in either the FME error registers port error registers, or both, as shown in the table below.  For more details on each of the registers, please refer to FME_CSR.xls
                           
                          Table 14-1: Error Registers
                           MMIO Region Area Register Description FME CoreFIM FME_ERROR FME Error Status Register 0.  Registers parity errors, underflow or overflow errors and access mismatches. FME CoreFIM FME_ERROR0_MASK FME Error Mask Register 0.  Write a 0 to mask errors in the FME Error Status Register 0. FME External PCIE0_ERROR PCIe0 Error Status Register. FME External PCIE0_ERROR_MASK PCIe0 Error Mask Register 0.  Write a 0 to mask errors in the PCIe0 Error Status Register 0. FME CoreFIM FME_FIRST_ERROR First FME Error Register. FME CoreFIM FME_NEXT_ERROR FME Next Error Register. FME CoreFIM RAS_NOFAT_ERR_STAT Reliability/Accessibility/Serviceability (RAS) Non-Fatal Error Status Register. FME CoreFIM RAS_NOFAT_ERR_MASK RAS Non-Fatal Error Mask Register. Write 0 to mask error fields in RAS_NOFAT_ERR_STAT Register. FME CoreFIM RAS_CATFAT_ERR_STAT RAS Catastrophic and Fatal Errors Status Register. FME CoreFIM RAS_CATFAT_ERR_MASK RAS Catastrophic and Fatal Errors Mask Register. Write 0 to mask error fields in the RAS_CATFAT_ERR_STAT Register. FME CoreFIM RAS_ERROR_INJ RAS error Injection Register. PORT CoreFIM PORT_ERROR Port Error Status Register. PORT CoreFIM PORT_FIRST_ERROR Port First Error Register . PORT CoreFIM PORT_MALFORMED_REQ0 Port Malformed Request Register 0.  Provides malformed request header LSBs. PORT CoreFIM PORT_MALFORMED_REQ1 Port Malformed Request Register 1.  Provides malformed request header MSBs."},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#141-fme-errors","title":"14.1 FME Errors","text":""},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1411-fme_error0","title":"14.1.1 FME_ERROR0","text":"
                          The FME_ERROR0 register flags CoreFIM FME errors in the Global Error (GLBL_ERROR) private feature. The error bits in this register are sticky bits. You can only clear these bits through software or through a system reset. Writing a 1 to the error field in the register clears the corresponding error bit. Writing a 1 to the corresponding bit in FME_ERROR0_MASK register masks the error.
                           
                          Table 14-2: FME Error Types
                           Error Type Description Fabric errors FIFO underflow/overflow condition in CoreFIM. These errors only occur if you have introduced bugs into the FIM or during very rare single event upset (SEU) or SEU-like events. Invalid port access A port can either be mapped to a PF or a VF, and not both. The checker reads the AfuAccessCtrl field in the FME CSR PORT0_OFFSET register to determine the access type of the Port. If it finds a PF is trying to access a port that is mapped to a VF or vice-versa, an error will be reported. Invalid AFU access An AFU can either be mapped to a PF or a VF, and not both. The checker reads the AfuAccessCtrl field in the FME CSR PORT0_OFFSET register to determine the access type of the AFU associated with the Port. If it finds a PF is trying to access an AFU that is mapped to a VF or vice-versa, an error is reported and a fake response is sent back to the requester to avoid a completion timeout on the host."},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1412-pcie0_error","title":"14.1.2 PCIE0_ERROR","text":"
                          The PCIe Avalon-ST to Arm\u00ae AMBA\u00ae 4 AXI4-Stream bridge monitors the PCIe link for errors and logs any such errors in the PCIE0_ERROR register (in PCIE0 feature region) and PCIE0_ERROR register in the GLBL_ERR private feature.    The error bits in the PCIE0_ERROR register are sticky bits that you can only clear through software or through a system reset. Writing a 1 to the error field in the register clears the corresponding error bit. Writing a 1 to the corresponding bit in PCIE0_ERROR0_MASK masks the error.  
                           
                          If you have other external FME features, you can add similar _ERROR registers to this space. Please refer to the following spreadsheet: https://github.com/OFS/ofs-f2000x-pl/ipss/pcie/rtl/PCIE_CSR.xls or the SystemVerilog file: https://github.com/OFS/ofs-f2000x-pl/ipss/pcie/rtl/pcie_csr.sv for more details on this register.  
                          NOTE
                           
                          The PCIE0_ERROR register is located in both the Global Error external feature memory space and a separate PCIe external feature memory space.  OPAE software supports the PCIe external feature memory space beginning at offset 0x40000 for OFS EA and going forward.  PCIe registers beginning at 0x4000 in the Global Error external feature memory space are there for backward compatibility to the Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1413-fme_first_error-fme_next_error","title":"14.1.3 FME_FIRST_ERROR, FME_NEXT_ERROR","text":"
                          The FME_FIRST_ERROR register flags which of the FME error reporting registers, such as FME_ERROR0, PCIE0_ERROR0, has reported the first error occurrence. The error fields of the first error register are then continuously logged into the FME_FIRST_ERROR register until a system reset or software clears all the errors in that first error register. Likewise, the FME_NEXT_ERROR indicates which of the FME error reporting registers (except the first error register) has reported the next occurrence of error after the first error register. The error fields of the next error register are continuously logged into the FME_NEXT_ERROR register until a system reset or software clears all the errors in the second error register.
                           
                          Please refer to fme_csr.sv for individual register field descriptions or the SystemVerilog file: fme_csr.sv
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1414-reliability-accessibility-serviceability-ras-error-status","title":"14.1.4 Reliability, Accessibility, Serviceability (RAS) Error Status","text":"
                          The RAS feature in CoreFIM labels errors as non-fatal, fatal or catastrophic based on their impact to the system.  * A non-fatal error usually originates from software or an AFU.  With a non-fatal error, the user application may still be able to recover from the error by performing a soft reset on the AFU, fixing the user application software if necessary, and clearing the error. On the other hand, a fatal or catastrophic error is non-recoverable and requires the platform to be reset. * Non-fatal errors are logged in the RAS_NOFAT_ERR_STAT register and fatal or catastrophic errors are logged in the RAS_CATFAT_ERR_STAT register.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14141-non-fatal-errors","title":"14.1.4.1 Non-Fatal Errors","text":"
                          The RAS_NOFAT_ERR_STAT is a read-only status register that is specifically added for the RAS feature. It logs the high-level status of non-fatal errors in the hardware.  Unlike the error bits in the PCIE0_ERROR and FME_ERROR0 registers which are RW1C (software can write a 1 to clear the error), the error bits in this register are read-only and can only be cleared by system reset. Software has an option to mask the error using RAS_NOFAT_ERR_MASK.
                           
                          Please refer to FME_CSR.xls for individual register field descriptions or the SystemVerilog file: fme_csr.sv
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14142-catastrophic-fatal-errors","title":"14.1.4.2 Catastrophic & Fatal Errors","text":"
                          The RAS_CATFAT_ERR_STAT is a read-only status register that is specifically added for the RAS feature. It captures the high-level status of errors that can only be recovered with a system reset. Therefore, the error bits in the RAS_CATFAT_ERR_STAT register are read-only and can only be cleared by system reset or masked through RAS_CATFAT_ERR_MASK. Please refer to FME_CSR.xls for individual register field descriptions or the SystemVerilog file: fme_csr.sv
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1415-ras-error-injection","title":"14.1.5 RAS Error Injection","text":"
                          For software testing purposes, you can inject non-fatal, fatal, and catastrophic errors into the platform through the RAS_ERROR_INJ register.  These errors are reported in the RAS_CATFAT_ERR_STAT and RAS_NOFAT_ERR_STAT registers. Please refer to FME_CSR.xls for individual register field descriptions or the SystemVerilog file: fme_csr.sv
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1416-fme-error-interrupts","title":"14.1.6 FME Error Interrupts","text":"
                          In an event of an FME error, the MSI-X module in the FIM generates an interrupt so the host can decide on the next course of action. The FIM does not stall upstream and downstream traffic after the FME error. However, a port error triggered by invalid request from a user AFU stalls all the traffic going from AFU to PCIe. The interrupt capability is discoverable by querying the NumbSuppInterrupt field of the PORT_CAPABILITY register in the Port private feature.  The MSI-X vector number is recorded in the InterruptVectorNumber field of the GLBL_ERROR_CAPABILITY register of the Global Error external feature.
                           
                          An FME error interrupt is generated in response to the FME_ERROR0, PCIE0_ERROR0, RAS_NOFAT_ERR_STAT, or RAS_CATFAT_ERR_STAT registers recording a new, unmasked, error per the rules defined by CoreFIM interrupt feature.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14161-msi-x-masking-pending-bit-array-pba-clearing","title":"14.1.6.1 MSI-X Masking & Pending Bit Array (PBA) Clearing","text":"
                          If the MSI-X vector corresponding to the FME error interrupt is masked, events are recorded in the PBA array.  Clearing the FME error status registers clears the corresponding MSI-X PBA entries.  If only some events are cleared, the normal interrupt triggering rules apply and a new pending interrupt is registered.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1417-fme-error-handling","title":"14.1.7 FME Error Handling","text":"
                          When the host receives an FME error interrupt, it must take the recommended actions described below to bring the system back to its normal operation.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14171-catastrophicfatal-error","title":"14.1.7.1 Catastrophic/Fatal Error","text":"
                          A system reset is mandatory for any catastrophic or fatal error.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14172-non-fatal-error","title":"14.1.7.2 Non-Fatal Error","text":"
                          When software receives a non-fatal error interrupt which does not require a system reset, it can take the following steps to clear the error after software handles the error: 1.  Set the *_ERROR_MASK register to all 1\u2019s to mask all errors 2. Clear the *_FIRST_ERROR register 3. Clear the *_ERROR register 4. Set *_ERROR_MASK register to all 0\u2019s to enable all errors
                           
                             
                          • Result: The *_ERROR & *_FIRST_ERROR registers begin capturing new errors.
                            •  
                           
                          NOTE
                           
                          A system reset can only clear RAS Error status registers.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#143-mmio-requests","title":"14.3 MMIO Requests","text":"
                          The FIM is designed to gracefully handle MMIO request scenarios.  
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1431-unsupported-functions-and-bar","title":"14.3.1 Unsupported Functions and BAR","text":"
                          The PCIe hard IP in the FIM guarantees that only TLP packets for the functions and BARs supported by the FIM (as configured in PCIe HIP IP instantiation) are sent to the FIM.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1432-mmio-request-decoding","title":"14.3.2 MMIO Request Decoding","text":"
                          The packet router and memory decoder in the FIM ensure that only legal MMIO requests are forwarded to the targeted MMIO region. Full address and BAR decoding is done both in the packet router and the memory decoder to ensure the requests are forwarded to the designated CSR region as defined in the MMIO Regions chapter.  Any unsolicited/illegal MMIO request is dropped, and an error is reported back to host through the FME error register.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1433-unused-fmeport-csr-regions","title":"14.3.3 Unused FME/Port CSR Regions","text":"
                          All the CSR slaves in FIM which are mapped to the FME or Port CSR regions must always respond to MMIO read requests targeting its associated CSR region. A CSR slave must return all 0s for MMIO reads to its unused CSR region such as a reserved space or a region where no CSR register is implemented for the address.
                           
                          The FIM ensures MMIO reads to FME or Port CSR regions that are not mapped to any CSR slave always gets a response of all 0s. The memory decoder module and fake responder module in the FIM provide this guaranteed response.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1434-unsupported-mmio-request","title":"14.3.4 Unsupported MMIO Request","text":"
                          Any MMIO request targeting FME or Port CSR regions with a length or address alignment that are not  supported by the FIM is dropped, and an error is logged in PCIE0_ERROR register. The MMIO checker module in the FIM guarantees this response. When an unsupported MMIO read request to the FIM CSR region is detected, the FIM sends back a CPL (completion without data) with error status (UR) back to host.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1435-afu-access-violation","title":"14.3.5 AFU Access Violation","text":"
                          AFU access violations refer to the scenarios where a PF is attempting to access the MMIO region of an AFU bound to a VF (virtualization enabled), or when a VF is trying to access the MMIO region of an AFU bound to a PF (virtualization disabled). When such a violation is detected, the FIM drops the request and logs the error in the FME_ERROR0 register. If the request is an MMIO read request, the FIM returns a fake response to the host.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1436-afu-mmio-response-timeout","title":"14.3.6 AFU MMIO Response Timeout","text":"
                          An AFU MMIO Response timeout functions in the same manner described in the MMIO Response Timeout section.
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#15-ofs-design-hierarchy","title":"15 OFS Design Hierarchy","text":"
                          Files for design, build and unit test simulation are found at https://github.com/OFS/ofs-f2000x-pl, release branch release/ofs-2023.3.  
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#151-design-guidance","title":"15.1 Design Guidance","text":"
                          The OFS FIM is designed with configurability and scalability in mind.  At a high level, these are the necessary steps for a user to customize the design.  Please refer to the FPGA Interface Manager Developer's Guide
                           
                          Table 15-2 Features
                           Step Description Comments 1 Re-configure PCIe HIP for additional VFs (if necessary) * PF0 is mandatory for running OPAE software* Only modification of the VFs (added or subtracted) by the user is recommended. * Default configuration supports 1 PF and 3 VFs. 2 Update Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF MUX-DEMUX configuration (if necessary) * The PF/VF MUX-DEMUX is parameterized for flexibility. You can change, add or delete PF or VF functions by updating the top_cfg_pkg.sv file. * You also have the option of keeping the default configuration and tying off the unused VFs if needed. 3 Update top level and AFU level as necessary * If you integrating additional external interfaces, make the edits at the top level (ofs_top.sv) and propagate the interface down to the AFU level (afu_top.sv and soc_afu_top.sv) 4 Add user implemented function(s) in AFU * All of your implemented functions must have the required Arm\u00ae AMBA\u00ae 4 AXI4-Stream interface for both the data path and the MMIO control path to CSRs. * All CSRs in the user-implemented function must have the required DFH layout. * See host exerciser CSRs for reference. 5 Update UVM testbench * The OFS full chip UVM environment is coded specifically for verifying the default configuration containing the host exercisers for the PCIe, memory, and Ethernet. * User must update the UVM testbench to reflect new RTL behavior for any customization changes. See The Simulation User Guide 
                          For more information on modifying the FIM, refer to the FPGA Interface Manager Developer's Guide
                          "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                          Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                           
                          OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/","title":"Platform Evaluation Script: Open FPGA Stack for Intel Agilex 7 SoC Attach FPGAs","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to user space."},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#1-overview","title":"1 Overview","text":""},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#11-about-this-document","title":"1.1 About this Document","text":"
                          This document serves as a set-up and user guide for the checkout and evaluation of an Intel IPU Platform F2000X-PL development platform using Open FPGA Stack (OFS). After reviewing the document, you will be able to:
                           
                             
                          • Set-up and modify the script to the your environment
                            •  
                           
                             
                          • Compile and simulate an OFS reference design
                            •  
                           
                             
                          • Run hardware and software tests to evaluate the complete OFS flow
                            •  
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#table-1-2-software-version-summary","title":"Table 1-2: Software Version Summary","text":"Component Version Description FPGA Platform Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL Intel platform you can use for your custom board development OFS FIM Source Code Branch: https://github.com/OFS/ofs-f2000x-pl, Tag: ofs-2023.3-1 OFS Shell RTL for Intel Agilex 7 FPGA (targeting https://github.com/OFS/ofs-f2000x-pl) OFS FIM Common Branch: https://github.com/OFS/ofs-fim-common/releases/tag/ofs-2023.3-2, Tag: https://github.com/OFS/ofs-fim-common/releases/tag/ofs-2023.3-2 Common RTL across all OFS-based platforms AFU Examples Branch: examples-afu , Tag:ofs-examples-https://github.com/OFS/examples-afu/releases/tag/ofs-2023.3-2 Tutorials and simple examples for the Accelerator Functional Unit region (workload region) OPAE SDK Branch: 2.10.0-1, Tag: 2.10.0-1 Open Programmable Acceleration Engine Software Development Kit Kernel Drivers Branch: ofs-2023.3-6.1-3, Tag: ofs-2023.3-6.1-3 OFS specific kernel drivers OPAE Simulation Branch: opae-sim, Tag: 2.10.0-1 Accelerator Simulation Environment for hardware/software co-simulation of your AFU (workload) Intel Quartus Prime Pro Edition Design Software 23.3 Intel\u00ae Quartus\u00ae Prime Pro Edition Linux Software tool for Intel FPGA Development Operating System Ubuntu 22.04 Operating system on which this script has been tested 
                          A download page containing the release and already-compiled FIM binary artifacts that you can use for immediate evaluation on the Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL can be found on the OFS ofs-2023.3-1 official release drop on GitHub.
                           
                           Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#2-introduction-to-ofs-evaluation-script","title":"2 Introduction to OFS Evaluation Script","text":"
                          By following the setup steps and using the OFS evaluation script you can quickly evaluate many features that the OFS framework provides and also leverage this script for your own development.  
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#21-pre-requisites","title":"2.1 Pre-Requisites","text":"
                          This script uses the following set of software tools which should be installed using the directory structure below. Tool versions can vary.
                           
                             
                          • Intel Quartus\u00ae Prime Pro Software
                            •  
                          • Synopsys\u00ae VCS Simulator
                            •  
                          • Siemens\u00ae Questa\u00ae Simulator
                            •  
                           
                          Figure 2-1 Folder Hierarchy for Software Tools
                           
                           
                             
                          1.  
                            You must create a directory named \"ofs-X.X.X\" where the X represents the current release number, for example ofs-2023.3-1. 
                             
                            2.  
                          2.  
                            You must clone the required OFS repositories as per Figure 2-2.  Please refer to the BKC table for locations. Please go [OFS Getting Started User Guide] for the instructions for the BKC installation.
                             
                            3.  
                          3.  
                            Once the repositories are cloned, contact your intel representative to receive the eval script evaluation script (ofs_f2000x_eval.sh) and copy it to the ofs-2023.3-1 directory location as shown in the example below:
                             
                            4.  
                           
                          Figure 2-2 Directory Structure for OFS Project
                           
                          ## ofs-2023.3-1\n##  -> examples-afu\n##  -> linux-dfl\n##  -> ofs-f2000x-pl\n##  -> opae-sdk\n##  -> opae-sim\n##  -> ofs_f2000x_eval.sh\n
                           
                             
                          1. Contact your intel representative to receive the README file named (README_ofs-agx7-pcie-attach_eval.txt) and copy it to the ofs-2023.3-1 directory location. The README informs the user which sections to modify in the script prior to building the FIM and running hardware, software and simulation tests.
                            2.  
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#22-f2000x-evaluation-script-modification","title":"2.2 f2000x Evaluation Script modification","text":"
                          To adapt this script to the user environment please follow the instructions below which explains which line numbers to change in the ofs_f2000x_eval.sh script 
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#user-directory-creation","title":"User Directory Creation","text":"
                          The user must create the top-level source directory and then clone the OFS repositories
                           
                          mkdir ofs-2023.3-1\n
                           
                          In the example above we have used ofs-2023.3-1 as the directory name
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#set-up-proxy-server-lines-67-69","title":"Set-Up Proxy Server (lines 67-69)","text":"
                          Please enter the location of your proxy server to allow access to external internet to build software packages.
                           
                          Note: Failing to add proxy server will prevent cloning of repositories and the user will be unable to build the OFS framework.
                           
                          export http_proxy=<user_proxy>\nexport https_proxy=<user_proxy>\nexport no_proxy=<user_proxy>\n
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#license-files-lines-72-74","title":"License Files (lines 72-74)","text":"
                          Please enter the the license file locations for the following tool variables
                           
                          export LM_LICENSE_FILE=<user_license>\nexport DW_LICENSE_FILE=<user_license>\nexport SNPSLMD_LICENSE_FILE=<user_license>\n
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#tools-location-line-87-88-89-90","title":"Tools Location (line 87, 88, 89, 90)","text":"
                          Set Location of Quartus, Synopsys and Questasim
                           
                          export QUARTUS_TOOLS_LOCATION=/home\nexport SYNOPSYS_TOOLS_LOCATION=/home\nexport QUESTASIM_TOOLS_LOCATION=/home\nexport ONEAPI_TOOLS_LOCATION=/opt\n
                           
                          In the example above /home is used as the base location of Quartus, Synopsys and Questasim tools, /opt is used for the oneAPI tools
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#quartus-tools-version-line-93","title":"Quartus Tools Version (line 93)","text":"
                          Set version of Quartus
                           
                          export QUARTUS_VERSION=23.3\n
                           
                          In the example above \"23.3\" is used as the Quartus tools version
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#opae-tools-line-108","title":"OPAE Tools (line 108)","text":"
                          change OPAE SDK VERSION
                           
                          export OPAE_SDK_VERSION=2.10.0-1\n
                           
                          In the example above \"2.10.0-1\" is used as the OPAE SDK tools version
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#pcie-bus-number","title":"PCIe (Bus Number)","text":"
                          The Bus number must be entered by the user after installing the hardware in the chosen server, in the example below \"b1\" is the Bus Number for a single card as defined in the evaluation script.
                           
                          export OFS_CARD0_BUS_NUMBER=b1\n
                           
                          The evaluation script uses the bus number as an identifier to interrogate the card. The command below will identify the accelerater card plugged into a server. 
                           
                          lspci | grep acc\n\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                           
                          The result identifies the card as being assigned \"b1\" as the bus number so the entry in the script changes to
                           
                          export OFS_CARD0_BUS_NUMBER=b1\n
                           
                          The user can also run the following command on the ofs_f2000x_eval.sh script to automatically change the bus number to b1 in the ofs_f2000x_eval.sh script.
                           
                          grep -rli 'b1' * | xargs -i@ sed -i 'b1' @
                           
                          if the bus number is 85 for example 
                           
                          85:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                           
                          the command to change to 85 in the evaluation script would be
                           
                          grep -rli 'b1' * | xargs -i@ sed -i '85' @
                           
                          The ofs_f2000x_eval.sh script has now been modified to the server set-up and the user can proceed to build, compile and simulate the OFS stack
                           
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#3-f2000x-evaluation-script","title":"3 f2000x Evaluation Script","text":""},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#31-overview","title":"3.1 Overview","text":"
                          The figure below shows a snapshot of the full evaluation script menu showing all 62 options and each one one of 11 sub-menus which focus on different areas of evaluation. Each of these menu options are described in the next section.
                           
                          Figure 3-1 ofs_f2000x_eval.sh Evaluation Menu
                           
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#311-ofs-tools-menu","title":"3.1.1 OFS TOOLS MENU","text":"
                          By selecting \"List of Documentation for OFS f2000x Project,\" a list of links to the latest OFS documentation appears. Note that these links will take you to documentation for the most recent release which may not correspond to the release version you are evaluating. To find the documentation specific to your release, ensure you clone the intel-ofs-docs tag that corresponds to your OFS version.
                           
                          By selecting \"Check Versions of Operating System and Quartus Premier Design Suite\", the tool verifies correct Operating System, Quartus version, kernel parameters, license files and paths to installed software tools.
                           
                           Menu Option Example Output 1 - List of Documentation for OFS f2000x Project  Open FPGA Stack Overview                  Guides you through the setup and build steps to evaluate the OFS solution                 https://ofs.github.io 2 - Check versions of Operating System and Quartus Premier Design Suite (QPDS) Checking Linux release                 Linux version 5.15.92-dfl-66b0076c2c-lts (oe-user@oe-host) (x86_64-ese-linux-gcc (GCC) 11.3.0, GNU ld (GNU Binutils) 2.38.20220708) #1 SMP PREEMPT Wed Feb 8 21:21:27 UTC 2023                  Checking RedHat release                 cat: /etc/redhat-release: No such file or directory                  Checking Ubuntu release                 cat: /etc/lsb-release: No such file or directory                  Checking Kernel parameters                 initrd=\\microcode.cpio LABEL=Boot root=PARTUUID=2beb0add-07bf-4736-bc31-9b60e7b78375  rootfstype=ext4  console=ttyS5,115200 iomem=relaxed intel_iommu=on pci=realloc default_hugepagesz=1G hugepagesz=1G hugepages=4 rootwait console=ttyS0,115200 console=tty0                  Checking Licenses                 LM_LICENSE_FILE is set to port@socket number:port@socket number                 DW_LICENSE_FILE is set to port@socket number:port@socket number                 SNPSLMD_LICENSE_FILE is set to port@socket number:port@socket number                  Checking Tool versions                 QUARTUS_HOME is set to /home/intelFPGA_pro/23.3/quartus                 QUARTUS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus                 IMPORT_IP_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../ip                 QSYS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../qsys/bin                  Checking QPDS Patches                 Quartus Prime Shell                 Version 23.3 Build XXX XX/XX/XXXX Patches X.XX SC Pro Edition                 Copyright (C) XXXX  Intel Corporation. All rights reserved."},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#312-ofs-hardware-menu","title":"3.1.2 OFS HARDWARE MENU","text":"
                          Identifies card by PCIe number, checks power, temperature and current firmware configuration. 
                           
                           Menu Option Example Output 3 - Identify Acceleration Development Platform (OFS) f2000x Hardware via PCIe PCIe card detected as                 15:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)                 Host Server is connected to SINGLE card configuration 4 - Identify the Board Management Controller (BMC) Version and check BMC sensors Intel IPU Platform F2000X-PL                 Board Management Controller NIOS FW version: 1.1.9                 Board Management Controller Build version: 1.1.9                 //****** BMC SENSORS ******//                 Object Id                        : 0xF000000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x5010302EB5E4B14                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 3e46f208-8b49-5899-9e3d-b8b6a4d25a4e 5 - Identify the FPGA Management Engine (FME) Version Intel IPU Platform F2000X-PL                 Board Management Controller NIOS FW version: 1.1.9                 Board Management Controller Build version: 1.1.9                 //****** FME ******//                 Object Id                        : 0xF000000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x5010302EB5E4B14                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 3e46f208-8b49-5899-9e3d-b8b6a4d25a4e                 Boot Page                        : user1                 User1 Image Info                 : 9e3db8b6a4d25a4e3e46f2088b495899                 User2 Image Info                 : 9e3db8b6a4d25a4e3e46f2088b495899                 Factory Image Info               : None 6 - Check Board Power and Temperature Intel IPU Platform F2000X-PL                 Board Management Controller NIOS FW version: 1.1.9                 Board Management Controller Build version: 1.1.9                 //****** POWER ******//                 Object Id                        : 0xF000000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x5010302EB5E4B14                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 3e46f208-8b49-5899-9e3d-b8b6a4d25a4e                 ( 1) QSFP (Primary) Supply Rail Voltage                 : N/A                 etc ......................                  Intel IPU Platform F2000X-PL                 Board Management Controller NIOS FW version: 1.1.9                 Board Management Controller Build version: 1.1.9                 //****** TEMP ******//                 Object Id                        : 0xF000000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x5010302EB5E4B14                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 3e46f208-8b49-5899-9e3d-b8b6a4d25a4e                 ( 1) FPGA FABRIC Remote Digital Temperature#3           : 33.00 Celsius                 etc ...................... 7 - Check Accelerator Port status //****** PORT ******//                 Object Id                        : 0xEF00000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00 8 - Check MAC and PHY status Intel IPU Platform F2000X-PL                 Board Management Controller NIOS FW version: 1.1.9                 Board Management Controller Build version: 1.1.9                 //****** MAC ******//                 Object Id                        : 0xF000000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x5010302EB5E4B14                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 3e46f208-8b49-5899-9e3d-b8b6a4d25a4e                 Number of MACs                   : 255                 mac info is not supported                  Intel IPU Platform F2000X-PL                 Board Management Controller NIOS FW version: 1.1.9                 Board Management Controller Build version: 1.1.9                 //****** PHY ******//                 Object Id                        : 0xF000000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x5010302EB5E4B14                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 3e46f208-8b49-5899-9e3d-b8b6a4d25a4e                 QSFP0                            : Connected                 QSFP1                            : Connected                  //****** HSSI information ******//                 HSSI version                     : 1.0                 Number of ports                  : 8                 Port0                            :25GbE        DOWN                 Port1                            :25GbE        DOWN                 Port2                            :25GbE        DOWN                 Port3                            :25GbE        DOWN                 Port4                            :25GbE        DOWN                 Port5                            :25GbE        DOWN                 Port6                            :25GbE        DOWN                 Port7                            :25GbE        DOWN"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#313-ofs-pfvf-mux-and-ofss-menu","title":"3.1.3 OFS PF/VF MUX and OFSS MENU","text":"
                          This menu reports the number of PF/VF functions in the reference example and also allows you to reduce the number to 1PF and 1VF to reduce resource utilisation and create larger area for your workload development. This selection is optional and if the user wants to implement the default number of PF's and VF's then option 9, 10 and 11 should not be used.  Additionally the code used to make the PF/VF modification can be leveraged to increase or modify the number of PF/VFs in the existing design within the limits that the PCIe Subsystem supports (8PF/2K VFs)
                           
                           Menu Option Description 9 - Check PF/VF Mux and OFSS Configuration for OFS f2000x Project  This menu selection displays the current configuration of all f2000x ofss files located in the following directory $OFS_ROOTDIR/tools/ofss_config               Check f2000x base config OFSS set up             [ip]             type = ofs              [settings]             platform = f2000x             fim = base_x16             family = agilex             part = AGFC023R25A2E2VR0             device_id = 6100               Check f2000x hssi_2x100 OFSS set up             [ip]             type = hssi              [settings]             output_name = hssi_ss             num_channels = 2             data_rate = 100GCAUI-4               Check f2000x hssi_2x100_caui2 OFSS set up             [ip]             type = hssi              [settings]             output_name = hssi_ss             num_channels = 2             data_rate = 100GAUI-2               Check f2000x hssi_8x10 OFSS set up             [ip]             type = hssi              [settings]             output_name = hssi_ss             num_channels = 8             data_rate = 10GbE              Check f2000x hssi_8x25 OFSS set up             [ip]             type = hssi              [settings]             output_name = hssi_ss             num_channels = 8             data_rate = 25GbE               Check f2000x IOPLL OFSS set up             [ip]             type = iopll              [settings]             output_name = sys_pll             instance_name = iopll_0              [p_clk]             freq = 470              Check f2000x Memory OFSS set up             [ip]             type = memory              [settings]             output_name = mem_ss_fm             preset = f2000x              Check f2000x PCIe Host OFSS set up             [ip]             type = pcie              [settings]             output_name = pcie_ss              [pf0]             bar0_address_width = 21              [pf1]              Check f2000x PCIe SoC OFSS set up             [ip]             type = pcie              [settings]             output_name = soc_pcie_ss              [pf0]             num_vfs = 3             bar0_address_width = 21             vf_bar0_address_width = 21  10 - Modify PF/VF Mux and OFSS Configuration for OFS f2000x Project As an example this menu selection modifies the pcie_host.ofss and pcie_soc.ofss file to 1 PF located in the following directory $OFS_ROOTDIR/tools/ofss_config/pcie             This option also displays the the modified pcie_host.ofss and pcie_soc.ofss files 11 - Build PF/VF Mux and OFSS Configuration for OFS f2000x Project If option 10 is not used then the default number of PF's and VF's is used to build the FIM, if option 10 is selected then only 1 VF is built to reduce logic utilisation"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#314-ofs-fimpr-build-menu","title":"3.1.4 OFS FIM/PR BUILD MENU","text":"
                          Builds FIM, Partial Reconfiguration Region and Remote Signal Tap
                           
                           Menu Option Description 12 - Check OFS software versions for OFS f2000x Project OFS_ROOTDIR is set to /home/user_area/ofs-X.X.X/ofs-f2000x-pl                 OPAE_SDK_REPO_BRANCH is set to release/X.X.X                 OPAE_SDK_ROOT is set to /home/user_area/ofs-X.X.X/ofs-f2000x-pl/../opae-sdk                 LD_LIBRARY_PATH is set to /home/user_area/ofs-X.X.X/ofs-f2000x-pl/../opae-sdk/lib64: 13 - Build FIM for f2000x Hardware This option builds the FIM based on the setting for the $OFS_PLATFORM, $FIM_SHELL environment variable. Check this variable in the following file ofs_f2000x_eval.sh 14 - Check FIM Identification of FIM for f2000x Hardware The FIM is identified by the following file fme-ifc-id.txt located at $OFS_ROOTDIR/$FIM_WORKDIR/syn/syn_top/ 15 - Build Partial Reconfiguration Tree for f2000x Hardware This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development 16 - Build Base FIM Identification(ID) into PR Build Tree template This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU workloads 17 - Build Partial Reconfiguration Tree for f2000x Hardware with Remote Signal Tap This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development for the Remote Signal Tap flow 18 - Build Base FIM Identification(ID) into PR Build Tree template with Remote Signal Tap This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree for Remote Signal Tap to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU workloads"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#315-ofs-hardware-programmingdiagnostic-menu","title":"3.1.5 OFS HARDWARE PROGRAMMING/DIAGNOSTIC MENU","text":"
                          The following submenu allows you to: * Program and check flash  * Perform a remote system update (RSU) of the FPGA image into the FPGA * Bind virtual functions to VFIO PCIe driver  * Run host exerciser (HE) commands such as loopback to test interfaces VFIO PCI driver binding * Read the control and status registers (CSRs) for bound modules that are part of the OFS reference design.
                           
                           Menu Option Description 19 - Program BMC Image into f2000x Hardware The user must place a new BMC flash file in the following directory $OFS_ROOTDIR/bmc_flash_files. Once the user executes this option a new BMC image will be programmed. A remote system upgrade command is initiated to store the new BMC image 20 - Check Boot Area Flash Image from f2000x Hardware This option checks which location area in FLASH the image will boot from, the default is user1              Boot Page : user1 21 - Program FIM Image into user1 area for f2000x Hardware This option programs the FIM image \"ofs_top_page1_unsigned_user1.bin\" into user1 area in flash 22 - Initiate Remote System Upgrade (RSU) from user1 Flash Image into f2000x Hardware This option initiates a Remote System Upgrade and soft reboots the server and re-scans the PCIe bus for the new image to be loaded              2022-11-10 11:26:24,307 - [[pci_address(0000:b1:00.0), pci_id(0x8086, 0xbcce)]] performing RSU operation             2022-11-10 11:26:24,310 - [[pci_address(0000:b0:02.0), pci_id(0x8086, 0x347a)]] removing device from PCIe bus             2022-11-10 11:26:24,357 - waiting 10 seconds for boot             2022-11-10 11:26:34,368 - rescanning PCIe bus: /sys/devices/pci0000:b0/pci_bus/0000:b0             2022-11-10 11:26:35,965 - RSU operation complete 23 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 24 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 25 - Create Virtual Functions (VF) and bind driver to vfio-pci f2000x Hardware This option creates vfio-pci driver binding for the PF's and VF's             Once the VF's have been bound to the driver the user can select menu option 23 to check that the new drivers are bound 26 - Verify FME Interrupts with hello_events The hello_events utility is used to verify FME interrupts. This tool injects FME errors and waits for error interrupts, then clears the errors 27 - Run HE-LB Test This option runs 5 tests              1) checks and generates traffic with the intention of exercising the path from the AFU to the Host at full bandwidth             2) run a loopback throughput test using one cacheline per request             3) run a loopback read test using four cachelines per request             4) run a loopback write test using four cachelines per request             5) run a loopback throughput test using four cachelines per request 28 - Run HE-MEM Test This option runs 2 tests              1) Checking and generating traffic with the intention of exercising the path from FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host             2) run a loopback throughput test using one cacheline per request 29 - Run HE-HSSI Test This option runs 1 test              HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs, and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic              1) Send traffic through the 10G AFU              30 - Run Traffic Generator AFU Test This option runs 3 tests              TG AFU has an OPAE application to access & exercise traffic, targeting a specific bank              1) Run the preconfigured write/read traffic test on channel 0             2) Target channel 1 with a 1MB single-word write only test for 1000 iterations             3) Target channel 2 with 4MB write/read test of max burst length for 10 iterations 31 - Read from CSR (Command and Status Registers) for f2000x Hardware This option reads from the following CSR's             HE-LB Command and Status Register Default Definitions             HE-MEM Command and Status Register Default Definitions             HE-HSSI Command and Status Register Default Definitions"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#316-ofs-hardware-afu-testing-menu","title":"3.1.6  OFS HARDWARE AFU TESTING MENU","text":"
                          This submenu tests partial reconfiguration by building and loading an memory-mapped I/O example AFU/workload, executes software from host, and tests remote signal tap.
                           
                           Menu Option Description 32 - Build and Compile host_chan_mmio example This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS (Green Bit Stream) ready for hardware programming 33 - Execute host_chan_mmio example This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test 34 - Modify host_chan_mmio example to insert Remote Signal Tap This option inserts a pre-defined host_chan_mmio.stp Signal Tap file into the OFS code to allow a user to debug the host_chan_mmio AFU example 35 - Build and Compile host_chan_mmio example with Remote Signal Tap This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS(Green Bit Stream) ready for hardware programming with Remote Signal tap enabled 36 - Execute host_chan_mmio example with Remote Signal Tap This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test. The user must open the Signal Tap window when running the host code to see the transactions in the Signal tap window"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#317-ofs-hardware-afu-bbb-testing-menu","title":"3.1.7 OFS HARDWARE AFU BBB TESTING MENU","text":"
                          This submenu tests partial reconfiguration using a hello_world example AFU/workload, executes sw from host
                           
                           Menu Option Description 37 - Build and Compile hello_world example This option builds the hello_ world example from the following repo $FPGA_BBB_CCI_SRC/samples/tutorial/afu_types/01_pim_ifc/$AFU_BBB_TEST_NAME, where AFU_BBB_NAME=hello_world. This produces a GBS(Green Bit Stream) ready for hardware programming 38 - Execute hello_world example This option builds the host code for hello_world example and programs the GBS file and then executes the test"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#319-ofs-unit-test-project-menu","title":"3.1.9 OFS UNIT TEST PROJECT MENU","text":"
                          Builds, compiles and runs standalone simulation block tests. More unit test examples are found at the following location ofs-f2000x-pl/sim/unit_test 
                           
                           Menu Option Result 39 - Generate Simulation files for Unit Test This option builds the simulation file set for running a unit test simulation 40 - Simulate Unit Test dfh_walker and log waveform This option runs the dfh_walker based on the environment variable \"UNIT_TEST_NAME=dfh_walker\" in the evaluation script. A user can change the test being run by modifying this variable"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#3110-ofs-uvm-project-menu","title":"3.1.10 OFS UVM PROJECT MENU","text":"
                          Builds, compiles and runs full chip simulation tests. The user should execute the options sequentially ie 41,42, 43 and 44
                           
                           Menu Option Description 41 - Check UVM software versions for f2000x Project DESIGNWARE_HOME is set to /home/synopsys/vip_common/vip_Q-2020.03A                 UVM_HOME  is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm                 VCS_HOME is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel                 VERDIR is set to /home/user_area/ofs-X.X.X/fim-f2000x-pl/verification                 VIPDIR is set to /home/user_area/ofs-X.X.X/fim-f2000x-pl/verification 42 - Compile UVM IP This option compiles the UVM IP  43 - Compile UVM RTL and Testbench This option compiles the UVM RTL and Testbench 44 - Simulate UVM dfh_walking_test and log waveform This option runs the dfh_walking test based on the environment variable \"UVM_TEST_NAME=dfh_walking_test\" in the evaluation script. A user can change the test being run by modifying this variable 45 - Simulate all UVM test cases (Regression Mode) This option runs the f2000x regression mode, cycling through all UVM tests defined in verification/tests/test_pkg.svh file"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#3111-ofs-build-all-project-menu","title":"3.1.11 OFS BUILD ALL PROJECT MENU","text":"
                          Builds the complete OFS flow, good for regression testing and overnight builds
                           
                          For this menu a user can run a sequence of tests (compilation, build and simulation) and executes them sequentially. After the script is successfully executed, a set of binary files is produced which a you can use to evaluate your hardware. Log files are also produced which checks whether the tests passed.
                           
                          A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 46 from the main menu the script will execute 24 tests ie (main menu options 2, 9, 12, 13, 14, 15, 16, 17, 18, 32, 34, 35, 37, 39, 40, 41, 42, 43 and 44. These 19 menu options are chosen to build the complete OFS flow covering build, compile and simulation.
                           
                           Menu Option Result 46 - Build and Simulate Complete f2000x Project Generating Log File with date and timestamp                 Log file written to /home/guest/ofs-2.3.1/log_files/f2000x_log_2022_11_10-093649/ofs_f2000x_eval.log"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#definition-of-multi-test-set-up","title":"Definition of Multi-Test Set-up","text":"
                          Menu Option 46 above in the evaluation script can be refined to tailor it to the users need and is principally defined by the variable below
                           
                          MULTI_TEST[A,B]=C
                           
                          where
                           
                          A= Total Number of menu options in script B= Can be changed to a number to select the test order C= Menu Option in Script
                           
                          Example 1 MULTI_TEST[46,0]=2
                           
                          A= 46 is the total number of options in the script B= 0 indicates that this is the first test to be run in the script C= Menu option in Script ie 2- List of Documentation for OFS f2000x Project
                           
                          Example 2 MULTI_TEST[46,0]=2 MULTI_TEST[46,1]=9
                           
                          In the example above two tests are run in order ie 0, and 1 and the following menu options are executed ie 2- List of Documentation for OFS f2000x Project and 9 - Check OFS software versions for OFS f2000x Project
                           
                          The user can also modify the build time by de-selecting options they do not wish to use, see below for a couple of use-case scenarios.
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#default-user-case","title":"Default User Case","text":"
                          A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 46 from the main menu the script will execute 24 tests ie (main menu options 2, 9, 12, 13, 14, 15, 16, 17, 18, 32, 34, 35, 37, 39, 40, 41, 42, 43 and 44. All other tests with an \"X\" indicates do not run that test
                           
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#user-case-for-ofs-fimpr-build-menu","title":"User Case for OFS FIM/PR BUILD MENU","text":"
                          In the example below when the user selects option 46 from the main menu the script will only run options from the OFS FIM/PR BUILD MENU (7 options, main menu options 12, 13, 14, 15, 16, 17 and 18). All other tests with an \"X\" indicates do not run that test.
                           
                           
                          "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#4-f2000x-common-test-scenarios","title":"4 f2000x Common Test Scenarios","text":"
                          This section will describe the most common compile build scenarios if a user wanted to evaluate an acceleration card on their server. The Pre-requisite column indcates the menu comamnds that must be run befere executing the test eg To run Test 5 then a user needs to have run option 13, 15 and 16 before running options 23, 24, 25, 32 and 33.
                           Test Test Scenario Pre-Requisite Menu Option Menu Option Test 1 FIM Build - 13 Test 2 Partial Reconfiguration Build 13 15, 16 Test 3 Program FIM and perform Remote System Upgrade 13 21, 22 Test 4 Bind PF and VF to vfio-pci drivers - 23, 24, 25 Test 5 Build, compile and test AFU on hardware 13, 15, 16 23, 24, 25, 32, 33 Test 6 Build, compile and test AFU Basic Building Blocks on hardware 13, 15, 16 23, 24, 25, 37, 38 Test 7 Build and Simulate Unit Tests - 39, 40 Test 8 Build and Simulate UVM Tests - 42, 43, 44, 45"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                          Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                           
                          OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/","title":"OFS Getting Started Guide for Intel Agilex 7 SoC Attach FPGAs","text":"
                          Last updated: February 03, 2024 
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#10-about-this-document","title":"1.0 About this Document","text":"
                          The purpose of this document is to help users get started in evaluating the 2023.3 version of the SoC Attach release targeting the Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL. After reviewing this document, a user shall be able to:
                           
                             
                          • Set up their server environment according to the Best Known Configuration (BKC)
                            •  
                          • Build a Yocto image with the OPAE SDK and Linux DFL Drivers included
                            •  
                          • Load and verify firmware targeting the SR and PR regions of the board, the BMC, and the ICX-D SoC NVMe and BIOS
                            •  
                          • Verify full stack functionality offered by the SoC Attach OFS solution
                            •  
                          • Know where to find additional information on other SoC Attach ingredients
                            •  
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#11-audience","title":"1.1 Audience","text":"
                          The information in this document is intended for customers evaluating the Intel IPU Platform F2000X-PL. The card is an acceleration development platform (ADP) intended to be used as a starting point for evaluation and development. This document will cover key topics related to initial bring up and development of the Intel IPU Platform F2000X-PL, with links for deeper dives on the topics discussed therein.
                           
                          Note: Code command blocks are used throughout the document. Commands that are intended for you to run are preceded with the symbol '$', and comments with '#'. Full command output may not be shown.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-1-terminology","title":"Table 1: Terminology","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to user space."},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-2-related-documentation","title":"Table 2: Related Documentation","text":"Name Location Platform Evaluation Script: Open FPGA Stack for Intel Agilex 7 FPGA link FPGA Interface Manager Technical Reference Manual for Intel Agilex 7 SoC Attach: Open FPGA Stack link Software Reference Manual: Open FPGA Stack link Intel\u00ae FPGA Interface Manager Developer Guide: Intel Agilex 7 SoC Attach: Open FPGA Stack link Intel\u00ae Accelerator Functional Unit Developer Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 FPGAs SoC Attach link Security User Guide: Intel Open FPGA Stack link Virtual machine User Guide: Open FPGA Stack + KVM link Docker User Guide: Intel\u00ae Open FPGA Stack link Release Notes link - under \"Important Notes\" Board Management Controller User Guide v1.1.9 (Pre-Release): Intel IPU Platform F2000X-PL Work with your Intel sales representative for access"},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-3-related-repositories","title":"Table 3: Related Repositories","text":"Name Location Intel-FPGA-BBB https://github.com/OPAE/intel-fpga-bbb.git OFS-PLATFORM-AFU-BBB https://github.com/OFS/ofs-platform-afu-bbb.git, tag: ofs-2023.3-1 AFU-EXAMPLES https://github.com/OFS/examples-afu.git, tag: ofs-2023.2-1 OPAE-SDK https://github.com/OFS/opae-sdk.git, tag: 2.10.0-1 LINUX-DFL https://github.com/OFS/linux-dfl.git, tag: ofs-2023.1-6.15-1 META-OFS https://github.com/OFS/meta-ofs, tag: ofs-2023.1-4"},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-4-software-and-firmware-component-version-summary-for-soc-attach","title":"Table 4: Software and Firmware Component Version Summary for SoC Attach","text":"Component Version Download link (where applicable) Available FIM Version(s) PR Interface ID: 3dac7126-3ce7-5fe8-b629-932096abb09b 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell Host Operating System Ubuntu 22.04 Official Release Page Host OPAE SDK 2.10.0-1 https://github.com/OFS/opae-sdk/releases/tag/2.5.0-3 SoC OS meta-intel-ese Reference Distro 1.0-ESE (kirkstone) 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell SoC Kernel Version 5.15.92-dfl-66b0076c2c-lts 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell SoC OPAE SDK 2.10.0-1 https://github.com/OFS/opae-sdk/releases/tag/2.5.0-3 SoC Linux DFL ofs-2023.1-6.15-1 https://github.com/OFS/linux-dfl/releases/tag/ofs-2023.1-5.15-1 SoC BMC and RTL 1.2.4 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell SoC BIOS 0ACRH608_REL 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell 
                          Not all components shown in Table 4 will have an update available upon release. The OPAE SDK and Linux DFL software stacks are incorporated into a Yocto image and do not need to be downloaded separately. Updates required for the 2023.3 OFS SoC Attach Release for Intel IPU Platform F2000X-PL are shown under Table 9 in section 2.0 Updating the Intel IPU Platform F2000X-PL.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#12-server-requirements","title":"1.2 Server Requirements","text":"
                          The following requirements must be met when purchasing a server to support the Intel IPU Platform F2000X-PL.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#121-host-server-specifications","title":"1.2.1 Host Server Specifications","text":"
                          The host server must meet the following minimal specifications:
                           
                             
                          • The server platform must contain at least 64GiB of RAM to to compile the Yocto or FIM images
                            •  
                          • If using the server platform to build a Yocto image, it is recommended to have at least 200 GB of free storage space
                            •  
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#122-host-bios","title":"1.2.2 Host BIOS","text":"
                          Te Host BIOS settings known to work with the Intel IPU Platform F2000X-PL:
                           
                             
                          • PCIe slot width must be x16
                            •  
                          • PCIe slot speed must be 4
                            •  
                          • PCIe slot must have iommu enabled
                            •  
                          • Intel VT for Directed I/O (VT-d) must be enabled
                            •  
                           
                          Specific BIOS paths are not listed here, as they can differ between BIOS vendors and versions.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#123-host-server-kernel-and-grub-configuration","title":"1.2.3 Host Server Kernel and GRUB Configuration","text":"
                          While many host Linux kernel and OS distributions may work with this design, only the following configuration(s) have been tested:
                           
                             
                          • Ubuntu 22.04, 6.1-lts
                            •  
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#13-server-forced-air-cooling","title":"1.3 Server Forced Air Cooling","text":"
                          The Intel IPU Platform F2000X-PL is a high-performance processing card with a passive heat sink to dissipate device heat and must be installed in a server with sufficient forced airflow cooling to keep all devices operating below maximum temperature. The table below lists the thermal terms and descriptions used in thermal analysis.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-5-thermal-terms-and-descriptions","title":"Table 5: Thermal Terms and Descriptions","text":"Term Description Cubic Feet per Minute (CFM) Volumetric airflow rate, in cubic feet per minute, of air passing through faceplate. Tj FPGA Junction Temperature TLA Local Ambient temperature. Temperature of forced air as it enters the Intel IPU Platform F2000X-PL. \u00a0 Note: In many systems, this is higher than the room ambient due to  heating effects of chassis components. 
                          Note: The FPGA junction temperature must not exceed 100\u00b0C. The case temperature of the QSFP modules must meet the module vendor's specification.
                           
                          Note: The table below provides the thermal targets for which the Intel IPU Platform F2000X-PL was designed. As a card manufacturer, you must qualify your own production cards.
                           
                          The maximum card inlet air temperatures must support continuous operation under the worst-case power scenario of 150W TDP.
                           
                          The airflow requirements for optimal heat sink performance at minimum is characteristic of CAT 3 servers or PCIe SIG Level 7 thermal profiles, in both, forward & reverse flow, see figure below:
                           
                           
                           
                          As the Intel IPU Platform F2000X-PL is a development platform, it is not integrated into the server baseband management controller closed loop cooling control. It is strongly recommended that you set your server's fan settings to run constantly at 100% with the server chassis lid closed to prevent unwanted Intel IPU Platform F2000X-PL thermal shutdown.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#14-external-connections","title":"1.4 External Connections","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-1-external-connections","title":"Figure 1: External Connections","text":"
                          The items listed Table 6 in are known to work for external connectivity. Specific links are given for convenience, other products may be used but have not been tested.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-6-external-connection-cables","title":"Table 6: External Connection Cables","text":"Item Part Number Link to source RS-232 to USB Adapter DTECH FTDI USB to TTL Serial Adapter, 3 m USB to TTL Serial USB to Ethernet Adapter, Aluminum 3 Port USB 3.0 Teknet USB Hub with Ethernet adapter Flash Drive, 64 GB or larger SanDisk QSFP DAC Cable \u00a0FS.com Generic 2m 100G QSP28 Passive Direct Attach Copper QSFP28 DAC (optional) Intel FPGA Download Cable II PL-USB2-BLASTER USB-Blaster II"},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#15-preparing-the-intel-ipu-platform-f2000x-pl-for-installation","title":"1.5 Preparing the Intel IPU Platform F2000X-PL for Installation","text":"
                          Turn the board over to back side and remove the Kapton tape covering switches SW2 and SW3 and make sure the switches are set as shown in Figure 1.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-7-switch-settings","title":"Table 7: Switch Settings","text":"Name Value SW3.1 off SW3.2 off SW3.2 on SW3.2 off SW2 off"},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-2-board-switch-settings","title":"Figure 2: Board Switch Settings","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#151-usb-to-serial-adapter","title":"1.5.1 USB to Serial Adapter","text":"
                          The Intel IPU Platform F2000X-PL has a serial UART for access located on back edge of the board. This connection is useful for making BIOS and boot settings and for monitoring the SoC. In most servers, you will need to remove a riser card and route the USB to serial cable and (optional) Intel FPGA USB Blaster through an unused PCIe slot above or below where the IPU is installed. See Figure 3 for an example of cable routing.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-3-cable-routing","title":"Figure 3: Cable Routing","text":"
                          The USB to serial connection is shown in Figure 4 where the White wire is TXD, Black wire is ground and Green wire is RXD.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-4-usb-to-serial-adapter-connection","title":"Figure 4: USB to Serial Adapter connection","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#152-ipu-jtag","title":"1.5.2 IPU JTAG","text":"
                          The Intel IPU Platform F2000X-PL provides a 10 pin JTAG header for FPGA and Cyclone 10 Board Management Controller development work using a Intel FPGA Download Cable II. This JTAG connection is optional for initial bring-up but is useful for manual image reprogramming and debug. See Figure 5 noting the orientation of the connection. The orientation of the USB Blaster II requires careful installation in a PCIe bay that has additional room in the adjacent bay. This may require you to either install the board over the PSU of the server, or to temporarily remove an adjacent riser while programming.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-5-usb-blaster-ii-connection","title":"Figure 5: USB Blaster II Connection","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-6-usb-blaster-ii-installation","title":"Figure 6: USB Blaster II Installation","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#153-power","title":"1.5.3 Power","text":"
                          The Intel IPU Platform F2000X-PL must receive power from both the 12 V and 3.3V PCIe slot and the 12 V Auxiliary 2\u00d74 power connector. The board does not power up if any of the 12 V and 3.3 V PCIe slot, or 12 V Auxiliary power sources are disconnected.
                           
                          PCIe specifications define 12 V Auxiliary power connector pin assignment. The Intel IPU Platform F2000X-PL implements an 8-position right angle (R/A) through-hole PCB header assembly on the top right side of the board as depicted in the picture below.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-7-12v-pcie-aux-connector-location","title":"Figure 7: 12V PCIe AUX Connector Location","text":"
                          Refer the table below for pinout details.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-8-12v-2x3-aux-connector-pin-out","title":"Table 8: 12V (2x3) AUX Connector Pin Out","text":"Pin Description 1 +12V 2 +12V 3 +12V 4 Sense 1 5 Ground 6 Sense 0 7 Ground 8 Ground 
                          See Auxiliary power connection in Figure 8.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-8-auxiliary-power-connection","title":"Figure 8: Auxiliary Power Connection","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#154-usb-hub-connection","title":"1.5.4 USB Hub Connection","text":"
                          The USB Hub is connected to the USB type A connector on the front panel. Additionally, attach a network connected Ethernet connection to the USB hub. See Figure 9.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-9-usb-hub-connection","title":"Figure 9: USB Hub Connection","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#155-creating-a-bootable-usb-flash-drive-for-the-soc","title":"1.5.5 Creating a Bootable USB Flash Drive for the SoC","text":"
                          Connect your flash drive to an available Linux host. In this section the USB will set up to be used as a secondary boot source for the SoC and will also be used to update the NVMe from which the ICX-D SoC boots in section 2.1 Updating the F2000X-PL ICX-D SoC NVMe.
                           
                          You will load the latest pre-compiled Yocto core-image-minimal WIC image into USB flash. This image can be downloaded from 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell, under assets, or compiled from meta-ofs. Compilation is discussed in section 4.0 Compiling a Custom Yocto SoC Image.
                           
                             
                          1.  
                            Insert a 64 GB or larger USB Flash Drive into the USB slot of a computer/server you can use to format the drive. The following instructions assume you are using some flavor of GNU+Linux. You need sudo access privileges on this machine.
                             
                            2.  
                          2.  
                            In a terminal window, find the device name of the USB flash drive and unmount the device:
                             
                            $ lsblk\n\nNAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT\nsda 8:0 0 1.8T 0 disk\n\u251c\u2500sda1 8:1 0 600M 0 part /boot/efi\n\u251c\u2500sda2 8:2 0 1G 0 part /boot\n\u2514\u2500sda3 8:3 0 1.8T 0 part\n\u251c\u2500rhel-root 253:0 0 50G 0 lvm /\n\u251c\u2500rhel-swap 253:1 0 4G 0 lvm \\[SWAP\\]\n\u2514\u2500rhel-home 253:6 0 1.7T 0 lvm /home\nsdb 8:16 0 447.1G 0 disk\n\u251c\u2500sdb1 8:17 0 600M 0 part\n\u251c\u2500sdb2 8:18 0 1G 0 part\n\u2514\u2500sdb3 8:19 0 445.5G 0 part\n\u251c\u2500fedora_localhost\\--live-swap 253:2 0 4G 0 lvm\n\u251c\u2500fedora_localhost\\--live-home 253:3 0 301G 0 lvm\n\u251c\u2500fedora_localhost\\--live-root 253:4 0 70G 0 lvm\n\u2514\u2500fedora_localhost\\--live-centos_root 253:5 0 70.5G 0 lvm\nsdd 8:48 1 57.3G 0 disk\n\u2514\u2500sdd1 8:49 1 57.3G 0 part\n
                             
                            In the above example, the 64 GB USB Flash device is designated sdd. Note, your device file name may be different. You are looking for an entry that matches the size of   your USB Flash. You can also check the output of dmesg after manually plugging in your USB Flash device to view the name the kernel has given it in an auto-generated event.
                             
                            3.  
                          3.  
                            Unmount the USB flash (if not already unmounted).
                             
                            $ sudo umount /dev/sdd1\numount: /dev/sdd1: not mounted.\n
                             
                            4.  
                          4.  
                            Download the Yocto WIC image. To prevent boot errors that may arise when using the same boot image loaded in both USB flash and on-board NVMe, you must choose an older version of the Yocto WIC to load onto the USB. Browse the tagged Yocto release images on GitHub and choose the second newest release image as the temporary USB boot target. In this example we will use the OFS 2023.1 RC3 release. You will also need to download the newest Yocto release image (core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic.gz).
                             
                            # Download an older version of the Yocto release image to use as the USB boot target, version 2023.1 RC3 shown here\n$ wget https://github.com/OFS/meta-ofs/releases/download/ofs-2023.1-3/core-image-full-cmdline-intel-corei7-64-20230505161810.rootfs.wic.gz\n$ wget https://github.com/OFS/meta-ofs/releases/download/ofs-2023.1-3/core-image-full-cmdline-intel-corei7-64-20230505161810.rootfs.wic.gz.sha256\n# Verify the checksum of the downloaded image\n$ sha256sum -c https://github.com/OFS/meta-ofs/releases/download/ofs-2023.1-3/core-image-full-cmdline-intel-corei7-64-20230505161810.rootfs.wic.gz.sha256\n# Uncompress the package\n$ gzip -d core-image-full-cmdline-intel-corei7-64-20230505161810.rootfs.wic.gz\n\n\n\n# Download the most recent Yocto release image, which will overwrite on-board NVMe\n$ wget https://github.com/OFS/meta-ofs/releases/download/ofs-2023.1-4/core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic\n$ wget https://github.com/OFS/meta-ofs/releases/download/ofs-2023.1-4/core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic.sha256\n# Verify the checksum of the downloaded image\nsha256sum -c core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic.sha256\n# Uncompress the package\n$ gzip -d core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic\n
                             
                            5.  
                          5.  
                            Copy core-image-full-cmdline-intel-corei7-64-20230505161810.rootfs.wic to the USB flash. This process may take several minutes.
                             
                            $ sudo dd if=core-image-full-cmdline-intel-corei7-64-20230505161810.rootfs.wic of=/dev/sdd1 bs=512k status=progress conv=sync \n$ sgdisk -e /dev/sdd\n
                             
                            6.  
                          6.  
                            Create a partition to store the Yocto image, which will be used to overwrite on-board NVMe as the default boot target.
                             
                                $ sudo fdisk /dev/sdd\n    Command (m for help): p\n    Command (m for help): n\n    Partition number (4-128, default 4): <<press enter>>\n    First sector (14617908-125045390, default 14618624): <<press enter>>\n    Last sector, +/-sectors or +/-size{K,M,G,T,P} (14618624-125045390, default\n    125045390): <<press enter>>\n    Created a new partition 4 of type 'Linux filesystem' and of size 92 GiB.\n    Command (m for help): p\nCommand (m for help): w\n
                             
                            7.  
                          7.  
                            Verify USB flash is partitioned.
                             
                            $ lsblk\n    NAME        MAJ:MIN RM   SIZE RO TYPE MOUNTPOINTS\n    sdd           8:0    1 114.6G  0 disk\n    |-sdd1        8:1    1   300M  0 part\n    |-sdd2        8:2    1  22.1G  0 part\n    |-sdd3        8:3    1    44M  0 part\n    `-sdd4        8:4    1  92.2G  0 part\n
                             
                            8.  
                          8.  
                            Format the new partition.
                             
                            $ mkfs -t ext4 /dev/sdd4\n$ mount /dev/sda4 /mnt\n
                             
                            9.  
                          9.  
                            Copy compressed core-image-minimal WIC into /mnt.
                             
                            $ cp core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic /mnt\n
                             
                            10.  
                           
                          Remove the USB flash from the Linux computer and install the flash drive in the USB hub attached to the Intel IPU Platform F2000X-PL.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#20-updating-the-intel-ipu-platform-f2000x-pl","title":"2.0 Updating the Intel IPU Platform F2000X-PL","text":"
                          Every Intel IPU Platform F2000X-PL ships with pre-programmed firmware for the FPGA user1, user2, and factory images, the Cyclone 10 BMC RTL and FW, the SoC NVMe, and the SoC BIOS. The combination of FW images in Table 4 compose the official 2023.3 OFS SoC Attach Release for Intel IPU Platform F2000X-PL. Upon initial receipt of the board from manufacturing you will need to update two of these regions of flash to conform with the best known configuration for SoC Attach.  As shown in Table 9, not all devices require firmware updates. To instruct users in the process of updating on-board Intel IPU Platform F2000X-PL firmware, examples are provided in this guide illustrating the firmware update process for all devices.  
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-9-intel-ipu-platform-f2000x-pl-fw-components","title":"Table 9: Intel IPU Platform F2000X-PL FW Components","text":"HW Component File Name Version Update Required (Yes/No) Download Location FPGA SR Image1 user1: ofs_top_page1_unsigned_user1.binuser2: ofs_top_page2_unsigned_user2.binfactory: ofs_top_page0_unsigned_factory.bin PR Interface ID: 3dac7126-3ce7-5fe8-b629-932096abb09b Yes 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell FPGA PR IMAGE2 ofs_pr_afu.green_region_unsigned.gbs N/A Yes Compiled with FIM ICX-D NVMe core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic N/A Yes 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell BMC RTL and FW AC_BMC_RSU_user_retail_1.2.4_unsigned.rsu 1.2.4 No 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell BIOS Version 0ACRH608_REL.BIN 0ACRH608_REL Yes 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell 
                          If a component does not have a required update, it will not have an entry on 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell.
                           
                          1To check the PR Interface ID of the currently programmed FIM and the BMC RTL and FW version, use fpgainfo fme from the SoC. 2Must be programmed if using AFU-enabled exercisers, not required otherwise.
                           
                          $ fpgainfo fme\nIntel IPU Platform F2000X-PL\n**Board Management Controller NIOS FW version: 1.2.4**\n**Board Management Controller Build version: 1.2.4**\n//****** FME ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\n**Bitstream Id                     : 360572756485162679**\nBitstream Version                : 5.0.1\n**Pr Interface Id                  : 3dac7126-3ce7-5fe8-b629-932096abb09b**\nBoot Page                        : user1\nUser1 Image Info                 : 9e3db8b6a4d25a4e3e46f2088b495899\nUser2 Image Info                 : None\nFactory Image Info               : None\n
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#21-updating-the-f2000x-pl-icx-d-soc-nvme","title":"2.1 Updating the F2000X-PL ICX-D SoC NVMe","text":"
                          The Intel IPU Platform F2000X-PL ships with a Yocto image pre-programmed in NVMe, which is not the same as the SoC Attach OFS image that we will be using. The latter provides only the OPAE SDK and Linux DFL drivers and is fully open source. This section will show how you can use an attached USB drive to load a new image into flash. You will use a serial terminal to install the new image - Minicom and PuTTy terminal emulators have both been tested. minicom is used for demonstration purposes as the serial terminal to access the ICX-D SoC UART connection in this section. Information on compiling your own Yocto image for use with the Intel IPU Platform F2000X-PL is discussed in section 4.0 Compiling a Custom Yocto SoC Image.
                           
                          Note: Username and password for the default SoC NVMe boot image are \"root\" and \"root@123\".
                           
                             
                          1.  
                            First, make sure to complete the steps in section 1.5.5 Creating a Bootable USB Flash Drive for the SoC, and attach the USB drive either directly into the rear of the Intel IPU Platform F2000X-PL, or into a USB Hub that itself is connected to the board.
                             
                            2.  
                          2.  
                            Ensure your Minicom terminal settings match those shown below. You must direct Minicom to the USB device created in /dev associated with your RS-232 to USB Adapter cable. This cable must be attached to a server that is separate from the one housing your Intel IPU Platform F2000X-PL. Check the server logs in dmesg to figure out which device is associated with your cable: [    7.637291] usb 1-4: FTDI USB Serial Device converter now attached to ttyUSB0. In this example the special character file /dev/ttyUSB0 is associated with our cable, and can be connected to using the following command: sudo minicom --color=on -D /dev/ttyUSB0.
                             
                             
                            3.  
                          3.  
                            Change the SoC boot order to boot from USB first. Reboot the server. From your     serial Minicom terminal, watch your screen and     press 'ESC' key to go into BIOS setup mode. Once BIOS setup comes     up as shown below, click the right arrow key six times to move from     'Main' set up menu to 'Boot' setup:
                             
                            Main setup menu:
                             
                             
                             
                            Your order of boot devices may differ. You need to move the USB flash    up to Boot Option #1 by first using the down arrow key to highlight    the USB device then use '+' key to move the USB device to #1 as    shown below:
                             
                             
                            Press 'F4' to save and exit.
                             
                            4.  
                          4.  
                            You will boot into Yocto automatically. Log in with username root and an empty password using Minicom. Take note of the IP address of the board, you can use this to log in without needing the serial cable.
                             
                             
                            Verify that you have booted from the USB and not the on-board NVMe lsblk -no pkname $(findmnt -n / | awk '{ print $2 }'). You should see a device matching /dev/sd*, and not nvme*n*p*. If you see nvme*n*p*, then review the previous steps.
                             
                            Record the IP address of the SoC at this time ip -4 addr. This will be used to log in remotely using SSH.
                             
                            5.  
                          5.  
                            Check that 4 partitions created in 1.5.5 Creating a Bootable USB Flash Drive for the SoC are visible to the SoC in /dev/sd*:
                             
                            $ lsblk -l\nroot@intel-corei7-64:/mnt# lsblk -l\nNAME      MAJ:MIN RM   SIZE RO TYPE MOUNTPOINTS\nsda         8:0    1 117.2G  0 disk\nsda1        8:1    1  38.5M  0 part /boot\nsda2        8:2    1    20G  0 part /\nsda3        8:3    1    44M  0 part [SWAP]\nsda4        8:4    1  97.1G  0 part /mnt\nnvme0n1   259:0    0  59.6G  0 disk\nnvme0n1p1 259:1    0   300M  0 part\nnvme0n1p2 259:2    0  22.1G  0 part\nnvme0n1p3 259:3    0    44M  0 part\n
                             
                            Mount partition 4, and cd into it.
                             
                            $ mount /dev/sda4 /mnt`\n$ cd /mnt\n
                             
                            6.  
                          6.  
                            Install the Yocto release image in the SoC NVMe.
                             
                            $ dd if=core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic of=/dev/nvme0n1 bs=1M status=progress conv=sync\n$ sync\n$ sgdisk -e /dev/nvme0n1\n
                             
                            The transfer from USB to NVMe will take several minutes.
                             
                            7.  
                          7.  
                            Reboot the SoC and update the SoC BIOS to boot from NVMe. Follow steps 2 and 3 from this section again, and this time move the NVME back to the front of the boot order. The NVMe is named UEFI OS (PCIe SSD) by the BIOS. Press F4 to save and exit.
                             
                            You can use wget to retrieve a new version of the Yocto release image from meta-ofs once the SoC's network connection is up. Use wget to copy the image to the SoC over the network under /mnt. You may need to delete previous Yocto images to save on space: $ wget --no-check-certificate --user <Git username> --ask-password https://github.com/OFS/meta-ofs/releases/download/ofs-2023.3-2/core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic.gz. Uncompress the newly retrieved file: gzip -d core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic.gz. This may take several minutes.
                             
                            8.  
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#211-setting-the-time","title":"2.1.1 Setting the Time","text":"
                             
                          1.  
                            Use Linux command to set system time using format: date --set=\"STRING\".
                             
                            date -s \"26 APRIL 2023 18:00:00\"\n
                             
                            2.  
                          2.  
                            Set HWCLOCK to current system time:
                             
                            hwclock --systohc\n
                             
                            Verify time is set properly
                             
                            date\n...\nhwclock --show\n...\n
                             
                            3.  
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#30-updating-the-intel-ipu-platform-f2000x-pl","title":"3.0 Updating the Intel IPU Platform F2000X-PL","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#31-preparing-the-intel-ipu-platform-f2000x-pl-soc-for-updates","title":"3.1 Preparing the Intel IPU Platform F2000X-PL SoC for Updates","text":"
                          Updating the Intel IPU Platform F2000X-PL firmware often requires reboots of the SoC or reconfiguration of the FPGA region. If there are processes connected from the host to the SoC that do not expect the downtime to occur, or if the host is not tolerant to a surprise PCie link down, the following instructions can be used to properly orchestrate updates with the host when reboots occur.
                           
                          Note: Intel IPU Platform F2000X-PL FPGA and BMC updates are initiated by commands issued on the IPU SoC. Issue the following commands from the host to remove any processes that would be impacted by this update. The instructions on properly removing the IPU from PCIe bus require the OPAE SDK to be installed on the host. Refer to section 6.0 Setting up the Host for this process.
                           
                             
                          1.  
                            From a host terminal shell, find PCIe Bus/Device/Function (BDF) address of your Intel IPU Platform F2000X-PL. Run the command lspci | grep bcce to print all boards with a DID that matches bcce.
                             
                            $ lspci | grep bcce\n31:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n31:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\n# In this example, 31:00.0 is the proper PCIe BDF of our device\n
                             
                            2.  
                          2.  
                            Shut down all VMs and software applications attached to any PFs/VFs on the host
                             
                            3.  
                          3. Issue the command sudo pci_device <<PCIe BDF>> unplug on the host to remove the PCIe device from the PCIe bus
                            4.  
                          4. Shut down all software applications on the SoC accessing non-management PFs/VFs
                            5.  
                          5. Issue your update command on the SoC, which will cause an SoC reboot and surprise PCIe link down on the host (ex. reboot, rsu bmc/bmcimg/fpga)
                            6.  
                          6. Once you have completed all firmware updates, you may restart application software on the SoC
                            7.  
                          7. Issue command sudo pci_device <<PCIe BDF>> plug on the host to rescan the PCIe bus and rebind the device to its native driver
                            8.  
                          8. Restart software applications on the host
                            9.  
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#32-updating-fim-bmc-and-afu-with-fpgasupdate","title":"3.2 Updating FIM, BMC and AFU with fpgasupdate","text":"
                          The fpgasupdate tool updates the Intel\u00ae C10 10 BMC image and firmware, root entry hash, FPGA Static Region (SR) and user image (PR). The fpgasupdate will only accept images that have been formatted using PACSign. If a root entry hash has been programmed onto the board, then the image will also need to be signed using the correct keys. Please refer to the Security User Guide: Intel Open FPGA Stack for information on creating signed images and on programming and managing the root entry hash. This repository requires special permissions to access - please reach out to your Intel representative to request. The fpgasupdate tool is used to program images into flash memory. The rsu tool is used to configure the FPGA/BMC with an image that is already stored in flash memory, or to switch between user1 and user2 images. All images received from an official Intel release will be \"unsigned\", as described in the Security User Guide: Intel Open FPGA Stack.
                           
                          Note: 'Unsigned' in this context means the image has passed through PACsign and has had the proper security blocks prepended using a set of 'dummy' keys. FIMs with image signing enabled will require all programmable images to pass through PACsign even if the currently programmed FIM/BMC do not require specific keys to authenticate.
                           
                          There are two regions of flash you may store FIM images for general usage, and one backup region. These locations are referred to as user1, user2, and factory. The factory region is not programmed by default and can only be updated once keys have been provisioned. The BMC FW and RTL will come pre-programmed with version 1.1.9.
                           
                          Updating the FIM from the SoC requires the SoC be running a Yocto image that includes the OPAE SDK and Linux DFL drivers. Updating the FIM using fpgasupdate also requires an OFS enabled FIM to be configured on the F2000X-PL, which it will ship with from manufacturing. You need to transfer any update files to the SoC over SSH. The OPAE SDK utility fpgasupdate will be used to update all ofthe board's programmable firmware . This utility will accept files of the form *.rsu, *.bin, and *.gbs, provided the proper security data blocks have been prepended by PACSign. The default configuration the IPU platform ships with will match the below:
                           
                          $ fpgainfo fme\nIntel IPU Platform F2000X-PL\n**Board Management Controller NIOS FW version: 1.2.4**\n**Board Management Controller Build version: 1.2.4**\n//****** FME ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\n**Bitstream Id                     : 360572756485162679**\nBitstream Version                : 5.0.1\n**Pr Interface Id                  : 3dac7126-3ce7-5fe8-b629-932096abb09b**\nBoot Page                        : user1\nUser1 Image Info                 : 9e3db8b6a4d25a4e3e46f2088b495899\nUser2 Image Info                 : None\nFactory Image Info               : None\n
                           
                          To load a new update image, you need to pass the IPU's PCIe BDF and the file name to fpgasupdate on the SoC. The below example will update the user1 image in flash:
                           
                          $ fpgasupdate ofs_top_page1_unsigned_user1.bin 15:00.0\n
                           
                          After loading an update image, rsu fpga/bmc/bmcimg can be used to reload the firmware and apply the change, as discussed below. An RSU of the BMC will always cause a reload of both the BMC and FPGA images.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#33-loading-images-with-rsu","title":"3.3 Loading images with rsu","text":"
                          RSU performs a Remote System Update operation on an Intel IPU Platform F2000X-PL, given its PCIe address. An rsu operation sends an instruction to the device to trigger a power cycle of the card only if run with bmcimg. rsu will force reconfiguration from flash for either the BMC or FPGA. PCIe Advanced Error Reporting (AER) is temporarily disabled for the card when RSU is in progress
                           
                          The Intel IPU Platform F2000X-PL contains two regions of flash you may store FIM images. These locations are referred to as user1 and user2. After an image has been programmed to either of these regions in flash using fpgasupdate, you may perform an rsu to reconfigure the Agilex 7 FPGA with the new image stored in flash. This operation will indicate to the BMC which region to configure the FPGA device from after power-on.
                           
                          If the factory image has been updated, Intel strongly recommends you immediately RSU to the factory image to ensure the image is functional.
                           
                          You can determine which region of flash was used to configure their FPGA device using the command fpgainfo fme and looking at the row labelled Boot Page.
                           
                          When loading a new FPGA SR image, use the command rsu fpga. When loading a new BMC image, use the command rsu bmc. When using the RSU command, you may select which image will be configured to the selected device. For example, when performing an RSU for the Intel Agilex 7 FPGA, you may select to configure the user1, user2, or factory image. When performing an RSU for the C10 BMC, you may select to configure the user or factory image. You may also use RSU to reconfigure the SDM on devices that support it. The RSU command sends an instruction to the BMC to reconfigure the selected device from an image stored in flash.
                           
                          $ rsu fpga --page=user1 15:00.0\n
                           
                          Useage:
                           
                          rsu bmc --page=(user|factory) [PCIE_ADDR]\nrsu fpga --page=(user1|user2|factory) [PCIE_ADDR]\nrsu sdm [PCIE_ADDR]\n
                           
                          You can use RSU to change which page in memory the FPGA will boot from by default.
                           
                          Synopsis:
                           
                          rsu fpgadefault --page=(user1|user2|factory) --fallback=<csv> 15:00.0\n
                           
                          Use to set the default FPGA boot sequence. The --page option determines the primary FPGA boot image. The --fallback option allows a comma-separated list of values to specify fallback images.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#34-updating-the-icx-d-soc-bios","title":"3.4 Updating the ICX-D SoC BIOS","text":"
                          The ICX-D SoC NVMe comes pre-programmed with BIOS v7 (0ACRH007). This version will need to be replaced with 0ACRH608_REL. BIOS update files come in the form 0ACRH\\<\\<version>>.bin, and can be downloaded on 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell. This update process is in-band, and requires you to download and install a BIOS UEFI utility from AMI called \"APTIO V AMI Firmware Update Utility\", available here. This package will install a utility in the UEFI shell called AfuEfix64.efi which will be used to overwrite the ICX-D BIOS.
                           
                             
                          1.  
                            Check your BIOS Version. Reboot the SoC and wait for the BIOS version to be shown. In this example, the BIOS will need to be updated.
                             
                             
                            2.  
                          2.  
                            Download both the ICX-D SoC Update image and APTIO V AMI Firmware Update Utility. Unzip the BIOS update image and locate your BIOS update binary. Unzip Aptio_V_AMI_Firmware_Update_Utility.zip, and then navigate to Aptio_V_AMI_Firmware_Update_Utility\\afu\\afuefi\\64 and unzip AfuEfi64.zip. The file we need from this package is called AfuEfix64.efi.
                             
                             
                            3.  
                          3.  
                            Copy both files over to the SoC into /boot/EFI using the SoC's IP.
                             
                            $ scp 0ACRH608_REL.BIN root@XX.XX.XX.XX:/boot/EFI\n$ scp AfuEfix64.efi root@XX.XX.XX.XX:/boot/EFI\n
                             
                            4.  
                          4.  
                            Reboot the SoC from a TTY Serial terminal. Watch your screen and press 'ESC' key to go into BIOS setup mode. Select 'Built-in EFI Shell'.
                             
                             
                            5.  
                          5.  
                            At EFI prompt enter the following:
                             
                            Shell> FS0:\nFS0:> cd EFI\nFS0:\\EFI\\> AfuEfix64.efi 0ACRH608_REL.BIN /p /n /b\n
                             
                            Press 'E'. When the update has completed type 'reset -w' to reboot.
                             
                             
                            6.  
                          6.  
                            Watch your screen and press 'ESC' key to go into BIOS setup mode. Verify BIOS version matches expectation.
                             
                             
                            7.  
                          7.  
                            Click the right arrow key six times to move from 'Main' set up menu to 'Boot' setup. Select NVMe as the primary boot source.
                             
                             
                            8.  
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#40-compiling-a-custom-yocto-soc-image","title":"4.0 Compiling a Custom Yocto SoC Image","text":"
                          Current Yocto image architecture for SoC Attach is based off of the IOTG Yoct-based ESE BSP and substitutes the Linux DFL kernel including the latest DFL drivers for FPGA devices along with the OPAE SDK user space. The image targets x86_64 SoC FPGA devices but should boot on most UEFI-based machines. The source code and documentation for this image is hosted on the meta-ofs repository.
                           
                          Build requirements exceed 100 GiB of disk space, depending on which image is built. As a reference point, on a system with two Intel(R) Xeon(R) E5-2699 v4 for a total of 44 CPU cores, the initial, non-incremental build takes less than an hour of wall time.
                           
                          The repo tool is needed to clone the various Yocto layer repositories used in this example.
                           
                          Note: If you are behind a firewall that prevents you from accessing references using the git:// protocol, you can use the following to redirect Git to use the corresponding https repositories for Yocto only: git config --global url.https://git.yoctoproject.org/.insteadOf git://git.yoctoproject.org/
                           
                          To compile the image as-is, use the following steps (as provided in meta-ofs):
                           
                             
                          1.  
                            Create and initialize the source directory.
                             
                            mkdir ofs-yocto && cd ofs-yocto\ngit clone --recurse-submodules --shallow-submodules https://github.com/OFS/meta-ofs\ncd meta-ofs\ngit checkout tags/ofs-2023.3-2\n
                             
                            2.  
                          2.  
                            Build packages and create an image.
                             
                            cd examples/iotg-yocto-ese\nTEMPLATECONF=$PWD/conf source openembedded-core/oe-init-build-env build\nbitbake mc:x86-2022-minimal:core-image-full-cmdline\n
                             
                            3.  
                           
                          The resulting GPT disk image is available in uncompressed (.wic) and compressed form (.wic.gz) in meta-ofs/examples/iotg-yocto-ese/build/tmp-x86-2021-minimal-glibc/deploy/images/intel-corei7-64/. With no changes the uncompressed image size is ~21 GB.
                           
                          The image type core-image-full-cmdline includes the familiar GNU core utilities, as opposed to core-image-minimal which uses BusyBox instead.
                           
                          The example build configuration files under build/conf/ are symlinked from examples/iotg-yocto-ese/. To customise the image, start by modifying local.conf and bblayers.conf.
                           
                          The uncompressed Yocto image can be loaded onto a flash drive as discussed in section 1.5.5 Creating a Bootable USB Flash Drive for the SoC and written to NVMe as the default boot target for the SoC as demonstrated in section 2.1 Updating the F2000X-PL ICX-D SoC NVMe.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#50-verifying-the-icx-d-soc-opae-sdk","title":"5.0 Verifying the ICX-D SoC OPAE SDK","text":"
                          The reference SoC Attach FIM and unaltered FIM compilations contain Host Exerciser Modules (HEMs). These are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Full supported functionality of the HEMs is documented in the host_exerciser opae.io GitHub page. SoC Attach supports HEMs run both with and without an AFU image programmed into the board's one supported PR region. This image is available on the offial SoC Attach GitHub Page, and is programmed using fpgasupdate as shown in section 3.2. A few select examples run from the SoC and their expected results will be shown below.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#51-checking-telemetry-with-fpgainfo","title":"5.1 Checking Telemetry with fpgainfo","text":"
                          The fpgainfo utility displays FPGA information derived from sysfs files.
                           
                          The command argument is one of the following: errors, power, temp, port, fme, bmc, phy, mac, and security. Some commands may also have other arguments or options that control their behavior.
                           
                          For systems with multiple FPGA devices, you can specify the BDF to limit the output to the FPGA resource with the corresponding PCIe configuration. If not specified, information displays for all resources for the given command.
                           
                          An example output for fpgainfo fme is shown below. Your IDs may not match what is shown here:
                           
                          $ fpgainfo fme\nIntel IPU Platform F2000X-PL\nBoard Management Controller NIOS FW version: 1.1.9\nBoard Management Controller Build version: 1.1.9\n//****** FME ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x50103023DFBFC8E\nBitstream Version                : 5.0.1\nPr Interface Id                  : bf74e494-ad12-5509-98ab-4105d27979f3\nBoot Page                        : user1\nUser1 Image Info                 : 98ab4105d27979f3bf74e494ad125509\nUser2 Image Info                 : None\nFactory Image Info               : None\n
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#52-host-exercisers","title":"5.2 Host Exercisers","text":"
                          Of these five tests listed below, the first three do not require an AFU be loaded into the board's PR region. They exercise data paths that pass exclusively through the FIM. The latter three tests exercise data through the AFU data path, and require SoC Attach release AFU Image to be configured using fpgasupdate.
                           
                             
                          • Run HE-MEM with 2 cachelines per request in mem mode, exercising the FPGA's connection to DDR. No AFU required.
                            Note: If you see the error message Allocate SRC Buffer, Test mem(1): FAIL, then you may need to manually allocate 2MiB Hugepages using the following: echo 20 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
                             
                            # Create VF device\n$ pci_device <PCIe Bus>00.0 vf 3\n# Bind VF0 to vfio-pci\n$ opaei.io init -d <PCIe Bus>:00.1 <username>:<username>\n# Check for HE-MEM Accelerator GUID 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n$ fpgainfo port -B <PCIe Bus>:00.0\n# Run desired HE-MEM test(s)\n$ host_exerciser --cls cl_1 --mode lpbk mem\nstarting test run, count of 1\nAPI version: 2\nAFU clock: 470 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 9948\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.096 GB/s\n    Test mem(1): PASS\n
                             
                            •  
                           
                             
                          • Generate traffic with HE-HSSI. No AFU required.
                            # Create VF device\n$ pci_device <PCIe Bus>00.0 vf 3\n# Bind VF2 to vfio-pci\n$ opaei.io init -d <PCIe Bus>:00.2 <username>:<username>\n# Check for HE-HSSI Accelerator GUID 823c334c-98bf-11ea-bb37-0242ac130002\n$ fpgainfo port -B <PCIe Bus>:00.0\n# Show number of configured ports\n$ fpgainfo phy | grep Port\n# Generate traffic for specific port number\n$ hssi --pci-address <PCIe Bus>:00.2 hssi_10g --num-packets 100 --port <port number>\n10G loopback test\n  Tx/Rx port: 1\n  Tx port: 1\n  Rx port: 1\n  eth_loopback: on\n  he_loopback: none\n  num_packets: 100\n  packet_length: 64\n  src_address: 11:22:33:44:55:66\n    (bits): 0x665544332211\n  dest_address: 77:88:99:aa:bb:cc\n    (bits): 0xccbbaa998877\n  random_length: fixed\n  random_payload: incremental\n...\n
                             
                            This command will generate a log file in the directory is was run from. Check TxPackets and RxPackets for any loss. Two more supported HSSI commands are hssistats, which provides MAC statistics, and hssimac, which provides maximum TX and RX frame size.
                             
                            •  
                           
                             
                          • Test memory traffic generation using MEM-TG. This will exercise and test available memory channels with a configurable memory pattern. Does not require an AFU image.
                            # Create VF device\n$ pci_device <PCIe Bus>00.0 vf 3\n# Bind VF2 to vfio-pci\n$ opaei.io init -d <PCIe Bus>:00.3 <username>:<username>\n# Check for MEM-TG Accelerator GUID 4dadea34-2c78-48cb-a3dc-5b831f5cecbb\n$ fpgainfo port -B <PCIe Bus>:00.0\n# Example MEM-TG Test\n$  mem_tg --loops 500 -w 1000 -r 0 -b 0x1 --stride 0x1 -m 0 tg_test\n
                             
                            •  
                           
                             
                          • HE-LPBK is designed to demo how AFUs move data between host memory and the FPGA. Will check latency, MMIO latency, MMIO bandwidth, and PCIe bandwidth.  LPBK workload requires the SoC Attach AFU be loaded into the board's PR slot.
                            # Create VF device\n$ pci_device <PCIe Bus>00.0 vf 3\n# Bind VF1 to vfio-pci\n$ opaei.io init -d <PCIe Bus>:00.1 <username>:<username>\n# Check for LPBK GUID 56e203e9-864f-49a7-b94b-12284c31e02b\n$ fpgainfo port -B <PCIe Bus>:00.0\n# Example loopback test\n$  host_exerciser --mode lpbk lpbk\n
                             
                            •  
                           
                             
                          • Exercise He-HSSI subsystem from the AFU. This test will generate and receieve packets from any of the 8 available ports. This HSSI workload requires the SoC Attach AFU be loaded into the board's PR slot.
                            # Create VF device\n$ pci_device <PCIe Bus>00.0 vf 3\n# Bind VF6 to vfio-pci\n$ opaei.io init -d <PCIe Bus>:00.2 <username>:<username>\n# Check for HSSI GUID 823c334c-98bf-11ea-bb37-0242ac130002\n$ fpgainfo port -B <PCIe Bus>:00.0\n# Geneate traffic for specific port\n$  hssi --pci-address <bus_num>:00.6 hssi_10g --num-packets 100 --port <port_num>\n
                             
                            This command will generate a log file in the directory is was run from. Check TxPackets and RxPackets for any loss. Two more supported HSSI commands are hssistats, which provides MAC statistics, and hssimac, which provides maximum TX and RX frame size.
                             
                            •  
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#53-additional-opae-sdk-utilities","title":"5.3 Additional OPAE SDK Utilities","text":"
                          This section will discuss OPAE SDK utilities that were not covered by previous sections. These commands are all available on the ICX-D SoC Yocto image by default. A full description and syntax breakdown for each command is located on the official OPAE SDK github.io repo.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-6-opae-sdk-utilities","title":"Table 6: OPAE SDK Utilities","text":"Utility Description fpgasupdate 3.2 Updating FIM, BMC and AFU with fpgasupdate rsu Section 3.3 Loading images with rsu host_exerciser Section 5.2 Host Exercisers hssi Section 5.2 Host Exercisers hssistats Section 5.2 Host Exercisers hssimac Section 5.2 Host Exercisers mem_tg Section 5.2 Host Exercisers usrclk userclk tool is used to set high and low clock frequency to an AFU. mmlink Remote signaltap is software tool used for debug RTL (AFU), effectively a signal trace capability that Quartus places into a green bitstream. Remote Signal Tap provides access the RST part of the Port MMIO space, and then runs the remote protocol on top. opaevfio The opaevfio command enables the binding/unbinding of a PCIe device to/from the vfio-pci device driver. See https://kernel.org/doc/Documentation/vfio.txt for a description of vfio-pci. opae.io An interactive Python environment packaged on top of libopaevfio.so, which provides user space access to PCIe devices via the vfio-pci driver. The main feature of opae.io is its built-in Python command interpreter, along with some Python bindings that provide a means to access Configuration and Status Registers (CSRs) that reside on the PCIe device. bitstreaminfo Prints bitstream metadata. fpgaconf Lower level programming utility that is called automatically by fpgasupdate. fpgaconf writes accelerator configuration bitstreams (also referred to as \"green bitstreams\") to an FPGA device recognized by OPAE. In the process, it also checks the green bitstream file for compatibility with the targeted FPGA and its current infrastructure bitstream (the \"blue bistream\"). fpgad Periodically monitors/reports the error status reflected in the device driver's error status sysfs files. Establishes the channel by which events are communicated to the OPAE application. Programs a NULL bitstream in response to AP6 event. fpgad is required to be running before API calls fpgaRegisterEvent and fpgaUnregisterEvent will succeed."},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#60-setting-up-the-host","title":"6.0 Setting up the Host","text":"
                          External SoC Attach supports testing Host to FPGA latency, MMIO latency, and MMIO bandwidth. This testing is accomplished using the utility host_exerciser on the host, whichis included as a part of OPAE. This section will cover the installation and verification flow for a host interacting with the SoC Attach workload.
                           
                          Review Section 1.2 Server Requirements for a list of changes required on the host to support an Intel IPU Platform F2000X-PL and for a list of supported OS distributions. Installation will require an active internet connetion to resolve dependencies.
                           
                          The following software checks may be run on the host to verify the FPGA has been detected and has auto-negotatiated the correct PCIe link width/speed. These commands do not require any packages to be installed. We are using PCIe BDF b1:00.0 as an example address.
                           
                          # Check that the board has enumerated successfully.\n# Your PCIe BDF may differ from what is shown below.\n$ lspci | grep accel\nb1:00.0 Processing accelerators: Intel Corporation Device bcce \nb1:00.1 Processing accelerators: Intel Corporation Device bcce\n\n# Check PCIe link status and speed. Width should be x16, and speed whould be 16GT/s\nsudo lspci -s b1:00.0 -vvv | grep LnkSta | grep -o -P 'Width.{0,4}'\nsudo lspci -s b1:00.0 -vvv | grep LnkSta | grep -o -P 'Speed.{0,7}'\n\nsudo lspci -s b1:00.1 -vvv | grep LnkSta | grep -o -P 'Width.{0,4}'\nsudo lspci -s b1:00.1 -vvv | grep LnkSta | grep -o -P 'Speed.{0,7}'\n
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#61-installing-the-opae-sdk-on-the-host","title":"6.1 Installing the OPAE SDK On the Host","text":"
                          The OPAE SDK software stack sits in user space on top of the OFS kernel drivers. It is a common software infrastructure layer that simplifies and streamlines integration of programmable accelerators such as FPGAs into software applications and environments. OPAE consists of a set of drivers, user-space libraries, and tools to discover, enumerate, share, query, access, manipulate, and reconfigure programmable accelerators. OPAE is designed to support a layered, common programming model across different platforms and devices. To learn more about OPAE, its documentation, code samples, an explanation of the available tools, and an overview of the software architecture, visit the opae.io page.
                           
                          The OPAE SDK source code is contained within a single GitHub repository hosted at the OPAE Github. This repository is open source and does not require any permissions to access.
                           
                             
                          1.  
                            Before Installing the newest version of OPAE you must remove any prior OPAE framework installations.
                             
                            $ sudo apt-get remove opae*\n
                             
                            2.  
                          2.  
                            The following system and Python3 package dependencies must be installed before OPAE may be built.
                             
                            $ sudo apt-get install bison flex git ssh pandoc devscripts debhelper cmake python3-dev libjson-c-dev uuid-dev libhwloc-dev doxygen libtbb-dev libncurses-dev libspdlog-dev libspdlog1 python3-pip libedit-dev pkg-config libcli11-dev libssl-dev dkms libelf-dev gawk openssl libudev-dev libpci-dev  libiberty-dev autoconf llvm\n\n$ python3 -m pip install setuptools pybind11 jsonschema\n
                             
                            3.  
                          3.  
                            Clone the OPAE SDK repo. In this example we will use the top level directory OFS for our package installs.
                             
                            $ mkdir OFS && cd OFS\n$ git init\n$ git clone https://github.com/OFS/opae-sdk.git\n$ cd opae-sdk\n$ git checkout tags/2.10.0-1\n\n# Verifying we are on the correct release tag\n$ git describe --tags\n2.10.0-1\n
                             
                            4.  
                          4.  
                            Navigate to the automatic DEB package build script location and execute.
                             
                            $ cd OFS/opae-sdk/packaging/opae/deb \n$ ./create\n\n# Verify all packages are present\n$ ls | grep opae.*.deb\nopae_2.5.0-1_amd64.deb\nopae-dbgsym_2.5.0-1_amd64.ddeb\nopae-devel_2.5.0-1_amd64.deb\nopae-devel-dbgsym_2.5.0-1_amd64.ddeb\nopae-extra-tools_2.5.0-1_amd64.deb\nopae-extra-tools-dbgsym_2.5.0-1_amd64.ddeb\n
                             
                            5.  
                          5.  
                            Install your newly built OPAE SDK packages.
                             
                            $ cd OFS/opae-sdk/packaging/opae/deb\n$ sudo dpkg -i opae*.deb\n
                             
                            The OPAE tools installed on the host are identical to those installed on the SoC as shown in sections 5.0 through 5.3. A set of pre-compiled OPAE SDK artifacts are included in this release. These can be downloaded from 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell and installed without building/configuring.
                             
                            $ tar xf opae-*.tar.gz\n$ sudo dpkg -i opae*.deb\n
                             
                            6.  
                          6.  
                            Enable iommu=on, pcie=realloc, and set hugepages as host kernel parameters.
                             
                            # Check if parameters are already enabled\n$ cat /proc/cmdline\n
                             
                            If you do not see intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200, then add them manually.
                             
                            $ sudo vim /etc/default/grub\n# Edit the value for GRUB_CMDLINE_LINUX, add the values at the end of the variable inside of the double quotes. Example: GRUB_CMDLINE_LINUX=\"crashkernel=auto resume=/dev/mapper/rhel00-swap rd.lvm.lv=rhel00/root rd.lvm.lv=rhel00/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\"\n# Save your changes, then apply them with the following\n$ sudo grub2-mkconfig\n$ sudo reboot now\n
                             
                            7.  
                           
                          After rebooting, check that proc/cmdline reflects your changes.
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#62-verifying-the-soc-attach-solution-on-the-host","title":"6.2 Verifying the SoC Attach Solution on the Host","text":"
                          The SoC Attach workload supports testing MMIO HW and Latency and PCIe BW and latency out of box. Execution of the host_exerciser binary on the host requires the OPAE SDK to be installed as shown in section 6.1 Installing the OPAE SDK On the Host. You will also need to have a proper SoC Attach FIM configured on your board as shown in section 3.2 Updating FIM, BMC and AFU with fpgasupdate.
                           
                             
                          1.  
                            Initialize PF attached to HSSI LPBK GUID with vfio-pci driver.
                             
                            $ sudo opae.io init -d 0000:b1:00.0 <username>:<username>\n$ sudo opae.io init -d 0000:b1:00.1 <username>:<username>\n
                             
                            2.  
                          2.  
                            Run host_exerciser loopback tests (only lpbk is supported). There are more methods of operation than are shown below - read the HE help message for more information.
                             
                            3.  
                           
                          # Example lpbk tests.\n$ sudo host_exerciser lpbk\n$ sudo host_exerciser --mode lpbk lpbk\n$ sudo host_exerciser --cls cl_4  lpbk\n$ sudo host_exerciser --perf true --cls cl_4  lpbk\n# Number of cachelines per request 4.\n$ sudo host_exerciser --mode lpbk --cls cl_4 lpbk\n$ sudo host_exerciser --perf true --mode lpbk --cls cl_4 lpbk\n# vNumber of cachelines per request 4.\n$ sudo host_exerciser --mode read --cls cl_4 lpbk\n$ sudo host_exerciser --perf true --mode read --cls cl_4 lpbk\n# Number of cachelines per request 4.\n$ sudo host_exerciser --mode write --cls cl_4 lpbk\n$ sudo host_exerciser --perf true --mode write --cls cl_4 lpbk\n# Number of cachelines per request 4.\n$ sudo host_exerciser --mode trput --cls cl_4 lpbk\n$ sudo host_exerciser --perf true --mode trput --cls cl_4 lpbk\n# Enable interleave requests in throughput mode\n$ sudo host_exerciser --mode trput --interleave 2 lpbk\n$ sudo host_exerciser --perf true --mode trput --interleave 2 lpbk\n#with delay option.\n$ sudo host_exerciser --mode read --delay true lpbk\n$ sudo host_exerciser --mode write --delay true lpbk\n# Test all modes of operation\n$ host_exerciser --testall=true lpbk\n
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#63-fpga-device-access-permissions","title":"6.3 FPGA Device Access Permissions","text":"
                          Access to FPGA accelerators and devices is controlled using file access permissions on the Intel\u00ae FPGA device files, /dev/dfl-fme.* and /dev/dfl-port.*, as well as to the files reachable through /sys/class/fpga_region/.
                           
                          In order to allow regular (non-root) users to access accelerators, you need to grant them read and write permissions on /dev/dfl-port.* (with * denoting the respective socket, i.e. 0 or 1). E.g.:
                           
                          sudo chmod a+rw /dev/dfl-port.0\n
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#64-memlock-limit","title":"6.4 Memlock limit","text":"
                          Depending on the requirements of your application, you may also want to increase the maximum amount of memory a user process is allowed to lock. The exact way to do this depends on your Linux distribution.
                           
                          You can check the current memlock limit using
                           
                          ulimit -l\n
                           
                          A way to permanently remove the limit for locked memory for a regular user is to add the following lines to your /etc/security/limits.conf:
                           
                          user1    hard   memlock           unlimited\nuser1    soft   memlock           unlimited\n
                           
                          This removes the limit on locked memory for user user1. To remove it for all users, you can replace user1 with *:
                           
                          *    hard   memlock           unlimited\n*    soft   memlock           unlimited\n
                           
                          Note that settings in the /etc/security/limits.conf file don't apply to services. To increase the locked memory limit for a service you need to modify the application's systemd service file and add the line:
                           
                          [Service]\nLimitMEMLOCK=infinity\n
                          "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                          Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                           
                          OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/","title":"Simulation User Guide: Open FPGA Stack for Intel Agilex 7 SoC Attach FPGAs","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to user space."},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#1-overview","title":"1 Overview","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#11-about-this-document","title":"1.1 About this Document","text":"
                          This document serves as a set-up and user guide for the UVM simulation tool using OFS. After reviewing the document, you will be able to:
                           
                             
                          • Set-up the UVM verification tool suite
                            •  
                           
                             
                          • Run pre-existing UVM unit tests and also create new UVM tests for your design
                            •  
                           
                           Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#2-introduction-to-uvm","title":"2 Introduction to UVM","text":"
                          The Functional Verification Environment for OFS is UVM (Universal Verification Methodology) compliant and provides configurable setup for verifying various FIM features in simulation.
                           
                          The purpose of this document is to demonstrate a full chip level and unit level UVM based verification environment for the current base shell FIM architecture as well as providing examples to extend the setup for different OFS variants by reusing the existing architecture and infrastructure for UVM based verification.
                           
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#3-universal-testbench-architecture","title":"3 Universal Testbench Architecture","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#31-overview","title":"3.1 Overview","text":"
                          The main idea behind UVM is to develop modular, reusable, and a scalable testbench structure by providing an API framework that can be deployed across multiple projects.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#32-core-verification-concepts","title":"3.2 Core Verification Concepts","text":"
                          The following is the list of verification components that will be used to design a UVM testbench architecture:
                           
                          \u2022   Sequencer \u2022   Driver \u2022   Monitor \u2022   Scoreboard \u2022   Virtual Sequencer \u2022   Interface \u2022   Verification Environment \u2022   TOP Testbench
                           
                          Figure 1 provides the general UVM Testbench and the verification components involved in the top-level architecture.
                           
                           
                          Figure 1 Typical UVM Testbench
                           
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4-ofs-testbench-architecture","title":"4 OFS Testbench Architecture","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#41-overview","title":"4.1 Overview","text":"
                          OFS (Open FPGA Stack) provides a UVM (Universal Verification Methodology) environment for the FIM with a modular, reusable, and scalable testbench structure via an API framework.
                           
                          The framework consists of a FIM Testbench which is UVM compliant and integrates third party VIPs from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this Testbench. UVM RAL(Register Abstraction Level) is used for CSR (Command and Status Registers) verification. 
                           
                          The qualified verification IPs will help to detect incorrect protocol behavior, help to focus on FIM features and accelerate the verification process.
                           
                          Verification components include:
                           
                          \u2022 FIM monitor to detect correct design behavior\n\u2022 FIM assertions for signal level integrity testing\n\u2022 Arm AMBA Arm\u00ae AMBA\u00ae 4 AXI4 scoreboards to check data integrity\n\u2022 FIM coverage to collect functional data\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#42-base-fim-dut","title":"4.2 Base FIM DUT","text":"
                          The hardware architecture of an Intel Agilex 7 FIM is based on the OFS hardware architecture. The following is the list of features and subsystems supported in the base shell.
                           
                          \u2022   PCIe Subsystem\n\u2022   HSSI Subsystem\n\u2022   Memory Subsystem\n\u2022   HPS Subsystem\n\u2022   FME\n\u2022   AFU with PR support\n\u2022   QSFP Controllers\n\u2022   PMCI Controller, MCTP\n
                           
                           
                          Figure 2 DUT Base Shell Diagram
                           
                          Figure 2 shows the high level architecture of an Intel Agilex 7 Base Shell. It has a Gen4x16, 100G Ethernet Datapath in a 2x4x25G configuration. The Intel Agilex 7 Base Shell is a shell that will enable a user to build other shell variants for a custom configuration. For the f2000x board there is one shell variant
                           
                          base_x16
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#43-full-chip-level-verification-archiecture-for-fim","title":"4.3 Full Chip Level Verification Archiecture for FIM","text":"
                          Figure 3 shows a graphical representation a full chip testbench that includes major RTL blocks depicted in a OFS Intel Agilex 7 based UVM environment 
                           
                           
                          Figure 3 OFS FIM Testbench
                           
                          The major connection is the interface between the Xeon CPU and the FPGA where the PCIe Verification IP is connected to PCIe Subsystem. Therefore, as a full chip simulation environment, PCIe host VIP is the sole VIP/BFM used. PCIe host VIP connects to PCIe device which resides in FPGA in serial mode.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#431-testbench-components","title":"4.3.1 Testbench components","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4311-tb-top","title":"4.3.1.1 TB TOP","text":"
                          TOP is the top level testbench and consists of a FIM DUT instance and top-level UVM Verification Environment instance. It binds RTL inputs with the verification environmnet interfaces to drive stimulus. It also has clock generation and reset driving logic.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4312-fim-verification-environment","title":"4.3.1.2 FIM Verification Environment","text":"
                          This is the top most verification environment class and consists of the protocol specific PCI Express and AXI UVM environment VIP instances, Monitors, Scoreboards, Assertions, Functional coverage Modules and other verification components. It instantiates Virtual sequencers to control stimuli for FIM traffic from different sequencers of the VIPs.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4313-synopsys-vips","title":"4.3.1.3 Synopsys VIPs","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#43131-pci-vip-as-host","title":"4.3.1.3.1 PCI VIP as Host","text":"
                          This is Synopsys Licensed PCI Express Gen4 VIP and acts as Root Port. The PCI Express link is connected to the DUT using TX-RX lanes. Agent is an active component and includes a sequencer to generate TLPs, Driver to drive it on a PCI Express link and Monitor to check the protocol correctness.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#43132-axi-streaming-vip-monitors","title":"4.3.1.3.2 AXI-Streaming VIP Monitors","text":"
                          This is Synopsys Licensed AXI streaming interface Verification IP used as a Passive Agent to monitor AXI-ST links at various points. Please refer to Figure 3 to see all the AXI-ST monitors connected to different modules.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#43133-axi4-memory-mapped-vip-monitors","title":"4.3.1.3.3 AXI4-Memory Mapped VIP Monitors","text":"
                          This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 memory mapped interface Verification IP used in passive mode to observe memory requests. For each master-slave pair, the verification environment has a VIP instantiated in passive mode for monitoring protocol violations and performing data integrity checks.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#43134-axi4-lite-vip-monitors","title":"4.3.1.3.4 AXI4-Lite VIP Monitors","text":"
                          This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4 Lite interface Verification IP used in passive mode to observe MMIO requests. For each master-slave pair, the verification environment has a VIP instantiated in passive mode for monitoring protocol violations and perfoming data integrity checks. Please refer to Figure 3 to see all the Arm\u00ae AMBA\u00ae 4 AXI4-Lite monitors connected to different modules.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#43135-axi4-lite-vip-as-pmci-master","title":"4.3.1.3.5 AXI4-Lite VIP as PMCI Master","text":"
                          This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 Lite interface Verification IP used to drive and observe MMIO requests as PMCI master. For each master-slave pair, the verification environment has a VIP instantiated in active mode and includes a sequencer to generate MMIO requests, driver to drive these requests on AXI-4 Lite interface to BPF and a monitor for observing protocol violations and data integrity checks. 
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4314-axi4-s-scoreboard","title":"4.3.1.4 AXI4-S Scoreboard","text":"
                          The Arm\u00ae AMBA\u00ae 4 AXI4-S scoreboard checks data integrity of source and sink components. It has input transactions from VIP monitors and a TLP to AXI converter for PCIe TLP packets. It makes sure the valid TLPs or AXI transactions are not missed while traversing from Host to AFU and reverse. The scoreboard will be disabled for error testing especially for invalid TLP requests and UR responses.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4315-virtual-sequencer","title":"4.3.1.5 Virtual Sequencer","text":"
                          The virtual sequencer is the top-level sequencer which controls Enumeration, MMIO Requests, downstream and Upstream traffic as well as HSSI and Memory transactions. It makes sure that the transactions are ordered correctly as per the design specification while running a UVM test simulation.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4316-fim-monitor","title":"4.3.1.6 FIM Monitor","text":"
                          The FIM monitor is used for checking the correctness of a specific FIM feature. As an example, a user can add interrupt related checks, error generation and clearing checks, Protocol checker impacts etc. in this component. 
                           
                          This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4317-fim-assertions","title":"4.3.1.7 FIM Assertions","text":"
                          The assertion component is used for checking the signal level integrity of a specific FIM feature or behavior. As an example, we can add interrupt signal triggering and clear assertions, Error detection to error register bit assertions, protocol checker and clearing logic, FLR behavior assertion etc. in this top-level module. There are alos assertions to make sure the inputs are driven correctly to catch errors in a users simulation. 
                           
                          This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4318-fim-functional-coverage","title":"4.3.1.8 FIM Functional Coverage","text":"
                          The FIM functional coverage component will have the functional coverage of each feature like interrupts, CSR's, MMIO requests, Byte align transactions, error reporting etc. to demonstrate a variety of FIM features. This will help us to find holes in the design as well as wide verification coverage to make sure thorough testing of a design. It will also provide a user what the FIM is capable of and how much code coverage they are testing.
                           
                          This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                           
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#5-uvm-verification-set-up","title":"5 UVM Verification set-up","text":"
                          To run the tutorial steps in this guide requires the following development environment:
                           Item Version Intel Quartus Prime Pro Intel Quartus Prime Pro 23.3 Simulator Synopsys VCS P-2019.06-SP2-5 or newer for UVM simulation of top level FIM Simulator (Questasim) Questasim 2023.2 or newer for UVM simulation of top level FIM"},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#51-uvm-prerequisite","title":"5.1 UVM Prerequisite","text":"
                          Retrieve OFS repositories.
                           
                          The OFS FIM source code is included in the OTCShare GitHub repository. Create a new directory to use as a clean starting point to store the retrieved files.  The following is a short description of each repository, followed by the git commands for cloning.  The instructions section uses the HTTPS git method for cloning repositories.  Cloning the repo using the HTTPS method requires a personal access token.  Please see this blog post for information about obtaining a personal access token Token authentication requirements for Git operations.
                           
                          Navigate to location for storage of OFS source, create the top-level source directory and clone OFS repositories.
                           
                          $ mkdir release/ofs-2023.3\n$ cd release/ofs-2023.3\n$ export OFS_BUILD_ROOT=$PWD\n$ git clone --branch --recurse-submodules https://github.com/ofs-f2000x-pl.git\n\nCloning into 'ofs-f2000x-pl'...'\nUsername for 'https://github.com': <<Enter your git hub username>>\nPassword for 'https://<<Your username>>': <<Enter your personal access token>>\nremote: Enumerating objects:  ....\n...\n...\nResolving deltas  ..., done.\n\n$ cd ofs-f2000x-pl\n$ git checkout tags/ofs-2023.3-1\n
                           
                          Verify that the correct tag/branch have been checked out        
                           
                          $ git describe --tags\n\n$ ofs-2023.3-1\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#52-license-requirements","title":"5.2 License Requirements","text":"
                          The FIM Testbench is UVM compliant and integrates third party VIP's from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this TB. UVM RAL (Register Abstraction Layer) is used for CSR Verification.
                           
                          The Qualified Verification IPs will help to detect incorrect protocol behavior easily, help to focus on FIM features and accelerate the verification process.
                           
                          \u2022 VCS & DVE\n\u2022 SNPS-Assertions\n\u2022 Verdi\n\u2022 VerdiCoverage\n\u2022 VerdiSimDB\n\u2022 VerdiTransactionDebugUltra\n\u2022 VIP-AMBA-AXI-SVT\n\u2022 VIP-AMBA-STREAM-SVT\n\u2022 VIP-PCIE-SVT\n\u2022 VIP-PCIE-TS-SVT\n\u2022 VIP-PCIE-G3-OPT-SVT\n\u2022 VIP-Ethernet-SVT\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#53-software-tools-requirements","title":"5.3 Software Tools Requirements","text":"
                          The following tools are required for successful UVM set-up
                           
                             
                          • Python 3.6.8
                            •  
                          • Synopsys PCIE and AMBA AXI UVM VIP Q-2020.03A License
                            •  
                          • VCS R-2020.12-SP2 License       Note: Makefile can be modified to use DVE instead of Verdi
                            •  
                           
                             
                          • VCS R-2020.12-SP2 License 
                            •  
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#54-creating-a-software-tools-script","title":"5.4 Creating a Software Tools Script","text":"
                          The UVM tool set-up is best done by creating a simple set-up script so all applicable tools are sourced before running the tests.
                           
                          The following environment variables can be pasted into a script and used prior to running the UVM verification environment
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#license-files","title":"License Files","text":"
                          export LM_LICENSE_FILE=\nexport SNPSLMD_LICENSE_FILE=\n
                           
                          The license environment variables LM_LICENSE_FILE and SNPSLMD_LICENSE_FILE can point to a server license on your system.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#general-environment-variables","title":"General Environment Variables","text":"
                          export IOFS_BUILD_ROOT=$PWD\nexport OFS_ROOTDIR=<user_path>/ofs-f2000x-pl\nexport WORKDIR=$OFS_ROOTDIR\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#quartus-tools","title":"Quartus Tools","text":"
                          export QUARTUS_HOME=<user_path>/intelFPGA_pro/23.3/quartus\nexport QUARTUS_ROOTDIR=$QUARTUS_HOME\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$PATH\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-verification-tools","title":"Synopsys Verification Tools","text":"
                          export DESIGNWARE_HOME=<user_path>/synopsys/vip_common/vip_Q-2020.03A\nexport PATH=$DESIGNWARE_HOME/bin:$PATH\nexport UVM_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport PATH=$VCS_HOME/bin:$PATH\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-verification-tools","title":"QuestaSIM Verification Tools","text":"
                          export MTI_HOME=<user_path>/mentor/questasim/2023.2/linux64\nexport PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\nexport QUESTA_HOME=$MTI_HOME\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#6-running-a-uvm-simulation-test-and-analysing-results","title":"6 Running a UVM Simulation Test and Analysing Results","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#61-simulation","title":"6.1 Simulation","text":"
                          The default simulator used in the simulation script is Synopsys VCS-MX. Users can refer to the options and adopt the options for other simulators. The script is a makefile that calls vlogan, vcs and simv for compilation, elaboration and simulation, respectively.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#62-file-structure","title":"6.2 File Structure","text":"
                          After cloning the repo, the verification and ofs-common directories contain all UVM verification related files. The directory structure is shown in Figure 4 below.
                           
                           
                          Figure 4 UVM Verification Directory File Structure
                           
                          ofs-f2000x-pl/verification/testbench has a testbench, uvm env, virtual sequencer, RAL etc. 
                           
                          ofs-f2000x-pl/tests contains all uvm tests and sequences. 
                           
                          Users can run the simulation under \"ofs-f2000x-pl/verification/scripts\" directory and the simulation result is outputted to a \"sim\" directory for Synopsys VCS or sim_msim for Questasim.
                           
                          The simulation result folder is named after the test name with increasing suffix number. If user runs the same test multiple times, the suffix is incremented by 1 each time.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#63-uvm-test-suite","title":"6.3 UVM Test Suite","text":"
                          The UVM environment contains a variety of tests that have been developed to test out the FIM portion of OFS.
                           
                          The table below has four columns which describe the \"Test Name\", \"DUT Scope\", \"Test Scenario\" and the \"Checking Criteria\".
                           
                          Tests are located at ofs-f2000x-pl/ofs-common/verification/fpga_family/agilex/tests
                           Test Name DUT Scope Test Scenario Checking Criteria afu_mmio_flr_pf0_test PF0 FLR Reset Apply FLR Reset for PF0 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF0 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf0_vf0_test PF0_VF0_FLR Reset Apply FLR Reset for PF0_VF0 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR       access are working fine afu_mmio_flr_pf0_vf1_test PF0_VF1_FLR Reset Apply FLR Reset for PF0_VF1 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf0_vf2_test PF0_VF2_FLR Reset Apply FLR Reset for PF0_VF2 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf1_vf0_test PF1_VF0_FLR Reset Apply FLR Reset for PF1_VF0 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf2_test PF2 FLR Reset Apply FLR Reset for PF2 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF2 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf3_test PF3 FLR Reset Apply FLR Reset for PF3 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF3 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf4_test PF4 FLR Reset Apply FLR Reset for PF4 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF4 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_stress_5bit_tag_test AFU-StressAFU-Stress To check the AFU Stress by sending traffic with 5bit tag from all PF/VF. i.e Send traffic on HE-LPK/HE-MEM and initiating MMIO access to other PF/VFs Data checking afu_stress_8bit_tag_test AFU-Stress To check the AFU Stress by sending traffic with 5bit tag from all PF/VF. i.e Send traffic on HE-LPK/HE-MEM and initiating MMIO access to other PF/VFs Data checking afu_stress_test AFU-Stress 1. Initiate transactions to all the supported PF/VF from PCIE VIP and ensure that traffic is sent to all blocks of the AFU.                 2. Ensure that CE/HE-LB/HE-MEM/HSSI/BPF/FME are seeing traffic.                 3. Ensure that HE-LB/HE-MEM/CE sends DMWR/DMRD requests to PCIE VIP.                 4. Ensure the Mux/DeMux blocks is able to handle the traffic based on the PF's/VF's and proper muxing/demuxing happens. Data checking bar_32b_test PCIe MMIO Path Set the BAR values to 32bit and test mmio access BAR address, Register Base Offset bar_64b_test PCIe MMIO Path Set the BAR values to 64bit and test mmio access BAR address, Register Base Offset dfh_walking_test DFH DFH walking offset checking, eol checking -> tb emif_csr_test EMIF CSR access data checking fme_csr_test FME CSR CSR accesses data checking fme_err_intr_test Interrupt FME Interrupt request using FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read fme_intr_test Interrupt FME interrupt request using RAS ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read fme_multi_err_intr_test Interrupt FME interrupt request using RAS ERROR and FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests generated from FME and RAS ERROR bits he_hssi_csr_test HE-HSSI CSR accesses data checking he_hssi_rx_lpbk_25G_10G_test HE-HSSI Sending back to back ethernet data traffic with 25G speed on HSSI RX Port0-7 lanes using Ethernet VIPs Enable the loopback mode in HE-HSSI and compare the pkts recived on HSSI TX Port(DATA CHECKING) he_hssi_tx_lpbk_P0_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port0  lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P1_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port1 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P2_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port2 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P3_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port3 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P4_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port4 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P5_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port5 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P6_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port6 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P7_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port7 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_lpbk_cont_test HE-LPBK Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking he_lpbk_csr_test HE-LPBK CSR accesses data checking he_lpbk_flr_rst_test HE-LPBK FLR RST is generated in HE_LPBK access data checking, counter checking he_lpbk_long_rst_test HE-LPBK Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_lpbk_long_test HE-LPBK Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_lpbk_multi_user_intr_test HE-LPBK generate user HE_LB interrupt interrupt checking he_lpbk_rd_cont_test HE-LPBK Read only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking he_lpbk_rd_test HE-LPBK Read only mode. Randomize num_lines, addresses, req_len counter checking he_lpbk_reqlen16_test HE-LPBK To check the behavior of HE_LPK block when req_length 16 and num_lines set to 1024 Cache lines in Loopback mode data checking, counter checking he_lpbk_reqlen1_test HE-LPBK Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_lpbk_reqlen2_test HE-LPBK Loopback mode. 128 CLs, req_len = 2CL, random addresses data checking, counter checking he_lpbk_reqlen4_test HE-LPBK Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_lpbk_reqlen8_test HE-LPBK Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_lpbk_rst_in_middle_test PCIe MMIO Path Set HE_LPK in  all the modes randomly and iterate the transactions in loop. At the end of every loop assert the Soft reset in the middle of the transactions Register Base Offset he_lpbk_test HE-LPBK Loopback mode.  Randomize num_lines, addresses, req_len data checking, counter checking he_lpbk_thruput_contmode_test HE-LPBK Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking he_lpbk_thruput_test HE-LPBK Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_lpbk_wr_cont_test HE-LPBK Write only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking he_lpbk_wr_test HE-LPBK Write only mode. Randomize num_lines, addresses, req_len counter checking he_mem_cont_test HE-MEM Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking, counter checking he_mem_csr_test HE-MEM CSR accesses data checking he_mem_lpbk_long_rst_test HE-MEM Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_mem_lpbk_long_test HE-MEM Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_mem_lpbk_reqlen16_test HE-MEM To check the behavior of HE_LPK block when req_length 16 and num_lines set to 1024 Cache lines in Loopback mode data checking, counter checking he_mem_lpbk_reqlen1_test HE-MEM Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_mem_lpbk_reqlen2_test HE-MEM Loopback mode. 128 CLs, req_len = 2CL, random addresses data checking, counter checking he_mem_lpbk_reqlen4_test HE-MEM Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_mem_lpbk_reqlen8_test HE-MEM Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_mem_lpbk_test HE-MEM Loopback mode.  Randomize num_lines, addresses, req_len data checking, counter checking he_mem_multi_user_intr_test Interrupt Back to back multiple User interrupt request from HE MEM Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read for multiple back to back request he_mem_rd_cont_test HE-MEM Read only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_rd_test HE-MEM Read only mode. Randomize num_lines, addresses, req_len counter checking he_mem_thruput_contmode_directed_test HE-MEM Set HE_MEM in Thrput Continuous mode and test data checking, counter checking he_mem_thruput_contmode_test HE-MEM Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_thruput_test HE-MEM Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_mem_user_intr_test Interrupt FME interrupt request using RAS ERROR and FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests generated from FME and RAS ERROR bits he_mem_wr_cont_test HE-MEM Write only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_wr_test HE-MEM Write only mode. Randomize num_lines, addresses, req_len counter checkingt he_random_test All HEs Enable all HEs and randomize modes data checking if in lpbk mode, counter checking helb_rd_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Read only mode Measure the performance helb_rd_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Read only mode Measure the performance helb_rd_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Read only mode Measure the performance helb_thruput_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Thruput mode Measure the performance helb_thruput_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Thruput mode Measure the performance helb_thruput_4cl_5bit_tag_test Performance Set HE_LPK in thruput mode and send traffic with req len 4 and num_lines set to 1024. Measure the Read/Write performance Measure the performance helb_thruput_4cl_8bit_tag_test Performance Set HE_LPK in thruput mode and send traffic with req len 4 and num_lines set to 1024. Measure the Read/Write performance Measure the performance helb_thruput_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Thruput mode Measure the performance helb_wr_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Write only mode Measure the performance helb_wr_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Write only mode Measure the performance helb_wr_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Write only mode Measure the performance hssi_ss_test HSSI CSR accesses data checking malformedtlp_pcie_rst_test Protocol Checker generate mllformed TLP protocol error and initiate pcie reset to clear the error Check for error malformedtlp_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MaxTagError_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transactions are completing. mem_tg_csr_test MEM-TG CSR access data checking mem_tg_traffic_gen_test MEM-TG This test checks the MEM_TG traffic generator flow  data checking mini_smoke_test All HEs shorter simpler version of random test for turn-in sanity check data checking if in lpbk mode, counter checking mix_intr_test Interrupt Mix interrupt testcase to send multiple FME and User interrupt request simultaneously Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests through different sources - FME and HE-MEM modules mmio_pcie_mrrs_128B_mps_128B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_128B_mps_256B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_256B_mps_128B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_256B_mps_256B_test PCIe - Max Payload/Max Read Req Size Random length mmio Read Checking valid possible combination of MPS & MRRS mmio_stress_nonblocking_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux with non-blocking MMIO reads data checking mmio_stress_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux data checking mmio_test PCIe MMIO Path MMIO targeting PF0(ST2MM, FME, PMCI, QSFP, HSSI SS), PF1, PF2,PF3, PF4, PF1.VF1, PF1.VF2 data checking mmio_unimp_test PCIe MMIO Path MMIO acccess to unimplemented addresses data checking MMIODataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MMIOInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MMIOTimedout_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. pcie_csr_test PCIESS Earlier msix registers were in fme block but now it has moved from fme to pciess. Hence coded a seperate test for msix data checking pcie_pmci_mctp_multi_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM multiple packets received from PMCI controller over APF and BPF to PCIe HIA subsystem pcie_pmci_mctp_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM (Vendor Defined Message) single packet received from PCIe HIA subsystem to PMCI controller over APF and BPF via AXI4-lite memory write request pmci_csr_test PMCI CSR CSR accesses data checking pmci_fme_csr_test PMCI FME CSR CSR accesses data checking pmci_pcie_mctp_multi_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM multiple packets received from PMCI controller over APF and BPF to PCIe HIA subsystem pmci_pcie_mctp_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM single packet received from PMCI controller over APF and BPF to PCIe HIA subsystem pmci_pciess_csr_test PMCI PCIESS CSR CSR accesses data checking pmci_qsfp_csr_test PMCI QSFP CSR CSR accesses data checking port_gasket_csr_test PORT GASKET CSR accesses data checking qsfp_csr_test QSFP CSR accesses data checking TxMWrDataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. TxMWrInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. uart_intr_test UART Checking Generates UART interrupt Check interrupt UnexpMMIORspErr_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. vdm_err_vid_test Vendor ID check generate a packet with undefined Vendor-ID from Host to PMCI_SS ID check 
                           
                          The next section describes how to compile and build the UVM environment prior to running each UVM test and analyinsg the results in the log files 
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#64-ip-compile","title":"6.4 IP Compile","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-vcs","title":"Synopsys VCS","text":"
                          To compile all IPs for the Synopsys VCS simulater:
                           
                              cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk cmplib_adp\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-tbd","title":"Questasim (TBD)","text":"
                          To compile all IPs for the Questasim simulater:
                           
                              cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk cmplib_adp\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#65-rtl-test-bench-compile","title":"6.5 RTL & Test Bench Compile","text":"
                          The RTL file list for compilation is located here: verification/scripts/rtl_comb.f
                           
                          The TB file list for compilation is located here: verification/scripts/ver_list.f
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-vcs_1","title":"Synopsys VCS","text":"
                          To compile RTL and Testbench for the Synopsys VCS simulater
                           
                              cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_adp DUMP=1\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-tbd_1","title":"Questasim (TBD)","text":"
                          To compile RTL and Testbench for the Questasim simulater
                           
                              cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_adp DUMP=1\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#66-ip-and-rtl-test-bench-compile","title":"6.6 IP and RTL & Test Bench Compile","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-vcs_2","title":"Synopsys VCS","text":"
                          If the user wants to compile all IPs and RTL Testbench in one command for Synopsys VCS then follow the procedure below
                           
                              cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_all DUMP=1\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-tbd_2","title":"Questasim (TBD)","text":"
                          If the user wants to compile all IPs and RTL Testbench in one command for Questasim then follow the procedure below
                           
                              cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_all DUMP=1\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-vcs_3","title":"Synopsys VCS","text":"
                          To run a simulation for Synopsys VCS:
                           
                              cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk run TESTNAME=ofs_mmio_test DUMP=1\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-tbd_3","title":"Questasim (TBD)","text":"
                          To run a simulation for Questasim:
                           
                              cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=ofs_mmio_test DUMP=1\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-vcs_4","title":"Synopsys VCS","text":"
                          To dump the waveform, \"DUMP=1\" must be added into the command line for Synopsys VCS build and simulation. 
                           
                              gmake -f Makefile_VCS.mk build_adp DUMP=1\n\n    gmake -f Makefile_VCS.mk run TESTNAME=<test_case_name> DUMP=1\n
                           
                          Or
                           
                              gmake -f Makefile_VCS.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-tbd_4","title":"Questasim (TBD)","text":"
                          To dump the waveform, \"DUMP=1\" must be added into the command line for Questasim build and simulation. 
                           
                              gmake -f Makefile_MSIM.mk build_adp DUMP=1\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=<test_case_name> DUMP=1\n
                           
                          Or
                           
                              gmake -f Makefile_MSIM.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                           
                          There are some optimizations in the Table below for convenience if you want to bypass some commands for both Synopsys VCS and Questasim:
                           Command (Synopsys VCS) Command (Questasim) Details gmake -f Makefile_VCS.mk build_all DUMP=1 gmake -f Makefile_MSIM.mk build_all DUMP=1 compile IP + compile RTL gmake -f Makefile_VCS.mk build_run TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk build_run TESTNAME= DUMP=1 compile RTL + run test gmake -f Makefile_VCS.mk do_it_all TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk do_it_all TESTNAME= DUMP=1 compile IP, RTL and run test gmake -f Makefile_VCS.mk rundb TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk rundb TESTNAME= DUMP=1 run test in sim dir + over-writes content"},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#67-uvm-regression-test","title":"6.7 UVM Regression Test","text":"
                          cd $VERDIR/scripts\n\npython uvm_regress.py -l -n 8 -p adp -k top_pkg -s vcs -c none\n\nFor Regression in VCS with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -p adp -k top_pkg -s vcs -c none\n\nResults are created in a sim directory ($VERDIR/sim) with individual testcase log dir\n\nFor Regression in MSIM with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -p adp -k top_pkg -s msim -c none\n
                           
                          Results are created in a sim directory ($VERDIR/sim_msim) with individual testcase log dir
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#68-uvm-waveform-and-transcript-analysis","title":"6.8 UVM Waveform and Transcript Analysis","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-vcs_5","title":"Synopsys VCS","text":"
                          Running Synopsys VCS UVM tests will generate a ofs-f2000x-pl/verification/sim directory
                           
                          \u2022 All build time logs are located at ofs-f2000x-pl/verification/sim\n\n\u2022 Each testcase will have separate directory inside sim ofs-f2000x-pl/verification/sim/<test_case_name>\n
                           
                          There are two tracker or log files that are available: runsim.log and trans.log. 
                           
                          runsim.log is the simulation log file generated from Synopsys VCS. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 5
                           
                           
                          Figure 5 runsim.log
                           
                          trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 6
                           
                           
                          Figure 6 trans.log
                           
                          The waveform generated is named as \"inter.vpd\". To open the waveform, go to simulation result directory and run 
                           
                              dve -full64 -vpd inter.vpd &\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-tbd_5","title":"Questasim (TBD)","text":"
                          Running Questasim UVM tests will generate a ofs-f2000x-pl/verification/sim_msim directory
                           
                          \u2022 All build time logs are at ofs-f2000x-pl/verification/sim_msim\n\n\u2022 Each testcase will have separate directory inside sim ofs-f2000x-pl/verification/sim_msim/<test_case_name>\n
                           
                          There are two tracker or log files that are available: runsim.log and trans.log. 
                           
                          runsim.log is the simulation log file generated from Questasim. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 7
                           
                           
                          Figure 7 runsim.log
                           
                          trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 8
                           
                           
                          Figure 8 trans.log
                           
                          The waveform generated is named as \"vsim.wlf\". To open the waveform, go to simulation result directory and run 
                           
                              vsim -view vsim.wlf &\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#69-uvm-coverage-analysis","title":"6.9 UVM Coverage Analysis","text":"
                          The following command allows to run a single testcase with coverage enabled
                           
                              gmake -f Makefile_VCS.mk cmplib_adp && gmake -f Makefile_VCS.mk build_adp DUMP=1 DEBUG=1 COV_FUNCTIONAL=1&& gmake -f Makefile_VCS.mk run TESTNAME=<TESTCASE-NAME> DUMP=1 DEBUG=1 COV_FUNCTIONAL=1 &\n
                           
                          The following command shows how to merge and generate the coverage report
                           
                              urg -dir <$VERDIR/sim/simv.vdb> <$VERDIR/sim/regression.vdb> -format both -dbname <regression_database_name>\n
                           
                          This will generate both urgreport directory and .vdb file Multiple regression.vdb from different regressions can be merged with the same command. 
                              e.g \"urg -dir <path1_till_simv.vdb> <path1_till_regression.vdb> <path2_till_regression.vdb> -report <dir> -format both -dbname <dirname>\"\n
                           
                          The following commands shows how to launch DVE and check the coverage reports
                           
                          To open DVE of a single regression or testcase, execute:\n\n    dve -full64 -cov -covdir simv.vdb regression.vdb &\n\nTo open DVE of a merged regression test, execute:\n\n    dve -full64 -cov -covdir <dirname.vdb> &\n
                           
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#7-csr-verification-using-uvm-ral","title":"7 CSR Verification using UVM RAL","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#71-overview","title":"7.1 Overview","text":"
                          The UVM Register Layer provides a standard base class library that enable users to implement the object-oriented model to access the DUT registers and memories. The UVM Register Layer is also referred to as UVM Register Abstraction Layer (UVM RAL). Design registers can be accessed independently of the physical bus interface. i.e. by calling read/write methods. This is shown in Figure 9 below.
                           
                           
                          Figure 9 RAL UVM Testbench
                           
                          The RAL register models for different CSR's mimics the design registers. All RAL files are located here.
                           
                              ofs-f2000x-pl/verification/testbench/ral\n
                           
                          The RAL model is generated through the Synopsys RALGEN tool and is used for CSR verification.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#72-ral-integration","title":"7.2 RAL Integration","text":"
                          For UVM RAL model integration to the environment, adapters for each CSR is implemented and integrated into the Testbench Environment. It is used to convert the PCIe bus sequence items into uvm_reg_bus_op and vice versa. The CSR test cases pick up all the registers from the respective CSR blocks and perform a default value, wr/rd check on them. 
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#73-ral-model-generation","title":"7.3 RAL Model Generation","text":"
                          Steps for RAL model generation
                           
                          Excel(xls) file containing the registers is required. Make sure there are separate xls files for each CSR and the worksheet name must contain the name \"reg_fields\".  
                           
                          Excel sheet snapshot example below for EMIF_CSR.xls located at /ipss/mem/rtl/adp
                           
                           
                          \u2022 Navigate to ofs-f2000x-pl/ofs-common/verification/common/scripts/ral\n\u2022 Copy the excel file (xls) to the above area\n\u2022 In the bash terminal run mk_ral.sh <Excel sheet name without extension > <output *.sv file name without ral_  prepended >\n\u2022 The above steps generate two ral *.sv files. File with _cov suffix is a coverage enabled ral model. \n\u2022 Copy *.sv files to ofs-f2000x-pl/verification/testbench/ral\n
                           
                           
                          \u2022 As an example to generate ral_ac_ce.sv from AC_CE_CSR.xls file the command is\n\n    mk_ral.sh AC_CE_CSR ac_ce\n
                           
                          This generates two ral models (ral_ac_ce.sv and ral_ac_ce_cov.sv)
                           
                          To add new registers
                           
                          \u2022 To create new registers, copy existing ones and modify the cells in the xls. Make sure the last line is also a copied blank line\n\u2022 Follow all the steps of RAL model generation\n
                           
                          To Generate a RAL model when a new xls sheet is created for a new component
                           
                          \u2022 Copy the relevant xls sheet to ofs-f2000x-pl/ofs-common/verification/common/scripts/ral\n\u2022 Follow all the steps of RAL model generation\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#74-top-level-verification-architecture-for-csr-testing","title":"7.4 Top Level Verification Architecture for CSR testing","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#741-testbench-components","title":"7.4.1 Testbench components","text":"
                          The testbench components for RAL are defined below
                           
                          \u2022 ral_reg_iofs_* (uvm_reg) generated by the steps as mentioned in section 5.3\n
                           
                          The uvm register class is written by extending the uvm_reg. A register represents a set of fields that are accessible as a single entity Each register contains any number of fields, which mirror the values of the corresponding elements in hardware
                           
                          \u2022 ral_block_iofs_* (uvm_block) generated in the same register file\n
                           
                          A register model is an instance of a register block, which may contain any number of registers, register files, memories, and other blocks
                           
                          \u2022 ral_block_ofs (uvm_block) \u2013 Contains all the CSR block instantiations\n\u2022 Reg2vip_*_adapter (uvm_reg_adapter) \u2013 This class defines an interface for converting between uvm_reg_bus_op and a specific bus transaction. For each CSR a respective adapter is present\n
                           
                          All the components are defined in ofs-f2000x-pl/ofs-common/verification/testbench
                           
                          Integration of components in testbench
                           
                          \u2022 The RAL blocks and adapters for each CSR is instantiated in tb_env\n\u2022 The bar range for each CSR is also set in the tb_env\n
                           
                          Sample Environment Integration snippets 
                           
                           
                           
                           
                           
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#8-modifying-uvm-testbench","title":"8 Modifying UVM Testbench","text":"
                          The next sections describe what needs to be considered when modifying the UVM, adding a new interface to the testbench and creating a new UVM test for a customised OFS Accelerator platform.
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#81-modifying-uvm-environment-for-new-shell-variant","title":"8.1 Modifying UVM environment for new Shell Variant","text":"
                          OFS f2000x comprises a shell based on PCIe Gen4x16 and is named base_x16
                           
                          This base_x16 shell is described by an RTL file list, IP File lists and setup scripts to complete the build flow
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#82-modifying-uvm-environment-and-setting-up-compile-and-run-flow","title":"8.2 Modifying UVM environment and setting up Compile and Run flow","text":"
                          All the variants can mostly reuse the existing UVM infrastructure to setup the build and run flow
                           
                          \u2022 Create directory under $OFS_BUILD_ROOT new variant e.g ofs-n9000\n\u2022 Change directory to $OFS_BUILD_ROOT/ofs-n9000/verification/scripts\n\u2022 modify Makefile it to point to the new RTL, IP and script files.\n
                           
                          Following these three steps above will enable the build and sim flow to run the existing UVM TB and tests with new IOFS n9000 variant.
                           
                          Adding a new interface requires signal connections in the testbench. An additional BFM or verification IP is needed to drive the new interface. The main testbench file tb_top.sv is found at the following location
                           
                              ofs-f2000x-pl/verification/testbench\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#83-adding-a-new-ral-directory","title":"8.3 Adding a new RAL directory","text":"
                          In case the design has many register changes and the user decides to generate all the new RAL models instead of reusing from existing base RAL models, the following steps will help to create and integrate a new RALDIR in the UVM environment.
                           
                          \u2022 Generate the new RAL files in desired directory. Preferably under the \"ofs-f2000x-pl/verification/common/testbench\" \n\u2022 By default, the Makefile points to base FIM RAL so set the RALDIR path in the Makefile to the new generated RAL file directory\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#84-modifying-tbtest-files-for-new-variant","title":"8.4 Modifying TB/Test files for new Variant","text":"
                          Create a define for the new variant. e.g 'define FIM_NEW. If you are modifying common files then add new logic under this define so other projects will not get affected with variant specific change.
                           
                          If there are more changes, please create separate \"testbench\" and \"test\" folders under this new variant. 
                           
                          \u2022 Extend variant specific env file from base env file to add variant specific changes.\n\u2022 Create new test/seq lib files in \"tests\" folder.\n\u2022 Create new variant package, add new TB/Tests/Sequence lib files and also import the base package files.\n
                           
                          If you are adding new files then make sure it's included in Makefile for the build+run flow. 
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#85-uvm-pcie-drivers","title":"8.5 UVM PCIe Drivers","text":"
                          The \"svt_pcie_driver_app_transaction_base_sequence\" is part of Synopsys PCIe VIP library. You can find the sequence definition in the following two directories
                           
                          \u2022 Navigate to \"$DESIGNWARE_HOME/vip/svt/pcie_svt/Q-2020.03/sverilog/src/vcs/svt_pcie_driver_app_transaction_sequence_collection.svp\" file. All the base and PCIe sequences are defined in this file.\n\n\u2022 When the OFS UVM build command is executed, it creates \"vip\" directory under \"$OFS_BUILD_ROOT/ofs-f2000x-pl/verification\". You can also find the same sequence file at \"$IOFS_BUILD_ROOT/ofs-f2000x/verification/vip/pcie_vip/src/sverilog/vcs/svt_pcie_driver_app_transaction_sequence_collection.sv\"\n
                          "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                          Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                           
                          OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                          "},{"location":"hw/ftile_devkit/","title":"F-Series (2xF-tile) Development Kit Collateral for OFS","text":"
                          This folder contains applicable collateral for OFS PCIe Attach reference shell targeting the F-Series (2xF-tile) Development Kit DK-DEV-AGF027F1.
                          "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/","title":"FPGA Interface Manager Developer Guide for Open FPGA Stack: Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) PCIe Attach","text":"
                          Last updated: February 03, 2024 
                          "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1-introduction","title":"1. Introduction","text":""},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#11-about-this-document","title":"1.1. About This Document","text":"
                          This document serves as a guide for OFS Agilex PCIe Attach developers targeting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile). The following topics are covered in this guide:
                           
                             
                          • Compiling the OFS Agilex PCIe Attach FIM design
                            •  
                          • Simulating the OFS Agilex PCIe Attach design
                            •  
                          • Customizing the OFS Agilex PCIe Attach FIM design
                            •  
                          • Configuring the FPGA with an OFS Agilex PCIe Attach FIM design
                            •  
                           
                          The FIM Development Walkthroughs Table lists all of the walkthroughs provided in this guide. These walkthroughs provide step-by-step instructions for performing different FIM Development tasks.
                           
                          Table: FIM Development Walkthroughs
                           Walkthrough Name Category Install Quartus Prime Pro Software Setup Install Git Large File Storage Extension Setup Clone FIM Repository Setup Set Development Environment Variables Setup Set Up Development Environment Setup Compile OFS FIM Compilation Manually Generate OFS Out-Of-Tree PR FIM Compilation Change the Compilation Seed Compilation Run Individual Unit Level Simulation Simulation Run Regression Unit Level Simulation Simulation Add a new module to the OFS FIM Customization Modify and run unit tests for a FIM that has a new module Customization Hardware test a FIM that has a new module Customization Debug the FIM with Signal Tap Customization Compile the FIM in preparation for designing your AFU Customization Resize the Partial Reconfiguration Region Customization Modify PCIe Configuration Using OFSS Customization Modify PCIe Configuration Using IP Presets Customization Create a Minimal FIM Customization Migrate to a Different Agilex Device Number Customization Modify the Ethernet Sub-System to 2x4x10GbE Customization Modify the Ethernet Sub-System to 3x4x10GbE Customization Remove the HPS Customization Set up JTAG Configuration Program the FPGA via JTAG Configuration"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#111-knowledge-pre-requisites","title":"1.1.1 Knowledge Pre-Requisites","text":"
                          It is recommended that you have the following knowledge and skills before using this developer guide.
                           
                             
                          • Basic understanding of OFS and the difference between OFS designs. Refer to the OFS Welcome Page.
                            •  
                          • Review the release notes for the Intel Agilex 7 PCIe Attach Reference Shells, with careful consideration of the Known Issues.
                            •  
                          • Review of Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)).
                            •  
                          • FPGA compilation flows using Intel\u00ae Quartus\u00ae Prime Pro Edition.
                            •  
                          • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
                            •  
                          • RTL (System Verilog) and coding practices to create synthesized logic.
                            •  
                          • RTL simulation tools.
                            •  
                          • Intel\u00ae Quartus\u00ae Prime Pro Edition Signal Tap Logic Analyzer tool software.
                            •  
                          "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#12-fim-development-theory","title":"1.2. FIM Development Theory","text":"
                          This section will help you understand how the OFS Agilex PCIe Attach FIM can be developed to fit your design goals.
                           
                          The Default FIM Features section provides general information about the default features of the OFS Agilex PCIe Attach FIM so you can become familiar with the default design. For more detailed information about the FIM architecture, refer to the OFS Agilex PCIe Attach FIM Technical Reference Manual.
                           
                          The Customization Options section then gives suggestions of how this default design can be customized. Step-by-step walkthroughs for many of the suggested customizations are later described in the FIM Customization section.
                           
                          FIM development for a new acceleration card generally consists of the following steps:
                           
                             
                          1. Install OFS and familiarize yourself with provided scripts and source code
                            2.  
                          2. Develop high level design with your specific functionality
                               
                            1. Determine requirements and key performance metrics
                              2.  
                            2. Select IP cores
                              3.  
                            3. Select FPGA device
                              4.  
                            4. Develop software memory map
                              5.  
                             
                            3.  
                          3. Select and implement FIM Physical interfaces including:
                               
                            1. External clock sources and creation of internal PLL clocks
                              2.  
                            2. General I/O
                              3.  
                            3. Ethernet modules
                              4.  
                            4. External memories
                              5.  
                            5. FPGA programming methodology
                              6.  
                             
                            4.  
                          4. Develop device physical implementation
                               
                            1. FPGA device pin assignment
                              2.  
                            2. Create logic lock regions
                              3.  
                            3. Create of timing constraints
                              4.  
                            4. Create Intel Quartus Prime Pro FIM test project and validate:
                                 
                              1. Placement
                                2.  
                              2. Timing constraints
                                3.  
                              3. Build script process
                                4.  
                              4. Review test FIM FPGA resource usage
                                5.  
                               
                              5.  
                             
                            5.  
                          5. Select FIM to AFU interfaces and development of PIM
                            6.  
                          6. Implement FIM design
                               
                            1. Develop RTL
                              2.  
                            2. Instantiate IPs
                              3.  
                            3. Develop test AFU to validate FIM
                              4.  
                            4. Develop unit and device level simulation
                              5.  
                            5. Develop timing constraints and build scripts
                              6.  
                            6. Perform timing closure and build validation
                              7.  
                             
                            7.  
                          7. Create FIM documentation to support AFU development and synthesis
                            8.  
                          8. Software Device Feature discovery
                            9.  
                          9. Integrate, validate, and debug hardware/software
                            10.  
                          10. Prepare for high volume production
                            11.  
                          "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#121-default-fim-features","title":"1.2.1 Default FIM Features","text":""},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1211-top-level","title":"1.2.1.1 Top Level","text":"
                          Figure: OFS Agilex PCIe Attach fseries-dk FIM Top-Level Diagram
                           
                          "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1212-interfaces","title":"1.2.1.2 Interfaces","text":"
                          The key interfaces in the OFS Agilex PCIe Attach design are listed in the Release Capabilities Table. It describes the capabilities of the fseries-dk hardware as well as the capabilities of the default OFS Agilex PCIe Attach design targeting the fseries-dk.
                           
                          Table: Release Capabilities
                           Interface fseries-dk Hardware Capabilities OFS Agilex PCIe Attach Default Design Implementation Host Interface[1] PCIe Gen4x8 PCIe Gen4x16 Network Interface 1 QSFP-56 cage1 QSFPDD-56 Cage 2x4x25GbE External Memory 3xDDR4 DIMMs sockets - 72-bits (1 available for HPS) 2xDDR4 - 2400MHz - 8Gb (1Gbx8) - 64-bits - No ECC1xDDR4 - 2400MHz - 16Gb (2Gbx8) - 40-bits - With ECC - For HPS 
                          [1] F-Tile ES version development kit has a form factor of PCIe Gen4x16, however it only supports PCIe Gen4x8. The OFS Agilex PCIe Attach design implements PCIe Gen4x16 with the assumption that it will train down to PCIe Gen4x8.
                          "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1213-subsystems","title":"1.2.1.3 Subsystems","text":"
                          The FIM Subsystems Table  describes the Platform Designer IP subsystems used in the OFS Agilex PCIe Attach fseries-dk FIM.
                           
                          Table: FIM Subsystems
                           Subsystem User Guide Document ID PCIe Subsystem PCIe Subsystem Intel FPGA IP User Guide for Intel Agilex OFS N/A Memory Subsystem Memory Subsystem Intel FPGA IP User Guide for Intel Agilex OFS 686148[1] Ethernet Subsystem Ethernet Subsystem Intel FPGA IP User Guide 773413[1] 
                          [1] You must log in to myIntel and request entitled access.
                          "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1214-host-exercisers","title":"1.2.1.4 Host Exercisers","text":"
                          The default AFU workload in the OFS Agilex PCIe Attach fseries-dk FIM contains several modules called Host Exercisers which are used to exercise the interfaces on the board. The Host Exerciser Descriptions Table describes these modules.
                           
                          Table: Host Exerciser Descriptions
                           Name Acronym Description OPAE Command Host Exerciser Loopback HE-LB Used to exercise and characterize host to FPGA data transfer. host_exerciser Host Exerciser Memory HE_MEM Used to exercise and characterize host to Memory data transfer. host_exerciser Host Exerciser Memory Traffic Generator HE_MEM_TG Used to exercise and test available memory channels with a configurable traffic pattern. mem_tg Host Exerciser High Speed Serial Interface HE-HSSI Used to exercise and characterize HSSI interfaces. hssi 
                          The host exercisers can be removed from the design at compile-time using command line arguments for the build script.
                          "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1215-module-access-via-apfbpf","title":"1.2.1.5 Module Access via APF/BPF","text":"
                          The OFS Agilex PCIe Attach fseries-dk FIM uses AXI4-Lite interconnect logic named the AFU Peripheral Fabric (APF) and Board Peripheral Fabric (BPF) to access the registers of the various modules in the design. The APF/BPF modules define master/slave interactions, namely between the host software and AFU and board peripherals. The APF Address Map Table describes the address mapping of the APF, followed by the BPF Address Map Table which describes the address mapping of the BPF.
                           
                          Table: APF Address Map
                           Address Size (Bytes) Feature 0x00000\u20130x3FFFF 256K Board Peripherals (See BPF Address Map table) 0x40000 \u2013 0x4FFFF 64K ST2MM 0x50000 \u2013 0x5FFFF 64K Reserved 0x60000 \u2013 0x60FFF 4K UART 0x61000 \u2013 0x6FFFF 4K Reserved 0x70000 \u2013 0x7FFFF 56K PR Gasket:4K= PR Gasket DFH, control and status4K= Port DFH4K=User Clock52K=Remote STP 0x80000 \u2013 0x80FFF 4K AFU Error Reporting 
                          Table: BPF Address Mapping
                           Address Size (Bytes) Feature 0x00000 - 0x0FFFF 64K FME 0x10000 - 0x10FFF 4K PCIe 0x11000 - 0x11FFF 4K Reserved 0x12000 - 0x12FFF 4K QSFP0 0x13000 - 0x13FFF 4K QSFP1 0x14000 - 0x14FFF 4K HSSI 0x15000 - 0x15FFF 4K EMIF"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#122-customization-options","title":"1.2.2 Customization Options","text":"
                          OFS is designed to be easily customizable to meet your design needs. The OFS FIM Customizations Table lists the general user flows for OFS Agilex PCIe Attach fseries-dk FIM development, along with example customizations for each user flow, plus links to step-by-step walkthroughs where available.
                           
                          Table: OFS FIM Customizations
                           Customization Walkthrough Name Add a new module to the OFS FIM Modify and run unit tests for a FIM that has a new module Hardware test a FIM that has a new module Debug the FIM with Signal Tap Compile the FIM in preparation for designing your AFU Resize the Partial Reconfiguration Region Modify PCIe Configuration Using OFSS Modify PCIe Configuration Using IP Presets Create a Minimal FIM Migrate to a Different Agilex Device Number Modify the Ethernet Sub-System to 2x4x10GbE Modify the Ethernet Sub-System to 3x4x10GbE Remove the HPS"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#13-development-environment","title":"1.3 Development Environment","text":"
                          This section describes the components required for OFS FIM development, and provides a walkthrough for setting up the environment on your development machine.
                           
                          Note that your development machine may be different than your deployment machine where the FPGA acceleration card is installed. FPGA development work and deployment work can be performed either on the same machine, or on different machines as desired. Please see the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up the environment for deployment machines.
                          "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#131-development-tools","title":"1.3.1 Development Tools","text":"
                          The Development Environment Table describes the Best Known Configuration (BKC) for the tools that are required for OFS FIM development.
                           
                          Table: Development Environment BKC
                           Component Version Installation Walkthrough Operating System RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6 N/A Intel Quartus Prime Software Quartus Prime Pro Version 23.3 for Linux + Patches 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe) Section 1.3.1.1 Python 3.6.8 or later N/A GCC 7.4.0 or later N/A cmake 3.15 or later N/A git with git-lfs 1.8.3.1 or later Section 1.3.1.2 FIM Source Files ofs-2023.3-2 Section 1.3.2.1"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1311-walkthrough-install-quartus-prime-pro-software","title":"1.3.1.1 Walkthrough: Install Quartus Prime Pro Software","text":"
                          Intel Quartus Prime Pro Version 23.3 is verified to work with the latest OFS release ofs-2023.3-2.  However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.
                           
                          Use RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6 for compatibility with your development flow and also testing your FIM design in your platform. 
                           
                          Prior to installing Quartus:
                           
                             
                          1.  
                            Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.
                             
                               
                            • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                              •  
                            • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.
                              •  
                             
                            2.  
                          2.  
                            Perform the following steps to satisfy the required dependencies.
                             
                            $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                             
                            Apply the following configurations.
                             
                            $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n
                             
                            3.  
                          3.  
                            Create the default installation path: /intelFPGA_pro/, where  is the default path of the Linux workstation, or as set by the system administrator and  is your Quartus version number. 
                            The installation path must satisfy the following requirements:
                             
                               
                            • Contain only alphanumeric characters
                              •  
                            • No special characters or symbols, such as !$%@^&*<>,
                              •  
                            • Only English characters
                              •  
                            • No spaces
                              •  
                             
                          4.  
                            Download your required Quartus Prime Pro Linux version here.
                             
                            5.  
                          5.  
                            Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe).
                             
                            6.  
                          6.  
                            After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                             
                            export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                             
                            For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:
                             
                            export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n
                             
                            7.  
                          7.  
                            Verify, Quartus is discoverable by opening a new shell:
                             
                            $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                             
                            8. "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1312-walkthrough-install-git-large-file-storage-extension","title":"1.3.1.2 Walkthrough: Install Git Large File Storage Extension","text":"
                            To install the Git Large File Storage (LFS) extension, execute the following commands:
                             
                               
                            1. Obtain Git LFS package     
                              curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\n
                              2.  
                            2. Install Git LFS package     
                              sudo dnf install git-lfs\n
                              3.  
                            3. Install Git LFS     
                              git lfs install\n
                              4.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#132-fim-source-files","title":"1.3.2 FIM Source Files","text":"
                            The source files for the OFS Agilex PCIe Attach FIM are provided in the following repository: https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2
                             
                            Some essential directories in the repository are described as follows:
                             
                            ofs-agx7-pcie-attach\n|  syn                      // Contains files related to synthesis\n|  |  board                     // Contains synthesis files for several cards, including the fseries-dk \n|  |  |  fseries-dk                 // Contains synthesis files for fseries-dk\n|  |  |  |  setup                       // Contains setup files, including pin constraints and location constraints\n|  |  |  |  syn_top                     // Contains Quartus project files\n|  verification             // Contains files for UVM testing\n|  ipss                     // Contains files for IP Sub-Systems\n|  |  qsfp                      // Contains source files for QSFP Sub-System\n|  |  hssi                      // Contains source files for HSSI Sub-System\n|  |  pmci                      // Contains source files for PMCI Sub-System (not used in F-Tile FIM)\n|  |  pcie                      // Contains source files for PCIe Sub-System\n|  |  mem                       // Contains source files for Memory Sub-System\n|  sim                      // Contains simulation files\n|  |  unit_test                 // Contains files for all unit tests\n|  |  |  scripts                    // Contains script to run regression unit tests\n|  license                  // Contains Quartus patch\n|  ofs-common               // Contains files which are common across OFS platforms\n|  |  verification              // Contains common UVM files\n|  |  scripts                   // Contains common scripts\n|  |  |  common\n|  |  |  |  syn                         // Contains common scripts for synthesis, including build script\n|  |  |  |  sim                         // Contains common scripts for simulation\n|  |  tools                     // Contains common tools files\n|  |  |  mk_csr_module              // Contains common files for CSR modules\n|  |  |  fabric_generation          // Contains common files for APF/BPF fabric generation\n|  |  |  ofss_config                // Contains common files for OFSS configuration tool\n|  |  |  |  ip_params                   // Contains default IP parameters for certain Sub-Systems when using OFSS\n|  |  src                       // Contains common source files, including host exercisers\n|  tools                    //\n|  |  ofss_config               // Contains top level OFSS files for each pre-made board configuration\n|  |  |  hssi                       // Contains OFSS files for Ethernet-SS configuraiton\n|  |  |  memory                     // Contains OFSS files for Memory-SS configuration\n|  |  |  pcie                       // Contains OFSS files for PCIe-SS configuration\n|  |  |  iopll                      // Contains OFSS files for IOPLL configuration\n|  src                      // Contains source files for Agilex PCIe Attach FIM\n|  |  pd_qsys                   // Contains source files related to APF/BPF fabric\n|  |  includes                  // Contains source file header files\n|  |  top                       // Contains top-level source files, including design top module\n|  |  afu_top                   // Contains top-level source files for AFU\n
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1321-walkthrough-clone-fim-repository","title":"1.3.2.1 Walkthrough: Clone FIM Repository","text":"
                            Perform the following steps to clone the OFS Agilex PCIe Attach FIM Repository:
                             
                               
                            1. Create a new directory to use as a clean starting point to store the retrieved files.     
                              mkdir OFS_BUILD_ROOT\ncd OFS_BUILD_ROOT\nexport OFS_BUILD_ROOT=$PWD\n
                              2.  
                            2. Clone GitHub repository using the HTTPS git method     
                              git clone --recurse-submodules https://github.com/OFS/ofs-agx7-pcie-attach.git\n
                              3.  
                            3. Check out the correct tag of the repository     
                              cd ofs-agx7-pcie-attach\ngit checkout --recurse-submodules tags/ofs-2023.3-2\n
                              4.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#133-environment-variables","title":"1.3.3 Environment Variables","text":"
                            The OFS FIM compilation and simulation scripts require certain environment variables be set prior to execution.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1331-walkthrough-set-development-environment-variables","title":"1.3.3.1 Walkthrough: Set Development Environment Variables","text":"
                            Perform the following steps to set the required environment variables. These environment variables must be set prior to simulation or compilation tasks so it is recommended that you create a script to set these variables.
                             
                               
                            1.  
                              Navigate to the top level directory of the cloned OFS FIM repository.
                               
                              cd ofs-agx7-pcie-attach\n
                               
                              2.  
                            2.  
                              Set project variables   
                              # Set OFS Root Directory - e.g. this is the top level directory of the cloned OFS FIM repository\nexport OFS_ROOTDIR=$PWD\n
                               
                              3.  
                            3.  
                              Set variables based on your development environment   
                              # Set proxies if required for your server\nexport http_proxy=<YOUR_HTTP_PROXY>\nexport https_proxy=<YOUR_HTTPS_PROXY>\nexport ftp_proxy=<YOUR_FTP_PROXY>\nexport socks_proxy=<YOUR_SOCKS_PROXY>\nexport no_proxy=<YOUR_NO_PROXY>\n\n# Set Quartus license path\nexport LM_LICENSE_FILE=<YOUR_LM_LICENSE_FILE>\n\n# Set Synopsys License path (if using Synopsys for simulation)\nexport DW_LICENSE_FILE=<YOUR_DW_LICENSE_FILE>\nexport SNPSLMD_LICENSE_FILE=<YOUR_SNPSLMD_LICENSE_FILE>\n\n# Set Quartus Installation Directory - e.g. $QUARTUS_ROOTDIR/bin contains Quartus executables\nexport QUARTUS_ROOTDIR=<YOUR_QUARTUS_INSTALLATION_DIRECTORY>\n\n# Set the Tools Directory - e.g. $TOOLS_LOCATION contains the 'synopsys' directory if you are using Synopsys. Refer to the $VCS_HOME variable for an example.\nexport TOOLS_LOCATION=<YOUR_TOOLS_LOCATION>\n
                               
                              4.  
                            4.  
                              Set generic environment variables
                               
                              # Set Work directory \nexport WORKDIR=$OFS_ROOTDIR\n\n# Set Quartus Tools variables\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport QUARTUS_VER_AC=$QUARTUS_ROOTDIR\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport IMPORT_IP_ROOTDIR=$IP_ROOTDIR\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\n\n# Set Verification Tools variables (if running simulations)\nexport DESIGNWARE_HOME=$TOOLS_LOCATION/synopsys/vip_common/vip_Q-2020.03A\nexport UVM_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport MTI_HOME=$QUARTUS_ROOTDIR/../questa_fse\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n\n# Set PATH to include compilation and simulation tools\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/../qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/bin:$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$DESIGNWARE_HOME/bin:$VCS_HOME/bin:$PATH\n
                               
                              5.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#134-walkthrough-set-up-development-environment","title":"1.3.4 Walkthrough: Set Up Development Environment","text":"
                            This walkthrough guides you through the process of setting up your development environment in preparation for FIM development. This flow only needs to be done once on your development machine.
                             
                               
                            1.  
                              Ensure that Quartus Prime Pro Version 23.3 for Linux with Intel Agilex FPGA device support is installed on your development machine. Refer to the Install Quartus Prime Pro Software section for step-by-step installation instructions.
                               
                                 
                              1. Verify version number
                                quartus_sh --version\n
                                 
                                Example Output:
                                 
                                Quartus Prime Shell\nVersion 23.3 Build 94 06/14/2023 SC Pro Edition\nCopyright (C) 2023  Intel Corporation. All rights reserved.\n
                                 
                                2.  
                               
                              2.  
                            2.  
                              Ensure that all support tools are installed on your development machine, and that they meet the version requirements.
                               
                                 
                              1.  
                                Python 3.6.8 or later
                                 
                                   
                                1.  
                                  Verify version number
                                   
                                  python --version\n
                                   
                                  Example Output:
                                   
                                  Python 3.6.8\n
                                   
                                  2.  
                                 
                                2.  
                              2.  
                                GCC 7.4.0 or later
                                 
                                   
                                1.  
                                  Verify version number
                                   
                                  gcc --version\n
                                   
                                  Example output:
                                   
                                  gcc (GCC) 7.4.0\n
                                   
                                  2.  
                                 
                                3.  
                              3.  
                                cmake 3.15 or later
                                 
                                   
                                1.  
                                  Verify version number
                                   
                                  cmake --version\n
                                   
                                  Example output:
                                   
                                  cmake version 3.15\n
                                   
                                  2.  
                                 
                                4.  
                              4.  
                                git with git-lfs 1.8.3.1 or later. Refer to the Install Git Large File Storage Extension section for step-by-step instructions on installing the Git Large File Storage (LFS) extension.
                                 
                                   
                                1.  
                                  Verify version number
                                   
                                  git --version\n
                                   
                                  Example output:
                                   
                                  git version 1.8.3.1\n
                                   
                                  2.  
                                 
                                5.  
                               
                              3.  
                            3.  
                              Clone the ofs-agx7-pcie-attach repository. Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              4.  
                            4.  
                              Install UART IP license patch .02.
                               
                                 
                              1.  
                                Navigate to the license directory
                                 
                                cd $IOFS_BUILD_ROOT/license\n
                                 
                                2.  
                              2.  
                                Install Patch 0.02
                                 
                                sudo ./quartus-0.0-0.02iofs-linux.run\n
                                 
                                3.  
                               
                              5.  
                            5.  
                              Install Quartus Patches 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe). All required patches are provided in the Assets of the OFS FIM Release Tag: https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2
                               
                              6.  
                            6.  
                              Verify that patches have been installed correctly. They should be listed in the output of the following command.
                               
                              quartus_sh --version\n
                               
                              7.  
                            7.  
                              Set required environment variables. Refer to the [Set Environment Variables] section for step-by-step instructions.
                               
                              8.  
                             
                            This concludes the walkthrough for setting up your development environment. At this point you are ready to begin FIM development.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2-fim-compilation","title":"2. FIM Compilation","text":"
                            This section describes the process of compiling OFS FIM designs using the provided build scripts. It contains two main sections:
                             
                               
                            • Compilation Theory - Describes the theory behind FIM compilation
                              •  
                            • Compilation Flows - Describes the process of compiling a FIM
                              •  
                             
                            The walkthroughs provided in this section are:
                             
                               
                            • Compile OFS FIM
                              •  
                            • Manually Generate OFS Out-Of-Tree PR FIM
                              •  
                            • Change the Compilation Seed
                              •  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#21-compilation-theory","title":"2.1 Compilation Theory","text":"
                            This sections describes the theory behind FIM compilation.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#211-fim-build-script","title":"2.1.1 FIM Build Script","text":"
                            The OFS Common Repository contains a script named build_top.sh which is used to build OFS FIM designs and generate output files that can be programmed to the board. After cloning the OFS FIM repository (with the ofs-common repository included), the build script can be found in the following location:
                             
                            $OFS_ROOTDIR/ofs-common/scripts/common/syn/build_top.sh\n
                             
                            The usage of the build_top.sh script is as follows:
                             
                            build_top.sh [-k] [-p] [-e] [--stage=<action>] [--ofss=<ip_config>] <build_target>[:<fim_options>] [<work_dir_name>]\n
                             Field Options Description Requirement -k None Keep. Preserves and rebuilds within an existing work tree instead of overwriting it. Optional -p None When set, and if the FIM supports partial reconfiguration, a PR template tree is generated at the end of the FIM build. The PR template tree is located in the top of the work directory but is relocatable and uses only relative paths. See $OFS_ROOTDIR/syn/common/scripts generate_pr_release.sh for details. Optional -e None Run only Quartus analysis and elaboration. It completes the setup stage, passes -end synthesis to the Quartus compilation flow and exits without running the finish stage. Optional --stage all | setup | compile | finish Controls which portion of the OFS build is run.\u00a0\u00a0- all: Run all build stages (default)\u00a0\u00a0- setup: Initialize a project in the work directory\u00a0\u00a0- compile: Run the Quartus compilation flow on a project that was already initialized with setup\u00a0\u00a0- finish: Complete OFS post-compilation tasks, such as generating flash images and, if -p is set, generating a release. Optional --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. This parameter is consumed during the setup stage and IP is updated only inside the work tree. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. Optional <build_target> fseries-dk Specifies which board is being targeted. Required <fim_options> flat | null_he_lb | null_he_hssi | null_he_mem | null_he_mem_tg | no_hssi Used to change how the FIM is built.\u00a0\u00a0\u2022 flat - Compiles a flat design (no PR assignments). This is useful for bringing up the design on a new board without dealing with PR complexity.\u00a0\u00a0\u2022 null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null.\u00a0\u00a0\u2022 null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null.\u00a0\u00a0\u2022 null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null.\u00a0\u00a0\u2022 null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null.\u00a0\u00a0\u2022 no_hssi - Removes the HSSI-SS from the design. More than one FIM option may be passed included in the <fim_options> list by concatenating them separated by commas. For example: <build_target>:flat,null_he_lb,null_he_hssi Optional <work_dir_name> String Specifies the name of the work directory in which the FIM will be built. If not specified, the default target is $OFS_ROOTDIR/work Optional 
                            Note: The he_null is a minimal block with CSRs that responds to PCIe MMIO requests in order to keep PCIe alive.
                             
                            The build script copies source files from the existing cloned repository into the specified work directory, which are then used for compilation. As such, any changes made in the base source files will be included in all subsequent builds, unless the -k option is used, in which case an existing work directories files are used as-is. Likewise, any changes made in a work directory is only applied to that work directory, and will not be updated in the base repository by default.
                             
                            Refer to Compile OFS FIM which provides step-by-step instructions for running the build_top.sh script with some of the different available options.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#212-ofss-file-usage","title":"2.1.2 OFSS File Usage","text":"
                            The OFS FIM build script can use OFSS files to easily customize design IP prior to compilation using preset configurations. The OFSS files specify certain parameters for different IPs. Using OFSS is provided as a convenience feature for building FIMs. The Provided OFSS Files table below describes the pre-made OFSS files for the fseries-dk that can be found in the $OFS_ROOTDIR/tools/ofss_config directory. Only the OFSS files listed in this table are compatible with the fseries-dk In order to compile an fseries-dk FIM, you must supply OFSS files corresponding to each IP that is present in your design.
                             
                            Table: Provided OFSS Files
                             OFSS File Name Location Type Description fseries-dk.ofss $OFS_ROOTDIR/tools/ofss_config Top Includes the following OFSS files [1]: \u00a0\u00a0\u2022 fseries-dk_base.ofss \u00a0\u00a0\u2022 pcie_host.ofss \u00a0\u00a0\u2022 iopll.ofss \u00a0\u00a0\u2022 memory_ftile.ofss fseries-dk_base.ofss $OFS_ROOTDIR/tools/ofss_config ofs Defines certain attributes of the design, including the platform name, device family, fim type, part number, and device ID. pcie_host.ofss $OFS_ROOTDIR/tools/ofss_config/pcie pcie Defines the PCIe Subsystem with the following configuration:\u00a0\u00a0\u2022 PF0 (3 VFs)\u00a0\u00a0\u2022 PF1 (0 VFs)\u00a0\u00a0\u2022 PF2 (0 VFs)\u00a0\u00a0\u2022 PF3 (0 VFs)\u00a0\u00a0\u2022 PF4 (0 VFs) pcie_1pf_1vf.ofss $OFS_ROOTDIR/tools/ofss_config/pcie pcie Defines the PCIe Subsystem with the following configuration:\u00a0\u00a0\u2022 PF0 (1 VF) iopll.ofss $OFS_ROOTDIR/tools/ofss_config/iopll iopll Sets the IOPLL frequency to 470 MHz memory_ftile.ofss $OFS_ROOTDIR/tools/ofss_config/memory memory Defines the memory IP preset file to be used during the build as fseries-dk hssi_8x25_ftile.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP preset file to be used during the build as fseries-dk 
                            [1] The fseries-dk.ofss file does not include an HSSI OFSS file by default. If you are using the fseries-dk.ofss file when building the FIM, you must also specify an HSSI OFSS file for F-tile. Refer to the Compile OFS FIM Section for examples of this flow.
                             
                            Note: Using OFSS is required for FIM builds targeting the F-tile Development Kit.
                             
                            There can typically be three sections contained within an OFSS file.
                             
                               
                            •  
                              [include]
                               
                                 
                              • This section of an OFSS file contains elements separated by a newline, where each element is the path to an OFSS file that is to be included for configuration by the OFSS Configuration Tool. Ensure that any environment variables (e.g. $OFS_ROOTDIR) are set correctly. The OFSS Config tool uses breadth first search to include all of the specified OFSS files; the ordering of OFSS files does not matter
                                •  
                               
                              •  
                             
                               
                            •  
                              [ip]
                               
                                 
                              • This section of an OFSS file contains a key value pair that allows the OFSS Config tool to determine which IP configuration is being passed in. The currently supported values of IP are ofs, iopll, pcie, memory, and hssi.
                                •  
                               
                              •  
                             
                               
                            •  
                              [settings]
                               
                                 
                              • This section of an OFSS file contains IP specific settings. Refer to existing IP OFSS file of different types to see what IP settings are set. For the IP type ofss, the settings will be information of the OFS device (platform, family, fim, part #, device_id)
                                •  
                               
                              •  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2121-platform-ofss-file","title":"2.1.2.1 Platform OFSS File","text":"
                            The <platform>.ofss file (e.g. $OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss) is the platform level OFSS wrapper file. This is typically the OFSS file that is provided to the build script. It only contains an include section which lists all other OFSS files that are to be used when the <platform>.ofss file is passed to the build script.
                             
                            An example structure of a <platform>.ofss file is as follows:
                             
                            [include]\n<PATH_TO_PLATFORM_BASE_OFSS_FILE>\n<PATH_TO_PCIE_OFSS_FILE>\n<PATH_TO_IOPLL_OFSS_FILE>\n<PATH_TO_MEMORY_OFSS_FILE>\n<PATH_TO_HSSI_OFSS_FILE>\n
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2122-ofs-ip-ofss-file","title":"2.1.2.2 OFS IP OFSS File","text":"
                            An OFSS file with IP type ofs (e.g. $OFS_ROOTDIR/tools/ofss_config/fseries-dk_base.ofss) contains board specific information for the target board.
                             
                            Currently supported configuration options for an OFSS file with IP type ofs are described in the OFS IP OFSS File Options table.
                             
                            Table: OFS IP OFSS File Options
                             Section Parameter fseries-dk Default Value [ip] type ofs [settings] platform N6001 family agilex fim base_x16 part AGFB027R24C2E2VR2 device_id 6001"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2123-pcie-ip-ofss-file","title":"2.1.2.3 PCIe IP OFSS File","text":"
                            An OFSS file with IP type pcie (e.g. $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss) is used to configure the PCIe-SS in the FIM.
                             
                            The PCIe OFSS file has a special section type ([pf*]) which is used to define physical functions (PFs) in the FIM. Each PF has a dedicated section, where the * character is replaced with the PF number. For example, [pf0], [pf1], etc. For reference FIM configurations, 0 virtual functions (VFs) on PF0 is not supported. This is because the PR region cannot be left unconnected. A NULL AFU may need to be instantiated in this special case. PFs must be consecutive. The PFVF Limitations table describes the supported number of PFs and VFs.
                             
                            Table: PF/VF Limitations
                             Parameter Value Min # of PFs 1 (on PF0) Max # of PFs 8 Min # of VFs 1 on PF0 Max # of VFs 2000 distributed across all PFs 
                            Currently supported configuration options for an OFSS file with IP type pcie are described in the PCIe IP OFSS File Options table.
                             
                            Table: PCIe IP OFSS File Options
                             Section Parameter Options Default Description [ip] type pcie N/A Specifies that this OFSS file configures the PCIe-SS [settings] output_name pcie_ss N/A Specifies the output name of the PCIe-SS IP preset String N/A OPTIONAL - Specifies the name of a PCIe-SS IP presets file to use when building the FIM. When used, a presets file will take priority over any other parameters set in this OFSS file. [pf*] num_vfs Integer 0 Specifies the number of Virtual Functions in the current PF bar0_address_width Integer 12 bar4_address_width Integer 14 vf_bar0_address_width Integer 12 ats_cap_enable 0 | 1 0 vf_ats_cap_enable 0 | 1 0 prs_ext_cap_enable 0 | 1 0 pasid_cap_enable 0 | 1 0 pci_type0_vendor_id 32'h Value 0x00008086 pci_type0_device_id 32'h Value 0x0000bcce revision_id 32'h Value 0x00000001 class_code 32'h Value 0x00120000 subsys_vendor_id 32'h Value 0x00008086 subsys_dev_id 32'h Value 0x00001771 sriov_vf_device_id 32'h Value 0x0000bccf exvf_subsysid 32'h Value 0x00001771 
                            The default values for all PCIe-SS parameters (that are not defined in the PCIe IP OFSS file) are defined in $OFS_ROOTDIR/ofs-common/tools/ofss_config/ip_params/pcie_ss_component_parameters.py. When using a PCIe IP OFSS file during compilation, the PCIe-SS IP that is used will be defined based on the values in the PCIe IP OFSS file plus the parameters defined in pcie_ss_component_parameters.py.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2124-iopll-ip-ofss-file","title":"2.1.2.4 IOPLL IP OFSS File","text":"
                            An OFSS file with IP type iopll (e.g. $OFS_ROOTDIR/tools/ofss_config/iopll/iopll.ofss) is used to configure the IOPLL in the FIM.
                             
                            The IOPLL OFSS file has a special section type ([p_clk]) which is used to define the IOPLL clock frequency.
                             
                            Currently supported configuration options for an OFSS file with IP type iopll are described in the IOPLL OFSS File Options table.
                             
                            Table: IOPLL OFSS File Options
                             Section Parameter Options Description [ip] type iopll Specifies that this OFSS file configures the IOPLL [settings] output_name sys_pll Specifies the output name of the IOPLL. instance_name iopll_0 Specifies the instance name of the IOPLL. [p_clk] freq Integer: 250 - 470 Specifies the IOPLL clock frequency in MHz. 
                            Note: The following frequencies have been tested on reference boards: 350MHz, 400MHz, 470MHz.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2125-memory-ip-ofss-file","title":"2.1.2.5 Memory IP OFSS File","text":"
                            An OFSS file with IP type memory (e.g. $OFS_ROOTDIR/tools/ofss_config/memory/memory_ftile.ofss) is used to configure the Memory-SS in the FIM.
                             
                            The Memory OFSS file specifies a preset value, which selects a presets file (.qprs) to configure the Memory-SS.
                             
                            Currently supported configuration options for an OFSS file with IP type memory are described in the Memory OFSS File Options table.
                             
                            Table: Memory OFSS File Options
                             Section Parameter Options Description [ip] type memory Specifies that this OFSS file configures the Memory-SS [settings] output_name mem_ss_fm Specifies the output name of the Memory-SS. preset fseries-dk | String[1] Specifies the name of the .qprs presets file that will be used to build the Memory-SS. 
                            [1] You may generate your own .qprs presets file with a unique name using Quartus. 
                             
                            Pre-provided Memory-SS presets files are located in the $OFS_ROOTDIR/ipss/mem/qip/presets directory.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2126-hssi-ip-ofss-file","title":"2.1.2.6 HSSI IP OFSS File","text":"
                            An OFSS file with IP type hssi (e.g. $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_8x25_ftile.ofss) is used to configure the Ethernet-SS in the FIM.
                             
                            Currently supported configuration options for an OFSS file with IP type hssi are described in the HSSI OFSS File Options table.
                             
                            Table: HSSI OFSS File Options
                             Section Parameter Options Description [ip] type hssi Specifies that this OFSS file configures the Ethernet-SS [settings] output_name hssi_ss Specifies the output name of the Ethernet-SS num_channels Integer Specifies the number of channels. data_rate 10GbE | 25GbE Specifies the data rate[1] preset fseries-dk | String[2] Specifies the name of the .qprs preset file that will be used to build the Ethernet-SS. This will overwrite the other settings in this OFSS file. 
                            [1] The presets file will take priority over the data_rate parameter, so this value will not take effect when using a presets file.
                             
                            [2] You may generate your own .qprs presets file with a unique name using Quartus. 
                             
                            Pre-provided Ethernet-SS presets are located in the  $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss/presets directory.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#213-ofs-build-script-outputs","title":"2.1.3 OFS Build Script Outputs","text":"
                            The output files resulting from running the the OFS FIM build_top.sh build script are copied to a single directory during the finish stage of the build script. The path for this directory is: $OFS_ROOTDIR/<WORK_DIRECTORY>/syn/board/fseries-dk/syn_top/output_files.
                             
                            The output files include programmable images and compilation reports. The OFS Build Script Output Descriptions table describes the images that are generated by the build script.
                             
                            Table: OFS Build Script Output Descriptions
                             File Name Description ofs_top.sof The FIM design SRAM Object File; a binary file of the compiled FIM image. ofs_top_hps.sof The build assembly process combines the FPGA ofs_top.sof programming file with u-boot-spl-dtb.hex to produce this file."},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#22-compilation-flows","title":"2.2 Compilation Flows","text":"
                            This section provides information for using the build script to generate different FIM types. Walkthroughs are provided for each compilation flow. These walkthroughs require that the development environment has been set up as described in the Set Up Development Environment section.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#221-flat-fim","title":"2.2.1 Flat FIM","text":"
                            A flat FIM is compiled such that there is no partial reconfiguration region, and the entire design is built as a flat design. This is useful for compiling new designs without worrying about the complexity introduced by partial reconfiguration. The flat compile removes the PR region and PR IP; thus, you cannot use the -p build flag when using the flat compile setting. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#222-in-tree-pr-fim","title":"2.2.2 In-Tree PR FIM","text":"
                            An In-Tree PR FIM is the default compilation if no compile flags or compile settings are used. This flow will compile the design with the partial reconfiguration region, but it will not create a relocatable PR directory tree to aid in AFU development. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#223-out-of-tree-pr-fim","title":"2.2.3 Out-of-Tree PR FIM","text":"
                            An Out-of-Tree PR FIM will compile the design with the partial reconfiguration region, and will create a relocatable PR directory tree to aid in AFU workload development. This is especially useful if you are developing a FIM to be used by another team developing AFU workloads. This is the recommended build flow in most cases. There are two ways to create the relocatable PR directory tree:
                             
                               
                            • Run the FIM build script with the -p option. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                              •  
                            • Run the generate_pr_release.sh script after running the FIM build script. Refer to the Manually Generate OFS Out-Of-Tree PR FIM Section step-by-step instructions for this flow.
                              •  
                             
                            In both cases, the generate_pr_release.sh is run to create the relocatable build tree. This script is located at $OFS_ROOTDIR/ofs-common/scripts/common/syn/generate_pr_release.sh. Usage for this script is as follows:
                             
                            generate_pr_release.sh -t <PATH_OF_RELOCATABLE_PR_TREE> <BOARD_TARGET> <WORK_DIRECTORY>\n
                             
                            The Generate PR Release Script Options table describes the options for the generate_pr_release.sh script.
                             
                            Table: Generate PR Release Script Options
                             Parameter Options Description <PATH_OF_RELOCATABLE_PR_TREE> String Specifies the location of the relocatable PR directory tree to be created. <BOARD_TARGET> fseries-dk Specifies the name of the board target. <WORK_DIRECTORY> String Specifies the existing work directory from which the relocatable PR directory tree will be created from. 
                            After generating the relocatable build tree, it is located in the $OFS_ROOTDIR/<WORK_DIRECTORY>/pr_build_template directory (or the directory you specified if generated separately). The contents of this directory have the following structure:
                             
                            \u251c\u2500\u2500 bin\n\u251c\u2500\u2500 \u251c\u2500\u2500 afu_synth\n\u251c\u2500\u2500 \u251c\u2500\u2500 qar_gen\n\u251c\u2500\u2500 \u251c\u2500\u2500 update_pim\n\u251c\u2500\u2500 \u251c\u2500\u2500 run.sh\n\u251c\u2500\u2500 \u251c\u2500\u2500 build_env_config\n\u251c\u2500\u2500 README\n\u251c\u2500\u2500 hw\n\u251c\u2500\u2500 \u251c\u2500\u2500 lib\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 build\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 fme-ifc-id.txt\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 platform\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 fme-platform-class.txt\n\u251c\u2500\u2500 \u251c\u2500\u2500 blue_bits\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 ofs_top_hps.sof\n\u2514\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 ofs_top.sof\n
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#223-he_null-fim","title":"2.2.3 HE_NULL FIM","text":"
                            An HE_NULL FIM refers to a design with one, some, or all of the Host Exercisers replaced by he_null blocks. The he_null is a minimal block with CSRs that responds to PCIe MMIO requests in order to keep PCIe alive. You may use any of the build flows (flat, in-tree, out-of-tree) with the HE_NULL compile options. The HE_NULL compile options are as follows:
                             
                               
                            • null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null
                              •  
                            • null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null
                              •  
                            • null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null
                              •  
                            • null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null
                              •  
                             
                            The Compile OFS FIM section gives step-by-step instructions for this flow.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#225-walkthrough-compile-ofs-fim","title":"2.2.5 Walkthrough: Compile OFS FIM","text":"
                            Perform the following steps to compile the OFS Agilex PCIe Attach FIM for fseries-dk:
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Navigate to the root directory.
                               
                              cd $OFS_ROOTDIR\n
                               
                              4.  
                            4.  
                              Run the build_top.sh script with the desired compile options using the fseries-dk OFSS presets. In this case you must use the fseries-dk.ofss file, and then specify the 8x25G HSSI configuration using the hssi_8x25_ftile.ofss (this is not necessary if using the no_hssi compile option).
                               
                                 
                              •  
                                Flat FIM
                                 
                                ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk:flat work_fseries-dk_flat\n
                                 
                                •  
                               
                                 
                              •  
                                In-Tree PR FIM
                                 
                                ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_in_tree_pr\n
                                 
                                •  
                               
                                 
                              •  
                                Out-of-Tree PR FIM
                                 
                                ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_oot_pr\n
                                 
                                •  
                               
                                 
                              •  
                                HE_NULL Flat FIM
                                 
                                ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/fseries-dk.ofss fseries-dk:flat,null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_fseries-dk_flat\n
                                 
                                •  
                               
                              5.  
                            5.  
                              Once the build script is complete, the build summary should report that the build is complete and passes timing. For example:
                               
                              ***********************************\n***\n***        OFS_PROJECT: ofs-agx7-pcie-attach\n***        OFS_BOARD: fseries-dk\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 6\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                               
                              6.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#226-walkthrough-manually-generate-ofs-out-of-tree-pr-fim","title":"2.2.6 Walkthrough: Manually Generate OFS Out-Of-Tree PR FIM","text":"
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Navigate to the root directory.
                               
                              cd $OFS_ROOTDIR\n
                               
                              4.  
                            4.  
                              Run the build_top.sh script with the desired compile options using the fseries-dk OFSS presets. In order to create the relocatable PR tree, you may not compile with the flat option. For example:
                               
                              ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk\n
                               
                              5.  
                            5.  
                              Run the generate_pr_release.sh script to create the relocatable PR tree.
                               
                              ./ofs-common/scripts/common/syn/generate_pr_release.sh -t work_fseries-dk/pr_build_template fseries-dk work_fseries-dk\n
                               
                              6.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#227-compilation-seed","title":"2.2.7 Compilation Seed","text":"
                            You may change the seed which is used by the build script during Quartus compilation to change the starting point of the fitter. Trying different seeds is useful when your design is failing timing by a small amount.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2271-walkthrough-change-the-compilation-seed","title":"2.2.7.1 Walkthrough: Change the Compilation Seed","text":"
                            Perform the following steps to change the compilation seed for the FIM build.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Edit the SEED assignment in the $OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_top.qsf file to your desired seed value. The value can be any non-negative integer value.
                               
                              set_global_assignment -name SEED 1\n
                               
                              4.  
                            4.  
                              Build the FIM. Refer to the Compile OFS FIM section for instructions.
                               
                              5.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#3-fim-simulation","title":"3. FIM Simulation","text":"
                            Unit level simulation of key components in the FIM is provided for verification of the following areas:
                             
                               
                            • Ethernet
                              •  
                            • PCIe
                              •  
                            • External Memory
                              •  
                            • Core FIM
                              •  
                             
                            The Unit Level simulations work with Synopsys VCS/VCSMX or Mentor Graphics Questasim simulators. The scripts to run each unit level simulation are located in $OFS_ROOTDIR/sim/unit_test. Each unit test directory contains a README which describes the test in detail.  Refer to the Supported Unit Tests table for a list of the supported unit tests.
                             
                            Note: The OFS Agilex PCIe Attach FIM for the F-Tile Development Kit does not support all of the unit tests that are provided in the unit_test directory.
                             
                            Table: Supported Unit Tests
                             Test Name Description bfm_test This is the unit test for PCIe BFM. The test uses HE-LB to perform memory loopback between FIM and the host. csr_test This is the unit test for FIM CSR access and AFU memory write/read dfh_walker This is the unit test for FME DFH walking flr This is the unit test for PCIe PF/VF FLR fme_csr_access This is the a unit test for the register access logic for $OFS_ROOTDIR/ofs-common/src/common/fme/fme_csr.sv fme_csr_directed This is the unit test for $OFS_ROOTDIR/ofs-common/src/common/fme/fme_csr.sv he_lb_test This is the unit test for HE_LPBK. The test uses HE-LB to perform memory loopback between FIM and the host. he_null_test This is the unit test for HE-NULL Exerciser. The test issues basic mmio Rd/Wr requests targetting HE-NULL CSRs. indirect_csr This is the unit test for axi4lite_indirect_csr_if module. pcie_csr_test This is the unit test for PCIE CSR access. pf_vf_access_test This is the unit test for PCIe PF/VF MMIO. Each function has a feature GUID at offset 0x8 with an associated register map. port_gasket_test This is the unit test for pg_csr block and it's connectivity to fabric. The test issues mmio Rd/Wr requests targetting the csrs in port_gasket. This test does not do any functional testing of partial reconfiguration, user clock or remote stp. remote_stp_test This is the unit test for remote stp. It covers mmio read access to remote_stp registers. uart_csr This is the unit test for UART CSR accesses."},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#31-simulation-file-generation","title":"3.1 Simulation File Generation","text":"
                            The simulation files must be generated prior to running unit level simulations. The script to generate simulation files is in the following location:
                             
                            $OFS_ROOTDIR/ofs-common/scripts/common/sim/gen_sim_files.sh\n
                             
                            The usage of the gen_sim_files.sh script is as follows:
                             
                            gen_sim_files.sh [--ofss=<ip_config>] <build_target>[:<fim_options>] [<device>] [<family>]\n
                             
                            The Gen Sim Files Script Options table describes the options for the gen_sim_files.sh script.
                             
                            Table: Gen Sim Files Script Options
                             Field Options Description Requirement --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. Platform Dependent[1] <build_target> fseries-dk | n6001 Specifies which board is being targeted. Required <fim_options> null_he_lb | null_he_hssi | null_he_mem | null_he_mem_tg Used to change how the FIM is built.\u00a0\u00a0- null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null.\u00a0\u00a0- null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null.\u00a0\u00a0- null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null.\u00a0\u00a0- null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null. More than one FIM option may be passed included in the <fim_options> list by concatenating them separated by commas. For example: <build_target>:null_he_lb,null_he_hssi Optional <device> string Specifies the device ID for the target FPGA. If not specified, the default device is parsed from the QSF file for the project. Optional <family> string Specifies the family for the target FPGA. If not specified, the default family is parsed from the QSF file for the project. Optional 
                            [1] Using OFSS is required for the F-Tile Development Kit.
                             
                            Refer to the Run Individual Unit Level Simulation section for an example of the simulation files generation flow.
                             
                            When running regression tests, you may use the -g command line argument to generate simulation files; refer to the Run Regression Unit Level Simulation section for step-by-step instructions.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#32-individual-unit-tests","title":"3.2 Individual Unit Tests","text":"
                            Each unit test may be run individually using the run_sim.sh script located in the following directory:
                             
                            $OFS_ROOTDIR/ofs-common/scripts/common/sim/run_sim.sh\n
                             
                            The usage for the run_sim.sh script is as follows:
                             
                            sh run_sim.sh TEST=<test> [VCSMX=<0|1> | MSIM=<0|1>]\n
                             
                            The Run Sim Script Options table describes the options for the run_sim.sh script.
                             
                            Table: Run Sim Script Options
                             Field Options Description TEST String Specify the name of the test to run, e.g. dfh_walker VCSMX 0 | 1 When set, the VCSMX simulator will be used MSIM 0 | 1 When set, the QuestaSim simulator will be used 
                            Note: The default simulator is VCS if neither VCSMX nor MSIM are set.
                             
                            The log for a unit test is stored in a transcript file in the simulation directory of the test that was run.
                             
                            $OFS_ROOTDIR/sim/unit_test/<TEST_NAME>/<SIMULATOR>/transcript\n
                             
                            For example, the log for the DFH walker test using VCSMX would be found at:
                             
                            $OFS_ROOTDIR/sim/unit_test/dfh_walker/sim_vcsmx/transcript\n
                             
                            The simulation waveform database is saved as vcdplus.vpd for post simulation review.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#321-walkthrough-run-individual-unit-level-simulation","title":"3.2.1 Walkthrough: Run Individual Unit Level Simulation","text":"
                            Perform the following steps to run an individual unit test.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Generate the simulation files for the fseries-dk
                               
                              cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk\n
                               
                              4.  
                            4.  
                              Navigate to the common simulation directory   
                              cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n
                               
                              5.  
                            5.  
                              Run the desired unit test using your desired simulator
                               
                                 
                              •  
                                Using VCS
                                 
                                sh run_sim.sh TEST=<test_name>\n
                                 
                                •  
                               
                                 
                              •  
                                Using VCSMX
                                 
                                sh run_sim.sh TEST=<test_name> VCSMX=1\n
                                 
                                •  
                               
                                 
                              •  
                                Using QuestaSim
                                 
                                sh run_sim.sh TEST=<test_name> MSIM=1\n
                                 
                                •  
                               
                                 
                              •  
                                For example, to run the DFH walker test using VCSMX:
                                 
                                sh run_sim.sh TEST=dfh_walker VCSMX=1\n
                                 
                                •  
                               
                              6.  
                            6.  
                              Once the test is complete, check the output for the simulation results. Review the log for detailed test results.
                               
                              Test status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/ofs-agx7-pcie-attach/sim/unit_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time         356233750000\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 356233750000 fs\nCPU Time:     57.730 seconds;       Data structure size:  47.2Mb\nTue Sep  5 09:44:19 2023\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                               
                              7.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#33-regression-unit-tests","title":"3.3 Regression Unit Tests","text":"
                            You may use the regression script regress_run.py to run some or all of the unit tests available with a single command. The regression script is in the following location:
                             
                            $OFS_ROOTDIR/sim/unit_test/scripts/regress_run.py\n
                             
                            The usage of the regression script is as follows:
                             
                            regress_run.py [-h] [-l] [-n <num_procs>] [-k <test_package>] [-s <simulator>] [-g] [--ofss <ip_config>] [-b <board_name>] [-e]\n
                             
                            The Regression Unit Test Script Options table describes the options for the regress_run.py script.
                             
                            Table: Regression Unit Test Script Options
                             Field Options Description -h | --help N/A Show the help message and exit -l | --local N/A Run regression locally -n | --n_procs Integer Maximum number of processes/tests to run in parallel when run locally. This has no effect on farm runs. -k | --pack all | fme | he | hssi | list | mem | pmci Test package to run during regression. The \"list\" option will look for a text file named \"list.txt\" in the \"unit_test\" directory for a text list of tests to run (top directory names). The default test package is all. -s | --sim vcs | vcsmx | msim Specifies the simulator used for the regression tests. The default simulator is vcs -g | --gen_sim_files N/A Generate simulation files. This only needs to be done once per repo update. This is the equivalent of running the gen_sim_files.sh script. -o | --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. -b | --board_name n6000 | n6001 | fseries-dk Specifies the board target -e | --email_list String Specifies email list to send results to multiple recipients 
                            The log for each unit test that is run by the regression script is stored in a transcript file in the simulation directory of the test that was run.
                             
                            $OFS_ROOTDIR/sim/unit_test/<TEST_NAME>/<SIMULATOR>/transcript\n
                             
                            For example, the log for the DFH walker test using VCSMX would be found at:
                             
                            $OFS_ROOTDIR/sim/unit_test/dfh_walker/sim_vcsmx/transcript\n
                             
                            The simulation waveform database is saved as vcdplus.vpd for post simulation review.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#331-walkthrough-run-regression-unit-level-simulation","title":"3.3.1 Walkthrough: Run Regression Unit Level Simulation","text":"
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Create a test list file to only run the unit level simulations that are supported for the fseries-dk FIM.
                               
                              touch $OFS_ROOTDIR/sim/unit_test/list.txt\n
                               
                              Copy the following list into the new file. You may remove tests from this list as desired.
                               
                              ./bfm_test/set_params.sh\n./csr_test/set_params.sh\n./dfh_walker/set_params.sh\n./flr/set_params.sh\n./fme_csr_access/set_params.sh\n./fme_csr_directed/set_params.sh\n./he_lb_test/set_params.sh\n./he_null_test/set_params.sh\n./hssi_csr_test/set_params.sh\n./hssi_kpi_test/set_params.sh\n./hssi_test/set_params.sh\n./indirect_csr/set_params.sh\n./pcie_csr_test/set_params.sh\n./pf_vf_access_test/set_params.sh\n./port_gasket_test/set_params.sh\n./qsfp_test/set_params.sh\n./remote_stp_test/set_params.sh\n./uart_csr/set_params.sh\n
                               
                              4.  
                            4.  
                              Run regression test with the your desired options. For example, to simulate with the options to generate simulation files, run locally, use 8 processes, run the specified test list, use VCSMX simulator, and target the fseries-dk:
                               
                              cd $OFS_ROOTDIR/sim/unit_test/scripts\n\npython regress_run.py --ofss $OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss -g -l -n 8 -k list -s vcsmx -b fseries-dk\n
                               
                              5.  
                            5.  
                              Once all tests are complete, check that the tests have passed.
                               
                              2023-08-30 15:00:50,256: Passing Unit Tests:13/13 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-08-30 15:00:50,256:    bfm_test:......... PASS -- Time Elapsed:0:22:23.940917\n2023-08-30 15:00:50,256:    csr_test:......... PASS -- Time Elapsed:0:22:46.262916\n2023-08-30 15:00:50,256:    dfh_walker:....... PASS -- Time Elapsed:0:22:16.544732\n2023-08-30 15:00:50,256:    flr:.............. PASS -- Time Elapsed:0:22:21.332386\n2023-08-30 15:00:50,256:    fme_csr_access:... PASS -- Time Elapsed:0:17:12.454034\n2023-08-30 15:00:50,256:    fme_csr_directed:. PASS -- Time Elapsed:0:17:22.947134\n2023-08-30 15:00:50,256:    he_lb_test:....... PASS -- Time Elapsed:0:28:38.962424\n2023-08-30 15:00:50,256:    indirect_csr:..... PASS -- Time Elapsed:0:21:15.387478\n2023-08-30 15:00:50,256:    pcie_csr_test:.... PASS -- Time Elapsed:0:22:33.838949\n2023-08-30 15:00:50,256:    pf_vf_access_test: PASS -- Time Elapsed:0:22:28.704149\n2023-08-30 15:00:50,256:    port_gasket_test:. PASS -- Time Elapsed:0:22:32.592301\n2023-08-30 15:00:50,256:    remote_stp_test:.. PASS -- Time Elapsed:0:22:01.485914\n2023-08-30 15:00:50,256:    uart_csr:......... PASS -- Time Elapsed:0:22:31.848882\n2023-08-30 15:00:50,256: Failing Unit Tests: 0/13 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-08-30 15:00:50,256: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-08-30 15:00:50,256:       Number of Unit test results captured: 13\n2023-08-30 15:00:50,256:       Number of Unit test results passing.: 13\n2023-08-30 15:00:50,256:       Number of Unit test results failing.:  0\n2023-08-30 15:00:50,256:     End Unit regression running at date/time................: 2023-08-30 15:00:50.256725\n2023-08-30 15:00:50,256:     Elapsed time for Unit regression run....................: 0:54:48.172625\n
                               
                              6.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#4-fim-customization","title":"4. FIM Customization","text":"
                            This section describes how to perform specific customizations of the FIM, and provides step-by-step walkthroughs for these customizations. Each walkthrough can be done independently. These walkthroughs require a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment. The FIM Customization Walkthroughs table lists the walkthroughs that are provided in this section. Some walkthroughs include steps for testing on hardware. Testing on hardware requires that you have a deployment environment set up. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                             
                            Table: FIM Customization Walkthroughs
                             Customization Walkthrough Name Add a new module to the OFS FIM Modify and run unit tests for a FIM that has a new module Hardware test a FIM that has a new module Debug the FIM with Signal Tap Compile the FIM in preparation for designing your AFU Resize the Partial Reconfiguration Region Modify PCIe Configuration Using OFSS Modify PCIe Configuration Using IP Presets Create a Minimal FIM Migrate to a Different Agilex Device Number Modify the Ethernet Sub-System to 2x4x10GbE Modify the Ethernet Sub-System to 3x4x10GbE Remove the HPS"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#41-adding-a-new-module-to-the-fim","title":"4.1 Adding a new module to the FIM","text":"
                            This section provides a information for adding a custom module to the FIM, simulating the new design, compiling the new design, implementing and testing the new design on hardware, and debugging the new design on hardware.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#411-hello-fim-theory-of-operation","title":"4.1.1 Hello FIM Theory of Operation","text":"
                            If you intend to add a new module to the FIM area, then you will need to inform the host software of the new module. The FIM exposes its functionalities to host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). This set of CSR registers and their operation is described in FIM MMIO Regions.
                             
                            See FPGA Device Feature List (DFL) Framework Overview for a description of the software process to read and process the linked list of Device Feature Header (DFH) CSRs within a FPGA.
                             
                            This example will add a hello_fim module to the design. The Hello FIM example adds a simple DFH register and 64bit scratchpad register connected to the Board Peripheral Fabric (BPF) that can be accessed by the Host. You can use this example as the basis for adding a new feature to your FIM.
                             
                            The Hello FIM design can be verified by Unit Level simulation, Universal Verification Methodology (UVM) simulation, and running in hardware on the fseries-dk card. The process for these are described in this section.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#4111-hello-fim-board-peripheral-fabric-bpf","title":"4.1.1.1 Hello FIM Board Peripheral Fabric (BPF)","text":"
                            The Hello FIM module will be connected to the Board Peripheral Fabric (BPF), and will be connected such that it can be mastered by the Host. The BPF is an interconnect generated by Platform Designer. The Hello FIM BPF Interface Diagram figure shows the APF/BPF Master/Slave interactions, as well as the added Hello FIM module.
                             
                            Figure: Hello FIM BPF Interface Diagram
                             
                             
                            The BPF fabric is defined in $OFS_ROOTDIR/src/pd_qsys/fabric/bpf.txt.
                             
                            We will add the Hello FIM module to an un-used address space in the MMIO region. The Hello FIM MMIO Address Layout table below shows the MMIO region for the Host with the Hello FIM module added at base address 0x16000.
                             
                            Table: Hello FIM MMIO Address Layout
                             Offset Feature CSR set 0x00000 FME AFU 0x10000 PCIe Interface 0x12000 QSFP Controller 0 0x13000 QSFP Controller 1 0x14000 HSSI Interface 0x15000 EMIF Interface 0x16000 Hello FIM"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#4112-hello-fim-csr","title":"4.1.1.2 Hello FIM CSR","text":"
                            The Hello FIM CSR will consist of the three registers shown in the Hello FIM CSR table below. The DFH and Hello FIM ID registers are read-only. The Scratchpad register supports read and write accesses.
                             
                            Table: Hello FIM CSR
                             Offset Attribute Description Default Value 0x016000 RO DFH(Device Feature Headers) register 0x30000006a0000100 0x016030 RW Scrachpad register 0x0 0x016038 RO Hello FIM ID register 0x6626070150000034"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#412-walkthrough-add-a-new-module-to-the-ofs-fim","title":"4.1.2 Walkthrough: Add a new module to the OFS FIM","text":"
                            Perform the following steps to add a new module to the OFS FIM that can be accessed by the Host.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Make hello_fim source directory
                               
                              mkdir $OFS_ROOTDIR/src/hello_fim\n
                               
                              4.  
                            4.  
                              Create hello_fim_top.sv file.
                               
                              touch $OFS_ROOTDIR/src/hello_fim/hello_fim_top.sv\n
                               
                              Copy the following code into hello_fim_top.sv:
                               
                              // ***************************************************************************\n//                               INTEL CONFIDENTIAL\n//\n//        Copyright (C) 2023 Intel Corporation All Rights Reserved.\n//\n// The source code contained or described herein and all  documents related to\n// the  source  code  (\"Material\")  are  owned  by  Intel  Corporation  or its\n// suppliers  or  licensors.    Title  to  the  Material  remains  with  Intel\n// Corporation or  its suppliers  and licensors.  The Material  contains trade\n// secrets  and  proprietary  and  confidential  information  of  Intel or its\n// suppliers and licensors.  The Material is protected  by worldwide copyright\n// and trade secret laws and treaty provisions. No part of the Material may be\n// used,   copied,   reproduced,   modified,   published,   uploaded,  posted,\n// transmitted,  distributed,  or  disclosed  in any way without Intel's prior\n// express written permission.\n//\n// No license under any patent,  copyright, trade secret or other intellectual\n// property  right  is  granted  to  or  conferred  upon  you by disclosure or\n// delivery  of  the  Materials, either expressly, by implication, inducement,\n// estoppel or otherwise.  Any license under such intellectual property rights\n// must be express and approved by Intel in writing.\n//\n// You will not, and will not allow any third party to modify, adapt, enhance, \n// disassemble, decompile, reverse engineer, change or create derivative works \n// from the Software except and only to the extent as specifically required by \n// mandatory applicable laws or any applicable third party license terms \n// accompanying the Software.\n//\n// -----------------------------------------------------------------------------\n// Engineer     : \n// Create Date  : September 2023\n// Module Name  : hello_fim_top.sv\n// Project      : OFS\n// -----------------------------------------------------------------------------\n//\n// Description: \n// This is a simple module that implements DFH registers and \n// AVMM address decoding logic.\n\nmodule hello_fim_top  #(\n   parameter ADDR_WIDTH  = 12, \n   parameter DATA_WIDTH = 64, \n   parameter bit [11:0] FEAT_ID = 12'h001,\n   parameter bit [3:0]  FEAT_VER = 4'h1,\n   parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n   parameter bit END_OF_LIST = 1'b0\n)(\n   input  logic    clk,\n   input  logic    reset,\n// -----------------------------------------------------------\n//  AXI4LITE Interface\n// -----------------------------------------------------------\n   ofs_fim_axi_lite_if.slave   csr_lite_if\n);\n\nimport ofs_fim_cfg_pkg::*;\nimport ofs_csr_pkg::*;\n\n//-------------------------------------\n// Signals\n//-------------------------------------\n   logic [ADDR_WIDTH-1:0]              csr_waddr;\n   logic [DATA_WIDTH-1:0]              csr_wdata;\n   logic [DATA_WIDTH/8-1:0]            csr_wstrb;\n   logic                               csr_write;\n   logic                               csr_slv_wready;\n   csr_access_type_t                   csr_write_type;\n\n   logic [ADDR_WIDTH-1:0]              csr_raddr;\n   logic                               csr_read;\n   logic                               csr_read_32b;\n   logic [DATA_WIDTH-1:0]              csr_readdata;\n   logic                               csr_readdata_valid;\n   logic [ADDR_WIDTH-1:0]              csr_addr;\n\n   logic [63:0]                        com_csr_writedata;\n   logic                               com_csr_read;\n   logic                               com_csr_write;\n   logic [63:0]                        com_csr_readdata;\n   logic                               com_csr_readdatavalid;\n   logic [5:0]                         com_csr_address;\n\n// AXI-M CSR interfaces\nofs_fim_axi_mmio_if #(\n   .AWID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .AWADDR_WIDTH (ADDR_WIDTH),\n   .WDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH),\n   .ARID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .ARADDR_WIDTH (ADDR_WIDTH),\n   .RDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH)\n) csr_if();\n\n// AXI4-lite to AXI-M adapter\naxi_lite2mmio axi_lite2mmio (\n   .clk       (clk),\n   .rst_n     (~reset),\n   .lite_if   (csr_lite_if),\n   .mmio_if   (csr_if)\n);\n\n//---------------------------------\n// Map AXI write/read request to CSR write/read,\n// and send the write/read response back\n//---------------------------------\nofs_fim_axi_csr_slave #(\n   .ADDR_WIDTH (ADDR_WIDTH),\n   .USE_SLV_READY (1'b1)\n\n   ) csr_slave (\n   .csr_if             (csr_if),\n\n   .csr_write          (csr_write),\n   .csr_waddr          (csr_waddr),\n   .csr_write_type     (csr_write_type),\n   .csr_wdata          (csr_wdata),\n   .csr_wstrb          (csr_wstrb),\n   .csr_slv_wready     (csr_slv_wready),\n   .csr_read           (csr_read),\n   .csr_raddr          (csr_raddr),\n   .csr_read_32b       (csr_read_32b),\n   .csr_readdata       (csr_readdata),\n   .csr_readdata_valid (csr_readdata_valid)\n);\n\n// Address mapping\nassign csr_addr             = csr_write ? csr_waddr : csr_raddr;\nassign com_csr_address      = csr_addr[5:0];  // byte address\nassign csr_slv_wready       = 1'b1 ;\n// Write data mapping\nassign com_csr_writedata    = csr_wdata;\n\n// Read-Write mapping\nalways_comb\nbegin\n   com_csr_read             = 1'b0;\n   com_csr_write            = 1'b0;\n   casez (csr_addr[11:6])\n      6'h00 : begin // Common CSR\n         com_csr_read       = csr_read;\n         com_csr_write      = csr_write;\n      end   \n      default: begin\n         com_csr_read       = 1'b0;\n         com_csr_write      = 1'b0;\n      end\n   endcase\nend\n\n// Read data mapping\nalways_comb begin\n   if (com_csr_readdatavalid) begin\n      csr_readdata       = com_csr_readdata;\n      csr_readdata_valid = 1'b1;\n   end\n   else begin\n      csr_readdata       = '0;\n      csr_readdata_valid = 1'b0;\n   end\nend\n\nhello_fim_com  #(\n   .FEAT_ID          (FEAT_ID),\n   .FEAT_VER         (FEAT_VER),\n   .NEXT_DFH_OFFSET  (NEXT_DFH_OFFSET),\n   .END_OF_LIST      (END_OF_LIST)\n) hello_fim_com_inst (\n   .clk                   (clk                     ),\n   .reset                 (reset                   ),\n   .writedata             (com_csr_writedata       ),\n   .read                  (com_csr_read            ),\n   .write                 (com_csr_write           ),\n   .byteenable            (4'hF                    ),\n   .readdata              (com_csr_readdata        ),\n   .readdatavalid         (com_csr_readdatavalid   ),\n   .address               (com_csr_address         )\n   );\nendmodule\n
                               
                              5.  
                            5.  
                              Create hello_fim_com.sv file.
                               
                              touch $OFS_ROOTDIR/src/hello_fim/hello_fim_com.sv\n
                               
                              Copy the following code to hello_fim_com.sv:
                               
                              module hello_fim_com #(\n  parameter bit [11:0] FEAT_ID = 12'h001,\n  parameter bit [3:0]  FEAT_VER = 4'h1,\n  parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n  parameter bit END_OF_LIST = 1'b0\n)(\ninput clk,\ninput reset,\ninput [63:0] writedata,\ninput read,\ninput write,\ninput [3:0] byteenable,\noutput reg [63:0] readdata,\noutput reg readdatavalid,\ninput [5:0] address\n);\n\nwire reset_n = !reset;  \nreg [63:0] rdata_comb;\nreg [63:0] scratch_reg;\n\nalways @(negedge reset_n ,posedge clk)  \n   if (!reset_n) readdata[63:0] <= 64'h0; else readdata[63:0] <= rdata_comb[63:0];\n\nalways @(negedge reset_n , posedge clk)\n   if (!reset_n) readdatavalid <= 1'b0; else readdatavalid <= read;\n\nwire wr = write;\nwire re = read;\nwire [5:0] addr = address[5:0];\nwire [63:0] din  = writedata [63:0];\nwire wr_scratch_reg = wr & (addr[5:0]  == 6'h30)? byteenable[0]:1'b0;\n\n// 64 bit scratch register\nalways @( negedge  reset_n,  posedge clk)\n   if (!reset_n)  begin\n      scratch_reg <= 64'h0;\n   end\n   else begin\n   if (wr_scratch_reg) begin \n      scratch_reg <=  din;  \n   end\nend\n\nalways @ (*)\nbegin\nrdata_comb = 64'h0000000000000000;\n   if(re) begin\n      case (addr)  \n        6'h00 : begin\n                rdata_comb [11:0]   = FEAT_ID ;  // dfh_feature_id  is reserved or a constant value, a read access gives the reset value\n                rdata_comb [15:12]  = FEAT_VER ;  // dfh_feature_rev    is reserved or a constant value, a read access gives the reset value\n                rdata_comb [39:16]  = NEXT_DFH_OFFSET ;  // dfh_dfh_ofst is reserved or a constant value, a read access gives the reset value\n                rdata_comb [40]     = END_OF_LIST ;        //dfh_end_of_list\n                rdata_comb [59:40]  = 20'h00000 ;  // dfh_rsvd1     is reserved or a constant value, a read access gives the reset value\n                rdata_comb [63:60]  = 4'h3 ;  // dfh_feat_type  is reserved or a constant value, a read access gives the reset value\n        end\n        6'h30 : begin\n                rdata_comb [63:0]   = scratch_reg; \n        end\n        6'h38 : begin\n                rdata_comb [63:0]       = 64'h6626_0701_5000_0034;\n        end\n        default : begin\n                rdata_comb = 64'h0000000000000000;\n        end\n      endcase\n   end\nend\n\nendmodule\n
                               
                              6.  
                            6.  
                              Create hello_fim_design_files.tcl file.
                               
                              touch $OFS_ROOTDIR/src/hello_fim/hello_fim_design_files.tcl\n
                               
                              Copy the following code into hello_fim_design_files.tcl
                               
                              # Copyright 2023 Intel Corporation.\n#\n# THIS SOFTWARE MAY CONTAIN PREPRODUCTION CODE AND IS PROVIDED BY THE\n# COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED\n# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE\n# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR\n# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\n# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR\n# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,\n# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE\n# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,\n# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# Hello FIM Files\n#--------------------\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_com.sv\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_top.sv\n
                               
                              7.  
                            7.  
                              Modify $OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_top.qsf to include Hello FIM module
                               
                              ######################################################\n# Verilog Macros\n######################################################\n.....\nset_global_assignment -name VERILOG_MACRO \"INCLUDE_HELLO_FIM\"     # Includes Hello FIM\n
                               
                              8.  
                            8.  
                              Modify $OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_top_sources.tcl to include Hello FIM design files
                               
                              ############################################\n# Design Files\n############################################\n...\n# Subsystems\n...\nset_global_assignment -name SOURCE_TCL_SCRIPT_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_design_files.tcl\n
                               
                              9.  
                            9.  
                              Modify $OFS_ROOTDIR/src/pd_qsys/fabric/fabric_design_files.tcl to include BPF Hello FIM Slave IP.
                               
                              #--------------------\n# BPF\n#--------------------\n...\nset_global_assignment -name IP_FILE   $::env(BUILD_ROOT_REL)/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip\n
                               
                              10.  
                            10.  
                              Modify $OFS_ROOTDIR/src/includes/fabric_width_pkg.sv to add Hello FIM slave information and update EMIF slave next offset.
                               
                              localparam bpf_hello_fim_slv_baseaddress = 'h16000;    // New\nlocalparam bpf_hello_fim_slv_address_width = 12;       // New\nlocalparam bpf_emif_slv_next_dfh_offset = 'h1000;      // Old value: 'hB000\nlocalparam bpf_hello_fim_slv_next_dfh_offset = 'hA000; // New\nlocalparam bpf_hello_fim_slv_eol = 'b0;                // New\n
                               
                              11.  
                            11.  
                              Modify $OFS_ROOTDIR/src/top/top.sv
                               
                                 
                              1.  
                                Add bpf_hello_fim_slv_if to AXI interfaces
                                 
                                // AXI4-lite interfaces\n...\nofs_fim_axi_lite_if #(.AWADDR_WIDTH(fabric_width_pkg::bpf_hello_fim_slv_address_width), .ARADDR_WIDTH(fabric_width_pkg::bpf_hello_fim_slv_address_width)) bpf_hello_fim_slv_if();\n
                                 
                                2.  
                              2.  
                                Add Hello FIM instantiation
                                 
                                //*******************************\n// Hello FIM Subsystem\n//*******************************\n\n`ifdef INCLUDE_HELLO_FIM\nhello_fim_top #(\n   .ADDR_WIDTH       (fabric_width_pkg::bpf_hello_fim_slv_address_width),\n   .DATA_WIDTH       (64),\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset),\n   .END_OF_LIST      (fabric_width_pkg::bpf_hello_fim_slv_eol)\n) hello_fim_top_inst (\n    .clk (clk_csr),\n    .reset(~rst_n_csr),\n    .csr_lite_if    (bpf_hello_fim_slv_if)\n);\n`else\ndummy_csr #(\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset),\n   .END_OF_LIST      (fabric_width_pkg::bpf_hello_fim_slv_eol)\n) hello_fim_dummy (\n   .clk         (clk_csr),\n   .rst_n       (rst_n_csr),\n   .csr_lite_if (bpf_hello_fim_slv_if)\n);\n\n`endif\n
                                 
                                3.  
                              3.  
                                Add interfaces for Hello FIM slv to bpf instantiation
                                 
                                bpf bpf (\n...\n  .bpf_hello_fim_slv_awaddr   (bpf_hello_fim_slv_if.awaddr  ),\n  .bpf_hello_fim_slv_awprot   (bpf_hello_fim_slv_if.awprot  ),\n  .bpf_hello_fim_slv_awvalid  (bpf_hello_fim_slv_if.awvalid ),\n  .bpf_hello_fim_slv_awready  (bpf_hello_fim_slv_if.awready ),\n  .bpf_hello_fim_slv_wdata    (bpf_hello_fim_slv_if.wdata   ),\n  .bpf_hello_fim_slv_wstrb    (bpf_hello_fim_slv_if.wstrb   ),\n  .bpf_hello_fim_slv_wvalid   (bpf_hello_fim_slv_if.wvalid  ),\n  .bpf_hello_fim_slv_wready   (bpf_hello_fim_slv_if.wready  ),\n  .bpf_hello_fim_slv_bresp    (bpf_hello_fim_slv_if.bresp   ),\n  .bpf_hello_fim_slv_bvalid   (bpf_hello_fim_slv_if.bvalid  ),\n  .bpf_hello_fim_slv_bready   (bpf_hello_fim_slv_if.bready  ),\n  .bpf_hello_fim_slv_araddr   (bpf_hello_fim_slv_if.araddr  ),\n  .bpf_hello_fim_slv_arprot   (bpf_hello_fim_slv_if.arprot  ),\n  .bpf_hello_fim_slv_arvalid  (bpf_hello_fim_slv_if.arvalid ),\n  .bpf_hello_fim_slv_arready  (bpf_hello_fim_slv_if.arready ),\n  .bpf_hello_fim_slv_rdata    (bpf_hello_fim_slv_if.rdata   ),\n  .bpf_hello_fim_slv_rresp    (bpf_hello_fim_slv_if.rresp   ),\n  .bpf_hello_fim_slv_rvalid   (bpf_hello_fim_slv_if.rvalid  ),\n  .bpf_hello_fim_slv_rready   (bpf_hello_fim_slv_if.rready  ),\n...\n);\n
                                 
                                4.  
                               
                              12.  
                            12.  
                              Modify $OFS_ROOTDIR/src/pd_qsys/fabric/bpf.txt to add the hello_fim module as a slave to the apf.
                               
                              # NAME   FABRIC      BASEADDRESS    ADDRESS_WIDTH SLAVES\napf         mst     n/a             18            fme,pcie,pmci,qsfp0,qsfp1,emif,hssi,hello_fim\n...\nhello_fim   slv     0x16000         12            n/a\n
                               
                              13.  
                            13.  
                              Execute helper script to generate BPF design files
                               
                              cd $OFS_ROOTDIR/src/pd_qsys/fabric/\n\nsh gen_fabrics.sh\n
                               
                              14.  
                            14.  
                              Once the script completes, the following new IP is created: $OFS_ROOTDIR/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip.
                               
                              15.  
                            15.  
                              [OPTIONAL] You may verify the BPF changes have been made correctly by opening bpf.qsys to analyze the BPF.
                               
                              cd $OFS_ROOTDIR/src/pd_qsys/fabric\n\nqsys-edit bpf.qsys --quartus-project=$OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_top.qpf\n
                               
                              Find the bpf_hello_fim_slv instance:
                               
                               
                              16.  
                            16.  
                              Compile the Hello FIM design
                               
                              cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_hello_fim\n
                               
                              17.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#413-walkthrough-modify-and-run-unit-tests-for-a-fim-that-has-a-new-module","title":"4.1.3 Walkthrough: Modify and run unit tests for a FIM that has a new module","text":"
                            Perform the following steps to modify the unit test files to support a FIM that has had a new module added to it.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                               
                            • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design in order to simulate.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Modify $OFS_ROOTDIR/sim/unit_test/dfh_walker/test_csr_defs.sv
                               
                                 
                              1.  
                                Add HELLO_FIM_IDX entry to t_dfh_idx enumeration.
                                 
                                ...\ntypedef enum {\n    FME_DFH_IDX,\n    THERM_MNGM_DFH_IDX,\n    GLBL_PERF_DFH_IDX,\n    GLBL_ERROR_DFH_IDX,\n    QSFP0_DFH_IDX,\n    QSFP1_DFH_IDX,\n    HSSI_DFH_IDX,\n    EMIF_DFH_IDX,\n    HELLO_FIM_DFH_IDX,  // New\n    PMCI_DFH_IDX,\n    ST2MM_DFH_IDX,\n    VUART_DFH_IDX,\n    PG_PR_DFH_IDX,\n    PG_PORT_DFH_IDX,\n    PG_USER_CLK_DFH_IDX,\n    PG_REMOTE_STP_DFH_IDX,\n    AFU_ERR_DFH_IDX,\n    MAX_DFH_IDX\n} t_dfh_idx;\n...\n
                                 
                                2.  
                              2.  
                                Add HELLO_FIM_DFH to get_dfh_names function.
                                 
                                ...\nfunction automatic dfh_name[MAX_DFH_IDX-1:0] get_dfh_names();\n...\n  dfh_names[PMCI_DFH_IDX]        = \"PMCI_DFH\";\n  dfh_names[HELLO_FIM_DFH_IDX]   = \"HELLO_FIM_DFH\";  // New\n  dfh_names[ST2MM_DFH_IDX]       = \"ST2MM_DFH\";\n...\nreturn dfh_names;\n...\n
                                 
                                3.  
                              3.  
                                Add expected DFH value for Hello FIM to the get_dfh_values function.
                                 
                                ...\nfunction automatic [MAX_DFH_IDX-1:0][63:0] get_dfh_values();\n...\n  dfh_values[PMCI_DFH_IDX]       = 64'h3_00000_xxxxxx_1012;\n  dfh_values[PMCI_DFH_IDX][39:16] = fabric_width_pkg::bpf_pmci_slv_next_dfh_offset;\n\n  dfh_values[HELLO_FIM_DFH_IDX]  = 64'h3_00000_xxxxxx_0100;  // New\n  dfh_values[HELLO_FIM_DFH_IDX][39:16] = fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset; // New\n\n  dfh_values[ST2MM_DFH_IDX]      = 64'h3_00000_xxxxxx_0014;\n  dfh_values[ST2MM_DFH_IDX][39:16] = fabric_width_pkg::apf_st2mm_slv_next_dfh_offset;\n...\nreturn dfh_values;\n...\n
                                 
                                4.  
                               
                              3.  
                            3.  
                              Generate simulation files
                               
                              cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk\n
                               
                              4.  
                            4.  
                              Run DFH Walker Simulation   
                              cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\nsh run_sim.sh TEST=dfh_walker\n
                               
                              5.  
                            5.  
                              Verify that the test passes, and that the output shows the Hello FIM in the DFH sequence
                               
                              ********************************************\n Running TEST(0) : test_dfh_walking\n********************************************\n...\n\nREAD64: address=0x00015000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x3000000010001009\n\nEMIF_DFH\n   Address   (0x15000)\n   DFH value (0x3000000010001009)\n\nREAD64: address=0x00016000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x30000000a0000100\n\nHELLO_FIM_DFH\n   Address   (0x16000)\n   DFH value (0x30000000a0000100)\n\nREAD64: address=0x00020000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x3000000200001012\n\nPMCI_DFH\n   Address   (0x20000)\n   DFH value (0x3000000200001012)\n\n...\n\nTest status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/ofs-agx7-pcie-attach/sim/unit_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time         356791250000\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 356791250000 fs\nCPU Time:     61.560 seconds;       Data structure size:  47.4Mb\nTue Aug 15 16:29:45 2023\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                               
                              6.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#414-walkthrough-hardware-test-a-fim-that-has-a-new-module","title":"4.1.4 Walkthrough: Hardware test a FIM that has a new module","text":"
                            Perform the following steps to program and hardware test a FIM that has had a new module added to it.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                              •  
                             
                               
                            • This walkthrough uses a FIM design that has been generated with a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for generating a Hello FIM design.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              [OPTIONAL] In the work directory where the FIM was compiled, determine the PR Interface ID of your design. You can use this value at the end of the walkthrough to verify that the design has been configured to the FPGA.
                               
                              cd $OFS_ROOTDIR/<work_directory>/syn/board/fseries-dk/syn_top/\n\ncat fme-ifc-id.txt\n
                               
                              Example output:
                               
                              5bcd682f-5093-5fc7-8cd2-ae8073e19452\n
                               
                              2.  
                            2.  
                              Switch to your deployment environment.
                               
                              3.  
                            3.  
                              Program the FPGA with the Hello FIM image. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                               
                              4.  
                            4.  
                              Run fpgainfo to determine the PCIe B:D.F of your board, and to verify the PR Interface ID matches the ID you found in Step 1.
                               
                              fpgainfo fme\n
                               
                              Example output:
                               
                              Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 360571655976424377 \nBitstream Version                : 5.0.1\nPr Interface Id                  : 5bcd682f-5093-5fc7-8cd2-ae8073e19452\nBoot Page                        : N/A\n
                               
                              Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                               
                              5.  
                            5.  
                              Initialize opae.io
                               
                              sudo opae.io init -d <B:D.F>\n
                               
                              For example:
                               
                              sudo opae.io init -d b1:00.0\n
                               
                              6.  
                            6.  
                              Run DFH walker. Note the value read back from offset 0x16000 indicates the DFH ID is 0x100 which matches the Hello FIM module.
                               
                              sudo opae.io walk -d <B:D.F>\n
                               
                              For example:
                               
                              sudo opae.io walk -d b1:00.0\n
                               
                              Example output:
                               
                              ...\noffset: 0x15000, value: 0x3000000010001009\n    dfh: id = 0x9, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x16000, value: 0x30000000a0000100\n    dfh: id = 0x100, rev = 0x0, next = 0xa000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x20000, value: 0x3000000200001012\n    dfh: id = 0x12, rev = 0x1, next = 0x20000, eol = 0x0, reserved = 0x0, feature_type = 0x3\n...\n
                               
                              7.  
                            7.  
                              Read all of the registers in the Hello FIM module
                               
                                 
                              1.  
                                Read the DFH Register
                                 
                                opae.io -d b1:00.0 -r 0 peek 0x16000\n
                                 
                                Example Output:
                                 
                                0x30000006a0000100\n
                                 
                                2.  
                              2.  
                                Read the Scratchpad Register
                                 
                                opae.io -d b1:00.0 -r 0 peek 0x16030\n
                                 
                                Example Output:
                                 
                                0x0\n
                                 
                                3.  
                              3.  
                                Read the ID Register
                                 
                                opae.io -d b1:00.0 -r 0 peek 0x16038\n
                                 
                                Example Output:
                                 
                                0x6626070150000034\n
                                 
                                4.  
                               
                              8.  
                            8.  
                              Verify the scratchpad register at 0x16030 by writing and reading back from it.
                               
                                 
                              1.  
                                Write to Scratchpad register
                                 
                                opae.io -d b1:00.0 -r 0 poke 0x16030 0x123456789abcdef\n
                                 
                                2.  
                              2.  
                                Read from Scratchpad register
                                 
                                opae.io -d b1:00.0 -r 0 peek 0x16030\n
                                 
                                Expected output:
                                 
                                0x123456789abcdef\n
                                 
                                3.  
                              3.  
                                Write to Scratchpad register
                                 
                                opae.io -d b1:00.0 -r 0 poke 0x16030 0xfedcba9876543210\n
                                 
                                4.  
                              4.  
                                Read from Scratchpad register
                                 
                                opae.io -d b1:00.0 -r 0 peek 0x16030\n
                                 
                                Expected output:
                                 
                                0xfedcba9876543210\n
                                 
                                5.  
                               
                              9.  
                            9.  
                              Release the opae.io tool
                               
                              opae.io release -d b1:00.0\n
                               
                              10.  
                            10.  
                              Confirm the driver has been set back to dfl-pci
                               
                              opae.io ls\n
                               
                              Example output:
                               
                              [0000:b1:00.0] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                               
                              11.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#415-walkthrough-debug-the-fim-with-signal-tap","title":"4.1.5 Walkthrough: Debug the FIM with Signal Tap","text":"
                            The following steps guide you through the process of adding a Signal Tap instance to your design. The added Signal Tap instance provides hardware to capture the desired internal signals and connect the stored trace information via JTAG. Please be aware that the added Signal Tap hardware will consume FPGA resources and may require additional floorplanning steps to accommodate these resources. Some areas of the FIM use logic lock regions and these regions may need to be re-sized.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                               
                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                              •  
                             
                               
                            • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design.
                              •  
                             
                            Perform the following steps in your development environment:
                             
                               
                            1.  
                              Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Synthesize the design using the -e build script option. You may skip this step if you are using a pre-synthesized design.
                               
                              cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -e --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_hello_fim_with_stp\n
                               
                              4.  
                            4.  
                              Open the design in Quartus. The Compilation Dashboard should show that the Analysis & Synthesis step has completed.
                               
                              quartus $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/fseries-dk/syn_top/ofs_top.qpf\n
                               
                               
                              5.  
                            5.  
                              Open Tools -> Signal Tap Logic Analyzer
                               
                               
                                 
                              1.  
                                Select the Default template and click Create
                                 
                                2.  
                              2.  
                                Assign the clock for sampling the Signal Tap instrumented signals of interest. Note that the clock selected should correspond to the signals you want to view for best trace fidelity. Different clocks can be used, however, there maybe issues with trace inaccuracy due to sampling time differences. This example instruments the hello_fim_top module previously intetegrated into the FIM. If unfamiliar with code, it is helpful to use the Quartus Project Navigator to find the block of interest and open the design instance for review.
                                 
                                   
                                1.  
                                  In the Signal Configuration -> Clock box of the Signal Tap Logic Analyzer window, click the \"...\" button
                                   
                                   
                                  2.  
                                2.  
                                  In the Node Finder tool that opens, type hello_fim_top_inst|clock into the Named field, then click Search. Select the clk signal from the Matching Nodes box and click the \">\" button to move it to the Nodes Found box. Click OK to close the Node Finder dialog.
                                   
                                   
                                  3.  
                                 
                                3.  
                              3.  
                                Update the sample depth and other Signal Tap settings as needed for your debugging criteria.
                                 
                                 
                                4.  
                              4.  
                                In the Signal Tap GUI add the nodes to be instrumented by double-clicking on the \"Double-click to add nodes\" legend.
                                 
                                 
                                5.  
                              5.  
                                This brings up the Node Finder to add the signals to be traced. In this example we will monitor the memory mapped interface to the Hello FIM. Select the signals that appear from the  search patterns hello_fim_top_inst|reset and hello_fim_top_inst|csr_lite_if\\*. Click Insert and close the Node Finder dialog.
                                 
                                 
                                6.  
                              6.  
                                To provide a unique name for your Signal Tap instance, select \"auto_signaltap_0\", right-click, and select Rename Instance (F2). Provide a descriptive name for your instance, for example, stp_for_hello_fim.
                                 
                                 
                                7.  
                              7.  
                                Save the newly created Signal Tap file, in the $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/fseries-dk/syn_top/ directory, and give it the same name as the instance. Ensure that the Add file to current project checkbox is ticked.
                                 
                                 
                                8.  
                              8.  
                                In the dialog that pops up, click \"Yes\" to add the newly created Signal Tap file to the project settings files.
                                 
                                 
                                This will aurtomatically add the following lines to $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/fseries-dk/syn_top/ofs_top.qsf:
                                 
                                set_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE stp_for_hello_fim.stp\nset_global_assignment -name SIGNALTAP_FILE stp_for_hello_fim.stp\n
                                 
                                9.  
                               
                              6.  
                            6.  
                              Close all Quartus GUIs.
                               
                              7.  
                            7.  
                              Compile the project with the Signal Tap file added to the project. Use the -k switch to perform the compilation using the files in a specified working directory and not the original ones from the cloned repository. 
                               
                              ./ofs-common/scripts/common/syn/build_top.sh -p -k --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_hello_fim_with_stp\n
                               
                              Alternatively, you can copy the ofs_top.qsf and stp_for_hello_fim.stp files from the Hello FIM with STP work directory to replace the original files in the cloned OFS repository. In this scenario, all further FIM compilation projects will include the Signal Tap instance integrated into the design. Execute the following commands for this alternative flow:
                               
                              Copy the modified file work_hello_fim_with_stp/syn/board/fseries-dk/syn_top/ofs_top.qsf to the source OFS repository, into syn/board/fseries-dk/syn_top/.
                               
                              cd $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/fseries-dk/syn_top\n\ncp ofs_top.qsf $OFS_ROOTDIR/syn/board/fseries-dk/syn_top\n\ncp stp_for_hello_fim.stp $OFS_ROOTDIR/syn/board/fseries-dk/syn_top\n
                               
                              Compile the FIM to create a new work directory.
                               
                              cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_hello_fim_with_stp_src_repo\n
                               
                              8.  
                            8.  
                              Ensure that the compile completes successfully and meets timing:
                               
                              ***********************************\n***\n***        OFS_PROJECT: n6001\n***        OFS_BOARD: fseries-dk\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 6\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                               
                              9.  
                            9.  
                              Set up a JTAG connection to the fseries-dk. Refer to the Set up JTAG section for step-by-step instructions.
                               
                              10.  
                            10.  
                              Copy the ofs_top.sof and stp_for_hello_fim.stp files to the machine which is connected to the fseries-dk via JTAG.
                               
                              11.  
                            11.  
                              From the JTAG connected machine, program the $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/fseries-dk/syn_top/output_files/ofs_top.sof image to the fseries-dk FPGA. Refer to the Program the FPGA via JTAG section for step-by-step programming instructions.
                               
                              12.  
                            12.  
                              Open the Quartus Signal Tap GUI
                               
                              $QUARTUS_ROOTDIR/bin/quartus_stpw\n
                               
                              13.  
                            13.  
                              In the Signal Tap Logic Analyzer window, select File -> Open, and choose the stp_for_hello_fim.stp file.
                               
                               
                              14.  
                            14.  
                              In the right pane of the Signal Tap GUI, in the Hardware: selection box select the cable for the fseries-dk. In the Device: selection box select the Agilex device.
                               
                               
                              15.  
                            15.  
                              If the Agilex Device is not displayed in the Device: list, click the 'Scan Chain' button to re-scan the JTAG device chain.
                               
                              16.  
                            16.  
                              Create the trigger conditions. In this example, we will capture data on a rising edge of the Read Address Valid signal.
                               
                               
                              17.  
                            17.  
                              Start analysis by selecting the 'stp_for_hello_fim' instance and pressing 'F5' or clicking the Run Analysis icon in the toolbar. You should see a green message indicating the Acquisition is in progress. Then, move to the Data Tab to observe the signals captured.
                               
                               
                              18.  
                            18.  
                              To generate traffic in the csr_lite_if signals of the Hello FIM module, walk the DFH list or peek/poke the Hello FIM registers.
                               
                              opae.io init -d 0000:b1:00.0\nopae.io walk -d 0000:b1:00.0\nopae.io release -d 0000:b1:00.0\n
                               
                              The signals should be captured on the rising edge of arvalid in this example. Zoom in to get a better view of the signals.
                               
                               
                              19.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#42-preparing-fim-for-afu-development","title":"4.2 Preparing FIM for AFU Development","text":"
                            To save area, the default Host Excercisers in the FIM can be replaced by a \"he_null\" blocks. There are a few things to note:
                             
                               
                            • \"he_null\" is a minimal block with registers that responds to PCIe MMIO request. MMIO responses are required to keep PCIe alive (end points enabled in PCIe-SS needs service downstream requests).
                              •  
                            • If an exerciser with other I/O connections such as \"he_mem\" or \"he_hssi\" is replaced, then then those I/O ports are simply tied off.
                              •  
                            • The options supported are null_he_lb, null_he_hssi, null_he_mem and null_he_mem_tg. Any combination, order or all can be enabled at the same time. 
                              •  
                            • Finer grain control is provided for you to, for example, turn off only the exercisers in the Static Region in order to save area.
                              •  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#421-walkthrough-compile-the-fim-in-preparation-for-designing-your-afu","title":"4.2.1 Walkthrough: Compile the FIM in preparation for designing your AFU","text":"
                            Perform the following steps to compile a FIM for where the exercisers are removed and replaced with an he_null module while keeping the PF/VF multiplexor connections.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the [Walkthrough: Set Up Development Environment] Section for instructions on setting up a development environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the FIM repository (or use an existing cloned repository). Refer to the [Walkthrough: Clone FIM Repository] section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the [Walkthrough: Set Development Environment Variables] section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Compile the FIM with the HE_NULL compile options
                               
                              cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss fseries-dk:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_fseries-dk\n
                               
                              4.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#43-partial-reconfiguration-region","title":"4.3 Partial Reconfiguration Region","text":"
                            To take advantage of the available resources in the Intel\u00ae Agilex\u00ae 7 FPGA for an AFU design, you can adjust the size of the AFU PR partition. An example reason for the changing the size of PR region is if you add more logic to the FIM region, then you may need to adjust the PR region to fit the additional logic into the static region.  Similarly, if you reduce logic in the FIM region, then you can adjust the PR region to provide more logic resources for the AFU.
                             
                            After the compilation of the FIM, the resources usage broken down by partitions as reported in the following file:
                             
                            $OFS_ROOTDIR/<WORDK_DIRECTORY>/syn/board/fseries-dk/syn_top/output_files/ofs_top.fit.rpt\n
                             
                            An example report of the resources usage by partitions defined for the FIM is shown as follows:
                             
                            +------------------------------------------------------------------------------------------+\n; Logic Lock Region Constraints                                                            ;\n+--------------------------------------+-------------------------+-------------------------+\n; Name                                 ; Place Region Constraint ; Route Region Constraint ;\n+--------------------------------------+-------------------------+-------------------------+\n; afu_top|port_gasket|pr_slot|afu_main ; (90, 40) to (350, 220)  ; (0, 0) to (385, 329)    ;\n+--------------------------------------+-------------------------+-------------------------+\n\n\n+----------------------------------------------------------------------------------------------+\n; Logic Lock Region Usage Summary                                                              ;\n+-------------------------------------------------------+--------------------------------------+\n; Statistic                                             ; afu_top|port_gasket|pr_slot|afu_main ;\n+-------------------------------------------------------+--------------------------------------+\n; ALMs needed [=A-B+C]                                  ; 48011.2 / 351140 ( 13 % )            ;\n;     [A] ALMs used in final placement                  ; 53324.4 / 351140 ( 15 % )            ;\n;     [B] Estimate of ALMs recoverable by dense packing ; 5452.3 / 351140 ( 1 % )              ;\n;     [C] Estimate of ALMs unavailable                  ; 139.0 / 351140 ( < 1 % )             ;\n; ALMs used for memory                                  ; 450.0                                ;\n; Combinational ALUTs                                   ; 67166                                ;\n; Dedicated Logic Registers                             ; 87533 / 1404560 ( 6 % )              ;\n; I/O Registers                                         ; 0                                    ;\n; Block Memory Bits                                     ; 1737568                              ;\n; M20Ks                                                 ; 137 / 5049 ( 2 % )                   ;\n; DSP Blocks needed [=A-B]                              ; 0 / 3439 ( 0 % )                     ;\n;     [A] DSP Blocks used in final placement            ; 0 / 3439 ( 0 % )                     ;\n;     [B] Estimate of DSPs recoverable by dense merging ; 0 / 3439 ( 0 % )                     ;\n; Pins                                                  ; 0                                    ;\n; IOPLLs                                                ; 0                                    ;\n;                                                       ;                                      ;\n; Region Placement                                      ; (90, 40) to (350, 220)               ;\n+-------------------------------------------------------+--------------------------------------+\n
                             
                            In this case, the default size for the afu_top|port_gasket|pr_slot|afu_main PR partition is large enough to hold the logic of the default AFU, which is mainly occupied by the Host Exercisers. However, larger designs might require additional resources.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#431-walkthrough-resize-the-partial-reconfiguration-region","title":"4.3.1 Walkthrough: Resize the Partial Reconfiguration Region","text":"
                            Perform the following steps to first analyze the PR logic lock regions in a default FIM design, then resize the PR region:
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              The OFS flow provides the TCL file $OFS_ROOTDIR/syn/board/fseries-dk/setup/pr_assignments.tcl which defines the PR partition where the AFU is allocated. Each region is a rectangle defined by the origin coordinate (X0, Y0) and the top right corner coordinate (X1, Y1).
                               
                              #####################################################\n# Main PR Partition -- green_region\n#####################################################\nset_instance_assignment -name PARTITION green_region -to afu_top|port_gasket|pr_slot|afu_main\nset_instance_assignment -name CORE_ONLY_PLACE_REGION ON -to afu_top|port_gasket|pr_slot|afu_main\nset_instance_assignment -name RESERVE_PLACE_REGION ON -to afu_top|port_gasket|pr_slot|afu_main\nset_instance_assignment -name PARTIAL_RECONFIGURATION_PARTITION ON -to afu_top|port_gasket|pr_slot|afu_main\n\n\nset_instance_assignment -name PLACE_REGION \"X90 Y40 X350 Y220\" -to afu_top|port_gasket|pr_slot|afu_main\nset_instance_assignment -name ROUTE_REGION \"X0 Y0 X385 Y329\" -to afu_top|port_gasket|pr_slot|afu_main\n
                               
                              4.  
                            4.  
                              [OPTIONAL] Use Quartus Chip Planner to visualize the default PR region allocation.
                               
                                 
                              1.  
                                Run the setup stage of the build script to create a work directory.
                                 
                                cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_resize_pr\n
                                 
                                2.  
                              2.  
                                Open the project in the Quartus GUI
                                 
                                quartus $OFS_ROOTDIR/work_fseries-dk_resize_pr/syn/board/fseries-dk/syn_top/ofs_top.qpf\n
                                 
                                3.  
                              3.  
                                In the Compilation Dashboard, click the arrow next to Compile Design to start the compilation. 
                                 
                                4.  
                              4.  
                                Once compilation is complete, click Tools -> Chip Planner to open the Chip Planner.
                                 
                                5.  
                              5.  
                                Analyze the regions shown. Note that the regions are made of rectangles described by an origin coordinate, region height, and region width. If you are modifying the regions, you will need to identify the coordinates of your desired region.
                                 
                                 
                                6.  
                              6.  
                                Close the Quartus GUI.
                                 
                                7.  
                               
                              5.  
                            5.  
                              Make changes to the partial reconfiguraton region in the $OFS_ROOTDIR/syn/board/fseries-dk/setup/pr_assignments.tcl file. You can modify the size and location of existing lock regions or add new ones and assign them to the AFU PR partition.
                               
                              6.  
                            6.  
                              Recompile your FIM and create the PR relocatable build tree using the following commands.
                               
                              cd $OFS_ROOTDIR    \n\nofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_resize_pr\n
                               
                              7.  
                            7.  
                              Analyze the resource utilization report $OFS_ROOTDIR/work_fseries-dk/syn/board/fseries-dk/syn_top/output_files/ofs_top.fit.rpt produced after recompiling the FIM.
                               
                              8.  
                            8.  
                              Perform further modification to the PR regions until the results are satisfactory. Make sure timing constraints are met.
                               
                              9.  
                             
                            For more information on how to optimize the floor plan of your Partial Reconfiguration design refer to the following online documentation.
                             
                               
                            • Analyzing and Optimizing the Design Floorplan
                              •  
                             
                               
                            • Partial Reconfiguration Design Flow - Step 3 - Floorplan the Design
                              •  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#44-pcie-configuration","title":"4.4 PCIe Configuration","text":"
                            The PCIe sub-system IP and PF/VF MUX can be modified either using the OFSS flow or the IP Presets flow. The OFSS flow supports a subset of all available PCIe Sub-system settings, while the IP Preset flow can make any available PCIe Sub-system settings change. With PCIe-SS modifcations related to the PFs and VFs, the PF/VF MUX logic is automatically configured based on the PCIe-SS configuration. The sections below describe each flow.
                             
                               
                            • PCIe Configuration Using OFSS
                              •  
                            • PCIe Configuration Using IP Presets
                              •  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#441-pfvf-mux-configuration","title":"4.4.1 PF/VF MUX Configuration","text":"
                            The default PF/VF MUX configuration for OFS PCIe Attach FIM for the fseries-dk can support up to 8 PFs and 2000 VFs distributed accross all PFs.
                             
                            For reference FIM configurations, 0 VFs on PF0 is not supported. This is because the PR region cannot be left unconnected. A NULL AFU may need to be instantiated in this special case. PFs must be consecutive. the PF/VF Limitations table describes the supported number of PFs and VFs.
                             
                            Table: PF/VF Limitations
                             Parameter Value Min # of PFs 1 (on PF0) Max # of PFs 8 Min # of VFs 1 on PF0 Max # of VFs 2000 distributed across all PFs 
                            New PF or VF instances will automatically have a null_afu module instantiated and connected to the new PF or VF.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#442-pcie-ss-configuration-registers","title":"4.4.2 PCIe-SS Configuration Registers","text":"
                            The PCIe-SS configuration registers contain the Vendor, Device and Subsystem Vendor ID registers which are used in PCIe add-in cards to uniquely identify the card for assignment to software drivers. OFS has these registers set with Intel values for out of the box usage. If you are using OFS for a PCIe add in card that your company manufactures, then update the PCIe Subsytem Subsystem ID and Vendor ID registers as described below and change OPAE provided software code to properly operate with your company's register values.
                             
                            The Vendor ID is assigned to your company by the PCI-SIG (Special Interest Group). The PCI-SIG is the only body officially licensed to give out IDs. You must be a member of PCI-SIG to request your own ID. Information about joining PCI-SIG is available here: PCI-SIG. You select the Subsystem Device ID for your PCIe add in card.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#443-pcie-configuration-using-ofss","title":"4.4.3 PCIe Configuration Using OFSS","text":"
                            The general flow for using OFSS to modify the PCIe Sub-system and PF/VF MUX is as follows:
                             
                               
                            1. Create or modify a PCIe OFSS file with the desired PCIe configuration. 
                              2.  
                            2. Call this PCIe OFSS file when running the FIM build script.
                              3.  
                             
                            The PCIe IP OFSS File Options table lists all of the configuration options supported by the OFSS flow. Any other customizations to the PCIe sub-system must be done with the IP Presets Flow.
                             
                            Table: PCIe IP OFSS File Options
                             Section Parameter Options Default Description [ip] type pcie N/A Specifies that this OFSS file configures the PCIe-SS [settings] output_name pcie_ss N/A Specifies the output name of the PCIe-SS IP preset String N/A OPTIONAL - Specifies the name of a PCIe-SS IP presets file to use when building the FIM. When used, a presets file will take priority over any other parameters set in this OFSS file. [pf*] num_vfs Integer 0 Specifies the number of Virtual Functions in the current PF bar0_address_width Integer 12 bar4_address_width Integer 14 vf_bar0_address_width Integer 12 ats_cap_enable 0 | 1 0 vf_ats_cap_enable 0 | 1 0 prs_ext_cap_enable 0 | 1 0 pasid_cap_enable 0 | 1 0 pci_type0_vendor_id 32'h Value 0x00008086 pci_type0_device_id 32'h Value 0x0000bcce revision_id 32'h Value 0x00000001 class_code 32'h Value 0x00120000 subsys_vendor_id 32'h Value 0x00008086 subsys_dev_id 32'h Value 0x00001771 sriov_vf_device_id 32'h Value 0x0000bccf exvf_subsysid 32'h Value 0x00001771"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#4431-walkthrough-modify-pcie-configuration-using-ofss","title":"4.4.3.1 Walkthrough: Modify PCIe Configuration Using OFSS","text":"
                            Perform the following steps to use OFSS files to configure the PCIe Sub-system and PF/VF MUX. In this example, we will add a PF with 2 VFs to the default configuration.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment to build the FIM. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                               
                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment to test the design. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              View the default OFS PCIe Attach FIM for the fseries-dk PF/VF configuration in the the $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss file.
                               
                              [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n\n[pf0]\nnum_vfs = 3\nbar0_address_width = 20\nvf_bar0_address_width = 20\n\n[pf1]\n\n[pf2]\nbar0_address_width = 18\n\n[pf3]\n\n[pf4]\n
                               
                              4.  
                            4.  
                              Create a new PCIe OFSS file from the existing pcie_host.ofss file
                               
                              cp $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_pfvf_mod.ofss\n
                               
                              5.  
                            5.  
                              Configure the new pcie_pfvf_mod.ofss with your desired PCIe settings. In this example we will add PF5 with 2 VFs.
                               
                              [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n\n[pf0]\nnum_vfs = 3\nbar0_address_width = 20\nvf_bar0_address_width = 20\n\n[pf1]\n\n[pf2]\nbar0_address_width = 18\n\n[pf3]\n\n[pf4]\n\n[pf5]\nnum_vfs = 2\n
                               
                              6.  
                            6.  
                              Edit the $OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss file to use the new PCIe configuration file pcie_pfvf_mod.ofss
                               
                              [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/fseries-dk_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_pfvf_mod.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory_ftile.ofss\n
                               
                              7.  
                            7.  
                              Compile the FIM. 
                               
                              cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_pfvf_mod\n
                               
                              8.  
                            8.  
                              Copy the resulting $OFS_ROOTDIR/work_fseries-dk_pfvf_mod/syn/board/fseries-dk/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                               
                              9.  
                            9.  
                              Switch to your deployment environment.
                               
                              10.  
                            10.  
                              Program the ofs_top.sof image to the fseries-dk FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                               
                              11.  
                            11.  
                              Verify the number of VFs on the newly added PF5. In this example, we defined 2 VFs on PF5 in Step 5.
                               
                              sudo lspci -vvv -s b1:00.5 | grep VF\n
                               
                              Example output:
                               
                              Initial VFs: 2, Total VFs: 2, Number of VFs: 0, Function Dependency Link: 05\nVF offset: 4, stride: 1, Device ID: bccf\nVF Migration: offset: 00000000, BIR: 0\n
                               
                              12.  
                            12.  
                              Verify communication with the newly added PF5. New PF/VF are seamlessly connected to their own CSR stub, which can be read at DFH Offset 0x0. You can bind to the function and perform opae.io peek commands to read from the stub CSR. Similarly, perform opae.io poke commands to write into the stub CSRs. Use this mechanism to verify that the new PF/VF Mux configuration allows to write and read back values from the stub CSRs. 
                               
                              The GUID for every new PF/VF CSR stub is the same.
                               
                              * NULL_GUID_L           = 64'haa31f54a3e403501\n* NULL_GUID_H           = 64'h3e7b60a0df2d4850\n
                               
                                 
                              1.  
                                Initialize the driver on PF5
                                 
                                sudo opae.io init -d b1:00.5\n
                                 
                                2.  
                              2.  
                                Read the GUID for the PF5 CSR stub.
                                 
                                sudo opae.io -d b1:00.5 -r 0 peek 0x8\nsudo opae.io -d b1:00.5 -r 0 peek 0x10\n
                                 
                                Example output:
                                 
                                0xaa31f54a3e403501\n0x3e7b60a0df2d4850\n
                                 
                                3.  
                               
                              13.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#444-pcie-configuration-using-ip-presets","title":"4.4.4 PCIe Configuration Using IP Presets","text":"
                            The general flow for using IP Presets to modify he PCIe Sub-system is as follows:
                             
                               
                            1. [OPTIONAL] Create a work directory using OFSS files if you wish to use OFSS configuration settings.
                              2.  
                            2. Open the PCIe-SS IP and make desired modifications.
                              3.  
                            3. Create an IP Presets file.
                              4.  
                            4. Create an PCIe OFSS file that uses the IP Presets file.
                              5.  
                            5. Build the FIM with the PCIe OFSS file from Step 4.
                              6.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#4441-walkthrough-modify-pcie-configuration-using-ip-presets","title":"4.4.4.1 Walkthrough: Modify PCIe Configuration Using IP Presets","text":"
                            Perform the following steps to use an IP Preset file to configure the PCIe Sub-system and PF/VF MUX. In this example, we will change the Revision ID of PF0. While this modification can be done with the OFSS flow, this walkthrough is intended to show the procedure for making any PCIe configuration change using IP presets.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment to build the FIM. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                               
                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment to test the design. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Run the setup stage of the build script using your desired OFSS configration to create a working directory for the fseries-dk design.
                               
                              ./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk\n
                               
                              4.  
                            4.  
                              Open the PCIe-SS in the work directory using Quartus Parameter Editor.
                               
                              qsys-edit $OFS_ROOTDIR/work_fseries-dk/ipss/pcie/qip/pcie_ss.ip\n
                               
                              5.  
                            5.  
                              In the IP Parameter Editor window, scroll down and select the PCIe Interfaces Port Settings -> Port 0 -> PCIe0 Device Identification Registers -> PCIe0 PF0 IDs tab. Modify the settings as desired. In this case, we are changing the Revision ID to 0x00000002. You may make any desired modifications.
                               
                               
                              6.  
                            6.  
                              Once you are satisfied with your modifcations, create a new IP Preset file.
                               
                                 
                              1.  
                                click New... in the Presets window.
                                 
                                 
                                2.  
                              2.  
                                In the New Preset window, set a unique Name for the preset; for example, fseries-dk-rev2.
                                 
                                 
                                3.  
                              3.  
                                Click the ... button to set the save location for the IP Preset file to $OFS_ROOTDIR/ipss/pcie/presets. Set the File Name to match the name selected in Step 9. Click OK.
                                 
                                 
                                4.  
                              4.  
                                In the New Preset window, click Save. Click No when prompted to add the file to the IP search path.
                                 
                                 
                                5.  
                               
                              7.  
                            7.  
                              Close IP Parameter Editor without saving or generating HDL.
                               
                              8.  
                            8.  
                              Create a new PCIe OFSS file in the $OFS_ROOTDIR/tools/ofss_config/pcie directory. For example:
                               
                              touch $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host_mod_preset.ofss\n
                               
                              Insert the following into the OFSS file to specify the IP Preset file created in Step 6. 
                               
                              [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\npreset = fseries-dk-rev2\n
                               
                              9.  
                            9.  
                              Edit the $OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss file to call new OFSS file created in the previous step.
                               
                              [include]  \"$OFS_ROOTDIR\"/tools/ofss_config/fseries-dk_base.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host_mod_preset.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory_ftile.ofss
                               
                              10.  
                            10.  
                              Compile the design with the modified fseries-dk.ofss file.
                               
                              cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_pcie_mod\n
                               
                              11.  
                            11.  
                              Copy the resulting $OFS_ROOTDIR/work_fseries-dk_pcie_mod/syn/board/fseries-dk/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                               
                              12.  
                            12.  
                              Switch to your deployment environment.
                               
                              13.  
                            13.  
                              Program the ofs_top.sof image to the fseries-dk FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                               
                              14.  
                            14.  
                              Use lspci to verify that the PCIe changes have been implemented. In this example, the Rev for PF0 is 02.
                               
                              lspci -nvmms 98:00.0\n
                               
                              Example output:
                               
                              Slot:   98:00.0\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nPhySlot:        1-1\nRev:    02\nNUMANode:       1\nIOMMUGroup:     8\n
                               
                              15.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#45-minimal-fim","title":"4.5 Minimal FIM","text":"
                            In a minimal FIM, the exercisers are removed. This minimal FIM is useful for HDL applications.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#451-walkthrough-create-a-minimal-fim","title":"4.5.1 Walkthrough: Create a Minimal FIM","text":"
                            Perform the following steps to create a Minimal FIM.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                               
                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Create a new platform OFSS file that will use a 1PF/1VF PCIe configuration.
                               
                              cp $OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss $OFS_ROOTDIR/tools/ofss_config/fseries-dk_1pf_1vf.ofss\n
                               
                              4.  
                            4.  
                              Edit the $OFS_ROOTDIR/tools/ofss_config/fseries-dk_1pf_1vf.ofss file to use the 1PF/1VF PCIe configuration file pcie_1pf_1vf.ofss
                               
                              [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/fseries-dk_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_1pf_1vf.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory_ftile.ofss\n
                               
                              5.  
                            5.  
                              Compile the FIM with Null HEs compile option, the No HSSI compile option, and 1PF/1VF configuration OFSS file.
                               
                              cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk_1pf_1vf.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg,no_hssi work_fseries-dk_minimal_fim\n
                               
                              6.  
                            6.  
                              Review the $OFS_ROOTDIR/work_fseries-dk_minimal_fim/syn/board/fseries-dk/syn_top/output_files/ofs_top.fit.rpt utilization report to see the utilization statistics for the Minimal fim. Refer to [Appendix A] Table A-4 for the expected utilization for this Minimal FIM.
                               
                              7.  
                            7.  
                              Copy the resulting $OFS_ROOTDIR/work_fseries-dk_minimal_fim/syn/board/fseries-dk/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                               
                              8.  
                            8.  
                              Switch to your deployment environment, if different than your development environment.
                               
                              9.  
                            9.  
                              Program the ofs_top.sof image to the fseries-dk FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                               
                              10.  
                            10.  
                              Use lspci to verify that the PCIe changes have been implemented.
                               
                              sudo lspci -vvv -s b1:00.0 | grep VF\n
                               
                              Example output: 
                              Initial VFs: 1, Total VFs: 1, Number of VFs: 0, Function Dependency Link: 00\nVF offset: 1, stride: 1, Device ID: bcce\nVF Migration: offset: 00000000, BIR: 0\n
                               
                              11.  
                            11.  
                              You may wish to adjust the PR logic lock regions to maximize the resources available for the AFU. Refer to the Resize the Partial Reconfiguration Region section for instructions.
                               
                              12.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#46-migrating-to-a-different-agilex-device-number","title":"4.6 Migrating to a Different Agilex Device Number","text":"
                            The following instructions enable a user to change the device part number of the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile). Please be aware that this release works with Intel\u00ae Agilex\u00ae 7 FPGA devices that have F-tile for PCIe and Ethernet.  Other tiles will take further work.
                             
                            You may wish to change the device part number for the following reasons
                             
                               
                            1. Migrate to same device package but with a different density
                              2.  
                            2. Migrate to a different package and with a different or same density
                              3.  
                             
                            The default device for the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) is AGFB027R24C2E2VR2
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#461-walkthrough-migrate-to-a-different-agilex-device-number","title":"4.6.1 Walkthrough: Migrate to a Different Agilex Device Number","text":"
                            Perform the following steps to migrate to a different Agilex Device. In this example, we will migrate from the default AGFB027R24C2E2VR2 device to AGFB027R31C2E2V. The package will change from R24C to R31C.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Modify the part field in the $OFS_ROOTDIR/tools/ofss_config/fseries-dk_base.ofss file to use AGFB027R31C2E2V.
                               
                              [ip]\ntype = ofs\n\n[settings]\nplatform = n6001\nfamily = agilex\nfim = base_x16\npart = AGFB027R31C2E2V \ndevice_id = 6001\n
                               
                              4.  
                            4.  
                              Modify the DEVICE field in the $OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_top.qsf file to use AGFB027R31C2E2V.
                               
                              ############################################################################################\n# FPGA Device\n############################################################################################\n\nset_global_assignment -name FAMILY \"Agilex 7\"\nset_global_assignment -name DEVICE AGFB027R31C2E2V \n
                               
                              5.  
                            5.  
                              Modify the DEVICE field in the $OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_pr_afu.qsf file to use AGFB027R31C2E2V.
                               
                              ############################################################################################\n# FPGA Device\n############################################################################################\n\nset_global_assignment -name FAMILY Agilex 7\nset_global_assignment -name DEVICE AGFB027R31C2E2V \n
                               
                              6.  
                            6.  
                              If the device you are migrating to uses the same package and pinout, you do not need to modify the pinout constraints. In this example, because we are migrating from package R24C to R31C, we need to modify the pinout to match the new device. If you would like Quartus to attempt to pin out the design automatically, you may remove all pin assignments. Typically, you will still need to set certain pins manually in order to guide Quartus for a successful compile (e.g. transceiver reference clocks).
                               
                                 
                              1.  
                                Commment out all pin assignments in the following files:     * $OFS_ROOTDIR/syn/board/fseries-dk/setup/emif_loc.tcl     * $OFS_ROOTDIR/syn/board/fseries-dk/setup/hps_loc.tcl     * $OFS_ROOTDIR/syn/board/fseries-dk/setup/top_loc.tcl
                                 
                                2.  
                              2.  
                                Identify the pins in the design that will be constrained. In this example we will manually constrain the QSFP reference clock and the PCIe reference clock to help guide the fitter. The Device Migration Pinout Table shows th pin assignments that will be set, along with the pin number for both the old R24C package and the new R31C package.
                                 
                                3.  
                               Net Name Pin Name AGF 027 R24C Pin # AGF 027 R31C Pin # \"qsfp_ref_clk(n)\" REFCLK_FGTL12C_Q1_RX_CH2n AV48 BD57 qsfp_ref_clk REFCLK_FGTL12C_Q1_RX_CH2p AW49 BB57 \"PCIE_REFCLK0(n)\" REFCLK_FGTR13A_Q1_RX_CH2n BU7 BP13 PCIE_REFCLK0 REFCLK_FGTR13A_Q1_RX_CH2p BR7 BN14 
                                 
                              1.  
                                Constrain the pins identified in Step 6.B in the $OFS_ROOTDIR/syn/board/fseries-dk/setup/top_loc.tcl file for the new pinout for the AGF 027 R31C package. 
                                 
                                set_location_assignment PIN_BB57 -to qsfp_ref_clk\nset_location_assignment PIN_BD57 -to \"qsfp_ref_clk(n)\"\n\nset_location_assignment PIN_BP13 -to \"PCIE_REFCLK0(n)\"\nset_location_assignment PIN_BN14 -to PCIE_REFCLK0\n
                                 
                                2.  
                              2.  
                                Uncomment the instance assignments related to he QSFP reference clock in the $OFS_ROOTDIR/syn/board/fseries-dk/setup/top_loc.tcl file.
                                 
                                set_instance_assignment -name IO_STANDARD \"CURRENT MODE LOGIC (CML)\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=disable_3p3v_tol\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=156250000\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=TRUE\" -to qsfp_ref_clk\n
                                 
                                3.  
                               
                              7.  
                            7.  
                              Compile a flat design. It is recommended to compile a flat design first before incorporating a PR region in the design.
                               
                              cd $OFS_ROOTDIR\n./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk:flat work_fseries-dk\n
                               
                              8.  
                            8.  
                              Verify that the build completes successfuly. If there are timing violation, try building with a different seed. Refer to the Change the Compilation Seed section for instructions on changing the build seed.
                               
                              9.  
                            9.  
                              When you are satisfied with the pinout, preserve it by hard-coding the desired pinout to followig files:
                               
                                 
                              • $OFS_ROOTDIR/syn/board/fseries-dk/setup/emif_loc.tcl
                                •  
                              • $OFS_ROOTDIR/syn/board/fseries-dk/setup/hps_loc.tcl
                                •  
                              • $OFS_ROOTDIR/syn/board/fseries-dk/setup/top_loc.tcl
                                •  
                               
                              10.  
                            10.  
                              When you are ready to re-incorporate PR into the design, modify the PR region to be compatible with the new device. Refer to the Resize the Partial Reconfiguration Region section for instructions.
                               
                              11.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#47-modify-the-ethernet-sub-system","title":"4.7 Modify the Ethernet Sub-System","text":"
                            This section describes the flows for modifying the Ethernet Sub-System.
                             
                            Note: The default HSSI-SS configuration for the fseries-dk is 2x4x25GbE.
                             
                            Note: The fseries-dk does not support 2x200GbE or 1x400GbE configurations.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#471-walkthrough-modify-the-ethernet-sub-system-to-2x4x10gbe","title":"4.7.1 Walkthrough: Modify the Ethernet Sub-System to 2x4x10GbE","text":"
                            Perform the following steps to modify the Ethernet Sub-System to 2x4x10GbE. In this walkthrough, we will create a new IP presets file that will be used during the build flow.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Run the setup stage of the build script to create a work directry for the fseries-dk design.
                               
                              ./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk\n
                               
                              4.  
                            4.  
                              Open the HSSI-SS IP in the work directory using qsys-edit
                               
                              qsys-edit $OFS_ROOTDIR/work_fseries-dk/ipss/hssi/qip/hssi_ss/hssi_ss.ip\n
                               
                              5.  
                            5.  
                              Change the PORT_PROFILE parameter from 25GbE to 10GbE for Ports 0 through 7.
                               
                               
                              6.  
                            6.  
                              Create an IP Presets file for this Ethernet Subsystem IP configuration.
                               
                                 
                              1.  
                                Click New in the bottom right corner of the Presets window.
                                 
                                 
                                2.  
                              2.  
                                In the New Preset window, set the Preset name. In this example, we will name it 2x4x10g-fseries-dk.
                                 
                                 
                                3.  
                              3.  
                                Click the ... button to select the save location of the Preset file.
                                 
                                4.  
                              4.  
                                In the Save As window, navigate to the $OFS_ROODIR/ipss/hssi/qip/hssi_ss/presets directory. Set the File Name to 2x4x10g-fseries-dk.qprs. Then click OK.
                                 
                                 
                                5.  
                              5.  
                                In the New Preset window, click Save. Click No when prompted to add the file to the IP search path.
                                 
                                6.  
                               
                              7.  
                            7.  
                              Close out of the IP Parameter Editor GUI. Do not save or generate the IP.
                               
                              8.  
                            8.  
                              Create a new HSSI OFSS file named $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_2x4x10_ftile.ofss with the following contents. This will call the IP Presets file generated in Step 6.
                               
                              [ip]\n  type = hssi\n\n  [settings]\n  output_name = hssi_ss\n  num_channels = 8\n  data_rate = 10GbE\n  preset = 2x4x10g-fseries-dk\n
                               
                              9.  
                            9.  
                              Run the FIM build script with the new HSSI OFSS file created in Step 8.
                               
                              ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_2x4x10_ftile.ofss fseries-dk work_fseries-dk_eth_2x4x10\n
                               
                              10.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#472-walkthrough-modify-the-ethernet-sub-system-to-3x4x10gbe","title":"4.7.2 Walkthrough: Modify the Ethernet Sub-System to 3x4x10GbE","text":"
                            Perform the following steps to modify the Ethernet Sub-System to 3x4x10GbE. In this walkthrough, we will create a new IP presets file that will be used during the build flow.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Run the setup stage of the build script to create a work directry for the fseries-dk design.
                               
                              ./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk\n
                               
                              4.  
                            4.  
                              Open the HSSI-SS IP in the work directory using qsys-edit
                               
                              qsys-edit $OFS_ROOTDIR/work_fseries-dk/ipss/hssi/qip/hssi_ss/hssi_ss.ip\n
                               
                              5.  
                            5.  
                              Change the NUM_ENABLED_PORTS parameter from 8 to 12.
                               
                              6.  
                            6.  
                              Change the PORT_PROFILE parameter from 25GbE to 10GbE for Ports 0 through 11.
                               
                               
                              7.  
                            7.  
                              Create an IP Presets file for this Ethernet Subsystem IP configuration.
                               
                                 
                              1.  
                                Click New in the bottom right corner of the Presets window.
                                 
                                 
                                2.  
                              2.  
                                In the New Preset window, set the Preset name. In this example, we will name it 3x4x10g-fseries-dk.
                                 
                                 
                                3.  
                              3.  
                                Click the ... button to select the save location of the Preset file.
                                 
                                4.  
                              4.  
                                In the Save As window, navigate to the $OFS_ROODIR/ipss/hssi/qip/hssi_ss/presets directory. Set the File Name to 3x4x10g-fseries-dk.qprs. Then click OK.
                                 
                                 
                                5.  
                              5.  
                                In the New Preset window, click Save. Click No when prompted to add the file to the IP search path.
                                 
                                6.  
                               
                              8.  
                            8.  
                              Close out of the IP Parameter Editor GUI. Do not save or generate the IP.
                               
                              9.  
                            9.  
                              Create a new HSSI OFSS file named $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_3x4x10_ftile.ofss with the following contents. This will call the IP Presets file generated in Step 6.
                               
                              [ip]\n  type = hssi\n\n  [settings]\n  output_name = hssi_ss\n  num_channels = 12\n  data_rate = 10GbE\n  preset = 3x4x10g-fseries-dk\n
                               
                              10.  
                            10.  
                              Change the number of QSFP ports from 2 to 3 in the $OFS_ROOTDIR/ipss/hssi/rtl/inc/ofs_fim_eth_plat_if_pkg.sv file.
                               
                              localparam NUM_QSFP_PORTS = 3; // QSFP cage on board\n
                               
                              11.  
                            11.  
                              Add the pin location assignments for the new QSFP channels in the $OFS_ROOTDIR/syn/board/fseries-dk/setup/top_loc.tcl file. In this example we will use channels 4 through 7 of the QSFPDD-56.
                               
                              set_location_assignment PIN_AC55 -to qsfp_serial[2].rx_p[0]\nset_location_assignment PIN_R55  -to qsfp_serial[2].rx_p[1]\nset_location_assignment PIN_AG55 -to qsfp_serial[2].rx_p[2]\nset_location_assignment PIN_W55  -to qsfp_serial[2].rx_p[3]\n\nset_location_assignment PIN_AD54 -to qsfp_serial[2].rx_n[0]\nset_location_assignment PIN_T54  -to qsfp_serial[2].rx_n[1]\nset_location_assignment PIN_AH54 -to qsfp_serial[2].rx_n[2]\nset_location_assignment PIN_Y54  -to qsfp_serial[2].rx_n[3]\n\nset_location_assignment PIN_AF52 -to qsfp_serial[2].tx_p[0]\nset_location_assignment PIN_W49  -to qsfp_serial[2].tx_p[1]\nset_location_assignment PIN_AK52 -to qsfp_serial[2].tx_p[2]\nset_location_assignment PIN_AB52  -to qsfp_serial[2].tx_p[3]\n\nset_location_assignment PIN_AE51 -to qsfp_serial[2].tx_n[0]\nset_location_assignment PIN_Y48  -to qsfp_serial[2].tx_n[1]\nset_location_assignment PIN_AJ51 -to qsfp_serial[2].tx_n[2]\nset_location_assignment PIN_AA51  -to qsfp_serial[2].tx_n[3]\n
                               
                              12.  
                            12.  
                              Build the FIM. 
                               
                              ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_3x4x10_ftile.ofss fseries-dk work_fseries-dk_eth_12x10g\n
                               
                              13.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#48-modifying-the-hps","title":"4.8 Modifying the HPS","text":"
                            This section describes ways to modify the HPS.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#481-walkthrough-remove-the-hps","title":"4.8.1 Walkthrough: Remove the HPS","text":"
                            Perform the following steps to remove the HPS from the FIM design.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                               
                              2.  
                            2.  
                              Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                               
                              3.  
                            3.  
                              Run the setup stage of the build script to create a work directry for the fseries-dk design.
                               
                              ./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk\n
                               
                              4.  
                            4.  
                              Create a Memory Sub-system IP presets file with the connection to the HPS removed.
                               
                                 
                              1.  
                                In the work directory created in Step 3, open the mem_ss.ip
                                 
                                qsys-edit $OFS_ROOTDIR/work_fseries-dk/ipss/mem/qip/mem_ss/mem_ss.ip\n
                                 
                                2.  
                              2.  
                                In the IP Parameter Editor window that opens, remove the entries corresponding to the HPS (row #2) in the Memory Interfaces and Application Interfaces sections. To do this, click the row to be removed, then click the minus (-) button.
                                 
                                 
                                3.  
                              3.  
                                In the Presets pane, click New... to create a new IP presets file.
                                 
                                 
                                4.  
                              4.  
                                In the New Preset window, create a unique preset name. For example, fseries-dk-mem-no-hps.
                                 
                                 
                                5.  
                              5.  
                                Click the ... button to select the save location of the IP presets file. In the Save As window, set the Look In field to the memory IP presets directory $OFS_ROOTDIR/ipss/mem/qip/presets. Set the File Name field to match the name selected in Step 4. Click OK.
                                 
                                 
                                6.  
                              6.  
                                Click Save in the New Preset window. Click No when prompted to add the file to the IP search path.
                                 
                                 
                                7.  
                              7.  
                                Close IP Parameter Editor without saving or generating HDL.
                                 
                                8.  
                               
                              5.  
                            5.  
                              Edit the Memory OFSS file $OFS_ROOTDIR/tools/ofss_config/memory/memory_ftile.ofss to use the IP presets file generated in Step 4.
                               
                              [ip]\ntype = memory\n\n[settings]\noutput_name = mem_ss\npreset = fseries-dk-mem-no-hps\n
                               
                              6.  
                            6.  
                              Edit $OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_top.qsf to comment out the INCLUDE_HPS macro definition.
                               
                              #set_global_assignment -name VERILOG_MACRO \"INCLUDE_HPS\"\n
                               
                              7.  
                            7.  
                              Build the FIM.
                               
                              ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_no_hps\n
                               
                              8.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#5-fpga-configuration","title":"5. FPGA Configuration","text":"
                            Configuring the Agilex FPGA on the fseries-dk is done by programming a SOF image to the FPGA via JTAG using Quartus Programer.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#51-walkthrough-set-up-jtag","title":"5.1 Walkthrough: Set up JTAG","text":"
                            The Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) has an on-board FPGA Download Cable II module which is used to program the FPGA via JTAG.
                             
                            Perform the following steps to establish a JTAG connection with the fseries-dk.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                              •  
                             
                               
                            • This walkthrough requires a workstation with Quartus Prime Pro Version 23.3 tools installed, specifically the jtagconfig tool.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Refer to the following figure for Steps 2 and 3.
                               
                               
                              2.  
                            2.  
                              Locate Single DIP Switch SW2 and 4-position DIP switch SW3 on the fseries-dk. These switches control the JTAG setup for the board. Ensure that both SW2 and SW3.3 are set to ON.
                               
                              3.  
                            3.  
                              Locate the J10 Micro-USB port on the fseries-dk. Connect a Micro-USB to USB-A cable between the J10 port and the workstation that has Quartus Prime Pro tools installed.
                               
                              4.  
                            4.  
                              Use the jtagconfig tool to check that the JTAG chain contains the AGFB027R24C2E2VR2 device.
                               
                              <QUARTUS_INSTALL_DIR>/23.3/quartus/bin/jtagconfig\n
                               
                              Example expected output:
                               
                              1) Agilex F-Series FPGA Dev Kit [1-6]\n0343B0DD   AGFB027R24C(.|R2|R0)/..\n020D10DD   VTAP10\n
                               
                              5.  
                             
                            This concludes the walkthrough for establishing a JTAG connection on the fseries-dk.
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#52-walkthrough-program-the-fpga-via-jtag","title":"5.2 Walkthrough: Program the FPGA via JTAG","text":"
                            This walkthrough describes the steps to program the Agilex FPGA on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) with a SOF image via JTAG.
                             
                            Pre-Requisites:
                             
                               
                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                              •  
                             
                               
                            • This walkthrough requires a SOF image which will be programmed to the Agilex FPGA. Refer to the Compile OFS FIM Section for step-by-step instructions for generating a SOF image.
                              •  
                             
                               
                            • This walkthrough requires a JTAG connection to the fseries-dk. Refer to the Set up JTAG section for step-by-step instructions.
                              •  
                             
                               
                            • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) is connected via JTAG.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Start in your deployment environment.
                               
                              2.  
                            2.  
                              If the card is already programmed with an OFS enabled design, determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                               
                              fpgainfo fme\n
                               
                              Example output:
                               
                              Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010202A8769764\nBitstream Version                : 5.0.1\nPr Interface Id                  : b541eb7c-3c7e-5678-a660-a54f71594b34\nBoot Page                        : N/A\n
                               
                              Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                               
                              3.  
                            3.  
                              Remove the card from PCIe prior to programming. This will disable AER for the PCIe root port to prevent a surprise link-down event during programming.
                               
                              sudo pci_device b1:00.0 unplug\n
                               
                              4.  
                            4.  
                              Switch to the machine with JTAG connection to the fseries-dk, if different than your deployment machine.
                               
                              5.  
                            5.  
                              Open the Quartus programmer GUI
                               
                              quartus_pgmw\n
                               
                               
                              6.  
                            6.  
                              Click Hardware Setup to open the Hardware Setup dialog window.
                               
                                 
                              1.  
                                In the Currently selected hardware field select the fseries-dk.
                                 
                                2.  
                              2.  
                                In the Hardware frequency field enter 16000000 Hz
                                 
                                 
                                3.  
                              3.  
                                Click Close
                                 
                                4.  
                               
                              7.  
                            7.  
                              In the Quartus Prime Programmer window, click Auto Detect.
                               
                              8.  
                            8.  
                              If prompted, select the AGFB027R24C2E2VR2 device. The JTAG chain should show the device.
                               
                               
                              9.  
                            9.  
                              Right click the AGFB027R24C2E2VR2 row and selct Change File.
                               
                               
                              10.  
                            10.  
                              In the Select New Programming File window that opens, select the .sof image you wish to program and click Open.
                               
                              11.  
                            11.  
                              Check the Program/Configure box for the AGFB027R24C2E2VR2 row, then click Start. Wait for the Progress bar to show 100% (Success).
                               
                               
                              12.  
                            12.  
                              Close the Quartus Programmer GUI.
                               
                              13.  
                            13.  
                              Switch to the deployment environment, if different than the JTAG connected machine.
                               
                              14.  
                            14.  
                              Replug the board PCIe
                               
                              sudo pci_device b1:00.0 plug\n
                               
                              15.  
                            15.  
                              Run fpgainfo fme to verify communication with the board, and to check the PR Interface ID.
                               
                              Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x501020241BF165B\nBitstream Version                : 5.0.1\nPr Interface Id                  : e7f69412-951f-5d1a-8cb7-8c778ac02055\nBoot Page                        : N/A\n
                               
                              Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                               
                              16.  
                            "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#appendix","title":"Appendix","text":""},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#appendix-a-resource-utilization-tables","title":"Appendix A: Resource Utilization Tables","text":"
                            Table A-1 Default Flat FIM Resource Utilization
                             Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 153105.5 16.77 624 4.7 afu_top 83365.9 9.13 274 2.06 auto_fab_0 1339.6 0.15 9 0.07 bpf 780.1 0.09 0 0.0 fme_top 658.5 0.07 6 0.05 hps_ss 0.0 0.0 0 0.0 hssi_wrapper 11681.9 1.28 81 0.61 mem_ss_top 5712.9 0.63 38 0.29 ofs_top_auto_tiles 8180.5 0.9 20 0.15 pcie_wrapper 39444.9 4.32 188 1.42 pmci_dummy_csr 668.2 0.07 0 0.0 qsfp_0 627.1 0.07 4 0.03 qsfp_1 624.6 0.07 4 0.03 rst_ctrl 17.1 0.0 0 0.0 sys_pll 0.4 0.0 0 0.0 
                            Table A-2 Default In-Tree PR FIM Resource Utilization
                             Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 158371.6 17.35 624 4.7 afu_top 88125.9 9.65 274 2.06 auto_fab_0 1305.6 0.14 9 0.07 bpf 782.5 0.09 0 0.0 fme_top 650.5 0.07 6 0.05 hps_ss 0.0 0.0 0 0.0 hssi_wrapper 11757.4 1.29 81 0.61 mem_ss_top 6155.1 0.67 38 0.29 ofs_top_auto_tiles 8263.6 0.91 20 0.15 pcie_wrapper 39393.5 4.32 188 1.42 pmci_dummy_csr 658.9 0.07 0 0.0 qsfp_0 625.0 0.07 4 0.03 qsfp_1 632.8 0.07 4 0.03 rst_ctrl 16.3 0.0 0 0.0 sys_pll 0.4 0.0 0 0.0 
                            Table A-3 Default Out-of-Tree FIM Resource Utilization
                             Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 158287.4 17.34 624 4.7 afu_top 88046.1 9.65 274 2.06 auto_fab_0 1313.2 0.14 9 0.07 bpf 784.7 0.09 0 0.0 fme_top 656.1 0.07 6 0.05 hps_ss 0.0 0.0 0 0.0 hssi_wrapper 11781.0 1.29 81 0.61 mem_ss_top 6164.6 0.68 38 0.29 ofs_top_auto_tiles 8224.2 0.9 20 0.15 pcie_wrapper 39365.6 4.31 188 1.42 pmci_dummy_csr 665.1 0.07 0 0.0 qsfp_0 635.1 0.07 4 0.03 qsfp_1 631.7 0.07 4 0.03 rst_ctrl 15.1 0.0 0 0.0 sys_pll 0.5 0.0 0 0.0 
                            Table A-4 Minimal FIM Resource Utilization
                             Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 101487.5 11.12 453 3.41 afu_top 31185.9 3.42 105 0.79 auto_fab_0 1309.6 0.14 9 0.07 bpf 797.6 0.09 0 0.0 fme_top 656.8 0.07 6 0.05 hps_ss 0.0 0.0 0 0.0 hssi_wrapper 11649.8 1.28 81 0.61 mem_ss_top 6447.4 0.71 38 0.29 ofs_top_auto_tiles 8288.5 0.91 20 0.15 pcie_wrapper 39208.5 4.3 186 1.4 pmci_dummy_csr 670.4 0.07 0 0.0 qsfp_0 623.6 0.07 4 0.03 qsfp_1 627.8 0.07 4 0.03 rst_ctrl 16.2 0.0 0 0.0 sys_pll 0.4 0.0 0 0.0"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#appendix-b-glossary","title":"Appendix B: Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                            Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                             
                            OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                            "},{"location":"hw/ftile_devkit/doc_modules/ftile_wt_program_fpga_via_jtag/","title":"Ftile wt program fpga via jtag","text":"
                            This walkthrough describes the steps to program the Agilex FPGA on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) with a SOF image via JTAG.
                             
                            Pre-Requisites:
                             
                               
                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 SoC Attach FPGAs (Intel Agilx 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                              •  
                             
                               
                            • This walkthrough requires a SOF image which will be programmed to the Agilex FPGA. Refer to the [Walkthrough: Compile OFS FIM] Section for step-by-step instructions for generating a SOF image.
                              •  
                             
                               
                            • This walkthrough requires a JTAG connection to the fseries-dk. Refer to the [Walkthrough: Set up JTAG] section for step-by-step instructions.
                              •  
                             
                               
                            • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) is connected via JTAG.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Start in your deployment environment.
                               
                              2.  
                            2.  
                              If the card is already programmed with an OFS enabled design, determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                               
                              sudo fpgainfo fme\n
                               
                              Example output:
                               
                              Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010202A8769764\nBitstream Version                : 5.0.1\nPr Interface Id                  : b541eb7c-3c7e-5678-a660-a54f71594b34\nBoot Page                        : N/A\n
                               
                              Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                               
                              3.  
                            3.  
                              Remove the card from PCIe prior to programming. This will disable AER for the PCIe root port to prevent a surprise link-down event during programming.
                               
                              sudo pci_device b1:00.0 unplug\n
                               
                              4.  
                            4.  
                              Switch to the machine with JTAG connection to the fseries-dk, if different than your deployment machine.
                               
                              5.  
                            5.  
                              Open the Quartus programmer GUI
                               
                              quartus_pgmw\n
                               
                               
                              6.  
                            6.  
                              Click Hardware Setup to open the Hardware Setup dialog window.
                               
                                 
                              1.  
                                In the Currently selected hardware field select the fseries-dk.
                                 
                                2.  
                              2.  
                                In the Hardware frequency field enter 16000000 Hz
                                 
                                 
                                3.  
                              3.  
                                Click Close
                                 
                                4.  
                               
                              7.  
                            7.  
                              In the Quartus Prime Programmer window, click Auto Detect.
                               
                              8.  
                            8.  
                              If prompted, select the AGFB027R24C2E2VR2 device. The JTAG chain should show the device.
                               
                               
                              9.  
                            9.  
                              Right click the AGFB027R24C2E2VR2 row and selct Change File.
                               
                               
                              10.  
                            10.  
                              In the Select New Programming File window that opens, select  ofs_top_hps.sof and click Open.
                               
                              11.  
                            11.  
                              Check the Program/Configure box for the AGFB027R24C2E2VR2 row, then click Start. Wait for the Progress bar to show 100% (Success).
                               
                               
                              12.  
                            12.  
                              Close the Quartus Programmer GUI.
                               
                              13.  
                            13.  
                              Switch to the deployment environment, if different than the JTAG connected machine.
                               
                              14.  
                            14.  
                              Replug the board PCIe
                               
                              sudo pci_device b1:00.0 plug\n
                               
                              15.  
                            15.  
                              Run fpgainfo fme to verify communication with the board, and to check the PR Interface ID.
                               
                              Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 360571655976424377\nBitstream Version                : 5.0.1\nPr Interface Id                  : d8fd88a7-8683-57ba-8be6-a1e058b7d4ed\nBoot Page                        : N/A\n
                               
                              Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                               
                              16.  
                            "},{"location":"hw/ftile_devkit/doc_modules/ftile_wt_set_up_jtag/","title":"Ftile wt set up jtag","text":"
                            The Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) has an on-board FPGA Download Cable II module which is used to program the FPGA via JTAG.
                             
                            Perform the following steps to establish a JTAG connection with the fseries-dk.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the [OFS Agilex PCIe Attach Getting Started Guide] for instructions on setting up a deployment environment.
                              •  
                             
                               
                            • This walkthrough requires a workstation with Quartus Prime Pro Version 23.3 tools installed, specifically the jtagconfig tool.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Refer to the following figure for Steps 2 and 3.
                               
                               
                              2.  
                            2.  
                              Locate Single DIP Switch SW2 and 4-position DIP switch SW3 on the fseries-dk. These switches control the JTAG setup for the board. Ensure that both SW2 and SW3.3 are set to ON.
                               
                              3.  
                            3.  
                              Locate the J10 Micro-USB port on the fseries-dk. Connect a Micro-USB to USB-A cable between the J10 port and the workstation that has Quartus Prime Pro tools installed.
                               
                              4.  
                            4.  
                              Use the jtagconfig tool to check that the JTAG chain contains the AGFB027R24C2E2VR2 device.
                               
                              <QUARTUS_INSTALL_DIR>/23.3/quartus/bin/jtagconfig\n
                               
                              Example expected output:
                               
                              1) Agilex F-Series FPGA Dev Kit [1-6]\n0343B0DD   AGFB027R24C(.|R2|R0)/..\n020D10DD   VTAP10\n
                               
                              5.  
                             
                            This concludes the walkthrough for establishing a JTAG connection on the fseries-dk.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/","title":"Getting Started Guide: Open FPGA Stack for Intel Agilex 7 FPGAs Targeting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)","text":"
                            Last updated: February 03, 2024 
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#10-about-this-document","title":"1.0 About This Document","text":"
                            The purpose of this document is to help users get started in evaluating the 2023.3 version of the PCIe Attach release targeting the F-tile Development Kit. After reviewing this document, a user shall be able to:
                             
                               
                            • Set up a server environment according to the Best Known Configuration (BKC)
                              •  
                            • Load and verify firmware targeting the FIM and AFU regions of the AGFB027R24C2E2VR2 FPGA
                              •  
                            • Verify full stack functionality offered by the PCIe Attach OFS solution
                              •  
                            • Learn where to find additional information on other PCIe Attach ingredients
                              •  
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#11-audience","title":"1.1 Audience","text":"
                            The information in this document is intended for customers evaluating the PCIe Attach shell targeting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile). This platform is a Development Kit  intended to be used as a starting point for evaluation and development of the Intel Agilex 7 FPGA F-Series with two F-Tiles.
                             
                            Note: Code command blocks are used throughout the document. Commands that are intended for you to run are preceded with the symbol '$', and comments with '#'. Full command output may not be shown.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-1-terminology","title":"Table 1: Terminology","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-2-software-and-component-version-summary-for-ofs-pcie-attach-targeting-the-f-tile-development-kit","title":"Table 2: Software and Component Version Summary for OFS PCIe Attach targeting the F-tile Development Kit","text":"
                            The OFS 2023.3 PCIe Attach release targeting the F-tile Development Kit is built upon tightly coupled software and Operating System version(s). The repositories listed below are used to manually build the Shell and the AFU portion of any potential workloads. Use this section as a general reference for the versions which compose this release. Specific instructions on building the FIM or AFU are discussed in their respective documents.
                             Component Version Download Link Quartus Quartus Prime Pro Version 23.3 https://www.intel.com/content/www/us/en/software-kit/782411/intel-quartus-prime-pro-edition-design-software-version-23-3-for-linux.html, patches: 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe) Host Operating System RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6 https://access.redhat.com/downloads/content/479/ver=/rhel---8/8.6/x86_64/product-software OneAPI-ASP ofs-2023.3-2 https://github.com/OFS/oneapi-asp/releases/tag/ofs-2023.3-2, patches: 0.02 OFS Platform AFU BBB ofs-2023.3-2 https://github.com/OFS/ofs-platform-afu-bbb/releases/tag/ofs-2023.3-2 OFS FIM Common Resources 2023.3 https://github.com/OFS/ofs-fim-common/releases/tag/ofs-2023.3-2 AFU Examples tag: ofs-2023.3-2 https://github.com/OFS/examples-afu/releases/tag/ofs-2023.3-2 OPAE-SIM tag: 2.10.0-1 https://github.com/OPAE/opae-sim"},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-3-programmable-firmware-version-summary-for-ofs-pcie-attach-targeting-the-f-tile-development-kit","title":"Table 3: Programmable Firmware Version Summary for OFS PCIe Attach targeting the F-tile Development Kit","text":"
                            OFS releases include pre-built binaries for the FPGA, OPAE SDK and Linux DFL which can be programmed out-of-box (OOB) and include known identifiers shown below. Installation of artifacts provided with this release will be discussed in their relevant sections.
                             Component Version Link FIM (shell) Pr Interface ID: 5bcd682f-5093-5fc7-8cd2-ae8073e19452 https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2 Host OPAE SDK https://github.com/OPAE/opae-sdk, tag: 2.10.0-1 https://github.com/OFS/opae-sdk/releases/tag/2.10.0-1 Host Linux DFL Drivers https://github.com/OPAE/linux-dfl, tag: ofs-2023.3-6.1-3 https://github.com/OFS/linux-dfl/releases/tag/ofs-2023.3-6.1-3"},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-4-hardware-bkc-for-ofs-pcie-attach-targeting-the-f-tile-development-kit","title":"Table 4: Hardware BKC for OFS PCIe Attach targeting the F-tile Development Kit","text":"
                            The following table highlights the hardware which composes the Best Known Configuation (BKC) for the OFS 2023.3 PCIe Attach release targeting F-tile Development Kit.
                             
                            Note: The Dell R750 server product line is known not to work with this release.
                             Component Link Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) https://www.intel.com/content/www/us/en/products/details/fpga/development-kits/agilex/agf027-and-agf023.html Intel FPGA Download Cable II https://www.intel.com/content/www/us/en/products/sku/215664/intel-fpga-download-cable-ii/specifications.html SuperMicro SYS-220HE-FTNR https://www.supermicro.com/en/products/system/hyper/2u/sys-220he-ftnr"},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#12-server-requirements","title":"1.2 Server Requirements","text":""},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#121-host-server-specifications","title":"1.2.1 Host Server Specifications","text":"
                            The host server must meet the following specifications:
                             
                               
                            • The server platform must contain 64 GB of RAM to run certain demos, and to compile FIM Images
                              •  
                            • The server platform must be able to fit, power, and cool an Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) as described by the product page
                              •  
                            • The server should be able to run PCIe at Gen 4 speeds to properly test designs and demos
                              •  
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#122-host-bios","title":"1.2.2 Host BIOS","text":"
                            These are the host BIOS settings known to work with the F-tile Development Kit. Information about the server's currently loaded firmware and BIOS settings can be found through its remote access controller, or by manually entering the BIOS by hitting a specific key during power on. Your specific server platform will include instructions on proper BIOS configuration and should be followed when altering settings.
                             
                               
                            • PCIe slot width must be set to x16
                              •  
                            • PCIe slot speed must be set to 4
                              •  
                            • PCIe slot must have iommu enabled
                              •  
                            • Intel VT for Directed I/O (VT-d) must be enabled
                              •  
                             
                            Specific BIOS paths are not listed here as they can differ between BIOS vendors and versions.
                             
                            In addition to BIOS settings required to support the operation of an OFS PCIe Attach solution targeting the F-tile Development Kit, server fan speed may need to be adjusted in the BIOS settings depending on local air temperature and air flow. The OFS PCIe Attach design does not automatically communicate cooling curve information with the on-board server management interface. Increasing air flow will mitigate possible thermal runaway and thermal throttling that may occur as a result.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#123-host-server-kernel-and-grub-configuration","title":"1.2.3 Host Server Kernel and GRUB Configuration","text":"
                            While many host Linux kernel and OS distributions may work with this design, only the following configuration(s) have been tested:
                             
                            OS: RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6  Kernel: 6.1-lts
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#13-preparing-the-f-tile-development-kit-for-installation-into-a-server","title":"1.3 Preparing the F-tile Development Kit for Installation into a Server","text":"
                               
                            1.  
                              The DK-DEV-AGF027F1ES (or it is called the F - tile Dev Kit, or FM86 Dev Kit) has LED light pipes on top of the QSFP cages.
                               
                               
                              These light pipes interfere with the server PCIe slot faceplate.
                               
                               
                              2.  
                            2.  
                              The light pipes can be easily removed by prying them off using a small screwdriver for leverage, then pushing the light pipes back to remove the retaining clips from the QSFP cage.
                               
                               
                               
                               
                              3.  
                            3.  
                              Board switch definitions can be found in the Intel Agilex\u00ae 7 F-Series FPGA (Two F-Tiles) Development Kit User Guide.
                               
                               
                              See the image below for SW1, SW4 and SW3.
                               
                               
                              Before inserting into a server, set SW5 to 'ON'.
                               
                               
                              4.  
                            4.  
                              Below shows a card installed into a PCIe riser with the light pipes removed.
                               
                               
                              5.  
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#14-f-tile-development-kit-jtag-setup","title":"1.4 F-tile Development Kit JTAG Setup","text":"
                            The Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) has an on-board FPGA Download Cable II module which is used to program the FPGA via JTAG.
                             
                            Perform the following steps to establish a JTAG connection with the fseries-dk.
                             
                            Pre-requisites:
                             
                               
                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the [OFS Agilex PCIe Attach Getting Started Guide] for instructions on setting up a deployment environment.
                              •  
                             
                               
                            • This walkthrough requires a workstation with Quartus Prime Pro Version 23.3 tools installed, specifically the jtagconfig tool.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Refer to the following figure for Steps 2 and 3.
                               
                               
                              2.  
                            2.  
                              Locate Single DIP Switch SW2 and 4-position DIP switch SW3 on the fseries-dk. These switches control the JTAG setup for the board. Ensure that both SW2 and SW3.3 are set to ON.
                               
                              3.  
                            3.  
                              Locate the J10 Micro-USB port on the fseries-dk. Connect a Micro-USB to USB-A cable between the J10 port and the workstation that has Quartus Prime Pro tools installed.
                               
                              4.  
                            4.  
                              Use the jtagconfig tool to check that the JTAG chain contains the AGFB027R24C2E2VR2 device.
                               
                              <QUARTUS_INSTALL_DIR>/23.3/quartus/bin/jtagconfig\n
                               
                              Example expected output:
                               
                              1) Agilex F-Series FPGA Dev Kit [1-6]\n0343B0DD   AGFB027R24C(.|R2|R0)/..\n020D10DD   VTAP10\n
                               
                              5.  
                             
                            This concludes the walkthrough for establishing a JTAG connection on the fseries-dk.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#15-upgrading-the-f-tile-development-kit-fim-via-jtag","title":"1.5 Upgrading the F-tile Development Kit FIM via JTAG","text":"
                            Intel provides a pre-built FIM that can be used out-of-box for platform bring-up. This shell design is available on the OFS 2023.3 Release Page. After programming the shell and installing both the OPAE SDK and Linux DFL kernel drivers (as shown in sections 3.0 OFS DFL Kernel Drivers and 4.0 OPAE Software Development Kit), you can confirm the correct FIM has been configured by checking the output of fpgainfo fme against the following table:
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-5-fim-version","title":"Table 5: FIM Version","text":"Identifier Value Pr Interface ID 5bcd682f-5093-5fc7-8cd2-ae8073e19452 Bitstream ID 360571655976424377 
                            You will need to download and unpack the artifact images for this release before upgrading your device. The file ofs_top_hps.sof is the base OFS FIM file. This file is loaded into the FPGA using the development kit built in USB Blaster. Please be aware this FPGA is not loaded into non-volatile storage, therefore if the server is power cycled, you will need to reload the FPGA .sof file.
                             
                            wget https://github.com/OFS/ofs-agx7-pcie-attach/releases/download/ofs-2023.3-2/fseries-images_ofs-2023-3-2.tar.gz\ntar xf fseries-dk-images.tar.gz\ncd fseries-dk-images/\n
                             
                            This walkthrough describes the steps to program the Agilex FPGA on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) with a SOF image via JTAG.
                             
                            Pre-Requisites:
                             
                               
                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 SoC Attach FPGAs (Intel Agilx 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                              •  
                             
                               
                            • This walkthrough requires a SOF image which will be programmed to the Agilex FPGA. Refer to the [Walkthrough: Compile OFS FIM] Section for step-by-step instructions for generating a SOF image.
                              •  
                             
                               
                            • This walkthrough requires a JTAG connection to the fseries-dk. Refer to the [Walkthrough: Set up JTAG] section for step-by-step instructions.
                              •  
                             
                               
                            • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) is connected via JTAG.
                              •  
                             
                            Steps:
                             
                               
                            1.  
                              Start in your deployment environment.
                               
                              2.  
                            2.  
                              If the card is already programmed with an OFS enabled design, determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                               
                              sudo fpgainfo fme\n
                               
                              Example output:
                               
                              Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010202A8769764\nBitstream Version                : 5.0.1\nPr Interface Id                  : b541eb7c-3c7e-5678-a660-a54f71594b34\nBoot Page                        : N/A\n
                               
                              Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                               
                              3.  
                            3.  
                              Remove the card from PCIe prior to programming. This will disable AER for the PCIe root port to prevent a surprise link-down event during programming.
                               
                              sudo pci_device b1:00.0 unplug\n
                               
                              4.  
                            4.  
                              Switch to the machine with JTAG connection to the fseries-dk, if different than your deployment machine.
                               
                              5.  
                            5.  
                              Open the Quartus programmer GUI
                               
                              quartus_pgmw\n
                               
                               
                              6.  
                            6.  
                              Click Hardware Setup to open the Hardware Setup dialog window.
                               
                                 
                              1.  
                                In the Currently selected hardware field select the fseries-dk.
                                 
                                2.  
                              2.  
                                In the Hardware frequency field enter 16000000 Hz
                                 
                                 
                                3.  
                              3.  
                                Click Close
                                 
                                4.  
                               
                              7.  
                            7.  
                              In the Quartus Prime Programmer window, click Auto Detect.
                               
                              8.  
                            8.  
                              If prompted, select the AGFB027R24C2E2VR2 device. The JTAG chain should show the device.
                               
                               
                              9.  
                            9.  
                              Right click the AGFB027R24C2E2VR2 row and selct Change File.
                               
                               
                              10.  
                            10.  
                              In the Select New Programming File window that opens, select  ofs_top_hps.sof and click Open.
                               
                              11.  
                            11.  
                              Check the Program/Configure box for the AGFB027R24C2E2VR2 row, then click Start. Wait for the Progress bar to show 100% (Success).
                               
                               
                              12.  
                            12.  
                              Close the Quartus Programmer GUI.
                               
                              13.  
                            13.  
                              Switch to the deployment environment, if different than the JTAG connected machine.
                               
                              14.  
                            14.  
                              Replug the board PCIe
                               
                              sudo pci_device b1:00.0 plug\n
                               
                              15.  
                            15.  
                              Run fpgainfo fme to verify communication with the board, and to check the PR Interface ID.
                               
                              Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 360571655976424377\nBitstream Version                : 5.0.1\nPr Interface Id                  : d8fd88a7-8683-57ba-8be6-a1e058b7d4ed\nBoot Page                        : N/A\n
                               
                              Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                               
                              16.  
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#16-f-tile-development-kit-installation-procedure","title":"1.6 F-tile Development Kit Installation Procedure","text":"
                            The following instructions will help ensure safe installation of the F-tile Development Kit into a supported server platform. Safety and Regulatory information can be found under the product page for this development kit. It is assumed you have previously removed the light pipes mounted above the platform's QSFP cages before attempting to slot into a server mounted riser.
                             
                               
                            1. Position the board over the selected connector on the motherboard
                              2.  
                            2. Press down gently and firmly seat the card in a PCIe slot. Depending on the server model being used, you may need to secure a retention screw or rotate retention clips over the development kit's faceplate.
                              3.  
                            3. Do not bend the card while inserting in a slot. Do not apply too much pressure while inserting.
                              4.  
                            4. Plug a standard 2x4 auxiliary power cord available from the server's ATX power supply or from the riser itself to the respective matching power connected on the board (J11). Both the PCIe slot and auxiliary PCIe power cable are required to power the entire board.
                              5.  
                            5. If you haven't already, follow the instructions in section 1.5 F-tile Development Kit JTAG Setup and connect a USB Blaster II Cable from the board to the server housing it.
                              6.  
                             
                            The EMIF subsystem and base FIM in this release do not work with the RDIMM memory installed in the Agilex 7 FPGA F-Series Development Kit. The RDIMM memory must be removed and replaced. The following RDIMM memory module(s) are known to work with this platform:
                             Amount Type Link 2 DDR4-2400 https://memory.net/product/mta9asf1g72az-2g3-micron-1x-8gb-ddr4-2400-udimm-pc4-19200t-e-single-rank-x8-module/"},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#20-ofs-stack-architecture-overview-for-reference-platform","title":"2.0 OFS Stack Architecture Overview for Reference Platform","text":""},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#21-hardware-components","title":"2.1 Hardware Components","text":"
                            The OFS hardware architecture decomposes all designs into a standard set of modules, interfaces, and capabilities. Although the OFS infrastructure provides a standard set of functionality and capability, the user is responsible for making the customizations to their specific design in compliance with the specifications outlined in the FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA.
                             
                            OFS is a hardware and software infrastructure that provides an efficient approach to developing a customer FPGA-based platform or workload using an Intel, 3rd party, or custom board.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#211-fpga-interface-manager","title":"2.1.1 FPGA Interface Manager","text":"
                            The FPGA Interface Manager (FIM), or shell of the FPGA provides platform management functionality, clocks, resets, and interface access to the host and peripheral features on the acceleration platform. The OFS architecture for Intel Agilex 7 FPGA provides modularity, configurability, and scalability. The primary components of the FPGA Interface Manager or shell of the reference design are:
                             
                               
                            • PCIe Subsystem - a hierarchical design that targets the P-tile PCIe hard IP and is configured to support Gen4 speeds and Arm AXI4-Stream Data Mover functional mode.
                              •  
                            • Ethernet Subsystem - provides portability to different Ethernet configurations across platforms and generations and reusability of the hardware framework and software stack.
                              •  
                            • Memory Subsystem - composed of 5 DDR4 channels; two HPS DDR4 banks, x40 (x32 Data and x8 ECC), 1200 MHz, 1GB each, and four Fabric DDR4 banks, x32 (no ECC), 1200 MHz, 4GB
                              •  
                            • Hard Processor System - 64-bit quad core ARM\u00ae Cortex*-A53 MPCore with integrated peripherals.
                              •  
                            • Reset Controller
                              •  
                            • FPGA Management Engine - Provides a way to manage the platform and enable acceleration functions on the platform.
                              •  
                            • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                              •  
                            • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                              •  
                            • Platform Management Controller Interface (PMCI) to the board management controller
                              •  
                             
                            The FPGA Management Engine (FME) provides management features for the platform and the loading/unloading of accelerators through partial reconfiguration. Each feature of the FME exposes itself to the kernel-level OFS drivers on the host through a Device Feature Header (DFH) register that is placed at the beginning of Control Status Register (CSR) space. Only one PCIe link can access the FME register space in a multi-host channel design architecture at a time.
                             
                            Note: For more information on the FIM and its external connections, refer to the FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#212-afu","title":"2.1.2 AFU","text":"
                            An AFU is an acceleration workload that interfaces to the FIM. The AFU boundary in this reference design comprises both static and partial reconfiguration (PR) regions. You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region. A port gasket within the design provides all the PR specific modules and logic required to support partial reconfiguration. Only one partial reconfiguration region is supported in this design.
                             
                            Like the FME, the port gasket exposes its capability to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space.
                             
                            You can compile your design in one of the following ways:
                             
                               
                            • Your entire AFU resides in a partial reconfiguration region of the FPGA.
                              •  
                            • The AFU is part of the static region and is compiled as a flat design.
                              •  
                            • Your AFU contains both static and PR regions.
                              •  
                             
                            In this design, the AFU region is comprised of:
                             
                               
                            • AFU Interface handler to verify transactions coming from AFU region.
                              •  
                            • PF/VF Mux to route transactions to and from corresponding AFU components: ST2MM module, Virtio LB stub, PCIe loopback host exerciser (HE-LB), HSSI host exerciser (HE-HSSI), Memory Host Exerciser (HE-MEM), Traffic Generator to memory (HE-MEM-TG), Port Gasket (PRG) and HPS Copy Engine.
                              •  
                            • AXI4 Streaming to Memory Map (ST2MM) Module that routes MMIO CSR accesses to FME and board peripherals.
                              •  
                            • Host exercisers to test PCIe, memory and HSSI interfaces (these can be removed from the AFU region after your FIM design is complete to provide more resource area for workloads)
                              •  
                            • Basic HPS Copy Engine to copy second-stage bootloader and Linux OS image from Host DDR to HPS DDR.
                              •  
                            • Port gasket and partial reconfiguration support.
                              •  
                            • Component for handling PLDM over MCTP over PCIe Vendor Defined Messages (VDM)
                              •  
                             
                            The AFU has the option to consume native packets from the host or interface channels or to instantiate a shim provided by the Platform Interface Manager (PIM) to translate between protocols.
                             
                            Note: For more information on the Platform Interface Manager and AFU development and testing, refer to the AFU Development Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#22-ofs-software-overview","title":"2.2 OFS Software Overview","text":"
                            The responsibility of the OFS kernel drivers is to act as the lowest software layer in the FPGA software stack, providing a minimalist driver implementation between the host software and functionality that has been implemented on the development platform. This leaves the implementation of IP-specific software in user-land, not the kernel. The OFS software stack also provides a mechanism for interface and feature discovery of FPGA platforms.
                             
                            The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space, and can be found on the OPAE SDK Github.
                             
                            The OFS drivers decompose implemented functionality, including external FIM features such as HSSI, EMIF and SPI, into sets of individual Device Features. Each Device Feature has its associated Device Feature Header (DFH), which enables a uniform discovery mechanism by software. A set of Device Features are exposed through the host interface in a Device Feature List (DFL). The OFS drivers discover and \"walk\" the Device Features in a Device Feature List and associate each Device Feature with its matching kernel driver.
                             
                            In this way the OFS software provides a clean and extensible framework for the creation and integration of additional functionalities and their features.
                             
                            Note: A deeper dive on available SW APIs and programming model is available in the Software Reference Manual: Intel\u00ae Open FPGA Stack, on kernel.org, and through the Linux DFL wiki pages.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#30-ofs-dfl-kernel-drivers","title":"3.0 OFS DFL Kernel Drivers","text":"
                            OFS DFL driver software provides the bottom-most API to FPGA platforms. Libraries such as OPAE and frameworks like DPDK are consumers of the APIs provided by OFS. Applications may be built on top of these frameworks and libraries. The OFS software does not cover any out-of-band management interfaces. OFS driver software is designed to be extendable, flexible, and provide for bare-metal and virtualized functionality. An in depth look at the various aspects of the driver architecture such as the API, an explanation of the DFL framework, and instructions on how to port DFL driver patches to other kernel distributions can be found on https://github.com/OPAE/linux-dfl/wiki.
                             
                            An in-depth review of the Linux device driver architecture can be found on opae.github.io.
                             
                            The DFL driver suite can be automatically installed using a supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3 Release Page.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#31-ofs-dfl-kernel-driver-installation-environment-setup","title":"3.1 OFS DFL Kernel Driver Installation Environment Setup","text":"
                            All OFS DFL kernel driver primary release code resides in the Linux DFL GitHub repository. This repository is open source and should not require any permissions to access. It includes a snapshot of the Linux kernel with the OFS driver included in /drivers/fpga/*. Downloading, configuration, and compilation will be discussed in this section. Refer back to section 1.2.3 Host Server Kernel and GRUB Configuration for a list of supported Operating System(s).
                             
                            You can choose to install the DFL kernel drivers by either using pre-built binaries created for the BKC OS, or by building them on your local server. If you decide to use the pre-built packages available on the OFS 2023.3 Release Page, skip to section 3.3 Installing the OFS DFL Kernel Drivers from Pre-Built Packages. Regardless of your choice you will need to follow the steps in this section to prepare your server for installation.
                             
                            This installation process assumes the user has access to an internet connection to clone specific GitHub repositories, and to satisfy package dependencies.
                             
                               
                            1.  
                              It is recommended you lock your Red Hat release version to 8.6 to prevent accidental upgrades. Update installed system packages to their latest versions.
                               
                              subscription-manager release --set=8.6\nsudo dnf update\nsubscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms\nsudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n
                               
                              2.  
                            2.  
                              Install the following package dependencies if building and installing drivers from source. If you do not require the use of a proxy to pull in downloads using dnf, you can safely remove those parameters from the following commands:
                               
                              If you require the use of a proxy, add it to DNF using by editing the following file\nsudo nano /etc/dnf/dnf.conf\n# Include your proxy by adding the following line, replacing the URL with your proxy's URL\n# proxy=http://proxy.server.com:port\n\nsudo dnf install  python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel python3-pybind11 numactl-devel\n\npython3 -m pip install --user jsonschema virtualenv pudb pyyaml setuptools pybind11\n\n# If setuptools and pybind11 were already installed\n\npython3 -m pip upgrade pybind11 setuptools\n
                               
                              3.  
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#32-building-and-installing-the-ofs-dfl-kernel-drivers-from-source","title":"3.2 Building and Installing the OFS DFL Kernel Drivers from Source","text":"
                            It is recommended you create an empty top level directory for your OFS related repositories to keep the working environment clean. All steps in this installation will use a generic top-level directory at /home/OFS/. If you have created a different top-level directory, replace this path with your custom path.
                             
                            1. Initialize an empty git repository and clone the DFL driver source code:
                             
                            ```bash\nmkdir /home/OFS/\ncd /home/OFS/\ngit init\ngit clone https://github.com/OFS/linux-dfl\ncd /home/OFS/linux-dfl\ngit checkout tags/ofs-2023.3-6.1-3\n```\n\n*Note: The linux-dfl repository is roughly 5 GB in size.*\n
                             
                            2. Verify that the correct tag/branch have been checked out.
                             
                            ```bash\ngit describe --tags\nofs-2023.3-6.1-3\n```\n\n*Note: If two different tagged releases are tied to the same commit, running git describe tags may report the other release's tag. This is why the match is made explicit.*\n
                             
                            3. Copy an existing kernel configuration file from /boot and apply the minimal required settings changes.
                             
                            ```bash\ncd /home/OFS/linux-dfl\ncp /boot/config-`uname -r` .config\ncat configs/dfl-config >> .config\necho 'CONFIG_LOCALVERSION=\"-dfl\"' >> .config\necho 'CONFIG_LOCALVERSION_AUTO=y' >> .config\nsed -i -r 's/CONFIG_SYSTEM_TRUSTED_KEYS=.*/CONFIG_SYSTEM_TRUSTED_KEYS=\"\"/' .config\nsed -i '/^CONFIG_DEBUG_INFO_BTF/ s/./#&/' .config\necho 'CONFIG_DEBUG_ATOMIC_SLEEP=y' >> .config\nexport LOCALVERSION=\nmake olddefconfig\n```\n
                             
                            4. The above command may report errors resembling symbol value 'm' invalid for CHELSIO_IPSEC_INLINE. These errors indicate that the nature of the config has changed between the currently executing kernel and the kernel being built. The option \"m\" for a particular kernel module is no longer a valid option, and the default behavior is to simply turn the option off. However, the option can likely be turned back on by setting it to 'y'. If the user wants to turn the option back on, change it to 'y' and re-run \"make olddefconfig\":
                             
                            ```bash\ncd /home/OFS/linux-dfl\necho 'CONFIG_CHELSIO_IPSEC_INLINE=y' >> .config\nmake olddefconfig\n```\n\n*Note: To use the built-in GUI menu for editing kernel configuration parameters, you can opt to run `make menuconfig`.*\n
                             
                            5. Linux kernel builds take advantage of multiple processors to parallelize the build process. Display how many processors are available with the nproc command, and then specify how many make threads to utilize with the -j option. Note that number of threads can exceed the number of processors. In this case, the number of threads is set to the number of processors in the system.
                             
                            ```bash\ncd /home/OFS/linux-dfl\nmake -j $(nproc)\n```\n
                             
                            6. You have two options to build the source:
                             
                            - Using the built-in install option from the kernel Makefile.\n- Locally building a set of RPM/DEP packages.\n\n\nThis first flow will directly install the kernel and kernel module files without the need to create a package first:\n\n```bash\ncd /home/OFS/linux-dfl\nsudo make modules_install -j $(nproc)\nsudo make install\n```\n\nIn this second flow, the OFS Makefile contains a few options for package creation:\n\n- rpm-pkg: Build both source and binary RPM kernel packages\n- binrpm-pkg: Build only the binary kernel RPM package\n- deb-pkg: Build both source and binary deb kernel packages\n- bindeb-pkg: Build only the binary kernel deb package\n\nIf you are concerned about the size of the resulting package and binaries, they can significantly reduce the size of the package and object files by using the make variable INSTALL_MOD_STRIP. If this is not a concern, feel free to skip this step. The below instructions will build a set of binary RPM packages:\n\n```bash\ncd /home/OFS/linux-dfl\nmake INSTALL_MOD_STRIP=1 binrpm-pkg\n```\n\nBy default, a directory is created in your home directory called `rpmbuild`. This directory will house all the kernel packages which have been built. You need to navigate to the newly built kernel packages and install them. The following files were generated using the build command executed in the previous step:\n\n```bash\ncd ~/rpmbuild/RPMS/x86_64\nls\nkernel-6.1.41_dfl.x86_64.rpm  kernel-headers-6.1.41_dfl.x86_64.rpm\nsudo dnf localinstall kernel*.rpm\n```\n
                             
                            7. The system will need to be rebooted in order for changes to take effect. After a reboot, select the newly built kernel as the boot target. This can be done pre-boot using the command grub2-reboot, which removes the requirement for user intervention. After boot, verify that the currently running kernel matches expectation.
                             
                            ```bash\nuname -r\n6.1.41-dfl\n```\n
                             
                            8. Verify the DFL drivers have been successfully installed by reading version information directly from /lib/modules. Recall that the name of the kernel built as a part of this section is 6.1.41-dfl. If the user set a different name for their kernel, change this path as needed:
                             
                            ```bash\ncd /usr/lib/modules/6.1.41-dfl/kernel/drivers/fpga\nls\ndfl-afu.ko     dfl-fme.ko      dfl-fme-region.ko  dfl.ko             dfl-pci.ko      fpga-mgr.ko     intel-m10-bmc-sec-update.ko\ndfl-fme-br.ko  dfl-fme-mgr.ko  dfl-hssi.ko        dfl-n3000-nios.ko  fpga-bridge.ko  fpga-region.ko\n```\n\nIf an OFS device that is compatible with these drivers is installed on the server, you can double check the driver versions by listing the currently loaded kernel modules with `lsmod`:\n\n\n```bash\nlsmod | grep dfl\nuio_dfl                20480  0\ndfl_emif               16384  0\nuio                    20480  1 uio_dfl\nptp_dfl_tod            16384  0\ndfl_intel_s10_iopll    20480  0\n8250_dfl               20480  0\ndfl_fme_region         20480  0\ndfl_fme_br             16384  0\ndfl_fme_mgr            20480  2\ndfl_fme                49152  0\ndfl_afu                36864  0\ndfl_pci                20480  0\ndfl                    40960  11 dfl_pci,uio_dfl,dfl_fme,intel_m10_bmc_pmci,dfl_fme_br,8250_dfl,qsfp_mem,ptp_dfl_tod,dfl_afu,dfl_intel_s10_iopll,dfl_emif\nfpga_region            20480  3 dfl_fme_region,dfl_fme,dfl\nfpga_bridge            20480  4 dfl_fme_region,fpga_region,dfl_fme,dfl_fme_br\nfpga_mgr               20480  4 dfl_fme_region,fpga_region,dfl_fme_mgr,dfl_fme\n```\n
                             
                            9. Two kernel parameters must be added to the boot command line for the newly installed kernel. First, open the file grub:
                             
                            ```bash\nsudo vim /etc/default/grub\n```\n
                             
                            10. In the variable GRUB_CMDLINE_LINUX add the following parameters in bold: GRUB_CMDLINE_LINUX=\"crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\".
                             
                            *Note: If you wish to instead set hugepages on a per session basis, you can perform the following steps. These settings will be lost on reboot.*\n\n```bash\nmkdir -p /mnt/huge \nmount -t hugetlbfs nodev /mnt/huge \necho 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages \necho 2048 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages \n```\n
                             
                            11. Save your edits, then apply them to the GRUB2 configuration file.
                             
                            ```bash\nsudo grub2-mkconfig\n```\n
                             
                            12. Warm reboot. Your kernel parameter changes should have taken affect.
                             
                            ```bash\ncat /proc/cmdline\nBOOT_IMAGE=(hd1,gpt2)/vmlinuz-6.1.41-dfl root=/dev/mapper/cl-root ro crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200 rhgb quiet\n```\n\nA list of all DFL drivers and their purpose is maintained on the [DFL Wiki](https://github.com/OFS/linux-dfl/wiki/FPGA-DFL-Driver-Modules#fpga-driver-modules).\n
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#33-installing-the-ofs-dfl-kernel-drivers-from-pre-built-packages","title":"3.3 Installing the OFS DFL Kernel Drivers from Pre-Built Packages","text":"
                            To use the pre-built Linux DFL packages, you first need to download the files from the OFS 2023.3 Release Page. You can choose to either install using the SRC RPMs, or to use the pre-built RPM packages targeting the official supported release platform.
                             
                            tar xf kernel-6.1.41_dfl-1.x86_64-<<version>>.tar.gz\n\nsudo dnf localinstall kernel-6.1.41_dfl_<<version>>.x86_64.rpm \\\nkernel-devel-6.1.41_dfl_<<version>>.x86_64.rpm \\\nkernel-headers-6.1.41_dfl_<<version>>.x86_64.rpm\n\n### OR\n\nsudo dnf localinstall kernel-6.1.41_dfl_<<version>>.src.rpm\n
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#40-opae-software-development-kit","title":"4.0 OPAE Software Development Kit","text":"
                            The OPAE SDK software stack sits in user space on top of the OFS kernel drivers. It is a common software infrastructure layer that simplifies and streamlines integration of programmable accelerators such as FPGAs into software applications and environments. OPAE consists of a set of drivers, user-space libraries, and tools to discover, enumerate, share, query, access, manipulate, and reconfigure programmable accelerators. OPAE is designed to support a layered, common programming model across different platforms and devices. To learn more about OPAE, its documentation, code samples, an explanation of the available tools, and an overview of the software architecture, visit opae.github.io.
                             
                            The OPAE SDK source code is contained within a single GitHub repository hosted at the OPAE Github. This repository is open source and does not require any permissions to access.
                             
                            You can choose to install the OPAE SDK by either using pre-built binaries created for the BKC OS, or by building them on your local server. If you decide to use the pre-built packages available on the OFS 2023.3 Release Page, skip to section 4.3 Installing the OPAE SDK with Pre-built Packages. Regardless of your choice you will need to follow the steps in this section to prepare your server for installation.
                             
                            You may also choose to use the supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3 Release Page.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#41-opae-sdk-installation-environment-setup","title":"4.1 OPAE SDK Installation Environment Setup","text":"
                            This installation process assumes you have access to an internet connection in order to pull specific GitHub repositories, and to satisfy package dependencies.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-6-opae-package-description","title":"Table 6: OPAE Package Description","text":"Package Name Description opae OPAE SDK is a collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. It provides a library implementing the OPAE C API for presenting a streamlined and easy-to-use interface for software applications to discover, access, and manage FPGA devices and accelerators using the OPAE software stack. opae-debuginfo This package provides debug information for package opae. Debug information is useful when developing applications that use this package or when debugging this package. opae-debugsource This package provides debug sources for package opae. Debug sources are useful when developing applications that use this package or when debugging this package. opae-devel OPAE headers, tools, sample source, and documentation opae-devel-debuginfo This package provides debug information for package opae-devel. Debug information is useful when developing applications that use this package or when debugging this package. opae-tools This package contains OPAE base tools binaries opae-extra-tools Additional OPAE tools opae-extra-tools-debuginfo This package provides debug information for package opae-extra-tools. Debug information is useful when developing applications that use this package or when debugging this package. 
                               
                            1.  
                              Remove any currently installed OPAE packages.
                               
                              sudo dnf remove opae*\n
                               
                              2.  
                            2.  
                              Initialize an empty git repository and clone the tagged OPAE SDK source code.
                               
                              cd /home/OFS/\ngit init\ngit clone https://github.com/OFS/opae-sdk opae-sdk\ncd /home/OFS/opae-sdk\ngit checkout tags/2.10.0-1\n
                               
                              3.  
                            3.  
                              Verify that the correct tag/branch have been checkout out.
                               
                              git describe --tags\n2.10.0-1\n
                               
                              4.  
                            4.  
                              Set up a temporary podman container to build OPAE, which will allow you to customize the python installation without affecting system packages.
                               
                              cd /home/OFS\npodman pull registry.access.redhat.com/ubi8:8.6\npodman run -ti -v \"$PWD\":/src:Z -w /src registry.access.redhat.com/ubi8:8.6\n\n# Everything after runs within container:\n\n# Enable EPEL\ndnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n\ndnf install --enablerepo=codeready-builder-for-rhel-8-x86_64-rpms -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel make\n\npip3 install --upgrade --prefix=/usr pip setuptools pybind11\n\n./opae-sdk/packaging/opae/rpm/create unrestricted\n\nexit\n
                               
                              The following packages will be built in the same directory as create:
                               
                              5.  
                            5.  
                              Install the packages you just created.
                               
                              cd /home/OFS/opae-sdk/packaging/opae/rpm\nrm -rf opae-2.10.0-1.el8.src.rpm \nsudo dnf localinstall -y opae*.rpm\n
                               
                              6.  
                            6.  
                              Check that all packages have been installed and match expectation:
                               
                              rpm -qa | grep opae\nopae-2.8.0-1.el8.x86_64.rpm\nopae-debuginfo-2.8.0-1.el8.x86_64.rpm\nopae-debugsource-2.8.0-1.el8.x86_64.rpm\nopae-devel-2.8.0-1.el8.x86_64.rpm\nopae-devel-debuginfo-2.8.0-1.el8.x86_64.rpm\nopae-extra-tools-2.8.0-1.el8.x86_64.rpm\nopae-extra-tools-debuginfo-2.8.0-1.el8.x86_64.rpm\n
                               
                              7.  
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#42-installing-the-opae-sdk-with-pre-built-packages","title":"4.2 Installing the OPAE SDK with Pre-Built Packages","text":"
                            You can skip the entire build process and use a set of pre-built binaries supplied by Intel. Visit the OFS 2023.3 Release Page and navigate to the bottom of the page, under the Assets tab you will see a file named opae-2.10.0-1.x86_64-<>_<>.tar.gz. Download this package and extract its contents: 
                            tar xf opae-2.10.0-1.x86_64-<<date>>_<<build>>.tar.gz\n
                             
                            For a fast installation you can delete the source RPM as it isn't necessary, and install all remaining OPAE RPMs:
                             
                            rm opae-*.src.rpm\nsudo dnf localinstall opae*.rpm\n
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#43-fpga-device-access-permissions","title":"4.3 FPGA Device Access Permissions","text":"
                            Access to FPGA accelerators and devices is controlled using file access permissions on the Intel\u00ae FPGA device files, /dev/dfl-fme.* and /dev/dfl-port.*, as well as to the files reachable through /sys/class/fpga_region/.
                             
                            In order to allow regular (non-root) users to access accelerators, you need to grant them read and write permissions on /dev/dfl-port.* (with * denoting the respective socket, i.e. 0 or 1). E.g.:
                             
                            sudo chmod a+rw /dev/dfl-port.0\n
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#44-memlock-limit","title":"4.4 Memlock limit","text":"
                            Depending on the requirements of your application, you may also want to increase the maximum amount of memory a user process is allowed to lock. The exact way to do this depends on your Linux distribution.
                             
                            You can check the current memlock limit using
                             
                            ulimit -l\n
                             
                            A way to permanently remove the limit for locked memory for a regular user is to add the following lines to your /etc/security/limits.conf:
                             
                            user1    hard   memlock           unlimited\nuser1    soft   memlock           unlimited\n
                             
                            This removes the limit on locked memory for user user1. To remove it for all users, you can replace user1 with *:
                             
                            *    hard   memlock           unlimited\n*    soft   memlock           unlimited\n
                             
                            Note that settings in the /etc/security/limits.conf file don't apply to services. To increase the locked memory limit for a service you need to modify the application's systemd service file and add the line:
                             
                            [Service]\nLimitMEMLOCK=infinity\n
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#45-opae-tools-overview","title":"4.5 OPAE Tools Overview","text":"
                            The following section offers a brief introduction including expected output values for the utilities included with OPAE. A full explanation of each command with a description of its syntax is available in the opae-sdk GitHub repo.
                             
                            A list of all tools included in the OPAE SDK release can be found on the OPAE FPGA Tools tab of ofs.github.io.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#451-board-management-with-fpgainfo","title":"4.5.1 Board Management with fpgainfo","text":"
                            The fpgainfo utility displays FPGA information derived from sysfs files.
                             
                            Displays FPGA information derived from sysfs files. The command argument is one of the following: errors, power, temp, port, fme, bmc, phy or mac, security. Some commands may also have other arguments or options that control their behavior.
                             
                            For systems with multiple FPGA devices, you can specify the BDF to limit the output to the FPGA resource with the corresponding PCIe configuration. If not specified, information displays for all resources for the given command.
                             
                            Note: Your Bitstream ID and PR Interface Id may not match the below examples.
                             
                            The following examples walk through sample outputs generated by fpgainfo. As the F-tile Development Kit does not contain a traditional BMC as used by other OFS products, those lines in fpgainfo's output will not return valid objects. The subcommand fpgainfo bmc will likewise fail to report telemetry data.
                             
                            Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x501020241BF165B\nBitstream Version                : 5.0.1\nPr Interface Id                  : b4eda250-cdb7-5891-a06e-13d28d09bc32\nBoot Page                        : N/A\n
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#452-updating-with-fpgasupdate","title":"4.5.2 Updating with fpgasupdate","text":"
                            The fpgasupdate tool is used to program AFU workloads into an open slot in a FIM. The fpgasupdate tool only accepts images that have been formatted using PACsign.
                             
                            As the F-tile Development Kit does not contain a traditional BMC, you do not have access to a factory, user1, and user2 programmed image for both the FIM and BMC FW and RTL. Only the programming of a GBS workload is supported for this release.
                             
                            The process of programming a SOF with a new FIM version is shown in section 1.5 F-tile Development Kit JTAG Setup
                             
                            sudo fpgasupdate ofs_pr_afu.gbs   <PCI ADDRESS>\n[2022-04-14 16:42:31.58] [WARNING ] Update starting. Please do not interrupt.                                           \n[2022-04-14 16:42:31.58] [INFO    ] updating from file ofs_pr_afu.gbs with size 19928064               \n[2022-04-14 16:42:31.60] [INFO    ] waiting for idle                                                                 \n[2022-04-14 16:42:31.60] [INFO    ] preparing image file                                                                \n[2022-04-14 16:42:38.61] [INFO    ] writing image file                                                                 \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [19928064/19928064 bytes][Elapsed Time: 0:00:16.01]                                       \n[2022-04-14 16:42:54.63] [INFO    ] programming image file                                                              \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588][Elapsed Time: 0:06:16.40]                                                                 \n[2022-04-14 16:49:11.03] [INFO    ] update of 0000:b1:00.0 complete                                                     \n[2022-04-14 16:49:11.03] [INFO    ] Secure update OK                                                                   \n[2022-04-14 16:49:11.03] [INFO    ] Total time: 0:06:39.45\n
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#453-verify-fme-interrupts-with-hello_events","title":"4.5.3 Verify FME Interrupts with hello_events","text":"
                            The hello_events utility is used to verify FME interrupts. This tool injects FME errors and waits for error interrupts, then clears the errors.
                             
                            Sample output from sudo hello_events.
                             
                            sudo hello_events\nWaiting for interrupts now...\ninjecting error\nFME Interrupt occurred\nSuccessfully tested Register/Unregister for FME events!\nclearing error\n
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#454-host-exercisor-modules","title":"4.5.4 Host Exercisor Modules","text":"
                            The reference FIM and unchanged FIM compilations contain Host Exerciser Modules (HEMs). These are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. There are three HEMs present in the Intel OFS Reference FIM - HE-LPBK, HE-MEM, and HE-HSSI. These exercisers are tied to three different VFs that must be enabled before they can be used. Execution of these exercisers requires you bind specific VF endpoint to vfio-pci. The host-side software looks for these endpoints to grab the correct FPGA resource.
                             
                            Refer to the Intel FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA for a full description of these modules.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-7-module-pfvf-mappings","title":"Table 7: Module PF/VF Mappings","text":"Module PF/VF ST2MM PF0 HE-MEM PF0-VF0 HE-HSSI PF0-VF1 HE-MEM_TG PF0-VF2 HE-LB Stub PF1-VF0 HE-LB PF2 VirtIO LB Stub PF3 HPS Copy Engine PF4"},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#4541-he-mem-he-lb","title":"4.5.4.1 HE-MEM / HE-LB","text":"
                            The host exerciser used to exercise and characterize the various host-FPGA interactions eg. MMIO, Data transfer from host to FPGA , PR, host to FPGA memory etc. The Host Exerciser Loopback (HE-LBK) AFU can move data between host memory and FPGA.
                             
                            HE-LBK supports: - Latency (AFU to Host memory read) - MMIO latency (Write+Read) - MMIO BW (64B MMIO writes) - BW (Read/Write, Read only, Wr only)
                             
                            The Host Exerciser Loopback Memory (HE-MEM) AFU is used to exercise use of FPGA connected DDR, data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host.
                             
                            HE-LB is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth. HE-MEM is used to exercise use of FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host. HE-MEM uses external DDR memory (i.e. EMIF) to store data. It has a customized version of the AVMM interface to communicate with the EMIF memory controller. Both exercisers rely on the user-space tool host_exerciser. When using the F-tile Development Kit SmartNIC Platform, optimal performance requires the exercisers be run at 400 MHz.
                             
                            Execution of these exercisers requires you to bind specific VF endpoint to vfio-pci. The following commands will bind the correct endpoint for a device with B/D/F 0000:b1:00.0 and run through a basic loopback test.
                             
                            Note: While running the opae.io init command listed below, the command has failed if no output is present after completion. Double check that Intel VT-D and IOMMU have been enabled in the kernel as discussed in section 3.0 OFS DFL Kernel Drivers.
                             
                            sudo pci_device  0000:b1:00.0 vf 3\n\nsudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci                                                             \nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci \niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188                                                                  \nAssigning /dev/vfio/188 to user                                                                 \nChanging permissions for /dev/vfio/188 to rw-rw----\n\n\nsudo host_exerciser --clock-mhz 400 lpbk\n    starting test run, count of 1\nAPI version: 1\nAFU clock from command line: 400 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Number of clocks: 5224\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1022\n    Bandwidth: 5.018 GB/s\n    Test lpbk(1): PASS\n
                             
                            The following example will run a loopback throughput test using one cache line per request.
                             
                            sudo pci_device  0000:b1:00.0 vf 3\n\nsudo opae.io init -d 0000:b1:00.2 user:user\n\nsudo host_exerciser --clock-mhz 400 --mode trput --cls cl_1 lpbk\n    starting test run, count of 1\nAPI version: 1\nAFU clock from command line: 400 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 512\n    Host Exerciser numWrites: 513\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Number of clocks: 3517\n    Total number of Reads sent: 512\n    Total number of Writes sent: 512\n    Bandwidth: 7.454 GB/s\n    Test lpbk(1): PASS\n
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#4542-traffic-generator-afu-test-application","title":"4.5.4.2 Traffic Generator AFU Test Application","text":"
                            Beginning in OPAE version 2.0.11-1+ the TG AFU has an OPAE application to access & exercise traffic, targeting a specific bank. The supported arguments for test configuration are:
                             
                               
                            • Number of test loops: --loops
                              •  
                            • Number of read transfers per test loop: -r,--read
                              •  
                            • Number of write transfers per test loop: -w,--write
                              •  
                            • Burst size of each transfer: -b,--bls
                              •  
                            • Address stride between each transfer: --stride
                              •  
                            • Target memory TG: -m,--mem-channel
                              •  
                             
                            Below are some example commands for how to execute the test application. To run the preconfigured write/read traffic test on channel 0:
                             
                            mem_tg tg_test\n
                             
                            Target channel 1 with a 1MB single-word write only test for 1000 iterations
                             
                            mem_tg --loops 1000 -r 0 -w 2000 -m 1 tg_test\n
                             
                            Target channel 2 with 4MB write/read test of max burst length for 10 iterations
                             
                            mem_tg --loops 10 -r 8 -w 8 --bls 255 -m 2 tg_test\n
                             
                            sudo mem_tg --loops 1000 -r 2000 -w 2000 --stride 2 --bls 2  -m 1 tg_test\n[2022-07-15 00:13:16.349] [tg_test] [info] starting test run, count of 1\nMemory channel clock frequency unknown. Assuming 300 MHz.\nTG PASS\nMem Clock Cycles: 17565035\nWrite BW: 4.37232 GB/s\nRead BW: 4.37232 GB/s\n
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#4543-he-hssi","title":"4.5.4.3 HE-HSSI","text":"
                            HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic.
                             
                            The hssi application provides a means of interacting with the 10G and with the 100G HSSI AFUs. In both 10G and 100G operating modes, the application initializes the AFU, completes the desired transfer as described by the mode- specific options, and displays the ethernet statistics by invoking ethtool --statistics INTERFACE.
                             
                            Due to Ethernet differential pair routing on the ES version of the Intel Agilex\u00ae 7 F-Series FPGA (Two F-Tiles) Development Kit, some differential pairs were swapped to improve signal routing.  To account for the pair swap, there is a requirement to run a script to invert the differential traces.  If you run the command \u201cfpgainfo phy B:d.f\u201d when the Ethernet ports are connected to known good sources and observe the following three ports are down as shown below:
                             
                            sudo fpgainfo phy b1:00.0\nIntel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** PHY ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x501020241BF165B\nBitstream Version                : 5.0.1\nPr Interface Id                  : b4eda250-cdb7-5891-a06e-13d28d09bc32\nQSFP0                            : Connected\nQSFP1                            : Connected\n//****** HSSI information ******//\nHSSI version                     : 2.0\nNumber of ports                  : 8\nPort0                            :25GbE        UP\nPort1                            :25GbE        UP\nPort2                            :25GbE        UP\nPort3                            :25GbE        DOWN\nPort4                            :25GbE        UP\nPort5                            :25GbE        UP\nPort6                            :25GbE        DOWN\nPort7                            :25GbE        DOWN\n
                             
                            Create the following script called \u201cset_tx_inverse_polarity.sh\u201d to set make Transceiver PAM register settings:
                             
                            #!/bin/sh\n\n# Port 3\nbase_addr=$(printf \"%08d\" \"0x500000\")\naddress=`expr $base_addr + 589884`\n#address=`expr $base_addr + 589884`\noffset=`expr $address/4`\n\nhex_number=$(printf \"0x%06x\" \"$(($offset))\")\necho $hex_number\ncmd_sts=$(printf \"%32x\" \"$(($offset2))\")\ncsraddr=\"${hex_number}0500000002\"\ncsraddr1=\"${hex_number}0600000001\"\ndata=1a040\necho $csraddr\nsudo opae.io poke  0x140b0 0x0001a26500000000\nsleep 1\nsudo opae.io poke  0x140a8 $csraddr\nsleep 1\nsudo opae.io peek  0x140a8\nsleep 1\n\nsudo opae.io poke  0x140b0 0x0001226500000000\nsleep 1\nsudo opae.io poke  0x140a8 $csraddr\nsleep 1\nsudo opae.io peek  0x140a8\n# Port 6\n\nbase_addr=$(printf \"%08d\" \"0xb00000\")\naddress=`expr $base_addr + 589884`\n#address=`expr $base_addr + 589884`\noffset=`expr $address/4`\n\nhex_number=$(printf \"0x%06x\" \"$(($offset))\")\necho $hex_number\ncmd_sts=$(printf \"%32x\" \"$(($offset2))\")\ncsraddr=\"${hex_number}0500000002\"\ncsraddr1=\"${hex_number}0600000001\"\ndata=1a040\necho $csraddr\nsudo opae.io poke  0x140b0 0x0001a16500000000\nsleep 1\nsudo opae.io poke  0x140a8 $csraddr\nsleep 1\nsudo opae.io peek  0x140a8\nsleep 1\n\nsudo opae.io poke  0x140b0 0x0001216500000000\nsleep 1\nsudo opae.io poke  0x140a8 $csraddr\nsleep 1\nsudo opae.io peek  0x140a8\n# Port 7\n\nbase_addr=$(printf \"%08d\" \"0x1100000\")\naddress=`expr $base_addr + 589884`\n#address=`expr $base_addr + 589884`\noffset=`expr $address/4`\n\nhex_number=$(printf \"0x%06x\" \"$(($offset))\")\necho $hex_number\ncmd_sts=$(printf \"%32x\" \"$(($offset2))\")\ncsraddr=\"${hex_number}0500000002\"\ncsraddr1=\"${hex_number}0600000001\"\ndata=1a040\necho $csraddr\nsudo opae.io poke  0x140b0 0x0001a26500000000\nsleep 1\nsudo opae.io poke  0x140a8 $csraddr\nsleep 1\nsudo opae.io peek  0x140a8\nsleep 1\n\nsudo opae.io poke  0x140b0 0x0001226500000000\nsleep 1\nsudo opae.io poke  0x140a8 $csraddr\nsleep 1\nsudo opae.io peek  0x140a8\n
                             
                            The script set_tx_inverse_polarity.sh requires the VFIO driver on PF0 to access the Transceiver registers.  You will use the opae.io command prior to running set_tx_inverse_polarity.sh to bind the VFIO driver.  Once the script completes, release the VFIO driver with opae.io release.  
                             
                            The listing below shows the script being run:
                             
                            sudo opae.io init -d 0000:b1:00.0 $USER\nUnbinding (0x8086,0xbcce) at 0000:b1:00.0 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.0 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.0 is 8\n\nsh set_tx_inverse_polarity.sh\n0x16400f\n0x16400f0500000002\n0x16400f0500000006\n0x16400f0500000006\n0x2e400f\n0x2e400f0500000002\n0x2e400f0500000006\n0x2e400f0500000006\n0x46400f\n0x46400f0500000002\n0x46400f0500000006\n0x46400f0500000006\n\nsudo opae.io release -d 0000:b1:00.0\nReleasing (0x8086,0xbcce) at 0000:b1:00.0 from vfio-pci\nRebinding (0x8086,0xbcce) at 0000:b1:00.0 to dfl-pci\n\nsudo fpgainfo phy b1:00.0\nIntel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** PHY ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x501020241BF165B\nBitstream Version                : 5.0.1\nPr Interface Id                  : 767712e5-b1d0-5777-aea9-592572a6817f\nQSFP0                            : Connected\nQSFP1                            : Connected\n//****** HSSI information ******//\nHSSI version                     : 2.0\nNumber of ports                  : 8\nPort0                            :25GbE        UP\nPort1                            :25GbE        UP\nPort2                            :25GbE        UP\nPort3                            :25GbE        UP\nPort4                            :25GbE        UP\nPort5                            :25GbE        UP\nPort6                            :25GbE        UP\nPort7                            :25GbE        UP\n
                             
                            The following example walks through the process of binding the VF corresponding with the HE-HSSI exerciser to vfio-pci, sending traffic, and verifying that traffic was received.
                            "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-8-accelerator-pfvf-and-guid-mappings","title":"Table 8: Accelerator PF/VF and GUID Mappings","text":"Component VF Accelerator GUID F Series Dev Kit base PF XXXX:XX:XX.0 N/A VirtIO Stub XXXX:XX:XX.1 3e7b60a0-df2d-4850-aa31-f54a3e403501 HE-MEM Stub XXXX:XX:XX.2 56e203e9-864f-49a7-b94b-12284c31e02b Copy Engine XXXX:XX:XX.4 44bfc10d-b42a-44e5-bd42-57dc93ea7f91 HE-MEM XXXX:XX:XX.5 8568ab4e-6ba5-4616-bb65-2a578330a8eb HE-HSSI XXXX:XX:XX.6 823c334c-98bf-11ea-bb37-0242ac130002 MEM-TG XXXX:XX:XX.7 4dadea34-2c78-48cb-a3dc-5b831f5cecbb 
                               
                            1.  
                              Create 3 VFs in the PR region.
                               
                              sudo pci_device b1:00.0 vf 3 \n
                               
                              2.  
                            2.  
                              Verify all 3 VFs were created.
                               
                              lspci -s b1:00 \nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01) \nb1:00.1 Processing accelerators: Intel Corporation Device bcce \nb1:00.2 Processing accelerators: Intel Corporation Device bcce \nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device \nb1:00.4 Processing accelerators: Intel Corporation Device bcce \nb1:00.5 Processing accelerators: Intel Corporation Device bccf \nb1:00.6 Processing accelerators: Intel Corporation Device bccf \nb1:00.7 Processing accelerators: Intel Corporation Device bccf \n
                               
                              3.  
                            3.  
                              Bind all the PF/VF endpoints to the vfio-pci driver.
                               
                              sudo opae.io init -d 0000:b1:00.1 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 187\nAssigning /dev/vfio/187 to user\nChanging permissions for /dev/vfio/187 to rw-rw----\n\nsudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188\nAssigning /dev/vfio/188 to user\nChanging permissions for /dev/vfio/188 to rw-rw----\n\n...\n\nsudo opae.io init -d 0000:b1:00.7 user:user\nBinding (0x8086,0xbccf) at 0000:b1:00.7 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.7 is 319\nAssigning /dev/vfio/319 to user\nChanging permissions for /dev/vfio/319 to rw-rw----\n
                               
                              4.  
                            4.  
                              Check that the accelerators are present using fpgainfo. Note your port configuration may differ from the below.
                               
                              sudo fpgainfo port \n//****** PORT ******//\nObject Id                        : 0xEC00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0xE0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.7\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 4dadea34-2c78-48cb-a3dc-5b831f5cecbb\n//****** PORT ******//\nObject Id                        : 0xC0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.6\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 823c334c-98bf-11ea-bb37-0242ac130002\n//****** PORT ******//\nObject Id                        : 0xA0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.5\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n//****** PORT ******//\nObject Id                        : 0x80B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.4\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 44bfc10d-b42a-44e5-bd42-57dc93ea7f91\n//****** PORT ******//\nObject Id                        : 0x40B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n//****** PORT ******//\nObject Id                        : 0x20B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 3e7b60a0-df2d-4850-aa31-f54a3e403501\n
                               
                              5.  
                            5.  
                              Check Ethernet PHY settings with fpgainfo.
                               
                              sudo fpgainfo phy -B 0xb1 \nIIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\n//****** HSSI information ******//\nHSSI version                     : 1.0\nNumber of ports                  : 8\nPort0                            :25GbE        DOWN\nPort1                            :25GbE        DOWN\nPort2                            :25GbE        DOWN\nPort3                            :25GbE        DOWN\nPort4                            :25GbE        DOWN\nPort5                            :25GbE        DOWN\nPort6                            :25GbE        DOWN\nPort7                            :25GbE        DOWN\n
                               
                              6.  
                            6.  
                              Set loopback mode.
                               
                              sudo hssiloopback --loopback enable  --pcie-address 0000:b1:00.0 \nargs Namespace(loopback='enable', pcie_address='0000:b1:00.0', port=0)\nsbdf: 0000:b1:00.0\nFPGA dev: {'segment': 0, 'bus': 177, 'dev': 0, 'func': 0, 'path': '/sys/class/fpga_region/region0', 'pcie_address': '0000:b1:00.0'}\nargs.hssi_grps{0: ['dfl_dev.6', ['/sys/bus/pci/devices/0000:b1:00.0/fpga_region/region0/dfl-fme.0/dfl_dev.6/uio/uio0']]}\nfpga uio dev:dfl_dev.6\n\n--------HSSI INFO START-------\nDFH                     :0x3000000010002015\nHSSI ID                 :0x15\nDFHv                    :0.5\nguidl                   :0x99a078ad18418b9d\nguidh                   :0x4118a7cbd9db4a9b\nHSSI version            :1.0\nFirmware Version        :1\nHSSI num ports          :8\nPort0                   :25GbE\nPort1                   :25GbE\nPort2                   :25GbE\nPort3                   :25GbE\nPort4                   :25GbE\nPort5                   :25GbE\nPort6                   :25GbE\nPort7                   :25GbE\n--------HSSI INFO END-------\n\nhssi loopback enabled to port0\n
                               
                              7.  
                            7.  
                              Send traffic through the 10G AFU.
                               
                              sudo hssi --pci-address b1:00.6 hssi_10g --num-packets 100       \n10G loopback test\n  port: 0\n  eth_loopback: on\n  he_loopback: none\n  num_packets: 100\n  packet_length: 64\n  src_address: 11:22:33:44:55:66\n    (bits): 0x665544332211\n  dest_address: 77:88:99:aa:bb:cc\n    (bits): 0xccbbaa998877\n  random_length: fixed\n  random_payload: incremental\n  rnd_seed0: 5eed0000\n  rnd_seed1: 5eed0001\n  rnd_seed2: 25eed\n  eth:\n\nNo eth interface, so not honoring --eth-loopback.\n0x40000           ETH_AFU_DFH: 0x1000010000001000\n0x40008          ETH_AFU_ID_L: 0xbb370242ac130002\n0x40010          ETH_AFU_ID_H: 0x823c334c98bf11ea\n0x40030      TRAFFIC_CTRL_CMD: 0x0000000000000000\n0x40038     TRAFFIC_CTRL_DATA: 0x0000000100000000\n0x40040 TRAFFIC_CTRL_PORT_SEL: 0x0000000000000000\n0x40048        AFU_SCRATCHPAD: 0x0000000045324511\n\n0x3c00         number_packets: 0x00000064\n0x3c01          random_length: 0x00000000\n0x3c02         random_payload: 0x00000000\n0x3c03                  start: 0x00000000\n0x3c04                   stop: 0x00000000\n0x3c05           source_addr0: 0x44332211\n0x3c06           source_addr1: 0x00006655\n0x3c07             dest_addr0: 0xaa998877\n0x3c08             dest_addr1: 0x0000ccbb\n0x3c09        packet_tx_count: 0x00000064\n0x3c0a              rnd_seed0: 0x5eed0000\n0x3c0b              rnd_seed1: 0x5eed0001\n0x3c0c              rnd_seed2: 0x00025eed\n0x3c0d             pkt_length: 0x00000040\n0x3cf4          tx_end_tstamp: 0x000003d2\n0x3d00                num_pkt: 0xffffffff\n0x3d01               pkt_good: 0x00000064\n0x3d02                pkt_bad: 0x00000000\n0x3d07            avst_rx_err: 0x00000000\n0x3d0b          rx_sta_tstamp: 0x00000103\n0x3d0c          rx_end_tstamp: 0x0000053b\n0x3e00               mac_loop: 0x00000000\n\nHSSI performance:\n        Selected clock frequency : 402.832 MHz\n        Latency minimum : 642.948 ns\n        Latency maximum : 896.155 ns\n        Achieved Tx throughput : 18.4528 GB/s\n        Achieved Rx throughput : 16.7101 GB/s\n\nNo eth interface, so not showing stats.\n
                               
                              8.  
                             
                            The hssi_loopback utility works in conjunction with a packet generator accelerator function unit (AFU) to test high-speed serial interface (HSSI) cards. hssi_loopback tests both external and internal loopbacks.
                             
                            The hssistats tool provides the MAC statistics.
                            "},{"location":"hw/iseries_devkit/","title":"I-Series (2xR-tile and 1xF-Tile) Development Kit Collateral for OFS","text":"
                            This folder contains applicable collateral for OFS PCIe Attach reference shell targeting the I-Series (2xR-tile and 1xF-Tile) Development Kit DK-DEV-AGI027RBES.
                            "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/","title":"FPGA Interface Manager Developer Guide for Open FPGA Stack: Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) PCIe Attach","text":"
                            Last updated: February 03, 2024 
                            "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1-introduction","title":"1. Introduction","text":""},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#11-about-this-document","title":"1.1. About This Document","text":"
                            This document serves as a guide for OFS Agilex PCIe Attach developers targeting the Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile). The following topics are covered in this guide:
                             
                               
                            • Compiling the OFS Agilex PCIe Attach FIM design
                              •  
                            • Simulating the OFS Agilex PCIe Attach design
                              •  
                            • Customizing the OFS Agilex PCIe Attach FIM design
                              •  
                            • Configuring the FPGA with an OFS Agilex PCIe Attach FIM design
                              •  
                             
                            The FIM Development Walkthroughs Table lists all of the walkthroughs provided in this guide. These walkthroughs provide step-by-step instructions for performing different FIM Development tasks.
                             
                            Table: FIM Development Walkthroughs
                             Walkthrough Name Category Install Quartus Prime Pro Software Setup Install Git Large File Storage Extension Setup Clone FIM Repository Setup Set Development Environment Variables Setup Set Up Development Environment Setup Compile OFS FIM Compilation Manually Generate OFS Out-Of-Tree PR FIM Compilation Change the Compilation Seed Compilation Running Individual Unit Level Simulation Simulation Running Regression Unit Level Simulation Simulation Add a new module to the OFS FIM Customization Modify and run unit tests for a FIM that has a new module Customization Hardware test a FIM that has a new module Customization Debug the FIM with Signal Tap Customization Compile the FIM in preparation for designing your AFU Customization Resize the Partial Reconfiguration Region Customization Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS Customization Modify PCIe Configuration Using IP Presets Customization Migrate to a Different Agilex Device Number Customization Modify the Ethernet Sub-System to 1x400GbE Customization Set up JTAG Configuration Program the FPGA via JTAG Configuration"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#111-knowledge-pre-requisites","title":"1.1.1 Knowledge Pre-Requisites","text":"
                            It is recommended that you have the following knowledge and skills before using this developer guide.
                             
                               
                            • Basic understanding of OFS and the difference between OFS designs. Refer to the OFS Welcome Page.
                              •  
                            • Review the release notes for the Intel Agilex 7 PCIe Attach Reference Shells, with careful consideration of the Known Issues.
                              •  
                            • Review of Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile))
                              •  
                            • FPGA compilation flows using Intel\u00ae Quartus\u00ae Prime Pro Edition.
                              •  
                            • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
                              •  
                            • RTL (System Verilog) and coding practices to create synthesized logic.
                              •  
                            • RTL simulation tools.
                              •  
                            • Intel\u00ae Quartus\u00ae Prime Pro Edition Signal Tap Logic Analyzer tool software.
                              •  
                            "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#12-fim-development-theory","title":"1.2. FIM Development Theory","text":"
                            This section will help you understand how the OFS Agilex PCIe Attach FIM can be developed to fit your design goals.
                             
                            The Default FIM Features section provides general information about the default features of the OFS Agilex PCIe Attach FIM so you can become familiar with the default design. For more detailed information about the FIM architecture, refer to the OFS Agilex PCIe Attach FIM Technical Reference Manual.
                             
                            The Customization Options section then gives suggestions of how this default design can be customized. Step-by-step walkthroughs for many of the suggested customizations are later described in the FIM Customization section.
                             
                            FIM development for a new acceleration card generally consists of the following steps:
                             
                               
                            1. Install OFS and familiarize yourself with provided scripts and source code
                              2.  
                            2. Develop high level design with your specific functionality
                                 
                              1. Determine requirements and key performance metrics
                                2.  
                              2. Select IP cores
                                3.  
                              3. Select FPGA device
                                4.  
                              4. Develop software memory map
                                5.  
                               
                              3.  
                            3. Select and implement FIM Physical interfaces including:
                                 
                              1. External clock sources and creation of internal PLL clocks
                                2.  
                              2. General I/O
                                3.  
                              3. Ethernet modules
                                4.  
                              4. External memories
                                5.  
                              5. FPGA programming methodology
                                6.  
                               
                              4.  
                            4. Develop device physical implementation
                                 
                              1. FPGA device pin assignment
                                2.  
                              2. Create logic lock regions
                                3.  
                              3. Create of timing constraints
                                4.  
                              4. Create Intel Quartus Prime Pro FIM test project and validate:
                                   
                                1. Placement
                                  2.  
                                2. Timing constraints
                                  3.  
                                3. Build script process
                                  4.  
                                4. Review test FIM FPGA resource usage
                                  5.  
                                 
                                5.  
                               
                              5.  
                            5. Select FIM to AFU interfaces and development of PIM
                              6.  
                            6. Implement FIM design
                                 
                              1. Develop RTL
                                2.  
                              2. Instantiate IPs
                                3.  
                              3. Develop test AFU to validate FIM
                                4.  
                              4. Develop unit and device level simulation
                                5.  
                              5. Develop timing constraints and build scripts
                                6.  
                              6. Perform timing closure and build validation
                                7.  
                               
                              7.  
                            7. Create FIM documentation to support AFU development and synthesis
                              8.  
                            8. Software Device Feature discovery
                              9.  
                            9. Integrate, validate, and debug hardware/software
                              10.  
                            10. Prepare for high volume production
                              11.  
                            "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#121-default-fim-features","title":"1.2.1 Default FIM Features","text":""},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1211-top-level","title":"1.2.1.1 Top Level","text":"
                            Figure: OFS Agilex PCIe Attach iseries-dk FIM Top-Level Diagram
                             
                            "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1212-interfaces","title":"1.2.1.2 Interfaces","text":"
                            The key interfaces in the OFS Agilex PCIe Attach design are listed in the Release Capabilities Table. It describes the capabilities of the iseries-dk hardware as well as the capabilities of the default OFS Agilex PCIe Attach design targeting the iseries-dk.
                             
                            Table: Release Capabilities
                             Interface iseries-dk Hardware Capabilities OFS Agilex PCIe Attach Default Design Implementation Host Interface[1] PCIe Gen5x16 Bifurcated PCIe Gen5x8 (Only 1 PCIe interface implemented) Network Interface 2 - QSFP-DD 3 Build Options: 1. QSFP 1,0 = 25 GbE2. QSFP 1,0 = 200 GbE  3. QSFP 0 = 400 GbE External Memory 2 - board mounted independent single rank DDR4-2666 8GB (1 Gb x 64 + 8b ECC)Two DIMM sockets where each socket is single memory channels or independent channels (Check Dev Kit OPN for support option).DIMM socket supports DDR4 x72 (ECC) and can operate up to DDR-3200 (depending on the speed of the FPGA used) 2xDDR4-2666 - 8GB (1Gb x 64 bits) - ECC not implemented 
                            [1] The I-Series development kit has a form factor of PCIe x16, however in this release the PCIe Subsystem only supports bifurcated PCIe Gen5 x 8.  The FIM only connects 1 of the 2 PCIe links.  Future releases will support Gen5 x 16.
                            "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1213-subsystems","title":"1.2.1.3 Subsystems","text":"
                            The FIM Subsystems Table  describes the Platform Designer IP subsystems used in the OFS Agilex PCIe Attach iseries-dk FIM.
                             
                            Table: FIM Subsystems
                             Subsystem User Guide Document ID PCIe Subsystem PCIe Subsystem Intel FPGA IP User Guide for Intel Agilex OFS N/A Memory Subsystem Memory Subsystem Intel FPGA IP User Guide for Intel Agilex OFS 686148[1] Ethernet Subsystem Ethernet Subsystem Intel FPGA IP User Guide 773413[1] 
                            [1] You must log in to myIntel and request entitled access.
                            "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1214-host-exercisers","title":"1.2.1.4 Host Exercisers","text":"
                            The default AFU workload in the OFS Agilex PCIe Attach iseries-dk FIM contains several modules called Host Exercisers which are used to exercise the interfaces on the board. The Host Exerciser Descriptions Table describes these modules.
                             
                            Table: Host Exerciser Descriptions
                             Name Acronym Description OPAE Command Host Exerciser Loopback HE-LB Used to exercise and characterize host to FPGA data transfer. host_exerciser Host Exerciser Memory HE_MEM Used to exercise and characterize host to Memory data transfer. host_exerciser Host Exerciser Memory Traffic Generator HE_MEM_TG Used to exercise and test available memory channels with a configurable traffic pattern. mem_tg Host Exerciser High Speed Serial Interface HE-HSSI Used to exercise and characterize HSSI interfaces. hssi 
                            The host exercisers can be removed from the design at compile-time using command line arguments for the build script.
                            "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1215-module-access-via-apfbpf","title":"1.2.1.5 Module Access via APF/BPF","text":"
                            The OFS Agilex PCIe Attach iseries-dk FIM uses AXI4-Lite interconnect logic named the AFU Peripheral Fabric (APF) and Board Peripheral Fabric (BPF) to access the registers of the various modules in the design. The APF/BPF modules define master/slave interactions, namely between the host software and AFU and board peripherals. The APF Address Map Table describes the address mapping of the APF, followed by the BPF Address Map Table which describes the address mapping of the BPF.
                             
                            Table: APF Address Map
                             Address Size (Bytes) Feature 0x00000\u20130x3FFFF 256K Board Peripherals (See BPF Address Map table) 0x40000 \u2013 0x4FFFF 64K ST2MM 0x50000 \u2013 0x5FFFF 64K Reserved 0x60000 \u2013 0x60FFF 4K UART (not used) 0x61000 \u2013 0x6FFFF 4K Reserved 0x70000 \u2013 0x7FFFF 56K PR Gasket:4K= PR Gasket DFH, control and status4K= Port DFH4K=User Clock52K=Remote STP 0x80000 \u2013 0x80FFF 4K AFU Error Reporting 
                            Table: BPF Address Mapping
                             Address Size (Bytes) Feature 0x00000 - 0x0FFFF 64K FME 0x10000 - 0x10FFF 4K PCIe 0x11000 - 0x11FFF 4K Reserved 0x12000 - 0x12FFF 4K QSFP0 0x13000 - 0x13FFF 4K QSFP1 0x14000 - 0x14FFF 4K HSSI 0x15000 - 0x15FFF 4K EMIF 0x20000 - 0x3FFFF 128K PMCI (note, PMCI is not implemented)"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#122-customization-options","title":"1.2.2 Customization Options","text":"
                            OFS is designed to be easily customizable to meet your design needs. The OFS FIM Customizations Table lists the general user flows for OFS Agilex PCIe Attach iseries-dk FIM development, along with example customizations for each user flow, plus links to step-by-step walkthroughs where available.
                             
                            Table: OFS FIM Customizations
                             Customization Walkthrough Name Add a new module to the OFS FIM Modify and run unit tests for a FIM that has a new module Hardware test a FIM that has a new module Debug the FIM with Signal Tap Compile the FIM in preparation for designing your AFU Resize the Partial Reconfiguration Region Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS Modify PCIe Configuration Using IP Presets Migrate to a Different Agilex Device Number Modify the Ethernet Sub-System to 1x400GbE"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#13-development-environment","title":"1.3 Development Environment","text":"
                            This section describes the components required for OFS FIM development, and provides a walkthrough for setting up the environment on your development machine.
                             
                            Note that your development machine may be different than your deployment machine where the FPGA acceleration card is installed. FPGA development work and deployment work can be performed either on the same machine, or on different machines as desired. Please see the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up the environment for deployment machines.
                            "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#131-development-tools","title":"1.3.1 Development Tools","text":"
                            The Development Environment Table describes the Best Known Configuration (BKC) for the tools that are required for OFS FIM development.
                             
                            Table: Development Environment BKC
                             Component Version Installation Walkthrough Operating System RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6 N/A Intel Quartus Prime Software Quartus Prime Pro Version 23.3 for Linux + Patches 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe) Section 1.3.1.1 Python 3.6.8 or later N/A GCC 7.4.0 or later N/A cmake 3.15 or later N/A git with git-lfs 1.8.3.1 or later Section 1.3.1.2 FIM Source Files ofs-2023.3-2 Section 1.3.2.1"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1311-walkthrough-install-quartus-prime-pro-software","title":"1.3.1.1 Walkthrough: Install Quartus Prime Pro Software","text":"
                            Intel Quartus Prime Pro Version 23.3 is verified to work with the latest OFS release ofs-2023.3-2.  However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.
                             
                            Use RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6 for compatibility with your development flow and also testing your FIM design in your platform. 
                             
                            Prior to installing Quartus:
                             
                               
                            1.  
                              Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.
                               
                                 
                              • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                                •  
                              • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.
                                •  
                               
                              2.  
                            2.  
                              Perform the following steps to satisfy the required dependencies.
                               
                              $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                               
                              Apply the following configurations.
                               
                              $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n
                               
                              3.  
                            3.  
                              Create the default installation path: /intelFPGA_pro/, where  is the default path of the Linux workstation, or as set by the system administrator and  is your Quartus version number. 
                              The installation path must satisfy the following requirements:
                               
                                 
                              • Contain only alphanumeric characters
                                •  
                              • No special characters or symbols, such as !$%@^&*<>,
                                •  
                              • Only English characters
                                •  
                              • No spaces
                                •  
                               
                            4.  
                              Download your required Quartus Prime Pro Linux version here.
                               
                              5.  
                            5.  
                              Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe).
                               
                              6.  
                            6.  
                              After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                               
                              export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                               
                              For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:
                               
                              export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n
                               
                              7.  
                            7.  
                              Verify, Quartus is discoverable by opening a new shell:
                               
                              $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                               
                              8. "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1312-walkthrough-install-git-large-file-storage-extension","title":"1.3.1.2 Walkthrough: Install Git Large File Storage Extension","text":"
                              To install the Git Large File Storage (LFS) extension, execute the following commands:
                               
                                 
                              1. Obtain Git LFS package     
                                curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\n
                                2.  
                              2. Install Git LFS package     
                                sudo dnf install git-lfs\n
                                3.  
                              3. Install Git LFS     
                                git lfs install\n
                                4.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#132-fim-source-files","title":"1.3.2 FIM Source Files","text":"
                              The source files for the OFS Agilex PCIe Attach FIM are provided in the following repository: https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2
                               
                              Some essential directories in the repository are described as follows:
                               
                              ofs-agx7-pcie-attach\n|  syn                      // Contains files related to synthesis\n|  |  board                     // Contains synthesis files for several cards, including the iseries-dk \n|  |  |  iseries-dk                 // Contains synthesis files for iseries-dk\n|  |  |  |  setup                       // Contains setup files, including pin constraints and location constraints\n|  |  |  |  syn_top                     // Contains Quartus project files\n|  verification             // Contains files for UVM testing\n|  ipss                     // Contains files for IP Sub-Systems\n|  |  qsfp                      // Contains source files for QSFP Sub-System\n|  |  hssi                      // Contains source files for HSSI Sub-System\n|  |  pmci                      // Contains source files for PMCI Sub-System (not used in F-Tile FIM)\n|  |  pcie                      // Contains source files for PCIe Sub-System\n|  |  mem                       // Contains source files for Memory Sub-System\n|  sim                      // Contains simulation files\n|  |  unit_test                 // Contains files for all unit tests\n|  |  |  scripts                    // Contains script to run regression unit tests\n|  license                  // Contains Quartus patch\n|  ofs-common               // Contains files which are common across OFS platforms\n|  |  verification              // Contains common UVM files\n|  |  scripts                   // Contains common scripts\n|  |  |  common\n|  |  |  |  syn                         // Contains common scripts for synthesis, including build script\n|  |  |  |  sim                         // Contains common scripts for simulation\n|  |  tools                     // Contains common tools files\n|  |  |  mk_csr_module              // Contains common files for CSR modules\n|  |  |  fabric_generation          // Contains common files for APF/BPF fabric generation\n|  |  |  ofss_config                // Contains common files for OFSS configuration tool\n|  |  |  |  ip_params                   // Contains default IP parameters for certain Sub-Systems when using OFSS\n|  |  src                       // Contains common source files, including host exercisers\n|  tools                    //\n|  |  ofss_config               // Contains top level OFSS files for each pre-made board configuration\n|  |  |  hssi                       // Contains OFSS files for Ethernet-SS configuraiton\n|  |  |  memory                     // Contains OFSS files for Memory-SS configuration\n|  |  |  pcie                       // Contains OFSS files for PCIe-SS configuration\n|  |  |  iopll                      // Contains OFSS files for IOPLL configuration\n|  src                      // Contains source files for Agilex PCIe Attach FIM\n|  |  pd_qsys                   // Contains source files related to APF/BPF fabric\n|  |  includes                  // Contains source file header files\n|  |  top                       // Contains top-level source files, including design top module\n|  |  afu_top                   // Contains top-level source files for AFU\n
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1321-walkthrough-clone-fim-repository","title":"1.3.2.1 Walkthrough: Clone FIM Repository","text":"
                              Perform the following steps to clone the OFS Agilex PCIe Attach FIM Repository:
                               
                                 
                              1. Create a new directory to use as a clean starting point to store the retrieved files.     
                                mkdir OFS_BUILD_ROOT\ncd OFS_BUILD_ROOT\nexport OFS_BUILD_ROOT=$PWD\n
                                2.  
                              2. Clone GitHub repository using the HTTPS git method     
                                git clone --recurse-submodules https://github.com/OFS/ofs-agx7-pcie-attach.git\n
                                3.  
                              3. Check out the correct tag of the repository     
                                cd ofs-agx7-pcie-attach\ngit checkout --recurse-submodules tags/ofs-2023.3-2\n
                                4.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#133-environment-variables","title":"1.3.3 Environment Variables","text":"
                              The OFS FIM compilation and simulation scripts require certain environment variables be set prior to execution.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1331-walkthrough-set-development-environment-variables","title":"1.3.3.1 Walkthrough: Set Development Environment Variables","text":"
                              Perform the following steps to set the required environment variables. These environment variables must be set prior to simulation or compilation tasks so it is recommended that you create a script to set these variables.
                               
                                 
                              1.  
                                Navigate to the top level directory of the cloned OFS FIM repository.
                                 
                                cd ofs-agx7-pcie-attach\n
                                 
                                2.  
                              2.  
                                Set project variables   
                                # Set OFS Root Directory - e.g. this is the top level directory of the cloned OFS FIM repository\nexport OFS_ROOTDIR=$PWD\n
                                 
                                3.  
                              3.  
                                Set variables based on your development environment   
                                # Set proxies if required for your server\nexport http_proxy=<YOUR_HTTP_PROXY>\nexport https_proxy=<YOUR_HTTPS_PROXY>\nexport ftp_proxy=<YOUR_FTP_PROXY>\nexport socks_proxy=<YOUR_SOCKS_PROXY>\nexport no_proxy=<YOUR_NO_PROXY>\n\n# Set Quartus license path\nexport LM_LICENSE_FILE=<YOUR_LM_LICENSE_FILE>\n\n# Set Synopsys License path (if using Synopsys for simulation)\nexport DW_LICENSE_FILE=<YOUR_DW_LICENSE_FILE>\nexport SNPSLMD_LICENSE_FILE=<YOUR_SNPSLMD_LICENSE_FILE>\n\n# Set Quartus Installation Directory - e.g. $QUARTUS_ROOTDIR/bin contains Quartus executables\nexport QUARTUS_ROOTDIR=<YOUR_QUARTUS_INSTALLATION_DIRECTORY>\n\n# Set the Tools Directory - e.g. $TOOLS_LOCATION contains the 'synopsys' directory if you are using Synopsys. Refer to the $VCS_HOME variable for an example.\nexport TOOLS_LOCATION=<YOUR_TOOLS_LOCATION>\n
                                 
                                4.  
                              4.  
                                Set generic environment variables
                                 
                                # Set Work directory \nexport WORKDIR=$OFS_ROOTDIR\n\n# Set Quartus Tools variables\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport QUARTUS_VER_AC=$QUARTUS_ROOTDIR\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport IMPORT_IP_ROOTDIR=$IP_ROOTDIR\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\n\n# Set Verification Tools variables (if running simulations)\nexport DESIGNWARE_HOME=$TOOLS_LOCATION/synopsys/vip_common/vip_Q-2020.03A\nexport UVM_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport MTI_HOME=$QUARTUS_ROOTDIR/../questa_fse\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n\n# Set PATH to include compilation and simulation tools\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/../qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/bin:$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$DESIGNWARE_HOME/bin:$VCS_HOME/bin:$PATH\n
                                 
                                5.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#134-walkthrough-set-up-development-environment","title":"1.3.4 Walkthrough: Set Up Development Environment","text":"
                              This walkthrough guides you through the process of setting up your development environment in preparation for FIM development. This flow only needs to be done once on your development machine.
                               
                                 
                              1.  
                                Ensure that Quartus Prime Pro Version 23.3 for Linux with Intel Agilex FPGA device support is installed on your development machine. Refer to the Install Quartus Prime Pro Software section for step-by-step installation instructions.
                                 
                                   
                                1. Verify version number
                                  quartus_sh --version\n
                                   
                                  Example Output:
                                   
                                  Quartus Prime Shell\nVersion 23.3 Build 94 06/14/2023 SC Pro Edition\nCopyright (C) 2023  Intel Corporation. All rights reserved.\n
                                   
                                  2.  
                                 
                                2.  
                              2.  
                                Ensure that all support tools are installed on your development machine, and that they meet the version requirements.
                                 
                                   
                                1.  
                                  Python 3.6.8 or later
                                   
                                     
                                  1.  
                                    Verify version number
                                     
                                    python --version\n
                                     
                                    Example Output:
                                     
                                    Python 3.6.8\n
                                     
                                    2.  
                                   
                                  2.  
                                2.  
                                  GCC 7.4.0 or later
                                   
                                     
                                  1.  
                                    Verify version number
                                     
                                    gcc --version\n
                                     
                                    Example output:
                                     
                                    gcc (GCC) 7.4.0\n
                                     
                                    2.  
                                   
                                  3.  
                                3.  
                                  cmake 3.15 or later
                                   
                                     
                                  1.  
                                    Verify version number
                                     
                                    cmake --version\n
                                     
                                    Example output:
                                     
                                    cmake version 3.15\n
                                     
                                    2.  
                                   
                                  4.  
                                4.  
                                  git with git-lfs 1.8.3.1 or later. Refer to the Install Git Large File Storage Extension section for step-by-step instructions on installing the Git Large File Storage (LFS) extension.
                                   
                                     
                                  1.  
                                    Verify version number
                                     
                                    git --version\n
                                     
                                    Example output:
                                     
                                    git version 1.8.3.1\n
                                     
                                    2.  
                                   
                                  5.  
                                 
                                3.  
                              3.  
                                Clone the ofs-agx7-pcie-attach repository. Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                4.  
                              4.  
                                Install UART IP license patch .02.
                                 
                                   
                                1.  
                                  Navigate to the license directory
                                   
                                  cd $IOFS_BUILD_ROOT/license\n
                                   
                                  2.  
                                2.  
                                  Install Patch 0.02
                                   
                                  sudo ./quartus-0.0-0.02iofs-linux.run\n
                                   
                                  3.  
                                 
                                5.  
                              5.  
                                Install Quartus Patches 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe). All required patches are provided in the Assets of the OFS FIM Release Tag: https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2
                                 
                                6.  
                              6.  
                                Verify that patches have been installed correctly. They should be listed in the output of the following command.
                                 
                                quartus_sh --version\n
                                 
                                7.  
                              7.  
                                Set required environment variables. Refer to the [Set Environment Variables] section for step-by-step instructions.
                                 
                                8.  
                               
                              This concludes the walkthrough for setting up your development environment. At this point you are ready to begin FIM development.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2-fim-compilation","title":"2. FIM Compilation","text":"
                              This section describes the process of compiling OFS FIM designs using the provided build scripts. It contains two main sections:
                               
                                 
                              • Compilation Theory - Describes the theory behind FIM compilation
                                •  
                              • Compilation Flows - Describes the process of compiling a FIM
                                •  
                               
                              The walkthroughs provided in this section are:
                               
                                 
                              • Compile OFS FIM
                                •  
                              • Manually Generate OFS Out-Of-Tree PR FIM
                                •  
                              • Change the Compilation Seed
                                •  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#21-compilation-theory","title":"2.1 Compilation Theory","text":"
                              This section describes the theory behind FIM compilation.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#211-fim-build-script","title":"2.1.1 FIM Build Script","text":"
                              The OFS Common Repository contains a script named build_top.sh which is used to build OFS FIM designs and generate output files that can be programmed to the board. After cloning the OFS FIM repository (with the ofs-common repository included), the build script can be found in the following location:
                               
                              $OFS_ROOTDIR/ofs-common/scripts/common/syn/build_top.sh\n
                               
                              The usage of the build_top.sh script is as follows:
                               
                              build_top.sh [-k] [-p] [-e] [--stage=<action>] [--ofss=<ip_config>] <build_target>[:<fim_options>] [<work_dir_name>]\n
                               Field Options Description Requirement -k None Keep. Preserves and rebuilds within an existing work tree instead of overwriting it. Optional -p None When set, and if the FIM supports partial reconfiguration, a PR template tree is generated at the end of the FIM build. The PR template tree is located in the top of the work directory but is relocatable and uses only relative paths. See $OFS_ROOTDIR/syn/common/scripts generate_pr_release.sh for details. Optional -e None Run only Quartus analysis and elaboration. It completes the setup stage, passes -end synthesis to the Quartus compilation flow and exits without running the finish stage. Optional --stage all | setup | compile | finish Controls which portion of the OFS build is run.\u00a0\u00a0- all: Run all build stages (default)\u00a0\u00a0- setup: Initialize a project in the work directory\u00a0\u00a0- compile: Run the Quartus compilation flow on a project that was already initialized with setup\u00a0\u00a0- finish: Complete OFS post-compilation tasks, such as generating flash images and, if -p is set, generating a release. Optional --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. This parameter is consumed during the setup stage and IP is updated only inside the work tree. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. Optional <build_target> iseries-dk Specifies which board is being targeted. Required <fim_options> flat | null_he_lb | null_he_hssi | null_he_mem | null_he_mem_tg | no_hssi Used to change how the FIM is built.\u00a0\u00a0\u2022 flat - Compiles a flat design (no PR assignments). This is useful for bringing up the design on a new board without dealing with PR complexity.\u00a0\u00a0\u2022 null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null.\u00a0\u00a0\u2022 null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null.\u00a0\u00a0\u2022 null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null.\u00a0\u00a0\u2022 null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null.\u00a0\u00a0\u2022 no_hssi - Removes the HSSI-SS from the design. More than one FIM option may be passed included in the <fim_options> list by concatenating them separated by commas. For example: <build_target>:flat,null_he_lb,null_he_hssi Optional <work_dir_name> String Specifies the name of the work directory in which the FIM will be built. If not specified, the default target is $OFS_ROOTDIR/work Optional 
                              Note: The he_null is a minimal block with CSRs that responds to PCIe MMIO requests in order to keep PCIe alive.
                               
                              The build script copies source files from the existing cloned repository into the specified work directory, which are then used for compilation. As such, any changes made in the base source files will be included in all subsequent builds, unless the -k option is used, in which case an existing work directories files are used as-is. Likewise, any changes made in a work directory is only applied to that work directory, and will not be updated in the base repository by default.
                               
                              Refer to Compile OFS FIM which provides step-by-step instructions for running the build_top.sh script with some of the different available options.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#212-ofss-file-usage","title":"2.1.2 OFSS File Usage","text":"
                              The OFS FIM build script can use OFSS files to easily customize design IP prior to compilation using preset configurations. The OFSS files specify certain parameters for different IPs. Pre-made OFSS files can be found in the $OFS_ROOTDIR/tools/ofss_config directory. Using OFSS is provided as a convenience tool for building pre-defined FIMs.
                               
                              Table: Provided OFSS Files
                               OFSS File Name Location Type Description iseries-dk.ofss $OFS_ROOTDIR/tools/ofss_config Top Includes the following OFSS files [1]: \u00a0\u00a0\u2022 iseries-dk_base.ofss \u00a0\u00a0\u2022 pcie_host.ofss \u00a0\u00a0\u2022 iopll.ofss \u00a0\u00a0\u2022 memory_ftile.ofss iseries-dk_base.ofss $OFS_ROOTDIR/tools/ofss_config ofs Defines certain attributes of the design, including the platform name, device family, fim type, part number, and device ID. pcie_host_rtile.ofss $OFS_ROOTDIR/tools/ofss_config/pcie pcie Defines the PCIe Subsystem with the following configuration:\u00a0\u00a0\u2022 PF0 (3 VFs)\u00a0\u00a0\u2022 PF1 (0 VFs)\u00a0\u00a0\u2022 PF2 (0 VFs)\u00a0\u00a0\u2022 PF3 (0 VFs)\u00a0\u00a0\u2022 PF4 (0 VFs) iopll.ofss $OFS_ROOTDIR/tools/ofss_config/iopll iopll Sets the IOPLL frequency to 470 MHz memory_rtile.ofss $OFS_ROOTDIR/tools/ofss_config/memory memory Defines the memory IP preset file to be used during the build as iseries-dk hssi_8x25_ftile.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP preset file to be used during the build as fseries-dk hssi_2x200_ftile.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP preset file to be used during the build as 200g-fseries-dk hssi_1x400_ftile.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP preset file to be used during the build as 400g-fseries-dk 
                              [1] The iseries-dk.ofss file does not include an HSSI OFSS file by default. If you are using the iseries-dk.ofss file when building the FIM, you must also specify an HSSI OFSS file for F-tile. Refer to the Compile OFS FIM Section for examples of this flow.
                               
                              Note: Using OFSS is required for FIM builds targeting the I-Series Development Kit.
                               
                              There are typically three sections contained within an OFSS file.
                               
                                 
                              •  
                                [include]
                                 
                                   
                                • This section of an OFSS file contains elements separated by a newline, where each element is the path to an OFSS file that is to be included for configuration by the OFSS Configuration Tool. Ensure that any environment variables (e.g. $OFS_ROOTDIR) is set correctly. The OFSS Config tool uses breadth first search to include all of the specified OFSS files; the ordering of OFSS files does not matter
                                  •  
                                 
                                •  
                               
                                 
                              •  
                                [ip]
                                 
                                   
                                • This section of an OFSS file contains a key value pair that allows the OFSS Config tool to determine which IP configuration is being passed in. The currently supported values of IP are ofs, iopll, pcie, memory, and hssi.
                                  •  
                                 
                                •  
                               
                                 
                              •  
                                [settings]
                                 
                                   
                                • This section of an OFSS file contains IP specific settings. Refer to an existing IP OFSS file to see what IP settings are set. For the IP type `ofss``, the settings will be information of the OFS device (platform, family, fim, part #, device_id)
                                  •  
                                 
                                •  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2121-platform-ofss-file","title":"2.1.2.1 Platform OFSS File","text":"
                              The <platform>.ofss file (e.g. $OFS_ROOTDIR/tools/ofss_config/iseries-dk.ofss) is the platform level OFSS wrapper file. This is typically the OFSS file that is provided to the build script. It only contains an include section which lists all other OFSS files that are to be used when the <platform>.ofss file is passed to the build script.
                               
                              The generic structure of a <platform>.ofss file is as follows:
                               
                              [include]\n<PATH_TO_PLATFORM_BASE_OFSS_FILE>\n<PATH_TO_PCIE_OFSS_FILE>\n<PATH_TO_IOPLL_OFSS_FILE>\n<PATH_TO_MEMORY_OFSS_FILE>\n<PATH_TO_HSSI_OFSS_FILE>\n
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2122-ofs-ip-ofss-file","title":"2.1.2.2 OFS IP OFSS File","text":"
                              An OFSS file with IP type ofs (e.g. $OFS_ROOTDIR/tools/ofss_config/iseries-dk_base.ofss) contains board specific information for the target board.
                               
                              Currently supported configuration options for an OFSS file with IP type ofs are described in the OFS IP OFSS File Options table.
                               
                              Table: OFS IP OFSS File Options
                               Section Parameter iseries-dk Default Value [ip] type ofs [settings] platform N6001 family agilex fim base_x16 part AGIB027R29A1E2VR3 device_id 6001"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2123-pcie-ip-ofss-file","title":"2.1.2.3 PCIe IP OFSS File","text":"
                              An OFSS file with IP type pcie (e.g. $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss) is used to configure the PCIe-SS in the FIM.
                               
                              The PCIe OFSS file has a special section type ([pf*]) which is used to define physical functions (PFs) in the FIM. Each PF has a dedicated section, where the * character is replaced with the PF number. For example, [pf0], [pf1], etc. For reference FIM configurations, 0 virtual functions (VFs) on PF0 is not supported. This is because the PR region cannot be left unconnected. A NULL AFU may need to be instantiated in this special case. PFs must be consecutive. The PFVF Limitations table describes the supported number of PFs and VFs.
                               
                              Table: PF/VF Limitations
                               Parameter Value Min # of PFs 1 (on PF0) Max # of PFs 8 Min # of VFs 1 on PF0 Max # of VFs 2000 distributed across all PFs 
                              Currently supported configuration options for an OFSS file with IP type pcie are described in the PCIe IP OFSS File Options table.
                               
                              Table: PCIe IP OFSS File Options
                               Section Parameter Options Default Description [ip] type pcie N/A Specifies that this OFSS file configures the PCIe-SS [settings] output_name pcie_ss N/A Specifies the output name of the PCIe-SS IP preset String N/A OPTIONAL - Specifies the name of a PCIe-SS IP presets file to use when building the FIM. When used, a presets file will take priority over any other parameters set in this OFSS file. [pf*] num_vfs Integer 0 Specifies the number of Virtual Functions in the current PF bar0_address_width Integer 12 bar4_address_width Integer 14 vf_bar0_address_width Integer 12 ats_cap_enable 0 | 1 0 vf_ats_cap_enable 0 | 1 0 prs_ext_cap_enable 0 | 1 0 pasid_cap_enable 0 | 1 0 pci_type0_vendor_id 32'h Value 0x00008086 pci_type0_device_id 32'h Value 0x0000bcce revision_id 32'h Value 0x00000001 class_code 32'h Value 0x00120000 subsys_vendor_id 32'h Value 0x00008086 subsys_dev_id 32'h Value 0x00001771 sriov_vf_device_id 32'h Value 0x0000bccf exvf_subsysid 32'h Value 0x00001771 
                              The default values for all PCIe-SS parameters (that are not defined in the PCIe IP OFSS file) are defined in $OFS_ROOTDIR/ofs-common/tools/ofss_config/ip_params/pcie_ss_component_parameters.py. When using a PCIe IP OFSS file during compilation, the PCIe-SS IP that is used will be defined based on the values in the PCIe IP OFSS file plus the parameters defined in pcie_ss_component_parameters.py.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2124-iopll-ip-ofss-file","title":"2.1.2.4 IOPLL IP OFSS File","text":"
                              An OFSS file with IP type iopll (e.g. $OFS_ROOTDIR/tools/ofss_config/iopll/iopll.ofss) is used to configure the IOPLL in the FIM.
                               
                              The IOPLL OFSS file has a special section type ([p_clk]) which is used to define the IOPLL clock frequency.
                               
                              Currently supported configuration options for an OFSS file with IP type iopll are described in the IOPLL OFSS File Options table.
                               
                              Table: IOPLL OFSS File Options
                               Section Parameter Options Description [ip] type iopll Specifies that this OFSS file configures the IOPLL [settings] output_name sys_pll Specifies the output name of the IOPLL. instance_name iopll_0 Specifies the instance name of the IOPLL. [p_clk] freq Integer: 250 - 470 Specifies the IOPLL clock frequency in MHz. 
                              Note: The following frequencies have been tested on reference boards: 350MHz, 400MHz, 470MHz.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2125-memory-ip-ofss-file","title":"2.1.2.5 Memory IP OFSS File","text":"
                              An OFSS file with IP type memory (e.g. $OFS_ROOTDIR/tools/ofss_config/memory/memory_iseries.ofss) is used to configure the Memory-SS in the FIM.
                               
                              The Memory OFSS file specifies a preset value, which selects a presets file (.qprs) to configure the Memory-SS.
                               
                              Currently supported configuration options for an OFSS file with IP type memory are described in the Memory OFSS File Options table.
                               
                              Table: Memory OFSS File Options
                               Section Parameter Options Description [ip] type memory Specifies that this OFSS file configures the Memory-SS [settings] output_name mem_ss_fm Specifies the output name of the Memory-SS. preset iseries-dk | String[1] Specifies the name of the .qprs presets file that will be used to build the Memory-SS. 
                              [1] You may generate your own .qprs presets file with a unique name using Quartus. 
                               
                              Pre-provided Memory-SS presets files are located in the $OFS_ROOTDIR/ipss/mem/qip/presets directory.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2126-hssi-ip-ofss-file","title":"2.1.2.6 HSSI IP OFSS File","text":"
                              An OFSS file with IP type hssi (e.g. $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_8x25_ftile.ofss) is used to configure the Ethernet-SS in the FIM.
                               
                              Currently supported configuration options for an OFSS file with IP type hssi are described in the HSSI OFSS File Options table.
                               
                              Table: HSSI OFSS File Options
                               Section Parameter Options Description [ip] type hssi Specifies that this OFSS file configures the Ethernet-SS [settings] output_name hssi_ss Specifies the output name of the Ethernet-SS num_channels Integer Specifies the number of channels. data_rate 10GbE | 25GbE | 200GAUI-4 | 400GAUI-8 Specifies the data rate[1] preset fseries-dk | 200g-fseries-dk | 400g-fseries-dk | String[2] Specifies the name of the .qprs preset file that will be used to build the Ethernet-SS. This will overwrite the other settings in this OFSS file. 
                              [1] The presets file will take priority over the data_rate parameter, so this value will not take effect when using a presets file.
                               
                              [2] You may generate your own .qprs presets file with a unique name using Quartus. 
                               
                              Pre-provided Ethernet-SS presets are located in the  $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss/presets directory.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#213-ofs-build-script-outputs","title":"2.1.3 OFS Build Script Outputs","text":"
                              The output files resulting from running the the OFS FIM build_top.sh build script are copied to a single directory during the finish stage of the build script. The path for this directory is: $OFS_ROOTDIR/<WORK_DIRECTORY>/syn/board/iseries-dk/syn_top/output_files.
                               
                              The output files include programmable images and compilation reports. The OFS Build Script Output Descriptions table describes the images that are generated by the build script.
                               
                              Table: OFS Build Script Output Descriptions
                               File Name Description ofs_top.sof The FIM design SRAM Object File; a binary file of the compiled FIM image."},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#22-compilation-flows","title":"2.2 Compilation Flows","text":"
                              This section provides information for using the build script to generate different FIM types. Walkthroughs are provided for each compilation flow. These walkthroughs require that the development environment has been set up as described in the Set Up Development Environment section.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#221-flat-fim","title":"2.2.1 Flat FIM","text":"
                              A flat FIM is compiled such that there is no partial reconfiguration region, and the entire design is built as a flat design. This is useful for compiling new designs without worrying about the complexity introduced by partial reconfiguration. The flat compile removes the PR region and PR IP; thus, you cannot use the -p build flag when using the flat compile setting. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#222-in-tree-pr-fim","title":"2.2.2 In-Tree PR FIM","text":"
                              An In-Tree PR FIM is the default compilation if no compile flags or compile settings are used. This flow will compile the design with the partial reconfiguration region, but it will not create a relocatable PR directory tree to aid in AFU development. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#223-out-of-tree-pr-fim","title":"2.2.3 Out-of-Tree PR FIM","text":"
                              An Out-of-Tree PR FIM will compile the design with the partial reconfiguration region, and will create a relocatable PR directory tree to aid in AFU workload development. This is especially useful if you are developing a FIM to be used by another team developing AFU workloads. This is the recommended build flow in most cases. There are two ways to create the relocatable PR directory tree:
                               
                                 
                              • Run the FIM build script with the -p option. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                                •  
                              • Run the generate_pr_release.sh script after running the FIM build script. Refer to the Manually Generate OFS Out-Of-Tree PR FIM Section step-by-step instructions for this flow.
                                •  
                               
                              In both cases, the generate_pr_release.sh is run to create the relocatable build tree. This script is located at $OFS_ROOTDIR/ofs-common/scripts/common/syn/generate_pr_release.sh. Usage for this script is as follows:
                               
                              generate_pr_release.sh -t <PATH_OF_RELOCATABLE_PR_TREE> <BOARD_TARGET> <WORK_DIRECTORY>\n
                               
                              The Generate PR Release Script Options table describes the options for the generate_pr_release.sh script.
                               
                              Table: Generate PR Release Script Options
                               Parameter Options Description <PATH_OF_RELOCATABLE_PR_TREE> String Specifies the location of the relocatable PR directory tree to be created. <BOARD_TARGET> iseries-dk Specifies the name of the board target. <WORK_DIRECTORY> String Specifies the existing work directory from which the relocatable PR directory tree will be created from. 
                              After generating the relocatable build tree, it is located in the $OFS_ROOTDIR/<WORK_DIRECTORY>/pr_build_template directory (or the directory you specified if generated separately). The contents of this directory have the following structure:
                               
                              \u251c\u2500\u2500 bin\n\u251c\u2500\u2500 \u251c\u2500\u2500 afu_synth\n\u251c\u2500\u2500 \u251c\u2500\u2500 qar_gen\n\u251c\u2500\u2500 \u251c\u2500\u2500 update_pim\n\u251c\u2500\u2500 \u251c\u2500\u2500 run.sh\n\u251c\u2500\u2500 \u251c\u2500\u2500 build_env_config\n\u251c\u2500\u2500 README\n\u251c\u2500\u2500 hw\n\u251c\u2500\u2500 \u251c\u2500\u2500 lib\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 build\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 fme-ifc-id.txt\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 platform\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 fme-platform-class.txt\n\u251c\u2500\u2500 \u251c\u2500\u2500 blue_bits\n\u2514\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 ofs_top.sof\n
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#224-he_null-fim","title":"2.2.4 HE_NULL FIM","text":"
                              An HE_NULL FIM refers to a design with one, some, or all of the Host Exercisers replaced by he_null blocks. The he_null is a minimal block with CSRs that responds to PCIe MMIO requests in order to keep PCIe alive. You may use any of the build flows (flat, in-tree, out-of-tree) with the HE_NULL compile options. The HE_NULL compile options are as follows:
                               
                                 
                              • null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null
                                •  
                              • null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null
                                •  
                              • null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null
                                •  
                              • null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null
                                •  
                               
                              The Compile OFS FIM section gives step-by-step instructions for this flow.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#225-walkthrough-compile-ofs-fim","title":"2.2.5 Walkthrough: Compile OFS FIM","text":"
                              Perform the following steps to compile the OFS Agilex PCIe Attach FIM for iseries-dk:
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                Navigate to the root directory.
                                 
                                cd $OFS_ROOTDIR\n
                                 
                                4.  
                              4.  
                                Run the build_top.sh script with the desired compile options using the iseries-dk OFSS presets. In the examples below, the iseries-dk.ofss file is used to call the OFS OFSS file, the IOPLL OFSS File, the PCIe OFSS file, and the Memory OFSS file. The HSSI OFSS file is specified in the command, in this case, the hssi_8x25_ftile.ofss file is used. This is not necessary if using the no_hssi compile option.
                                 
                                   
                                •  
                                  Flat FIM
                                   
                                  ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk:flat work_iseries-dk_flat\n
                                   
                                  •  
                                 
                                   
                                •  
                                  In-Tree PR FIM
                                   
                                  ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk_in_tree_pr\n
                                   
                                  •  
                                 
                                   
                                •  
                                  Out-of-Tree PR FIM
                                   
                                  ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk_oot_pr\n
                                   
                                  •  
                                 
                                   
                                •  
                                  HE_NULL Flat FIM
                                   
                                  ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk:flat,null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_iseries-dk_flat\n
                                   
                                  •  
                                 
                                5.  
                              5.  
                                Once the build script is complete, the build summary should report that the build is complete and passes timing. For example:
                                 
                                ***********************************\n***\n***        OFS_PROJECT: ofs-agx7-pcie-attach\n***        OFS_BOARD: iseries-dk\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 1\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                 
                                6.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#226-walkthrough-manually-generate-ofs-out-of-tree-pr-fim","title":"2.2.6 Walkthrough: Manually Generate OFS Out-Of-Tree PR FIM","text":"
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                Navigate to the root directory.
                                 
                                cd $OFS_ROOTDIR\n
                                 
                                4.  
                              4.  
                                Run the build_top.sh script with the desired compile options using the iseries-dk OFSS presets. In order to create the relocatable PR tree, you may not compile with the flat option. For example:
                                 
                                ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk\n
                                 
                                5.  
                              5.  
                                Run the generate_pr_release.sh script to create the relocatable PR tree.
                                 
                                ./ofs-common/scripts/common/syn/generate_pr_release.sh -t work_iseries-dk/pr_build_template iseries-dk work_iseries-dk\n
                                 
                                6.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#227-compilation-seed","title":"2.2.7 Compilation Seed","text":"
                              You may change the seed which is used by the build script during Quartus compilation to change the starting point of the fitter. Trying different seeds is useful when your design is failing timing by a small amount.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2271-walkthrough-change-the-compilation-seed","title":"2.2.7.1 Walkthrough: Change the Compilation Seed","text":"
                              Perform the following steps to change the compilation seed for the FIM build.
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                Edit the SEED assignment in the $OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_top.qsf file to your desired seed value. The value can be any non-negative integer value.
                                 
                                set_global_assignment -name SEED 2\n
                                 
                                4.  
                              4.  
                                Build the FIM. Refer to the Compile OFS FIM section for instructions.
                                 
                                5.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#3-fim-simulation","title":"3. FIM Simulation","text":"
                              Unit level simulation of key components in the FIM is provided for verification of the following areas:
                               
                                 
                              • Ethernet
                                •  
                              • PCIe
                                •  
                              • External Memory
                                •  
                              • Core FIM
                                •  
                               
                              The Unit Level simulations work with Synopsys VCS/VCSMX or Mentor Graphics Questasim simulators. The scripts to run each unit level simulation are located in $OFS_ROOTDIR/sim/unit_test. Each unit test directory contains a README which describes the test in detail.  Refer to the Supported Unit Tests table for a list of the supported unit tests.
                               
                              Note: The OFS Agilex PCIe Attach FIM for the F-Tile Development Kit does not support all of the unit tests that are provided in the unit_test directory.
                               
                              Table: Supported Unit Tests
                               Test Name Description bfm_test This is the unit test for PCIe BFM. The test uses HE-LB to perform memory loopback between FIM and the host. csr_test This is the unit test for FIM CSR access and AFU memory write/read dfh_walker This is the unit test for FME DFH walking flr This is the unit test for PCIe PF/VF FLR fme_csr_access This is the a unit test for the register access logic for $OFS_ROOTDIR/ofs-common/src/common/fme/fme_csr.sv fme_csr_directed This is the unit test for $OFS_ROOTDIR/ofs-common/src/common/fme/fme_csr.sv he_lb_test This is the unit test for HE_LPBK. The test uses HE-LB to perform memory loopback between FIM and the host. he_null_test This is the unit test for HE-NULL Exerciser. The test issues basic mmio Rd/Wr requests targetting HE-NULL CSRs. indirect_csr This is the unit test for axi4lite_indirect_csr_if module. pcie_csr_test This is the unit test for PCIE CSR access. pf_vf_access_test This is the unit test for PCIe PF/VF MMIO. Each function has a feature GUID at offset 0x8 with an associated register map. port_gasket_test This is the unit test for pg_csr block and it's connectivity to fabric. The test issues mmio Rd/Wr requests targetting the csrs in port_gasket. This test does not do any functional testing of partial reconfiguration, user clock or remote stp. remote_stp_test This is the unit test for remote stp. It covers mmio read access to remote_stp registers. uart_csr This is the unit test for UART CSR accesses."},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#31-simulation-file-generation","title":"3.1 Simulation File Generation","text":"
                              The simulation files must be generated prior to running unit level simulations. The script to generate simulation files is in the following location:
                               
                              $OFS_ROOTDIR/ofs-common/scripts/common/sim/gen_sim_files.sh\n
                               
                              The usage of the gen_sim_files.sh script is as follows:
                               
                              gen_sim_files.sh [--ofss=<ip_config>] <build_target>[:<fim_options>] [<device>] [<family>]\n
                               
                              The Gen Sim Files Script Options table describes the options for the gen_sim_files.sh script.
                               
                              Table: Gen Sim Files Script Options
                               Field Options Description Requirement --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. Platform Dependent[1] <build_target> iseries-dk Specifies which board is being targeted. Required <fim_options> null_he_lb | null_he_hssi | null_he_mem | null_he_mem_tg Used to change how the FIM is built.\u00a0\u00a0- null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null.\u00a0\u00a0- null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null.\u00a0\u00a0- null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null.\u00a0\u00a0- null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null. More than one FIM option may be passed included in the <fim_options> list by concatenating them separated by commas. For example: <build_target>:null_he_lb,null_he_hssi Optional <device> string Specifies the device ID for the target FPGA. If not specified, the default device is parsed from the QSF file for the project. Optional <family> string Specifies the family for the target FPGA. If not specified, the default family is parsed from the QSF file for the project. Optional 
                              [1] Using OFSS is required for the F-Tile Development Kit.
                               
                              Refer to the Running Individual Unit Level Simulation section for an example of the simulation files generation flow.
                               
                              When running regression tests, you may use the -g command line argument to generate simulation files; refer to the Running Regression Unit Level Simulation section for step-by-step instructions.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#32-individual-unit-tests","title":"3.2 Individual Unit Tests","text":"
                              Each unit test may be run individually using the run_sim.sh script located in the following directory:
                               
                              $OFS_ROOTDIR/ofs-common/scripts/common/sim/run_sim.sh\n
                               
                              The usage for the run_sim.sh script is as follows:
                               
                              sh run_sim.sh TEST=<test> [VCSMX=<0|1> | MSIM=<0|1>]\n
                               
                              The Run Sim Script Options table describes the options for the run_sim.sh script.
                               
                              Table: Run Sim Script Options
                               Field Options Description TEST String Specify the name of the test to run, e.g. dfh_walker VCSMX 0 | 1 When set, the VCSMX simulator will be used MSIM 0 | 1 When set, the QuestaSim simulator will be used 
                              Note: The default simulator is VCS if neither VCSMX nor MSIM are set.
                               
                              The log for a unit test is stored in a transcript file in the simulation directory of the test that was run.
                               
                              $OFS_ROOTDIR/sim/unit_test/<TEST_NAME>/<SIMULATOR>/transcript\n
                               
                              For example, the log for the DFH walker test using VCSMX would be found at:
                               
                              $OFS_ROOTDIR/sim/unit_test/dfh_walker/sim_vcsmx/transcript\n
                               
                              The simulation waveform database is saved as vcdplus.vpd for post simulation review.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#321-walkthrough-running-individual-unit-level-simulation","title":"3.2.1 Walkthrough: Running Individual Unit Level Simulation","text":"
                              Perform the following steps to run an individual unit test.
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Clone the FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                Generate the simulation files for the iseries-dk
                                 
                                cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/iseries-dk.ofss iseries-dk\n
                                 
                                4.  
                              4.  
                                Navigate to the common simulation directory   
                                cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n
                                 
                                5.  
                              5.  
                                Run the desired unit test using your desired simulator
                                 
                                   
                                •  
                                  Using VCS
                                   
                                  sh run_sim.sh TEST=<test_name>\n
                                   
                                  •  
                                 
                                   
                                •  
                                  Using VCSMX
                                   
                                  sh run_sim.sh TEST=<test_name> VCSMX=1\n
                                   
                                  •  
                                 
                                   
                                •  
                                  Using QuestaSim
                                   
                                  sh run_sim.sh TEST=<test_name> MSIM=1\n
                                   
                                  •  
                                 
                                   
                                •  
                                  For example, to run the DFH walker test using VCSMX:
                                   
                                  sh run_sim.sh TEST=dfh_walker VCSMX=1\n
                                   
                                  •  
                                 
                                6.  
                              6.  
                                Once the test is complete, check the output for the simulation results. Review the log for detailed test results.
                                 
                                Test status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/ofs-agx7-pcie-attach/sim/unit_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time         356233750000\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 356233750000 fs\nCPU Time:     57.730 seconds;       Data structure size:  47.2Mb\nTue Sep  5 09:44:19 2023\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                                 
                                7.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#33-regression-unit-tests","title":"3.3 Regression Unit Tests","text":"
                              You may use the regression script regress_run.py to run some or all of the unit tests available with a single command. The regression script is in the following location:
                               
                              $OFS_ROOTDIR/sim/unit_test/scripts/regress_run.py\n
                               
                              The usage of the regression script is as follows:
                               
                              regress_run.py [-h] [-l] [-n <num_procs>] [-k <test_package>] [-s <simulator>] [-g] [--ofss <ip_config>] [-b <board_name>] [-e]\n
                               
                              The Regression Unit Test Script Options table describes the options for the regress_run.py script.
                               
                              Table: Regression Unit Test Script Options
                               Field Options Description -h | --help N/A Show the help message and exit -l | --local N/A Run regression locally -n | --n_procs Integer Maximum number of processes/tests to run in parallel when run locally. This has no effect on farm runs. -k | --pack all | fme | he | hssi | list | mem | pmci Test package to run during regression. The \"list\" option will look for a text file named \"list.txt\" in the \"unit_test\" directory for a text list of tests to run (top directory names). The default test package is all. -s | --sim vcs | vcsmx | msim Specifies the simulator used for the regression tests. The default simulator is vcs -g | --gen_sim_files N/A Generate simulation files. This only needs to be done once per repo update. This is the equivalent of running the gen_sim_files.sh script. -o | --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. -b | --board_name iseries-dk Specifies the board target -e | --email_list String Specifies email list to send results to multiple recipients 
                              The log for each unit test that is run by the regression script is stored in a transcript file in the simulation directory of the test that was run.
                               
                              $OFS_ROOTDIR/sim/unit_test/<TEST_NAME>/<SIMULATOR>/transcript\n
                               
                              For example, the log for the DFH walker test using VCSMX would be found at:
                               
                              $OFS_ROOTDIR/sim/unit_test/dfh_walker/sim_vcsmx/transcript\n
                               
                              The simulation waveform database is saved as vcdplus.vpd for post simulation review.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#331-walkthrough-running-regression-unit-level-simulation","title":"3.3.1 Walkthrough: Running Regression Unit Level Simulation","text":"
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                Create a test list file to only run the unit level simulations that are supported for the iseries-dk FIM.
                                 
                                touch $OFS_ROOTDIR/sim/unit_test/list.txt\n
                                 
                                Copy the following list into the new file. You may remove tests from this list as desired.
                                 
                                ./bfm_test/set_params.sh\n./csr_test/set_params.sh\n./dfh_walker/set_params.sh\n./flr/set_params.sh\n./fme_csr_access/set_params.sh\n./fme_csr_directed/set_params.sh\n./he_lb_test/set_params.sh\n./he_null_test/set_params.sh\n./hssi_csr_test/set_params.sh\n./hssi_kpi_test/set_params.sh\n./hssi_test/set_params.sh\n./indirect_csr/set_params.sh\n./pcie_csr_test/set_params.sh\n./pf_vf_access_test/set_params.sh\n./port_gasket_test/set_params.sh\n./qsfp_test/set_params.sh\n./remote_stp_test/set_params.sh\n./uart_csr/set_params.sh\n
                                 
                                4.  
                              4.  
                                Run regression test with the your desired options. For example, to simulate with the options to generate simulation files, run locally, use 8 processes, run the specified test list, use VCSMX simulator, and target the iseries-dk:
                                 
                                cd $OFS_ROOTDIR/sim/unit_test/scripts\n\npython regress_run.py --ofss $OFS_ROOTDIR/tools/ofss_config/iseries-dk.ofss -g -l -n 8 -k list -s vcsmx -b iseries-dk\n
                                 
                                5.  
                              5.  
                                Once all tests are complete, check that the tests have passed.
                                 
                                2023-08-30 15:00:50,256: Passing Unit Tests:13/13 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-08-30 15:00:50,256:    bfm_test:......... PASS -- Time Elapsed:0:22:23.940917\n2023-08-30 15:00:50,256:    csr_test:......... PASS -- Time Elapsed:0:22:46.262916\n2023-08-30 15:00:50,256:    dfh_walker:....... PASS -- Time Elapsed:0:22:16.544732\n2023-08-30 15:00:50,256:    flr:.............. PASS -- Time Elapsed:0:22:21.332386\n2023-08-30 15:00:50,256:    fme_csr_access:... PASS -- Time Elapsed:0:17:12.454034\n2023-08-30 15:00:50,256:    fme_csr_directed:. PASS -- Time Elapsed:0:17:22.947134\n2023-08-30 15:00:50,256:    he_lb_test:....... PASS -- Time Elapsed:0:28:38.962424\n2023-08-30 15:00:50,256:    indirect_csr:..... PASS -- Time Elapsed:0:21:15.387478\n2023-08-30 15:00:50,256:    pcie_csr_test:.... PASS -- Time Elapsed:0:22:33.838949\n2023-08-30 15:00:50,256:    pf_vf_access_test: PASS -- Time Elapsed:0:22:28.704149\n2023-08-30 15:00:50,256:    port_gasket_test:. PASS -- Time Elapsed:0:22:32.592301\n2023-08-30 15:00:50,256:    remote_stp_test:.. PASS -- Time Elapsed:0:22:01.485914\n2023-08-30 15:00:50,256:    uart_csr:......... PASS -- Time Elapsed:0:22:31.848882\n2023-08-30 15:00:50,256: Failing Unit Tests: 0/13 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-08-30 15:00:50,256: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-08-30 15:00:50,256:       Number of Unit test results captured: 13\n2023-08-30 15:00:50,256:       Number of Unit test results passing.: 13\n2023-08-30 15:00:50,256:       Number of Unit test results failing.:  0\n2023-08-30 15:00:50,256:     End Unit regression running at date/time................: 2023-08-30 15:00:50.256725\n2023-08-30 15:00:50,256:     Elapsed time for Unit regression run....................: 0:54:48.172625\n
                                 
                                6.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#4-fim-customization","title":"4. FIM Customization","text":"
                              This section describes how to perform specific customizations of the FIM, and provides step-by-step walkthroughs for these customizations. Each walkthrough can be done independently. These walkthroughs require a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment. The FIM Customization Walkthroughs table lists the walkthroughs that are provided in this section. Some walkthroughs include steps for testing on hardware. Testing on hardware requires that you have a deployment environment set up. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                               
                              Table: FIM Customization Walkthroughs
                               Customization Walkthrough Name Add a new module to the OFS FIM Modify and run unit tests for a FIM that has a new module Hardware test a FIM that has a new module Debug the FIM with Signal Tap Compile the FIM in preparation for designing your AFU Resize the Partial Reconfiguration Region Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS Modify PCIe Configuration Using IP Presets Migrate to a Different Agilex Device Number Modify the Ethernet Sub-System to 1x400GbE"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#41-adding-a-new-module-to-the-fim","title":"4.1 Adding a new module to the FIM","text":"
                              This section provides a information for adding a custom module to the FIM, simulating the new design, compiling the new design, implementing and testing the new design on hardware, and debugging the new design on hardware.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#411-hello-fim-theory-of-operation","title":"4.1.1 Hello FIM Theory of Operation","text":"
                              If you intend to add a new module to the FIM area, then you will need to inform the host software of the new module. The FIM exposes its functionalities to host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). This set of CSR registers and their operation is described in FIM MMIO Regions.
                               
                              See FPGA Device Feature List (DFL) Framework Overview for a description of the software process to read and process the linked list of Device Feature Header (DFH) CSRs within a FPGA.
                               
                              This example will add a hello_fim module to the design. The Hello FIM example adds a simple DFH register and 64bit scratchpad register connected to the Board Peripheral Fabric (BPF) that can be accessed by the Host. You can use this example as the basis for adding a new feature to your FIM.
                               
                              The Hello FIM design can be verified by Unit Level simulation, Universal Verification Methodology (UVM) simulation, and running in hardware on the iseries-dk card. The process for these are described in this section.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#4111-hello-fim-board-peripheral-fabric-bpf","title":"4.1.1.1 Hello FIM Board Peripheral Fabric (BPF)","text":"
                              The Hello FIM module will be connected to the Board Peripheral Fabric (BPF), and will be connected such that it can be mastered by the Host. The BPF is an interconnect generated by Platform Designer. The Hello FIM BPF Interface Diagram figure shows the APF/BPF Master/Slave interactions, as well as the added Hello FIM module.
                               
                              Figure: Hello FIM BPF Interface Diagram
                               
                               
                              The BPF fabric is defined in $OFS_ROOTDIR/src/pd_qsys/fabric/bpf.txt.
                               
                              We will add the Hello FIM module to an un-used address space in the MMIO region. The Hello FIM MMIO Address Layout table below shows the MMIO region for the Host with the Hello FIM module added at base address 0x16000.
                               
                              Table: Hello FIM MMIO Address Layout
                               Offset Feature CSR set 0x00000 FME AFU 0x10000 PCIe Interface 0x12000 QSFP Controller 0 0x13000 QSFP Controller 1 0x14000 HSSI Interface 0x15000 EMIF Interface 0x16000 Hello FIM"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#4112-hello-fim-csr","title":"4.1.1.2 Hello FIM CSR","text":"
                              The Hello FIM CSR will consist of the three registers shown in the Hello FIM CSR table below. The DFH and Hello FIM ID registers are read-only. The Scratchpad register supports read and write accesses.
                               
                              Table: Hello FIM CSR
                               Offset Attribute Description Default Value 0x016000 RO DFH(Device Feature Headers) register 0x30000006a0000100 0x016030 RW Scrachpad register 0x0 0x016038 RO Hello FIM ID register 0x6626070150000034"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#412-walkthrough-add-a-new-module-to-the-ofs-fim","title":"4.1.2 Walkthrough: Add a new module to the OFS FIM","text":"
                              Perform the following steps to add a new module to the OFS FIM that can be accessed by the Host.
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Clone the FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                Make hello_fim source directory
                                 
                                mkdir $OFS_ROOTDIR/src/hello_fim\n
                                 
                                4.  
                              4.  
                                Create hello_fim_top.sv file.
                                 
                                touch $OFS_ROOTDIR/src/hello_fim/hello_fim_top.sv\n
                                 
                                Copy the following code into hello_fim_top.sv:
                                 
                                // ***************************************************************************\n//                               INTEL CONFIDENTIAL\n//\n//        Copyright (C) 2023 Intel Corporation All Rights Reserved.\n//\n// The source code contained or described herein and all  documents related to\n// the  source  code  (\"Material\")  are  owned  by  Intel  Corporation  or its\n// suppliers  or  licensors.    Title  to  the  Material  remains  with  Intel\n// Corporation or  its suppliers  and licensors.  The Material  contains trade\n// secrets  and  proprietary  and  confidential  information  of  Intel or its\n// suppliers and licensors.  The Material is protected  by worldwide copyright\n// and trade secret laws and treaty provisions. No part of the Material may be\n// used,   copied,   reproduced,   modified,   published,   uploaded,  posted,\n// transmitted,  distributed,  or  disclosed  in any way without Intel's prior\n// express written permission.\n//\n// No license under any patent,  copyright, trade secret or other intellectual\n// property  right  is  granted  to  or  conferred  upon  you by disclosure or\n// delivery  of  the  Materials, either expressly, by implication, inducement,\n// estoppel or otherwise.  Any license under such intellectual property rights\n// must be express and approved by Intel in writing.\n//\n// You will not, and will not allow any third party to modify, adapt, enhance, \n// disassemble, decompile, reverse engineer, change or create derivative works \n// from the Software except and only to the extent as specifically required by \n// mandatory applicable laws or any applicable third party license terms \n// accompanying the Software.\n//\n// -----------------------------------------------------------------------------\n// Engineer     : \n// Create Date  : September 2023\n// Module Name  : hello_fim_top.sv\n// Project      : OFS\n// -----------------------------------------------------------------------------\n//\n// Description: \n// This is a simple module that implements DFH registers and \n// AVMM address decoding logic.\n\nmodule hello_fim_top  #(\n   parameter ADDR_WIDTH  = 12, \n   parameter DATA_WIDTH = 64, \n   parameter bit [11:0] FEAT_ID = 12'h001,\n   parameter bit [3:0]  FEAT_VER = 4'h1,\n   parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n   parameter bit END_OF_LIST = 1'b0\n)(\n   input  logic    clk,\n   input  logic    reset,\n// -----------------------------------------------------------\n//  AXI4LITE Interface\n// -----------------------------------------------------------\n   ofs_fim_axi_lite_if.slave   csr_lite_if\n);\n\nimport ofs_fim_cfg_pkg::*;\nimport ofs_csr_pkg::*;\n\n//-------------------------------------\n// Signals\n//-------------------------------------\n   logic [ADDR_WIDTH-1:0]              csr_waddr;\n   logic [DATA_WIDTH-1:0]              csr_wdata;\n   logic [DATA_WIDTH/8-1:0]            csr_wstrb;\n   logic                               csr_write;\n   logic                               csr_slv_wready;\n   csr_access_type_t                   csr_write_type;\n\n   logic [ADDR_WIDTH-1:0]              csr_raddr;\n   logic                               csr_read;\n   logic                               csr_read_32b;\n   logic [DATA_WIDTH-1:0]              csr_readdata;\n   logic                               csr_readdata_valid;\n   logic [ADDR_WIDTH-1:0]              csr_addr;\n\n   logic [63:0]                        com_csr_writedata;\n   logic                               com_csr_read;\n   logic                               com_csr_write;\n   logic [63:0]                        com_csr_readdata;\n   logic                               com_csr_readdatavalid;\n   logic [5:0]                         com_csr_address;\n\n// AXI-M CSR interfaces\nofs_fim_axi_mmio_if #(\n   .AWID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .AWADDR_WIDTH (ADDR_WIDTH),\n   .WDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH),\n   .ARID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .ARADDR_WIDTH (ADDR_WIDTH),\n   .RDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH)\n) csr_if();\n\n// AXI4-lite to AXI-M adapter\naxi_lite2mmio axi_lite2mmio (\n   .clk       (clk),\n   .rst_n     (~reset),\n   .lite_if   (csr_lite_if),\n   .mmio_if   (csr_if)\n);\n\n//---------------------------------\n// Map AXI write/read request to CSR write/read,\n// and send the write/read response back\n//---------------------------------\nofs_fim_axi_csr_slave #(\n   .ADDR_WIDTH (ADDR_WIDTH),\n   .USE_SLV_READY (1'b1)\n\n   ) csr_slave (\n   .csr_if             (csr_if),\n\n   .csr_write          (csr_write),\n   .csr_waddr          (csr_waddr),\n   .csr_write_type     (csr_write_type),\n   .csr_wdata          (csr_wdata),\n   .csr_wstrb          (csr_wstrb),\n   .csr_slv_wready     (csr_slv_wready),\n   .csr_read           (csr_read),\n   .csr_raddr          (csr_raddr),\n   .csr_read_32b       (csr_read_32b),\n   .csr_readdata       (csr_readdata),\n   .csr_readdata_valid (csr_readdata_valid)\n);\n\n// Address mapping\nassign csr_addr             = csr_write ? csr_waddr : csr_raddr;\nassign com_csr_address      = csr_addr[5:0];  // byte address\nassign csr_slv_wready       = 1'b1 ;\n// Write data mapping\nassign com_csr_writedata    = csr_wdata;\n\n// Read-Write mapping\nalways_comb\nbegin\n   com_csr_read             = 1'b0;\n   com_csr_write            = 1'b0;\n   casez (csr_addr[11:6])\n      6'h00 : begin // Common CSR\n         com_csr_read       = csr_read;\n         com_csr_write      = csr_write;\n      end   \n      default: begin\n         com_csr_read       = 1'b0;\n         com_csr_write      = 1'b0;\n      end\n   endcase\nend\n\n// Read data mapping\nalways_comb begin\n   if (com_csr_readdatavalid) begin\n      csr_readdata       = com_csr_readdata;\n      csr_readdata_valid = 1'b1;\n   end\n   else begin\n      csr_readdata       = '0;\n      csr_readdata_valid = 1'b0;\n   end\nend\n\nhello_fim_com  #(\n   .FEAT_ID          (FEAT_ID),\n   .FEAT_VER         (FEAT_VER),\n   .NEXT_DFH_OFFSET  (NEXT_DFH_OFFSET),\n   .END_OF_LIST      (END_OF_LIST)\n) hello_fim_com_inst (\n   .clk                   (clk                     ),\n   .reset                 (reset                   ),\n   .writedata             (com_csr_writedata       ),\n   .read                  (com_csr_read            ),\n   .write                 (com_csr_write           ),\n   .byteenable            (4'hF                    ),\n   .readdata              (com_csr_readdata        ),\n   .readdatavalid         (com_csr_readdatavalid   ),\n   .address               (com_csr_address         )\n   );\nendmodule\n
                                 
                                5.  
                              5.  
                                Create hello_fim_com.sv file.
                                 
                                touch $OFS_ROOTDIR/src/hello_fim/hello_fim_com.sv\n
                                 
                                Copy the following code to hello_fim_com.sv:
                                 
                                module hello_fim_com #(\n  parameter bit [11:0] FEAT_ID = 12'h001,\n  parameter bit [3:0]  FEAT_VER = 4'h1,\n  parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n  parameter bit END_OF_LIST = 1'b0\n)(\ninput clk,\ninput reset,\ninput [63:0] writedata,\ninput read,\ninput write,\ninput [3:0] byteenable,\noutput reg [63:0] readdata,\noutput reg readdatavalid,\ninput [5:0] address\n);\n\nwire reset_n = !reset;  \nreg [63:0] rdata_comb;\nreg [63:0] scratch_reg;\n\nalways @(negedge reset_n ,posedge clk)  \n   if (!reset_n) readdata[63:0] <= 64'h0; else readdata[63:0] <= rdata_comb[63:0];\n\nalways @(negedge reset_n , posedge clk)\n   if (!reset_n) readdatavalid <= 1'b0; else readdatavalid <= read;\n\nwire wr = write;\nwire re = read;\nwire [5:0] addr = address[5:0];\nwire [63:0] din  = writedata [63:0];\nwire wr_scratch_reg = wr & (addr[5:0]  == 6'h30)? byteenable[0]:1'b0;\n\n// 64 bit scratch register\nalways @( negedge  reset_n,  posedge clk)\n   if (!reset_n)  begin\n      scratch_reg <= 64'h0;\n   end\n   else begin\n   if (wr_scratch_reg) begin \n      scratch_reg <=  din;  \n   end\nend\n\nalways @ (*)\nbegin\nrdata_comb = 64'h0000000000000000;\n   if(re) begin\n      case (addr)  \n        6'h00 : begin\n                rdata_comb [11:0]   = FEAT_ID ;  // dfh_feature_id  is reserved or a constant value, a read access gives the reset value\n                rdata_comb [15:12]  = FEAT_VER ;  // dfh_feature_rev    is reserved or a constant value, a read access gives the reset value\n                rdata_comb [39:16]  = NEXT_DFH_OFFSET ;  // dfh_dfh_ofst is reserved or a constant value, a read access gives the reset value\n                rdata_comb [40]     = END_OF_LIST ;        //dfh_end_of_list\n                rdata_comb [59:40]  = 20'h00000 ;  // dfh_rsvd1     is reserved or a constant value, a read access gives the reset value\n                rdata_comb [63:60]  = 4'h3 ;  // dfh_feat_type  is reserved or a constant value, a read access gives the reset value\n        end\n        6'h30 : begin\n                rdata_comb [63:0]   = scratch_reg; \n        end\n        6'h38 : begin\n                rdata_comb [63:0]       = 64'h6626_0701_5000_0034;\n        end\n        default : begin\n                rdata_comb = 64'h0000000000000000;\n        end\n      endcase\n   end\nend\n\nendmodule\n
                                 
                                6.  
                              6.  
                                Create hello_fim_design_files.tcl file.
                                 
                                touch $OFS_ROOTDIR/src/hello_fim/hello_fim_design_files.tcl\n
                                 
                                Copy the following code into hello_fim_design_files.tcl
                                 
                                # Copyright 2023 Intel Corporation.\n#\n# THIS SOFTWARE MAY CONTAIN PREPRODUCTION CODE AND IS PROVIDED BY THE\n# COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED\n# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE\n# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR\n# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\n# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR\n# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,\n# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE\n# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,\n# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# Hello FIM Files\n#--------------------\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_com.sv\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_top.sv\n
                                 
                                7.  
                              7.  
                                Modify $OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_top.qsf to include Hello FIM module
                                 
                                ######################################################\n# Verilog Macros\n######################################################\n.....\nset_global_assignment -name VERILOG_MACRO \"INCLUDE_HELLO_FIM\"     # Includes Hello FIM\n
                                 
                                8.  
                              8.  
                                Modify $OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_top_sources.tcl to include Hello FIM design files
                                 
                                ############################################\n# Design Files\n############################################\n...\n# Subsystems\n...\nset_global_assignment -name SOURCE_TCL_SCRIPT_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_design_files.tclset_global_assignment -name SOURCE_TCL_SCRIPT_FILE ../setup/hello_fim_design_files.tcl\n
                                 
                                9.  
                              9.  
                                Modify $OFS_ROOTDIR/src/pd_qsys/fabric/fabric_design_files.tcl to include BPF Hello FIM Slave IP.
                                 
                                #--------------------\n# BPF\n#--------------------\n...\nset_global_assignment -name IP_FILE   $::env(BUILD_ROOT_REL)/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip\n
                                 
                                10.  
                              10.  
                                Modify $OFS_ROOTDIR/src/includes/fabric_width_pkg.sv to add Hello FIM slave information and update EMIF slave next offset.
                                 
                                localparam bpf_hello_fim_slv_baseaddress = 'h16000;    // New\nlocalparam bpf_hello_fim_slv_address_width = 12;       // New\nlocalparam bpf_emif_slv_next_dfh_offset = 'h1000;      // Old value: 'hB000\nlocalparam bpf_hello_fim_slv_next_dfh_offset = 'hA000; // New\nlocalparam bpf_hello_fim_slv_eol = 'b0;                // New\n
                                 
                                11.  
                              11.  
                                Modify $OFS_ROOTDIR/src/top/top.sv
                                 
                                   
                                1.  
                                  Add bpf_hello_fim_slv_if to AXI interfaces
                                   
                                  // AXI4-lite interfaces\n...\nofs_fim_axi_lite_if #(.AWADDR_WIDTH(fabric_width_pkg::bpf_hello_fim_slv_address_width), .ARADDR_WIDTH(fabric_width_pkg::bpf_hello_fim_slv_address_width)) bpf_hello_fim_slv_if();\n
                                   
                                  2.  
                                2.  
                                  Add Hello FIM instantiation
                                   
                                  //*******************************\n// Hello FIM Subsystem\n//*******************************\n\n`ifdef INCLUDE_HELLO_FIM\nhello_fim_top #(\n   .ADDR_WIDTH       (fabric_width_pkg::bpf_hello_fim_slv_address_width),\n   .DATA_WIDTH       (64),\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset),\n   .END_OF_LIST      (fabric_width_pkg::bpf_hello_fim_slv_eol)\n) hello_fim_top_inst (\n    .clk (clk_csr),\n    .reset(~rst_n_csr),\n    .csr_lite_if    (bpf_hello_fim_slv_if)\n);\n`else\ndummy_csr #(\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset),\n   .END_OF_LIST      (fabric_width_pkg::bpf_hello_fim_slv_eol)\n) hello_fim_dummy (\n   .clk         (clk_csr),\n   .rst_n       (rst_n_csr),\n   .csr_lite_if (bpf_hello_fim_slv_if)\n);\n\n`endif\n
                                   
                                  3.  
                                3.  
                                  Add interfaces for Hello FIM slv to bpf instantiation
                                   
                                  bpf bpf (\n...\n  .bpf_hello_fim_slv_awaddr   (bpf_hello_fim_slv_if.awaddr  ),\n  .bpf_hello_fim_slv_awprot   (bpf_hello_fim_slv_if.awprot  ),\n  .bpf_hello_fim_slv_awvalid  (bpf_hello_fim_slv_if.awvalid ),\n  .bpf_hello_fim_slv_awready  (bpf_hello_fim_slv_if.awready ),\n  .bpf_hello_fim_slv_wdata    (bpf_hello_fim_slv_if.wdata   ),\n  .bpf_hello_fim_slv_wstrb    (bpf_hello_fim_slv_if.wstrb   ),\n  .bpf_hello_fim_slv_wvalid   (bpf_hello_fim_slv_if.wvalid  ),\n  .bpf_hello_fim_slv_wready   (bpf_hello_fim_slv_if.wready  ),\n  .bpf_hello_fim_slv_bresp    (bpf_hello_fim_slv_if.bresp   ),\n  .bpf_hello_fim_slv_bvalid   (bpf_hello_fim_slv_if.bvalid  ),\n  .bpf_hello_fim_slv_bready   (bpf_hello_fim_slv_if.bready  ),\n  .bpf_hello_fim_slv_araddr   (bpf_hello_fim_slv_if.araddr  ),\n  .bpf_hello_fim_slv_arprot   (bpf_hello_fim_slv_if.arprot  ),\n  .bpf_hello_fim_slv_arvalid  (bpf_hello_fim_slv_if.arvalid ),\n  .bpf_hello_fim_slv_arready  (bpf_hello_fim_slv_if.arready ),\n  .bpf_hello_fim_slv_rdata    (bpf_hello_fim_slv_if.rdata   ),\n  .bpf_hello_fim_slv_rresp    (bpf_hello_fim_slv_if.rresp   ),\n  .bpf_hello_fim_slv_rvalid   (bpf_hello_fim_slv_if.rvalid  ),\n  .bpf_hello_fim_slv_rready   (bpf_hello_fim_slv_if.rready  ),\n...\n);\n
                                   
                                  4.  
                                 
                                12.  
                              12.  
                                Modify $OFS_ROOTDIR/src/pd_qsys/fabric/bpf.txt to add the hello_fim module as a slave to the apf.
                                 
                                # NAME   FABRIC      BASEADDRESS    ADDRESS_WIDTH SLAVES\napf         mst     n/a             18            fme,pcie,pmci,qsfp0,qsfp1,emif,hssi,hello_fim\n...\nhello_fim   slv     0x16000         12            n/a\n
                                 
                                13.  
                              13.  
                                Execute helper script to generate BPF design files
                                 
                                cd $OFS_ROOTDIR/src/pd_qsys/fabric/\n\nsh gen_fabrics.sh\n
                                 
                                14.  
                              14.  
                                Once the script completes, the following new IP is created: $OFS_ROOTDIR/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip.
                                 
                                15.  
                              15.  
                                [OPTIONAL] You may verify the BPF changes have been made correctly by opening bpf.qsys to analyze the BPF.
                                 
                                cd $OFS_ROOTDIR/src/pd_qsys/fabric\n\nqsys-edit bpf.qsys --quartus-project=$OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_top.qpf\n
                                 
                                Find the bpf_hello_fim_slv instance:
                                 
                                 
                                16.  
                              16.  
                                Compile the Hello FIM design
                                 
                                cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk_hello_fim\n
                                 
                                17.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#413-walkthrough-modify-and-run-unit-tests-for-a-fim-that-has-a-new-module","title":"4.1.3 Walkthrough: Modify and run unit tests for a FIM that has a new module","text":"
                              Perform the following steps to modify the unit test files to support a FIM that has had a new module added to it.
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                •  
                               
                                 
                              • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design in order to simulate.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Modify $OFS_ROOTDIR/sim/unit_test/dfh_walker/test_csr_defs.sv
                                 
                                   
                                1.  
                                  Add HELLO_FIM_IDX entry to t_dfh_idx enumeration.
                                   
                                  ...\ntypedef enum {\n    FME_DFH_IDX,\n    THERM_MNGM_DFH_IDX,\n    GLBL_PERF_DFH_IDX,\n    GLBL_ERROR_DFH_IDX,\n    QSFP0_DFH_IDX,\n    QSFP1_DFH_IDX,\n    HSSI_DFH_IDX,\n    EMIF_DFH_IDX,\n    HELLO_FIM_DFH_IDX,  // New\n    PMCI_DFH_IDX,\n    ST2MM_DFH_IDX,\n    VUART_DFH_IDX,\n    PG_PR_DFH_IDX,\n    PG_PORT_DFH_IDX,\n    PG_USER_CLK_DFH_IDX,\n    PG_REMOTE_STP_DFH_IDX,\n    AFU_ERR_DFH_IDX,\n    MAX_DFH_IDX\n} t_dfh_idx;\n...\n
                                   
                                  2.  
                                2.  
                                  Add HELLO_FIM_DFH to get_dfh_names function.
                                   
                                  ...\nfunction automatic dfh_name[MAX_DFH_IDX-1:0] get_dfh_names();\n...\n  dfh_names[PMCI_DFH_IDX]        = \"PMCI_DFH\";\n  dfh_names[HELLO_FIM_DFH_IDX]   = \"HELLO_FIM_DFH\";  // New\n  dfh_names[ST2MM_DFH_IDX]       = \"ST2MM_DFH\";\n...\nreturn dfh_names;\n...\n
                                   
                                  3.  
                                3.  
                                  Add expected DFH value for Hello FIM to the get_dfh_values function.
                                   
                                  ...\nfunction automatic [MAX_DFH_IDX-1:0][63:0] get_dfh_values();\n...\n  dfh_values[PMCI_DFH_IDX]       = 64'h3_00000_xxxxxx_1012;\n  dfh_values[PMCI_DFH_IDX][39:16] = fabric_width_pkg::bpf_pmci_slv_next_dfh_offset;\n\n  dfh_values[HELLO_FIM_DFH_IDX]  = 64'h3_00000_xxxxxx_0100;  // New\n  dfh_values[HELLO_FIM_DFH_IDX][39:16] = fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset; // New\n\n  dfh_values[ST2MM_DFH_IDX]      = 64'h3_00000_xxxxxx_0014;\n  dfh_values[ST2MM_DFH_IDX][39:16] = fabric_width_pkg::apf_st2mm_slv_next_dfh_offset;\n...\nreturn dfh_values;\n...\n
                                   
                                  4.  
                                 
                                3.  
                              3.  
                                Generate simulation files
                                 
                                cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/iseries-dk.ofss iseries-dk\n
                                 
                                4.  
                              4.  
                                Run DFH Walker Simulation   
                                cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\nsh run_sim.sh TEST=dfh_walker\n
                                 
                                5.  
                              5.  
                                Verify that the test passes, and that the output shows the Hello FIM in the DFH sequence
                                 
                                ********************************************\n Running TEST(0) : test_dfh_walking\n********************************************\n\n...\n\nTX: Tag Search Comparison: CPLD Tag: 00a   Active Tag: 00a\nTX: PU Completion ADDED to transaction!\nTX: Match found for PU CPLD!\nRead64  Method Data Transaction: Address: 0000_0000_0001_5000   Data: 3000_0000_1000_1009\nEMIF_DFH\n   Address   (0x15000)\n   DFH value (0x3000000010001009)\n\nTX: Tag Search Comparison: CPLD Tag: 00b   Active Tag: 00b\nTX: PU Completion ADDED to transaction!\nTX: Match found for PU CPLD!\nRead64  Method Data Transaction: Address: 0000_0000_0001_6000   Data: 3000_0000_a000_0100\nHELLO_FIM_DFH\n   Address   (0x16000)\n   DFH value (0x30000000a0000100)\n\nTX: Tag Search Comparison: CPLD Tag: 00c   Active Tag: 00c\nTX: PU Completion ADDED to transaction!\nTX: Match found for PU CPLD!\nRead64  Method Data Transaction: Address: 0000_0000_0002_0000   Data: 3000_0002_0000_1012\nPMCI_DFH\n   Address   (0x20000)\n   DFH value (0x3000000200001012)\n\n...\n\nTest status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\n$finish called from file \"/home/applications.fpga.ofs.fim-n6001/sim/unit_test/dfh_walker/unit_test.sv\", line 236.\n$finish at simulation time 13720.141ns\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 13720141000 fs\nCPU Time:      3.290 seconds;       Data structure size:   5.8Mb\nMon Dec  4 09:27:50 2023\nTotal of 5 minutes elapsed for dfh_walker\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                                 
                                6.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#414-walkthrough-hardware-test-a-fim-that-has-a-new-module","title":"4.1.4 Walkthrough: Hardware test a FIM that has a new module","text":"
                              Perform the following steps to program and hardware test a FIM that has had a new module added to it.
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                •  
                               
                                 
                              • This walkthrough uses a FIM design that has been generated with a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for generating a Hello FIM design.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                [OPTIONAL] In the work directory where the FIM was compiled, determine the PR Interface ID of your design. You can use this value at the end of the walkthrough to verify that the design has been configured to the FPGA.
                                 
                                cd $OFS_ROOTDIR/<work_directory>/syn/board/iseries-dk/syn_top/\n\ncat fme-ifc-id.txt\n
                                 
                                Example output:
                                 
                                5bcd682f-5093-5fc7-8cd2-ae8073e19452\n
                                 
                                2.  
                              2.  
                                Switch to your deployment environment.
                                 
                                3.  
                              3.  
                                Program the FPGA with the Hello FIM image. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                                 
                                4.  
                              4.  
                                Run fpgainfo to determine the PCIe B:D.F of your board, and to verify the PR Interface ID matches the ID you found in Step 1.
                                 
                                fpgainfo fme\n
                                 
                                Example output:
                                 
                                Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:b1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : TBD \nBitstream Version                : 5.0.1\nPr Interface Id                  : 5bcd682f-5093-5fc7-8cd2-ae8073e19452\nBoot Page                        : N/A\n
                                 
                                Note: The errors related to the BMC are the result of the OFS BMC not being present on the iseries-dk design. These will be removed in a future release.
                                 
                                5.  
                              5.  
                                Initialize opae.io
                                 
                                sudo opae.io init -d <B:D.F>\n
                                 
                                For example:
                                 
                                sudo opae.io init -d b1:00.0\n
                                 
                                6.  
                              6.  
                                Run DFH walker. Note the value read back from offset 0x16000 indicates the DFH ID is 0x100 which matches the Hello FIM module.
                                 
                                sudo opae.io walk -d <B:D.F>\n
                                 
                                For example:
                                 
                                sudo opae.io walk -d b1:00.0\n
                                 
                                Example output:
                                 
                                ...\noffset: 0x15000, value: 0x3000000010001009\n    dfh: id = 0x9, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x16000, value: 0x30000000a0000100\n    dfh: id = 0x100, rev = 0x0, next = 0xa000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x20000, value: 0x3000000200001012\n    dfh: id = 0x12, rev = 0x1, next = 0x20000, eol = 0x0, reserved = 0x0, feature_type = 0x3\n...\n
                                 
                                7.  
                              7.  
                                Read all of the registers in the Hello FIM module
                                 
                                   
                                1.  
                                  Read the DFH Register
                                   
                                  opae.io -d b1:00.0 -r 0 peek 0x16000\n
                                   
                                  Example Output:
                                   
                                  0x30000006a0000100\n
                                   
                                  2.  
                                2.  
                                  Read the Scratchpad Register
                                   
                                  opae.io -d b1:00.0 -r 0 peek 0x16030\n
                                   
                                  Example Output:
                                   
                                  0x0\n
                                   
                                  3.  
                                3.  
                                  Read the ID Register
                                   
                                  opae.io -d b1:00.0 -r 0 peek 0x16038\n
                                   
                                  Example Output:
                                   
                                  0x6626070150000034\n
                                   
                                  4.  
                                 
                                8.  
                              8.  
                                Verify the scratchpad register at 0x16030 by writing and reading back from it.
                                 
                                   
                                1.  
                                  Write to Scratchpad register
                                   
                                  opae.io -d b1:00.0 -r 0 poke 0x16030 0x123456789abcdef\n
                                   
                                  2.  
                                2.  
                                  Read from Scratchpad register
                                   
                                  opae.io -d b1:00.0 -r 0 peek 0x16030\n
                                   
                                  Expected output:
                                   
                                  0x123456789abcdef\n
                                   
                                  3.  
                                3.  
                                  Write to Scratchpad register
                                   
                                  opae.io -d b1:00.0 -r 0 poke 0x16030 0xfedcba9876543210\n
                                   
                                  4.  
                                4.  
                                  Read from Scratchpad register
                                   
                                  opae.io -d b1:00.0 -r 0 peek 0x16030\n
                                   
                                  Expected output:
                                   
                                  0xfedcba9876543210\n
                                   
                                  5.  
                                 
                                9.  
                              9.  
                                Release the opae.io tool
                                 
                                opae.io release -d b1:00.0\n
                                 
                                10.  
                              10.  
                                Confirm the driver has been set back to dfl-pci
                                 
                                opae.io ls\n
                                 
                                Example output:
                                 
                                [0000:b1:00.0] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                                 
                                11.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#415-walkthrough-debug-the-fim-with-signal-tap","title":"4.1.5 Walkthrough: Debug the FIM with Signal Tap","text":"
                              The following steps guide you through the process of adding a Signal Tap instance to your design. The added Signal Tap instance provides hardware to capture the desired internal signals and connect the stored trace information via JTAG. Please be aware that the added Signal Tap hardware will consume FPGA resources and may require additional floorplanning steps to accommodate these resources. Some areas of the FIM use logic lock regions and these regions may need to be re-sized.
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                •  
                               
                                 
                              • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                •  
                               
                                 
                              • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design.
                                •  
                               
                              Perform the following steps in your development environment:
                               
                                 
                              1.  
                                Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                Synthesize the design using the -e build script option. You may skip this step if you are using a pre-synthesized design.
                                 
                                cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -e --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_hello_fim_with_stp\n
                                 
                                4.  
                              4.  
                                Open the design in Quartus. The Compilation Dashboard should show that the Analysis & Synthesis step has completed.
                                 
                                quartus $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/iseries-dk/syn_top/ofs_top.qpf\n
                                 
                                 
                                5.  
                              5.  
                                Open Tools -> Signal Tap Logic Analyzer
                                 
                                 
                                   
                                1.  
                                  Select the Default template and click Create
                                   
                                  2.  
                                2.  
                                  Assign the clock for sampling the Signal Tap instrumented signals of interest. Note that the clock selected should correspond to the signals you want to view for best trace fidelity. Different clocks can be used, however, there maybe issues with trace inaccuracy due to sampling time differences. This example instruments the hello_fim_top module previously intetegrated into the FIM. If unfamiliar with code, it is helpful to use the Quartus Project Navigator to find the block of interest and open the design instance for review.
                                   
                                     
                                  1.  
                                    In the Signal Configuration -> Clock box of the Signal Tap Logic Analyzer window, click the \"...\" button
                                     
                                     
                                    2.  
                                  2.  
                                    In the Node Finder tool that opens, select hello_fim_top_inst in the Look in: field. Type clk into the Named field, then click Search. Select the hello_fim_top_inst|clk signal from the Matching Nodes box and click the \">\" button to move it to the Nodes Found box. Click OK to close the Node Finder dialog.
                                     
                                     
                                    3.  
                                   
                                  3.  
                                3.  
                                  Update the sample depth and other Signal Tap settings as needed for your debugging criteria.
                                   
                                   
                                  4.  
                                4.  
                                  In the Signal Tap GUI add the nodes to be instrumented by double-clicking on the \"Double-click to add nodes\" legend.
                                   
                                   
                                  5.  
                                5.  
                                  This brings up the Node Finder to add the signals to be traced. In this example we will monitor the memory mapped interface to the Hello FIM. Enter csr_lite_if* in the Named: field. Select hello_fim_top_inst in the Look in: field.
                                   
                                  6.  
                                6.  
                                  Set the Object type: field to net_bus then click Search. Select the signals that appear in the Matching Nodes column, then click the > button.
                                   
                                   
                                  7.  
                                7.  
                                  Set the Object type: field to net then click Search. Select the non-bus signals that appear in the Matching Nodes column, then click the > button. Click Insert and close the Node Finder dialog.
                                   
                                   
                                  8.  
                                8.  
                                  To provide a unique name for your Signal Tap instance, select \"auto_signaltap_0\", right-click, and select Rename Instance (F2). Provide a descriptive name for your instance, for example, stp_for_hello_fim.
                                   
                                   
                                  9.  
                                9.  
                                  Save the newly created Signal Tap file, in the $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/iseries-dk/syn_top/ directory, and give it the same name as the instance. Ensure that the Add file to current project checkbox is ticked.
                                   
                                   
                                  10.  
                                10.  
                                  In the dialog that pops up, click \"Yes\" to add the newly created Signal Tap file to the project settings files.
                                   
                                   
                                  This will aurtomatically add the following lines to $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/iseries-dk/syn_top/ofs_top.qsf:
                                   
                                  set_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE STP_For_Hello_FIM.stp\nset_global_assignment -name SIGNALTAP_FILE STP_For_Hello_FIM.stp\n
                                   
                                  11.  
                                 
                                6.  
                              6.  
                                Close all Quartus GUIs.
                                 
                                7.  
                              7.  
                                Compile the project with the Signal Tap file added to the project. Use the -k switch to perform the compilation using the files in a specified working directory and not the original ones from the cloned repository. 
                                 
                                ./ofs-common/scripts/common/syn/build_top.sh -p -k --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_hello_fim_with_stp\n
                                 
                                8.  
                              8.  
                                Ensure that the compile completes successfully and meets timing:
                                 
                                ***********************************\n***\n***        OFS_PROJECT: ofs-agx7-pcie-attach\n***        OFS_BOARD: iseries-dk\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 1\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                 
                                9.  
                              9.  
                                Set up a JTAG connection to the iseries-dk. Refer to Set up JTAG section for step-by-step instructions.
                                 
                                10.  
                              10.  
                                Copy the ofs_top.sof and stp_for_hello_fim.stp files to the machine which is connected to the iseries-dk via JTAG.
                                 
                                11.  
                              11.  
                                From the JTAG connected machine, program the $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/iseries-dk/syn_top/output_files/ofs_top.sof image to the iseries-dk FPGA. Refer to the Program the FPGA via JTAG section for step-by-step programming instructions.
                                 
                                12.  
                              12.  
                                Open the Quartus Signal Tap GUI
                                 
                                $QUARTUS_ROOTDIR/bin/quartus_stpw\n
                                 
                                13.  
                              13.  
                                In the Signal Tap Logic Analyzer window, select File -> Open, and choose the stp_for_hello_fim.stp file.
                                 
                                 
                                14.  
                              14.  
                                In the right pane of the Signal Tap GUI, in the Hardware: selection box select the cable for the iseries-dk. In the Device: selection box select the Agilex device.
                                 
                                 
                                15.  
                              15.  
                                If the Agilex Device is not displayed in the Device: list, click the 'Scan Chain' button to re-scan the JTAG device chain.
                                 
                                16.  
                              16.  
                                Create the trigger conditions. In this example, we will capture data on a rising edge of the Read Address Valid signal.
                                 
                                 
                                17.  
                              17.  
                                Start analysis by selecting the 'stp_for_hello_fim' instance and pressing 'F5' or clicking the Run Analysis icon in the toolbar. You should see a green message indicating the Acquisition is in progress. Then, move to the Data Tab to observe the signals captured.
                                 
                                 
                                18.  
                              18.  
                                To generate traffic in the csr_lite_if signals of the Hello FIM module, walk the DFH list or peek/poke the Hello FIM registers.
                                 
                                opae.io init -d 0000:b1:00.0\nopae.io walk -d 0000:b1:00.0\nopae.io release -d 0000:b1:00.0\n
                                 
                                The signals should be captured on the rising edge of arvalid in this example. Zoom in to get a better view of the signals.
                                 
                                 
                                19.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#42-preparing-fim-for-afu-development","title":"4.2 Preparing FIM for AFU Development","text":"
                              To save area, the default Host Excercisers in the FIM can be replaced by a \"he_null\" blocks. There are a few things to note:
                               
                                 
                              • \"he_null\" is a minimal block with registers that responds to PCIe MMIO request. MMIO responses are required to keep PCIe alive (end points enabled in PCIe-SS needs service downstream requests).
                                •  
                              • If an exerciser with other I/O connections such as \"he_mem\" or \"he_hssi\" is replaced, then then those I/O ports are simply tied off.
                                •  
                              • The options supported are null_he_lb, null_he_hssi, null_he_mem and null_he_mem_tg. Any combination, order or all can be enabled at the same time. 
                                •  
                              • Finer grain control is provided for you to, for example, turn off only the exercisers in the Static Region in order to save area.
                                •  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#421-walkthrough-compile-the-fim-in-preparation-for-designing-your-afu","title":"4.2.1 Walkthrough: Compile the FIM in preparation for designing your AFU","text":"
                              Perform the following steps to compile a FIM for where the exercisers are removed and replaced with an he_null module while keeping the PF/VF multiplexor connections.
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the [Walkthrough: Set Up Development Environment] Section for instructions on setting up a development environment.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Clone the FIM repository (or use an existing cloned repository). Refer to the [Walkthrough: Clone FIM Repository] section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the [Walkthrough: Set Development Environment Variables] section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                Compile the FIM with the HE_NULL compile options
                                 
                                cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss iseries-dk:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_iseries-dk\n
                                 
                                4.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#43-partial-reconfiguration-region","title":"4.3 Partial Reconfiguration Region","text":"
                              To take advantage of the available resources in the Intel\u00ae Agilex\u00ae 7 FPGA for an AFU design, you can adjust the size of the AFU PR partition. An example reason for the changing the size of PR region is if you add more logic to the FIM region, then you may need to adjust the PR region to fit the additional logic into the static region.  Similarly, if you reduce logic in the FIM region, then you can adjust the PR region to provide more logic resources for the AFU.
                               
                              After the compilation of the FIM, the resources usage broken down by partitions as reported in the following file:
                               
                              $OFS_ROOTDIR/<WORDK_DIRECTORY>/syn/board/iseries-dk/syn_top/output_files/ofs_top.fit.rpt\n
                               
                              An example report of the resources usage by partitions defined for the FIM is shown as follows:
                               
                              +------------------------------------------------------------------------------------------+\n; Logic Lock Region Constraints                                                            ;\n+--------------------------------------+-------------------------+-------------------------+\n; Name                                 ; Place Region Constraint ; Route Region Constraint ;\n+--------------------------------------+-------------------------+-------------------------+\n; afu_top|port_gasket|pr_slot|afu_main ; (90, 40) to (350, 220)  ; (0, 0) to (385, 329)    ;\n+--------------------------------------+-------------------------+-------------------------+\n\n\n+----------------------------------------------------------------------------------------------+\n; Logic Lock Region Usage Summary                                                              ;\n+-------------------------------------------------------+--------------------------------------+\n; Statistic                                             ; afu_top|port_gasket|pr_slot|afu_main ;\n+-------------------------------------------------------+--------------------------------------+\n; ALMs needed [=A-B+C]                                  ; 48011.2 / 351140 ( 13 % )            ;\n;     [A] ALMs used in final placement                  ; 53324.4 / 351140 ( 15 % )            ;\n;     [B] Estimate of ALMs recoverable by dense packing ; 5452.3 / 351140 ( 1 % )              ;\n;     [C] Estimate of ALMs unavailable                  ; 139.0 / 351140 ( < 1 % )             ;\n; ALMs used for memory                                  ; 450.0                                ;\n; Combinational ALUTs                                   ; 67166                                ;\n; Dedicated Logic Registers                             ; 87533 / 1404560 ( 6 % )              ;\n; I/O Registers                                         ; 0                                    ;\n; Block Memory Bits                                     ; 1737568                              ;\n; M20Ks                                                 ; 137 / 5049 ( 2 % )                   ;\n; DSP Blocks needed [=A-B]                              ; 0 / 3439 ( 0 % )                     ;\n;     [A] DSP Blocks used in final placement            ; 0 / 3439 ( 0 % )                     ;\n;     [B] Estimate of DSPs recoverable by dense merging ; 0 / 3439 ( 0 % )                     ;\n; Pins                                                  ; 0                                    ;\n; IOPLLs                                                ; 0                                    ;\n;                                                       ;                                      ;\n; Region Placement                                      ; (90, 40) to (350, 220)               ;\n+-------------------------------------------------------+--------------------------------------+\n
                               
                              In this case, the default size for the afu_top|port_gasket|pr_slot|afu_main PR partition is large enough to hold the logic of the default AFU, which is mainly occupied by the Host Exercisers. However, larger designs might require additional resources.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#431-walkthrough-resize-the-partial-reconfiguration-region","title":"4.3.1 Walkthrough: Resize the Partial Reconfiguration Region","text":"
                              Perform the following steps to first analyze the PR logic lock regions in a default FIM design, then resize the PR region:
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                The OFS flow provides the TCL file $OFS_ROOTDIR/syn/board/iseries-dk/setup/pr_assignments.tcl which defines the PR partition where the AFU is allocated. Each region is a rectangle defined by the origin coordinate (X0, Y0) and the top right corner coordinate (X1, Y1).
                                 
                                #####################################################\n# Main PR Partition -- green_region\n#####################################################\nset_instance_assignment -name PARTITION green_region -to afu_top|pg_afu.port_gasket|pr_slot|afu_main\nset_instance_assignment -name CORE_ONLY_PLACE_REGION ON -to afu_top|pg_afu.port_gasket|pr_slot|afu_main\nset_instance_assignment -name RESERVE_PLACE_REGION ON -to afu_top|pg_afu.port_gasket|pr_slot|afu_main\nset_instance_assignment -name PARTIAL_RECONFIGURATION_PARTITION ON -to afu_top|pg_afu.port_gasket|pr_slot|afu_main\n\n\nset_instance_assignment -name PLACE_REGION \"X90 Y40 X350 Y220\" -to afu_top|pg_afu.port_gasket|pr_slot|afu_main\nset_instance_assignment -name ROUTE_REGION \"X0 Y0 X385 Y329\" -to afu_top|pg_afu.port_gasket|pr_slot|afu_main\n
                                 
                                4.  
                              4.  
                                [OPTIONAL] Use Quartus Chip Planner to visualize the default PR region allocation.
                                 
                                   
                                1.  
                                  Compile the design.
                                   
                                  ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk\n
                                   
                                  2.  
                                2.  
                                  Once the design is compiled, open it in Quartus.
                                   
                                  quartus $OFS_ROOTDIR/work_iseries-dk/syn/board/iseries-dk/syn_top/ofs_top.qpf\n
                                   
                                  3.  
                                3.  
                                  Switch to ofs_top.
                                   
                                   
                                  4.  
                                4.  
                                  click Tools -> Chip Planner to open the Chip Planner.
                                   
                                  5.  
                                5.  
                                  Analyze the regions shown. Note that the regions are made of rectangles described by an origin coordinate, region height, and region width. If you are modifying the regions, you will need to identify the coordinates of your desired region.
                                   
                                   
                                  6.  
                                6.  
                                  Close the Quartus GUI.
                                   
                                  7.  
                                 
                                5.  
                              5.  
                                Make changes to the partial reconfiguraton region in the $OFS_ROOTDIR/syn/board/iseries-dk/setup/pr_assignments.tcl file. You can modify the size and location of existing lock regions or add new ones and assign them to the AFU PR partition.
                                 
                                6.  
                              6.  
                                Recompile your FIM and create the PR relocatable build tree using the following commands.
                                 
                                cd $OFS_ROOTDIR    \n\nofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk_resize_pr\n
                                 
                                7.  
                              7.  
                                Analyze the resource utilization report $OFS_ROOTDIR/work_iseries-dk/syn/board/iseries-dk/syn_top/output_files/ofs_top.fit.rpt produced after recompiling the FIM.
                                 
                                8.  
                              8.  
                                Perform further modification to the PR regions until the results are satisfactory. Make sure timing constraints are met.
                                 
                                9.  
                               
                              For more information on how to optimize the floor plan of your Partial Reconfiguration design refer to the following online documentation.
                               
                                 
                              • Analyzing and Optimizing the Design Floorplan
                                •  
                               
                                 
                              • Partial Reconfiguration Design Flow - Step 3 - Floorplan the Design
                                •  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#44-pcie-configuration","title":"4.4 PCIe Configuration","text":"
                              The PCIe Subsystem can be easily modified using OFS provided script and the PCIe subsystem IP core.  In this section both the PCIe SR-IOV configuration and PCIe configuration registers will be modified.  You can use this process for setting up your specific settings.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#441-pcie-ss-configuration-registers","title":"4.4.1 PCIe-SS Configuration Registers","text":"
                              The PCIe-SS configuration registers contain the Vendor, Device and Subsystem Vendor ID registers which are used in PCIe add-in cards to uniquely identify the card for assignment to software drivers. OFS has these registers set with Intel values for out of the box usage. If you are using OFS for a PCIe add in card that your company manufactures, then update the PCIe Subsytem Subsystem ID and Vendor ID registers as described below and change OPAE provided software code to properly operate with your company's register values.
                               
                              The Vendor ID is assigned to your company by the PCI-SIG (Special Interest Group). The PCI-SIG is the only body officially licensed to give out IDs. You must be a member of PCI-SIG to request your own ID. Information about joining PCI-SIG is available here: PCI-SIG. You select the Subsystem Device ID for your PCIe add in card.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#442-pfvf-mux-configuration","title":"4.4.2 PF/VF MUX Configuration","text":"
                              The default PF/VF MUX configuration for OFS PCIe Attach FIM for the iseries-dk can support up to 8 PFs and 2000 VFs distributed accross all PFs.
                               
                              For reference FIM configurations, 0 VFs on PF0 is not supported. This is because the PR region cannot be left unconnected. A NULL AFU may need to be instantiated in this special case. PFs must be consecutive. the PF/VF Limitations table describes the supported number of PFs and VFs.
                               
                              Table: PF/VF Limitations
                               Parameter Value Min # of PFs 1 (on PF0) Max # of PFs 8 Min # of VFs 1 on PF0 Max # of VFs 2000 distributed across all PFs 
                              Please be aware, as the number of VFs goes up, timing closure can become more difficult.
                               
                              The scripts provided in ${OFS_ROOTDIR}/ofs-common/tools/ofss_config allows you to easily reconfigure the number of PFs and VFs, bar addresses, vendor/device ID values and more.  The PCIe Subsystem IP parameters that can be modified can be seen by reviewing ${OFS_ROOTDIR}/ofs-common/tools/ofss_config/ip_params/pcie_ss_component_parameters.py 
                               
                              New PF or VF instances will automatically have a null_afu module instantiated and connected to the new PF or VF.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#443-pcie-configuration-using-ofss","title":"4.4.3 PCIe Configuration Using OFSS","text":"
                              The general flow for using OFSS to modify the PCIe Sub-system and PF/VF MUX is as follows:
                               
                                 
                              1. Create or modify a PCIe OFSS file with the desired PCIe configuration. 
                                2.  
                              2. Call this PCIe OFSS file when running the FIM build script.
                                3.  
                               
                              The PCIe IP OFSS File Options table lists all of the configuration options supported by the OFSS flow. Any other customizations to the PCIe sub-system must be done with the IP Presets Flow.
                               
                              Table: PCIe IP OFSS File Options
                               Section Parameter Options Default Description [ip] type pcie N/A Specifies that this OFSS file configures the PCIe-SS [settings] output_name pcie_ss N/A Specifies the output name of the PCIe-SS IP preset String N/A OPTIONAL - Specifies the name of a PCIe-SS IP presets file to use when building the FIM. When used, a presets file will take priority over any other parameters set in this OFSS file. pcie_gen 4 | 5 [1] N/A Specifies the PCIe generation pcie_instances 1 | 2 [1] N/A Specifies the number of PCIe instances [pf*] num_vfs Integer 0 Specifies the number of Virtual Functions in the current PF bar0_address_width Integer 12 bar4_address_width Integer 14 vf_bar0_address_width Integer 12 ats_cap_enable 0 | 1 0 vf_ats_cap_enable 0 | 1 0 prs_ext_cap_enable 0 | 1 0 pasid_cap_enable 0 | 1 0 pci_type0_vendor_id 32'h Value 0x00008086 pci_type0_device_id 32'h Value 0x0000bcce revision_id 32'h Value 0x00000001 class_code 32'h Value 0x00120000 subsys_vendor_id 32'h Value 0x00008086 subsys_dev_id 32'h Value 0x00001771 sriov_vf_device_id 32'h Value 0x0000bccf exvf_subsysid 32'h Value 0x00001771 
                              [1] For the iseries-dk Gen5 2x8 is supported. Gen5 1x16 is not supported. Gen4 1x16 has not been validated.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#4431-walkthrough-modify-the-pcie-sub-system-and-pfvf-mux-configuration-using-ofss","title":"4.4.3.1 Walkthrough: Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS","text":"
                              Perform the following steps to use OFSS files to configure the PCIe Sub-system and PF/VF MUX. In this example both the PCIe SR-IOV configuration and PCIe configuration registers will be modified.
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                •  
                               
                                 
                              • To demonstrate updated PCIe PF/VF in hardware, use an OFS Agilex I-Series PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                View the default OFS PCIe Attach FIM for the iseries-dk PF/VF configuration in the the $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host_rtile.ofss file.
                                 
                                [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n# pcie_ss top_topology_hw.tcl parameter using the variables below\n# assuming x16 lane, following will map to  \"Gen<pcie_gen> <pcie_instances>x<16/pcie_instances>\"\n# i.e. for R-tile PCIe SS example \"Gen5 2x8\". For combinations supported.\n# Note:\n# - \"Gen5 2x8\" is supported at the moment\n# - \"Gen5 1x16\" is not supported in PCIe SS\n# - \"Gen4 1x16\" should work but not validated\npcie_gen = 5\npcie_instances = 2\n\n[pf0]\nnum_vfs = 3\nbar0_address_width = 20\nvf_bar0_address_width = 20\n\n[pf1]\n\n[pf2]\nbar0_address_width = 18\n\n[pf3]\n\n[pf4]\n
                                 
                                4.  
                              4.  
                                Create a new PCIe OFSS file from the existing pcie_host_rtile.ofss file
                                 
                                cp $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host_rtile.ofss $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host_rtile_pfvf_new.ofss\n
                                 
                                5.  
                              5.  
                                Configure the new pcie_host_rtile_pfvf_new.ofss with updated PF/VF settings. In this example the following changes are made:
                                 
                                   
                                1. PF0: 1 VF is added by changing num_vfs from 3 to 4.
                                  2.  
                                2. PF1, PF2: Vendor, device, subsystem vendor and subsystem device IDs are changed.
                                  3.  
                                3. PF3: 1 VF is added by inserting num_vfs = 1
                                  4.  
                                4. PF5, PF6 and PF7 are added to use the maximum supported number (8) PFs
                                  5.  
                                 
                                The block diagram of the updated afu_top is shown below:
                                 
                                 
                                You can use this example as a template for creating PCI settings for your specific requirements.
                                 
                                [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n# pcie_ss top_topology_hw.tcl parameter using the variables below\n# assuming x16 lane, following will map to  \"Gen<pcie_gen> <pcie_instances>x<16/pcie_instances>\"\n# i.e. for R-tile PCIe SS example \"Gen5 2x8\". For combinations supported.\n# Note:\n# - \"Gen5 2x8\" is supported at the moment\n# - \"Gen5 1x16\" is not supported in PCIe SS\n# - \"Gen4 1x16\" should work but not validated\npcie_gen = 5\npcie_instances = 2\n\n[pf0]\nnum_vfs = 4\nbar0_address_width = 20\nvf_bar0_address_width = 20\n\n[pf1]\npci_type0_vendor_id = \"0x00001172\"\npci_type0_device_id = \"0x0000c001\"\nsubsys_vendor_id = \"0x00001172\"\nsubsys_dev_id = \"0x0000bad1\"\n\n[pf2]\nbar0_address_width = 18\npci_type0_vendor_id = \"0x00001172\"\npci_type0_device_id = \"0x00001122\"\nsubsys_vendor_id = \"0x00001172\"\nsubsys_dev_id = \"0x0000ddcc\"\n\n[pf3]\nnum_vfs = 1\n[pf4]\n\n[pf5]\n\n[pf6]\n\n[pf7]\n
                                 
                                6.  
                              6.  
                                Edit the $OFS_ROOTDIR/tools/ofss_config/tools/ofss_config/iseries-dk.ofss file to use the new PCIe configuration file pcie_pfvf_new.ofss
                                 
                                [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/iseries-dk_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host_rtile_pfvf_new.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory_rtile.ofss\n
                                 
                                7.  
                              7.  
                                Compile the FIM. 
                                 
                                cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk_pf8\n
                                 
                                8.  
                              8.  
                                If needed, copy the resulting $OFS_ROOTDIR/work_iseries-dk_pfvf_pf8/syn/board/iseries-dk/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                                 
                                9.  
                              9.  
                                Switch to your deployment environment.
                                 
                                10.  
                              10.  
                                Program the ofs_top.sof image to the iseries-dk FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                                 
                                11.  
                              11.  
                                Verify the 8 PFs are present with proper device ID.
                                 
                                $ lspci | grep bcce\nad:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nad:00.3 Processing accelerators: Intel Corporation Device bcce (rev 01)\nad:00.4 Processing accelerators: Intel Corporation Device bcce (rev 01)\nad:00.5 Processing accelerators: Intel Corporation Device bcce (rev 01)\nad:00.6 Processing accelerators: Intel Corporation Device bcce (rev 01)\nad:00.7 Processing accelerators: Intel Corporation Device bcce (rev 01)\n\n  $ lspci | grep c001\nad:00.1 Processing accelerators: Altera Corporation Device c001 (rev 01)\n\n $ lspci | grep 1122\nad:00.2 Processing accelerators: Altera Corporation Device 1122 (rev 01)\n
                                 
                                12.  
                              12.  
                                The PCIe Subssytem is setup in bifurcation mode so the unused PCIe channel will enumerate.  You can see this enumeration:
                                 
                                $ lspci | grep Altera\nac:00.0 Non-VGA unclassified device: Altera Corporation Device 0000 (rev 01)\nad:00.1 Processing accelerators: Altera Corporation Device c001 (rev 01)\nad:00.2 Processing accelerators: Altera Corporation Device 1122 (rev 01)\n\nNote, ac:00.0 is unused PCIe channel.\n
                                 
                                13.  
                              13.  
                                Verify the new VFs can be added.  Use the OPAE SDK command pci_device to create VFs.  Verify PF 0 and PF 3 have proper number of VFs and have device ID of bccf.
                                 
                                $ sudo pci_device ad:00.0 vf 4\n$ sudo pci_device ad:00.3 vf 1\n$ sudo lspci -vvv -s ad:00.0 | grep VF\n                Initial VFs: 4, Total VFs: 4, Number of VFs: 4, Function Dependency Link: 00\n                VF offset: 8, stride: 1, Device ID: bccf\n                VF Migration: offset: 00000000, BIR: 0\n$ sudo lspci -vvv -s ad:00.3 | grep VF\n                Initial VFs: 1, Total VFs: 1, Number of VFs: 1, Function Dependency Link: 03\n                VF offset: 9, stride: 1, Device ID: bccf\n                VF Migration: offset: 00000000, BIR: 0\n
                                 
                                14.  
                              14.  
                                Verify communication with the newly added PF5. The OFSS script creates a Null AFU with a basic set of command/status (CSR) registers connected to new PF/VF instances.  This basic set of CSRs can be used as the starting point for your new function.  In this step, the PF/VF is bound to the VFIO driver and the OPAE SDK commands opae.io peek is used to read the CSR registers in the Null AFU instance coonected to PF5.  
                                 
                                You can use this mechanism to verify access to your newly developed AFU.
                                 
                                The GUID for every new PF/VF that has the automatically instantiated null_afu is:
                                 
                                * NULL_GUID_L           = 64'haa31f54a3e403501\n* NULL_GUID_H           = 64'h3e7b60a0df2d4850\n
                                 
                                   
                                1.  
                                  Initialize the driver on PF5
                                   
                                  sudo opae.io init -d ad:00.5 $USER\n
                                   
                                  Example output:
                                   
                                  Unbinding (0x8086,0xbcce) at 0000:ad:00.5 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:ad:00.5 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:ad:00.5 is 414\nAssigning /dev/vfio/414 to $USER\nChanging permissions for /dev/vfio/414 to rw-rw----\n
                                   
                                  2.  
                                2.  
                                  Read the GUID for the PF5 CSR stub.
                                   
                                  opae.io -d ad:00.5 -r 0 peek 0x8\n
                                   
                                  Example output:
                                   
                                  0xaa31f54a3e403501\n
                                   
                                  opae.io -d ad:00.5 -r 0 peek 0x10\n
                                   
                                  Example output:
                                   
                                  0x3e7b60a0df2d4850\n
                                   
                                  3.  
                                 
                                15.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#444-pcie-configuration-using-ip-presets","title":"4.4.4 PCIe Configuration Using IP Presets","text":"
                              The general flow for using IP Presets to modify he PCIe Sub-system is as follows:
                               
                                 
                              1. [OPTIONAL] Create a work directory using OFSS files if you wish to use OFSS configuration settings.
                                2.  
                              2. Open the PCIe-SS IP and make desired modifications.
                                3.  
                              3. Create an IP Presets file.
                                4.  
                              4. Create an PCIe OFSS file that uses the IP Presets file.
                                5.  
                              5. Build the FIM with the PCIe OFSS file from Step 4.
                                6.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#4441-walkthrough-modify-pcie-configuration-using-ip-presets","title":"4.4.4.1 Walkthrough: Modify PCIe Configuration Using IP Presets","text":"
                              Perform the following steps to use OFSS files to configure the PCIe Sub-system and PF/VF MUX. In this example, we will change the Revision ID of PF0. While this modification can be done with the OFSS flow, this walkthrough is intended to show the procedure for making any PCIe configuration change using IP presets.
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                •  
                               
                                 
                              • To demonstrate updated PCIe PF/VF in hardware, use an OFS Agilex I-Series PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                Run the setup stage of the build script using your desired OFSS configration to create a working directory for the iseries-dk design.
                                 
                                ./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk\n
                                 
                                4.  
                              4.  
                                Open the PCIe-SS in the work directory using Quartus Parameter Editor.
                                 
                                qsys-edit $OFS_ROOTDIR/work_iseries-dk/ipss/pcie/qip/pcie_ss.ip\n
                                 
                                5.  
                              5.  
                                In the IP Parameter Editor window, scroll down and select the PCIe Interfaces Port Settings -> Port 0 -> PCIe0 Device Identification Registers -> PCIe0 PF0 IDs tab. Modify the settings as desired. In this case, we are changing the Revision ID to 0x00000002. You may make any desired modifications.
                                 
                                 
                                6.  
                              6.  
                                Once you are satisfied with your modifcations, create a new IP Preset file.
                                 
                                   
                                1.  
                                  click New... in the Presets window.
                                   
                                   
                                  2.  
                                2.  
                                  In the New Preset window, set a unique Name for the preset; for example, iseries-dk-rev2.
                                   
                                   
                                  3.  
                                3.  
                                  Click the ... button to set the save location for the IP Preset file to $OFS_ROOTDIR/ipss/pcie/presets. Set the File Name to match the name selected in Step 9. Click OK.
                                   
                                   
                                  4.  
                                4.  
                                  In the New Preset window, click Save. Click No when prompted to add the file to the IP search path.
                                   
                                   
                                  5.  
                                 
                                7.  
                              7.  
                                Close IP Parameter Editor without saving or generating HDL.
                                 
                                8.  
                              8.  
                                Create a new PCIe OFSS file in the $OFS_ROOTDIR/tools/ofss_config/pcie directory. For example:
                                 
                                touch $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host_rtile_mod_preset.ofss\n
                                 
                                Insert the following into the OFSS file to specify the IP Preset file created in Step 8. 
                                 
                                [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\npreset = iseries-dk-rev2\n
                                 
                                9.  
                              9.  
                                Edit the $OFS_ROOTDIR/tools/ofss_config/iseries-dk.ofss file to call new OFSS file created in Step 10.
                                 
                                [include]  \"$OFS_ROOTDIR\"/tools/ofss_config/iseries-dk_base.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host_rtile_mod_preset.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory_rtile.ofss
                                 
                                10.  
                              10.  
                                Compile the design with the modified iseries-dk.ofss file.
                                 
                                cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk_pcie_mod\n
                                 
                                11.  
                              11.  
                                Copy the resulting $OFS_ROOTDIR/work_iseries-dk_pcie_mod/syn/board/iseries-dk/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                                 
                                12.  
                              12.  
                                Switch to your deployment environment.
                                 
                                13.  
                              13.  
                                Program the ofs_top.sof image to the iseries-dk FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                                 
                                14.  
                              14.  
                                Determing the PCIe B:D.F of your board. You may use the OPAE command fpgainfo fme to determine this.
                                 
                                fpgainfo fme\n
                                 
                                Example output:
                                 
                                Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:84:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x501020226F0E360\nBitstream Version                : 5.0.1\nPr Interface Id                  : 7428037b-5408-52f2-8b1d-707870ec518a\nBoot Page                        : N/A\n
                                 
                                Note: The errors related to the BMC are the result of the OFS BMC not being present on the iseries-dk design. These will be removed in a future release.
                                 
                                15.  
                              15.  
                                Use lspci with the PCIe B:D.F of your board to verify that the PCIe changes have been implemented. In this example, the Rev for PF0 is 02.
                                 
                                lspci -nvmms 84:00.0\n
                                 
                                Example output:
                                 
                                Slot:   84:00.0\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nRev:    02\nNUMANode:       1\nIOMMUGroup:     70\n
                                 
                                16.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#45-minimal-fim","title":"4.5 Minimal FIM","text":"
                              In a minimal FIM, the exercisers are removed and the PCIe SR-IOV PF/VF infrastructure is reduced to 1 PF and 1 VF. This minimal FIM is useful for oneAPI BSP applications.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#451-create-a-minimal-fim","title":"4.5.1 Create a Minimal FIM","text":"
                              Perform the following steps to create a Minimal FIM.
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                •  
                               
                                 
                              • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                Create a new platform OFSS file that will use a 1PF/1VF PCIe configuration.
                                 
                                cp $OFS_ROOTDIR/tools/ofss_config/pcie_host_rtile.ofss $OFS_ROOTDIR/tools/ofss_config/pcie_host_rtile_1pf_1vf.ofss\n
                                 
                                4.  
                              4.  
                                Edit the $OFS_ROOTDIR/tools/ofss_config/pcie_host_rtile.ofss_1pf_1vf.ofss file to use the 1PF/1VF PCIe configuration as shown below:
                                 
                                [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n# pcie_ss top_topology_hw.tcl parameter using the variables below\n# assuming x16 lane, following will map to  \"Gen<pcie_gen> <pcie_instances>x<16/pcie_instances>\"\n# i.e. for R-tile PCIe SS example \"Gen5 2x8\". For combinations supported.\n# Note:\n# - \"Gen5 2x8\" is supported at the moment\n# - \"Gen5 1x16\" is not supported in PCIe SS\n# - \"Gen4 1x16\" should work but not validated\npcie_gen = 5\npcie_instances = 2\n\n[pf0]\nnum_vfs = 1\nbar0_address_width = 20\nvf_bar0_address_width = 20\n
                                 
                                5.  
                              5.  
                                Edit $OFS_ROOTDIR/tools/ofss_config/iseries-dk.ofss to use new PCIe file pcie_host_rtile_1pf_1vf.ofss as shown below:
                                 
                                [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/iseries-dk_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host_rtile_1pf_1vf.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory_rtile.ofss\n
                                 
                                6.  
                              6.  
                                Compile the FIM with Null HEs compile option, the No HSSI compile option, and 1PF/1VF configuration OFSS file.
                                 
                                cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg,no_hssi work_iseries-dk_min\n
                                 
                                7.  
                              7.  
                                Review the $OFS_ROOTDIR/work_iseries-dk_min/syn/board/iseries/syn_top/output_files/ofs_top.fit.rpt utilization report to see the utilization statistics for the Minimal fim. Refer to [Appendix A] Table A-4 for the expected utilization for this Minimal FIM.
                                 
                                8.  
                              8.  
                                Copy the resulting $OFS_ROOTDIR/work_iseries-dk_min/syn/board/iseries/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                                 
                                9.  
                              9.  
                                Switch to your deployment environment, if different than your development environment.
                                 
                                10.  
                              10.  
                                Program the ofs_top.sof image to the iseries-dk FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                                 
                                11.  
                              11.  
                                Use lspci to verify that the PCIe changes have been implemented.
                                 
                                sudo lspci -vvv -s b1:00.0 | grep VF\n
                                 
                                Example output: 
                                Initial VFs: 1, Total VFs: 1, Number of VFs: 0, Function Dependency Link: 00\nVF offset: 1, stride: 1, Device ID: bcce\nVF Migration: offset: 00000000, BIR: 0\n
                                 
                                12.  
                              12.  
                                You may wish to adjust the PR logic lock regions to maximize the resources available for the AFU. Refer to the Resize the Partial Reconfiguration Region section for instructions.
                                 
                                13.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#46-migrating-to-a-different-agilex-device-number","title":"4.6 Migrating to a Different Agilex Device Number","text":"
                              The following instructions enable a user to change the device part number of the Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile). Please be aware that this release works with Intel\u00ae Agilex\u00ae 7 FPGA devices that have F-tile for PCIe and Ethernet.  Other tiles will take further work.
                               
                              You may wish to change the device part number for the following reasons
                               
                                 
                              1. Migrate to same device package but with a different density
                                2.  
                              2. Migrate to a different package and with a different or same density
                                3.  
                               
                              The default device for the Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) is AGIB027R29A1E2VR3
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#461-walkthrough-migrate-to-a-different-agilex-device-number","title":"4.6.1 Walkthrough: Migrate to a Different Agilex Device Number","text":"
                              Perform the following steps to migrate to a different Agilex Device. In this example, we will migrate from the default AGIB027R29A1E2VR3 device to AGIB027R31A1E2VB. The package will change from R29A to R31A.
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                Modify the part field in the $OFS_ROOTDIR/tools/ofss_config/iseries-dk_base.ofss file to use AGIB027R31A1E2VB.
                                 
                                [ip]\ntype = ofs\n\n[settings]\nplatform = n6001\nfamily = agilex\nfim = base_x16\npart = AGIB027R31A1E2VB \ndevice_id = 6001\n
                                 
                                4.  
                              4.  
                                Modify the DEVICE field in the $OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_top.qsf file to use AGIB027R31A1E2VB.
                                 
                                ############################################################################################\n# FPGA Device\n############################################################################################\n\nset_global_assignment -name FAMILY \"Agilex 7\"\nset_global_assignment -name DEVICE AGIB027R31A1E2VB \n
                                 
                                5.  
                              5.  
                                Modify the DEVICE field in the $OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_pr_afu.qsf file to use AGIB027R31A1E2VB.
                                 
                                ############################################################################################\n# FPGA Device\n############################################################################################\n\nset_global_assignment -name FAMILY Agilex 7\nset_global_assignment -name DEVICE AGIB027R31A1E2VB \n
                                 
                                6.  
                              6.  
                                If the device you are migrating to uses the same package and pinout, you do not need to modify the pinout constraints. In this example, because we are migrating from package R29A to R31A, we need to modify the pinout to match the new device. If you would like Quartus to attempt to pin out the design automatically, you may remove all pin assignments. Typically, you will still need to set certain pins manually in order to guide Quartus for a successful compile (e.g. transceiver reference clocks).
                                 
                                   
                                1.  
                                  Commment out all pin assignments in the following files:     * $OFS_ROOTDIR/syn/board/iseries-dk/setup/emif_loc.tcl     * $OFS_ROOTDIR/syn/board/iseries-dk/setup/top_loc.tcl
                                   
                                  2.  
                                2.  
                                  Identify the pins in the design that will be constrained. In this example we will manually constrain the QSFP reference clock and the PCIe reference clock to help guide the fitter. The Device Migration Pinout Table shows th pin assignments that will be set, along with the pin number for both the old R24C package and the new R31C package.
                                   Net Name R29A Pin Name R31A Pin Name AGI 027 R29A Pin # AGI 027 R31A Pin # qsfp_ref_clk REFCLK_FGTL12A_Q2_RX_CH4p REFCLK_FGTL12C_Q2_RX_CH4p JD74 AM57 PCIE_REFCLK0 REFCLK_GXRL14C_CH0p REFCLK_GXRL14A_CH0p DR68 BU56 PCIE_REFCLK1 REFCLK_GXRL14C_CH1p REFCLK_GXRL14A_CH1p CU68 BP57 
                                  3.  
                                3.  
                                  Constrain the pins identified in Step 6.B in the $OFS_ROOTDIR/syn/board/iseries-dk/setup/top_loc.tcl file for the new pinout for the AGF 027 R31C package. 
                                   
                                  set_location_assignment PIN_AM57 -to qsfp_ref_clk\n\nset_location_assignment PIN_BU56 -to PCIE_REFCLK0\nset_location_assignment PIN_BP57 -to PCIE_REFCLK1\n
                                   
                                  4.  
                                4.  
                                  Uncomment the instance assignments related to he QSFP and PCIe reference clocks in the $OFS_ROOTDIR/syn/board/iseries-dk/setup/top_loc.tcl file.
                                   
                                  set_instance_assignment -name IO_STANDARD \"CURRENT MODE LOGIC (CML)\" -to qsfp_ref_clk\n\nset_instance_assignment -name IO_STANDARD HCSL -to PCIE_REFCLK0\nset_instance_assignment -name IO_STANDARD HCSL -to PCIE_REFCLK1\n
                                   
                                  5.  
                                 
                                7.  
                              7.  
                                Compile a flat design. It is recommended to compile a flat design first before incorporating a PR region in the design.
                                 
                                cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk:flat work_iseries-dk_lrgdev\n
                                 
                                8.  
                              8.  
                                Verify that the build completes successfuly. If there are timing violation, try building with a different seed. Refer to the Change the Compilation Seed section for instructions on changing the build seed.
                                 
                                9.  
                              9.  
                                When you are satisfied with the pinout, preserve it by hard-coding the desired pinout to followig files:
                                 
                                   
                                • $OFS_ROOTDIR/syn/board/iseries-dk/setup/emif_loc.tcl
                                  •  
                                • $OFS_ROOTDIR/syn/board/iseries-dk/setup/top_loc.tcl
                                  •  
                                 
                                10.  
                              10.  
                                When you are ready to re-incorporate PR into the design, modify the PR region to be compatible with the new device. Refer to the Resize the Partial Reconfiguration Region section for instructions.
                                 
                                11.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#47-modify-the-ethernet-sub-system","title":"4.7 Modify the Ethernet Sub-System","text":"
                              This section describes the flows for modifying the Ethernet Sub-System.
                               
                              Note: The default HSSI-SS configuration for the iseries-dk is 2x4x25GbE.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#471-walkthrough-modify-the-ethernet-sub-system-to-1x400gbe","title":"4.7.1 Walkthrough: Modify the Ethernet Sub-System to 1x400GbE","text":"
                              OFS provides a preconfigured ofss file so the build script produces a FIM with a 1x400GbE Ethernet subsystem set for 400 GAUI-8.  You can build this system with the following:
                               
                                 
                              1.  
                                Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                 
                                2.  
                              2.  
                                Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                 
                                3.  
                              3.  
                                Build the I-Series FIM with 1x400GbE FIM:
                                 
                                ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_1x400_ftile.ofss iseries-dk work_iseries-dk_400\n
                                 
                                4.  
                               
                              The resulting FIM with 1x400GbE has the following MAC settings:
                               
                              Table: 1x400GbE MAC Settings
                               MAC Setting Value Client Interface Segmented FEC mode IEEE 802.3 RS(544.514) (CL 134) Auto Negotiation and link training Disabled Maximum Frame Size 1518 
                              You can change the MAC settings by opening the Ethernet Subsystem IP in IP Parameter Editor, update the setting and then save the update as a new preset.  You will then edit the ofss_config/hssi/hssi_1x400_ftile.ofss to use the new preset.
                               
                              The following steps describe the steps to change the 400 GbE MAC frame size to 9600 bytes.  Note, the 2x200 and 8x25 GbE MAC implementations can be changed using this process.
                               
                                 
                              1.  
                                Invoke IP Parameter editor to make changes to the Ethernet Subsystem IP.
                                 
                                cd $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss\nqsys-edit hssi_ss.ip --quartus-project=$OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_top.qpf\n
                                 
                                2.  
                              2.  
                                In IP Parameter editor, load the 400g-fseries-dk preset by selecting and then click Apply
                                 
                                 
                                3.  
                              3.  
                                In the Device 0 Configuration tab, go to the F-Tile IP Configuration tab and scroll down to P8 MAC Options - P8 Basic and change the TX and RX maximum framesize to 9600.
                                 
                                 
                                4.  
                              4.  
                                Click New in the Presets panel and in the New Preset pop up window, Name the new preset 400g-fseries-dk-9600 and in File enter $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss/presets/400g-fseries-dk-9600.qprs and click Save
                                 
                                 
                                5.  
                              5.  
                                Close IP Parameter Editor and do not save changes to hssi_ss.ip.  The new preset captured the changes and this new preset will be used in the following updates to re-generate the Ethernet IP subsystem with the updated frame size.
                                 
                                6.  
                              6.  
                                Create a new ofss file for the new preset.
                                 
                                cp $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_1x400_ftile.ofss $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_1x400_ftile_9600.ofss\n
                                 
                                7.  
                              7.  
                                Edit $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_1x400_ftile_9600.ofss to use the new 400g-fseries-dk-9600 preset as listed below:
                                 
                                [ip]\ntype = hssi\n\n[settings]\noutput_name = hssi_ss\nnum_channels = 1\ndata_rate = 400GAUI-8\npreset = 400g-fseries-dk-9600\n
                                 
                                8.  
                              8.  
                                Build the FIM with 9600 byte frame size by using the new hssi_1x400_ftile_9600.ofss file
                                 
                                ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_1x400_ftile_9600.ofss iseries-dk work_iseries-dk_400_9600\n
                                 
                                9.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#5-fpga-configuration","title":"5. FPGA Configuration","text":"
                              Configuring the Agilex FPGA on the iseries-dk is done by programming a SOF image to the FPGA the embedded USvia JTAG using Quartus Programer.
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#51-walkthrough-set-up-jtag","title":"5.1 Walkthrough: Set up JTAG","text":"
                              The Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) has an on-board FPGA Download Cable II module which is used to program the FPGA via JTAG.
                               
                              Perform the following steps to establish a JTAG connection with the iseries-dk.
                               
                              Pre-requisites:
                               
                                 
                              • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                •  
                               
                                 
                              • This walkthrough requires a workstation with Quartus Prime Pro Version 23.3 tools installed, specifically the jtagconfig tool.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Refer to the following figure showing the location of the on-board Intel FPGA Download Cable II micro USB connector.
                                 
                                 
                                2.  
                              2.  
                                Verify all switches are set to default as defined in Intel Agilex\u00ae 7 FPGA I-Series Development Kit User Guide.
                                 
                                3.  
                              3.  
                                Connect a Micro-USB to USB-A cable between the front panel J8 micro USB port and either the deployment server or an external computer that has Quartus Prime Pro Programming tools installed.
                                 
                                4.  
                              4.  
                                Use the jtagconfig tool to check that the JTAG chain contains the AGIB027R29A1E2VR3 device.
                                 
                                <QUARTUS_INSTALL_DIR>/23.3/quartus/bin/jtagconfig\n
                                 
                                Example expected output:
                                 
                                1) AGI FPGA Development Kit [1-13]\n  034BB0DD   AGIB027R29A(.|R2|R3)/..\n  020D10DD   VTAP10\n
                                 
                                5.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#52-walkthrough-program-the-fpga-via-jtag","title":"5.2 Walkthrough: Program the FPGA via JTAG","text":"
                              This walkthrough describes the steps to program the Agilex FPGA on the Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) with a SOF image via JTAG.
                               
                              Pre-Requisites:
                               
                                 
                              • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                •  
                               
                                 
                              • This walkthrough requires a SOF image which will be programmed to the Agilex FPGA. Refer to the Compile OFS FIM Section for step-by-step instructions for generating a SOF image.
                                •  
                               
                                 
                              • This walkthrough requires a JTAG connection to the iseries-dk. Refer to the Set up JTAG section for step-by-step instructions.
                                •  
                               
                                 
                              • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) is connected via JTAG.
                                •  
                               
                              Steps:
                               
                                 
                              1.  
                                Start in your deployment environment.
                                 
                                2.  
                              2.  
                                If the card is already programmed with an OFS enabled design, determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                                 
                                fpgainfo fme\n
                                 
                                Example output:
                                 
                                Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:84:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010202FAB46E6A\nBitstream Version                : 5.0.1\nPr Interface Id                  : b00b675b-a674-5849-9195-f662c92f5250\nBoot Page                        : N/A\n
                                 
                                Note: The errors related to the BMC are the result of the OFS BMC not being present on the iseries-dk design. These will be removed in a future release.
                                 
                                3.  
                              3.  
                                Remove the card from PCIe prior to programming. This will disable AER for the PCIe root port to prevent a surprise link-down event during programming.
                                 
                                sudo pci_device 84:00.0 unplug\n
                                 
                                4.  
                              4.  
                                Switch to the machine with JTAG connection to the iseries-dk, if different than your deployment machine.
                                 
                                5.  
                              5.  
                                Open the Quartus programmer GUI
                                 
                                quartus_pgmw\n
                                 
                                 
                                6.  
                              6.  
                                Click Hardware Setup to open the Hardware Setup dialog window.
                                 
                                   
                                1.  
                                  In the Currently selected hardware field select the iseries-dk.
                                   
                                  2.  
                                2.  
                                  In the Hardware frequency field enter 16000000 Hz
                                   
                                   
                                  3.  
                                3.  
                                  Click Close
                                   
                                  4.  
                                 
                                7.  
                              7.  
                                In the Quartus Prime Programmer window, click Auto Detect.
                                 
                                8.  
                              8.  
                                If prompted, select the AGIB027R29A1E2VR3 device. The JTAG chain should show the device.
                                 
                                 
                                9.  
                              9.  
                                Right click the AGIB027R29A1E2VR3 row and selct Change File.
                                 
                                 
                                10.  
                              10.  
                                In the Select New Programming File window that opens, select the .sof image you wish to program and click Open.
                                 
                                11.  
                              11.  
                                Check the Program/Configure box for the AGIB027R29A1E2VR3 row, then click Start. Wait for the Progress bar to show 100% (Success).
                                 
                                 
                                12.  
                              12.  
                                Close the Quartus Programmer GUI.
                                 
                                13.  
                              13.  
                                Switch to the deployment environment, if different than the JTAG connected machine.
                                 
                                14.  
                              14.  
                                Replug the board PCIe
                                 
                                sudo pci_device 84:00.0 plug\n
                                 
                                15.  
                              15.  
                                Run fpgainfo fme to verify communication with the board, and to check the PR Interface ID.
                                 
                                Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:84:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010202FAB46E6A\nBitstream Version                : 5.0.1\nPr Interface Id                  : ce31fe41-1d9b-572d-9ff1-9deae5c98b61\nBoot Page                        : N/A\n
                                 
                                Note: The errors related to the BMC are the result of the OFS BMC not being present on the iseries-dk design. These will be removed in a future release.
                                 
                                16.  
                              "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#appendix","title":"Appendix","text":""},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#appendix-a-resource-utilization-tables","title":"Appendix A: Resource Utilization Tables","text":"
                              Table A-1 Default Flat FIM Resource Utilization
                               Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 171138.4 18.75 767 5.78 afu_top 78519.0 8.6 249 1.88 auto_fab_0 1347.1 0.15 9 0.07 bpf 686.4 0.08 0 0.0 fme_top 633.0 0.07 6 0.05 hssi_wrapper 11887.5 1.3 81 0.61 mem_ss_top 5205.5 0.57 30 0.23 ofs_top_auto_tiles 4237.3 0.46 10 0.08 pcie_wrapper 66661.9 7.3 374 2.82 pmci_dummy_csr 670.5 0.07 0 0.0 qsfp_0 634.7 0.07 4 0.03 qsfp_1 633.6 0.07 4 0.03 rst_ctrl 17.3 0.0 0 0.0 sys_pll 0.5 0.0 0 0.0 
                              Table A-2 Default Out-of-Tree FIM Resource Utilization
                               Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 176794.4 19.37 767 5.78 afu_top 84201.6 9.22 249 1.88 auto_fab_0 1308.1 0.14 9 0.07 bpf 726.8 0.08 0 0.0 fme_top 630.2 0.07 6 0.05 hssi_wrapper 11957.6 1.31 81 0.61 mem_ss_top 5565.0 0.61 30 0.23 ofs_top_auto_tiles 4257.4 0.47 10 0.08 pcie_wrapper 66195.2 7.25 374 2.82 pmci_dummy_csr 671.0 0.07 0 0.0 qsfp_0 627.8 0.07 4 0.03 qsfp_1 631.0 0.07 4 0.03 rst_ctrl 18.6 0.0 0 0.0 sys_pll 0.5 0.0 0 0.0 
                              Table A-3 Minimal FIM Resource Utilization
                               Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 104858.0 11.49 524 3.95 afu_top 28211.9 3.09 107 0.81 auto_fab_0 1279.7 0.14 9 0.07 bpf 709.0 0.08 0 0.0 fme_top 617.9 0.07 6 0.05 hssi_dummy_csr 672.0 0.07 0 0.0 mem_ss_top 5513.7 0.6 30 0.23 pcie_wrapper 65808.3 7.21 372 2.8 pmci_dummy_csr 677.4 0.07 0 0.0 qsfp0_dummy_csr 677.3 0.07 0 0.0 qsfp1_dummy_csr 671.2 0.07 0 0.0 rst_ctrl 16.5 0.0 0 0.0 sys_pll 0.4 0.0 0 0.0"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#appendix-b-glossary","title":"Appendix B: Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                              Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                               
                              OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/","title":"Getting Started Guide: Open FPGA Stack for Intel Agilex 7 FPGAs Targeting the Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile)","text":"
                              Last updated: February 03, 2024 
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#10-about-this-document","title":"1.0 About This Document","text":"
                              The purpose of this document is to help users get started in evaluating the 2023.3 version of the PCIe Attach release targeting the I-Series Development Kit. After reviewing this document, a user shall be able to:
                               
                                 
                              • Set up a server environment according to the Best Known Configuration (BKC)
                                •  
                              • Load and verify firmware targeting the FIM and AFU regions of the Agilex FPGA
                                •  
                              • Verify full stack functionality offered by the PCIe Attach OFS solution
                                •  
                              • Learn where to find additional information on other PCIe Attach ingredients
                                •  
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#11-audience","title":"1.1 Audience","text":"
                              The information in this document is intended for customers evaluating the PCIe Attach shell targeting Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile). This platform is a Development Kit intended to be used as a starting point for evaluation and development.
                               
                              Note: Code command blocks are used throughout the document. Commands that are intended for you to run are preceded with the symbol '$', and comments with '#'. Full command output may not be shown.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-1-terminology","title":"Table 1: Terminology","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-2-software-and-component-version-summary-for-ofs-pcie-attach-targeting-the-i-series-development-kit","title":"Table 2: Software and Component Version Summary for OFS PCIe Attach targeting the I-Series Development Kit","text":"
                              The OFS 2023.3 PCIe Attach release is built upon tightly coupled software and Operating System version(s). The repositories listed below are used to manually build the Shell and the AFU portion of any potential workloads. Use this section as a general reference for the versions which compose this release. Specific instructions on building the FIM or AFU are discussed in their respective documents.
                               Component Version Quartus https://www.intel.com/content/www/us/en/software-kit/782411/intel-quartus-prime-pro-edition-design-software-version-23-3-for-linux.html, patches: 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe) Host Operating System https://access.redhat.com/downloads/content/479/ver=/rhel---8/8.6/x86_64/product-software OneAPI-ASP https://github.com/OFS/oneapi-asp/releases/tag/ofs-2023.3-2, patches: 0.02 OFS Platform AFU BBB https://github.com/OFS/ofs-platform-afu-bbb/releases/tag/ofs-2023.3-2 OFS FIM Common Resources https://github.com/OFS/ofs-fim-common/releases/tag/ofs-2023.3-2 AFU Examples https://github.com/OFS/examples-afu/releases/tag/ofs-2023.3-2 OPAE-SIM https://github.com/OPAE/opae-sim"},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-3-programmable-firmware-version-summary-for-ofs-pcie-attach-targeting-the-i-series-development-kit","title":"Table 3: Programmable Firmware Version Summary for OFS PCIe Attach Targeting the I-Series Development Kit","text":"
                              OFS releases include pre-built binaries for the FPGA, OPAE SDK and Linux DFL which can be programmed out-of-box (OOB) and include known identifiers shown below. Installation of artifacts provided with this release will be discussed in their relevant sections.
                               Component Version Link FIM (shell) Pr Interface ID: TBD https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2 Host OPAE SDK https://github.com/OPAE/opae-sdk, tag: 2.10.0-1 https://github.com/OFS/opae-sdk/releases/tag/2.10.0-1 Host Linux DFL Drivers https://github.com/OPAE/linux-dfl, tag: ofs-2023.3-6.1-3 https://github.com/OFS/linux-dfl/releases/tag/ofs-2023.3-6.1-3"},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-4-hardware-bkc-for-ofs-pcie-attach","title":"Table 4: Hardware BKC for OFS PCIe Attach","text":"
                              The following table highlights the hardware which composes the Best Known Configuation (BKC) for the OFS 2023.3 PCIe Attach release. The Intel FPGA Download Cable II is not required when using the I Series Dev Kit, as the device has an on-board blaster.
                               Component Link Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) https://www.intel.com/content/www/us/en/products/details/fpga/development-kits/agilex/agi027.html (optional) Intel FPGA Download Cable II https://www.intel.com/content/www/us/en/products/sku/215664/intel-fpga-download-cable-ii/specifications.html"},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#12-server-requirements","title":"1.2 Server Requirements","text":""},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#121-host-server-specifications","title":"1.2.1 Host Server Specifications","text":"
                              The host server must meet the following specifications:
                               
                                 
                              • The server platform should contain 128 GB of RAM to run certain demos, and to compile FIM Images
                                •  
                              • The server platform must be able to fit, power, and cool an Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) as described by the product page
                                •  
                              • The server should be able to run PCIe at Gen 5 speeds to properly test designs and demos
                                •  
                              • The server must be able to properly power the I Series Dev Kit using an auxillary power cable as described in section 3.1 of the Intel Agilex\u00ae 7 FPGA I-Series Development Kit User Guide
                                •  
                               
                              Note: Be sure to double check that purchased servers support the proper PCIe aux power cables.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#122-host-bios","title":"1.2.2 Host BIOS","text":"
                              These are the host BIOS settings known to work with the I-Series Development Kit. Information about the server's currently loaded firmware and BIOS settings can be found through its remote access controller, or by manually entering the BIOS by hitting a specific key during power on. Your specific server platform will include instructions on proper BIOS configuration and should be followed when altering settings.
                               
                                 
                              • PCIe slot must be bifurcated x8 / x8
                                •  
                              • PCIe slot speed must be set to Gen 5
                                •  
                              • PCIe slot must have iommu enabled
                                •  
                              • Intel VT for Directed I/O (VT-d) must be enabled
                                •  
                               
                              Specific BIOS paths are not listed here as they can differ between BIOS vendors and versions.
                               
                              In addition to BIOS settings required to support the operation of an OFS PCIe Attach solution targeting the I-Series Development Kit, server fan speed may need to be adjusted in the BIOS settings depending on local air temperature and air flow. The OFS PCIe Attach design does not automatically communicate cooling curve information with the on-board server management interface. Increasing air flow will mitigate possible thermal runaway and thermal throttling that may occur as a result. Refer to Board Thermal Requirements for more information.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#123-host-server-kernel-and-grub-configuration","title":"1.2.3 Host Server Kernel and GRUB Configuration","text":"
                              While many host Linux kernel and OS distributions may work with this design, only the following configuration(s) have been tested:
                               
                                 
                              • OS: RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6
                                •  
                              • Kernel: 6.1-lts
                                •  
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#13-preparing-the-i-series-development-kit-for-installation-into-a-server","title":"1.3 Preparing the I-Series Development Kit for Installation into a Server","text":"
                              Safety and Regulatory information can be found under the product page for this development kit.
                               
                              Ensure your device's switch configuration matches the default found in the Intel Agilex\u00ae 7 FPGA I-Series Development Kit User Guide. Physical installation of the board is covered in section 3.1 Applying Power to the Development Board.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#14-i-series-development-kit-jtag-setup","title":"1.4 I-Series Development Kit JTAG Setup","text":"
                              A specific JTAG driver needs to be installed on the host OS. Follow the instructions under the driver setup for Red Hat 5+ on Intel\u00ae FPGA Download Cable (formerly USB-Blaster) Driver for Linux*.
                               
                              View the JTAG Chain after installing the proper driver and Quartus 23.3
                               
                              cd ~/intelFPGA_pro/quartus/bin\n./jtagconfig -D\n1) AGI FPGA Development Kit [1-13]\n   (JTAG Server Version 23.3.0 Build 104 09/20/2023 SC Pro Edition)\n  034BB0DD   AGIB027R29A(.|R2|R3)/.. (IR=10)\n  020D10DD   VTAP10 (IR=10)\n    Design hash    27AA3E0B7CE0A5B9F366\n    + Node 08586E00  (110:11) #0\n    + Node 0C006E00  JTAG UART #0\n    + Node 19104600  Nios II #0\n    + Node 30006E02  Signal Tap #2\n    + Node 30006E01  Signal Tap #1\n    + Node 30006E00  Signal Tap #0\n\n  Captured DR after reset = (0069761BB020D10DD) [65]\n  Captured IR after reset = (000D55) [21]\n  Captured Bypass after reset = (2) [3]\n  Captured Bypass chain = (0) [3]\n  JTAG clock speed auto-adjustment is enabled. To disable, set JtagClockAutoAdjust parameter to 0\n  JTAG clock speed 24 MHz\n
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#15-upgrading-the-i-series-development-kit-fim-via-jtag","title":"1.5 Upgrading the I-Series Development Kit FIM via JTAG","text":"
                              Intel provides a pre-built FIM that can be used out-of-box for platform bring-up. This shell design is available on the OFS 2023.3 Release Page. After programming the shell and installing both the OPAE SDK and Linux DFL kernel drivers (as shown in sections 3.0 OFS DFL Kernel Drivers and 4.0 OPAE Software Development Kit), you can confirm the correct FIM has been configured by checking the output of fpgainfo fme against the following table:
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-5-fim-version","title":"Table 5: FIM Version","text":"Identifier Value Pr Interface ID 5bcd682f-5093-5fc7-8cd2-ae8073e19452 Bitstream ID TBD 
                                 
                              1.  
                                Download and unpack the artifacts from the 2023.3 release page. The file ofs_top_hps.sof is the base OFS FIM file. This file is loaded into the FPGA using the development kit built in USB Blaster. Please be aware this FPGA is not loaded into non-volatile storage, therefore if the server is power cycled, you will need to reload the FPGA .sof file.
                                 
                                wget https://github.com/OFS/ofs-agx7-pcie-attach/releases/download/ofs-2023.3-2/iseries-dk-images.tar.gz\ntar xf iseries-dk-images.tar.gz\ncd iseries-dk-images/\n
                                 
                                2.  
                              2.  
                                Remove the card from the PCIe bus to prevent a surprise link down. This step is not necessary if no image is currently loaded.
                                 
                                sudo pci_device <PCIe BDF> unplug\n
                                 
                                3.  
                              3.  
                                Start the Quartus Prime Programmer GUI interface, quartus_pgmw &, located in the bin directory of your Quartus installation. Select \"Hardware Setup\", double click the AGI FPGA Development Kit hardware item and change the hardware frequency to 16MHz.
                                 
                                 
                                4.  
                              4.  
                                On the home screen select \"Auto Detect\" and accept the default device.
                                 
                                5.  
                              5.  
                                Left click on the line for device AGIB027R29A (the Agilex device) and hit \"Change File\". Load ofs_top.sof from the artifacts directory and check \"Program / Configure\". Hit start.
                                 
                                 
                                6.  
                              6.  
                                Re-add the card to the PCIe bus. This step is not necessary if no image was loaded beforehand.
                                 
                                sudo pci_device <BDF> plug\n
                                 
                                7.  
                              7.  
                                If this is the first time you've loaded an image into the board, you will need to restart (warm boot) the server (not power cycle / cold boot).
                                 
                                8.  
                              8.  
                                Verify the PR Interface ID for your image matches expectation. When loading a FIM from the pre-compiled binary included in the artifacts archive, this ID will match the one listed in Table 5.
                                 
                                9.  
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#20-ofs-stack-architecture-overview-for-reference-platform","title":"2.0 OFS Stack Architecture Overview for Reference Platform","text":""},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#21-hardware-components","title":"2.1 Hardware Components","text":"
                              The OFS hardware architecture decomposes all designs into a standard set of modules, interfaces, and capabilities. Although the OFS infrastructure provides a standard set of functionality and capability, the user is responsible for making the customizations to their specific design in compliance with the specifications outlined in the FPGA Interface Manager Technical Reference Manual for Intel Agilex PCIe Attach: Open FPGA Stack.
                               
                              OFS is a hardware and software infrastructure that provides an efficient approach to developing a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#211-fpga-interface-manager","title":"2.1.1 FPGA Interface Manager","text":"
                              The FPGA Interface Manager (FIM), or shell of the FPGA provides platform management functionality, clocks, resets, and interface access to the host and peripheral features on the acceleration platform. The OFS architecture for Intel Agilex 7 FPGA provides modularity, configurability, and scalability. The primary components of the FPGA Interface Manager or shell of the reference design are:
                               
                                 
                              • PCIe Subsystem - a hierarchical design that targets the P-tile PCIe hard IP and is configured to support bifurcated Gen 5 speeds
                                •  
                              • Ethernet Subsystem - provides portability to different Ethernet configurations across platforms and generations and reusability of the hardware framework and software stack.
                                •  
                              • Memory Subsystem - 2 x 8 GB DDR4 DIMMs, supporting 2666 MHz speeds, 64-bit width (no ECC)
                                •  
                              • Reset Controller
                                •  
                              • FPGA Management Engine - Provides a way to manage the platform and enable acceleration functions on the platform.
                                •  
                              • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                                •  
                              • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                                •  
                              • Platform Management Controller Interface (PMCI) to the board management controller
                                •  
                               
                              The FPGA Management Engine (FME) provides management features for the platform and the loading/unloading of accelerators through partial reconfiguration. Each feature of the FME exposes itself to the kernel-level OFS drivers on the host through a Device Feature Header (DFH) register that is placed at the beginning of Control Status Register (CSR) space. Only one PCIe link can access the FME register space in a multi-host channel design architecture at a time.
                               
                              Note: For more information on the FIM and its external connections, refer to the FPGA Interface Manager Technical Reference Manual for Intel Agilex PCIe Attach: Open FPGA Stack.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#212-afu","title":"2.1.2 AFU","text":"
                              An AFU is an acceleration workload that interfaces to the FIM. The AFU boundary in this reference design comprises both static and partial reconfiguration (PR) regions. You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region. A port gasket within the design provides all the PR specific modules and logic required to support partial reconfiguration. Only one partial reconfiguration region is supported in this design.
                               
                              Like the FME, the port gasket exposes its capability to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space.
                               
                              You can compile your design in one of the following ways:
                               
                                 
                              • Your entire AFU resides in a partial reconfiguration region of the FPGA.
                                •  
                              • The AFU is part of the static region and is compiled as a flat design.
                                •  
                              • Your AFU contains both static and PR regions.
                                •  
                               
                              In this design, the AFU region is comprised of:
                               
                                 
                              • AFU Interface handler to verify transactions coming from AFU region.
                                •  
                              • PF/VF Mux to route transactions to and from corresponding AFU components: ST2MM module, Virtio LB stub, PCIe loopback host exerciser (HE-LB), HSSI host exerciser (HE-HSSI), Memory Host Exerciser (HE-MEM), Traffic Generator to memory (HE-MEM-TG), Port Gasket (PRG) and HPS Copy Engine.
                                •  
                              • AXI4 Streaming to Memory Map (ST2MM) Module that routes MMIO CSR accesses to FME and board peripherals.
                                •  
                              • Host exercisers to test PCIe, memory and HSSI interfaces (these can be removed from the AFU region after your FIM design is complete to provide more resource area for workloads)
                                •  
                              • Basic HPS Copy Engine to copy second-stage bootloader and Linux OS image from Host DDR to HPS DDR.
                                •  
                              • Port gasket and partial reconfiguration support.
                                •  
                              • Component for handling PLDM over MCTP over PCIe Vendor Defined Messages (VDM)
                                •  
                               
                              The AFU has the option to consume native packets from the host or interface channels or to instantiate a shim provided by the Platform Interface Manager (PIM) to translate between protocols.
                               
                              Note: For more information on the Platform Interface Manager and AFU development and testing, refer to the AFU Development Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#22-ofs-software-overview","title":"2.2 OFS Software Overview","text":"
                              The responsibility of the OFS kernel drivers is to act as the lowest software layer in the FPGA software stack, providing a minimalist driver implementation between the host software and functionality that has been implemented on the development platform. This leaves the implementation of IP-specific software in user-land, not the kernel. The OFS software stack also provides a mechanism for interface and feature discovery of FPGA platforms.
                               
                              The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space, and can be found on the OPAE SDK Github.
                               
                              The OFS drivers decompose implemented functionality, including external FIM features such as HSSI, EMIF and SPI, into sets of individual Device Features. Each Device Feature has its associated Device Feature Header (DFH), which enables a uniform discovery mechanism by software. A set of Device Features are exposed through the host interface in a Device Feature List (DFL). The OFS drivers discover and \"walk\" the Device Features in a Device Feature List and associate each Device Feature with its matching kernel driver.
                               
                              In this way the OFS software provides a clean and extensible framework for the creation and integration of additional functionalities and their features.
                               
                              Note: A deeper dive on available SW APIs and programming model is available in the Software Reference Manual: Intel\u00ae Open FPGA Stack, on kernel.org, and through the Linux DFL wiki pages.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#30-ofs-dfl-kernel-drivers","title":"3.0 OFS DFL Kernel Drivers","text":"
                              OFS DFL driver software provides the bottom-most API to FPGA platforms. Libraries such as OPAE and frameworks like DPDK are consumers of the APIs provided by OFS. Applications may be built on top of these frameworks and libraries. The OFS software does not cover any out-of-band management interfaces. OFS driver software is designed to be extendable, flexible, and provide for bare-metal and virtualized functionality. An in depth look at the various aspects of the driver architecture such as the API, an explanation of the DFL framework, and instructions on how to port DFL driver patches to other kernel distributions can be found on the wiki.
                               
                              An in-depth review of the Linux device driver architecture can be found on opae.github.io.
                               
                              The DFL driver suite can be automatically installed using a supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3 Release Page.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#31-ofs-dfl-kernel-driver-installation-environment-setup","title":"3.1 OFS DFL Kernel Driver Installation Environment Setup","text":"
                              All OFS DFL kernel driver primary release code resides in the Linux DFL GitHub repository. This repository is open source and should not require any permissions to access. It includes a snapshot of the Linux kernel with the OFS driver included in /drivers/fpga/*. Downloading, configuration, and compilation will be discussed in this section. Refer back to section 1.2.3 Host Server Kernel and GRUB Configuration for a list of supported Operating System(s).
                               
                              You can choose to install the DFL kernel drivers by either using pre-built binaries created for the BKC OS, or by building them on your local server. If you decide to use the pre-built packages available on the OFS 2023.3 Release Page, skip to section 3.3 Installing the OFS DFL Kernel Drivers from Pre-Built Packages. Regardless of your choice you will need to follow the steps in this section to prepare your server for installation.
                               
                              This installation process assumes the user has access to an internet connection to clone specific GitHub repositories, and to satisfy package dependencies.
                               
                              1. It is recommended you lock your Red Hat release version to 8.6 to prevent accidental upgrades. Update installed system packages to their latest versions.
                               
                              subscription-manager release --set=8.6\nsudo dnf update\nsubscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms\nsudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n
                               
                              2. Install the following package dependencies if building and installing drivers from source. If you do not require the use of a proxy to pull in downloads using dnf, you can safely remove those parameters from the following commands:
                               
                              If you require the use of a proxy, add it to DNF using by editing the following file\nsudo nano /etc/dnf/dnf.conf\n# Include your proxy by adding the following line, replacing the URL with your proxy's URL\n# proxy=http://proxy.server.com:port\n\nsudo dnf install  python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel python3-pybind11 numactl-devel\n\npython3 -m pip install --user jsonschema virtualenv pudb pyyaml setuptools pybind11\n\n# If setuptools and pybind11 were already installed\n\npython3 -m pip upgrade pybind11 setuptools\n
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#32-building-and-installing-the-ofs-dfl-kernel-drivers-from-source","title":"3.2 Building and Installing the OFS DFL Kernel Drivers from Source","text":"
                              It is recommended you create an empty top level directory for your OFS related repositories to keep the working environment clean. All steps in this installation will use a generic top-level directory at /home/OFS/. If you have created a different top-level directory, replace this path with your custom path.
                               
                              1. Initialize an empty git repository and clone the DFL driver source code:
                               
                              mkdir /home/OFS/\ncd /home/OFS/\ngit init\ngit clone https://github.com/OFS/linux-dfl\ncd /home/OFS/linux-dfl\ngit checkout tags/ofs-2023.3-6.1-3\n
                               
                              Note: The linux-dfl repository is roughly 5 GB in size.
                               
                              2. Verify that the correct tag/branch have been checked out.
                               
                              git describe --tags\nofs-2023.3-6.1-3\n
                               
                              Note: If two different tagged releases are tied to the same commit, running git describe tags may report the other release's tag. This is why the match is made explicit.
                               
                              3. Copy an existing kernel configuration file from /boot and apply the minimal required settings changes.
                               
                              cd /home/OFS/linux-dfl\ncp /boot/config-`uname -r` .config\ncat configs/dfl-config >> .config\necho 'CONFIG_LOCALVERSION=\"-dfl\"' >> .config\necho 'CONFIG_LOCALVERSION_AUTO=y' >> .config\nsed -i -r 's/CONFIG_SYSTEM_TRUSTED_KEYS=.*/CONFIG_SYSTEM_TRUSTED_KEYS=\"\"/' .config\nsed -i '/^CONFIG_DEBUG_INFO_BTF/ s/./#&/' .config\necho 'CONFIG_DEBUG_ATOMIC_SLEEP=y' >> .config\nexport LOCALVERSION=\nmake olddefconfig\n
                               
                              Note: You can add a unique identifier to the kernel name by changing CONFIG_LOCALVERSION in .config. By default this is set to '-dfl'. This is useful if needing to differentiate between multiple installs. For example, you can add -dfl-2023.3
                               
                              4. The above command may report errors resembling symbol value 'm' invalid for CHELSIO_IPSEC_INLINE. These errors indicate that the nature of the config has changed between the currently executing kernel and the kernel being built. The option \"m\" for a particular kernel module is no longer a valid option, and the default behavior is to simply turn the option off. However, the option can likely be turned back on by setting it to 'y'. If the user wants to turn the option back on, change it to 'y' and re-run \"make olddefconfig\":
                               
                              cd /home/OFS/linux-dfl\necho 'CONFIG_CHELSIO_IPSEC_INLINE=y' >> .config\nmake olddefconfig\n
                               
                              Note: To use the built-in GUI menu for editing kernel configuration parameters, you can opt to run make menuconfig.
                               
                              5. Linux kernel builds take advantage of multiple processors to parallelize the build process. Display how many processors are available with the nproc command, and then specify how many make threads to utilize with the -j option. Note that number of threads can exceed the number of processors. In this case, the number of threads is set to the number of processors in the system.
                               
                              cd /home/OFS/linux-dfl\nmake -j $(nproc)\n
                               
                              6. The DFL Makefile contains a few options for package creation:
                               
                                 
                              • rpm-pkg: Build both source and binary RPM kernel packages
                                •  
                              • binrpm-pkg: Build only the binary kernel RPM package
                                •  
                              • deb-pkg: Build both source and binary deb kernel packages
                                •  
                              • bindeb-pkg: Build only the binary kernel deb package
                                •  
                               
                              If you are concerned about the size of the resulting package and binaries, they can significantly reduce the size of the package and object files by using the make variable INSTALL_MOD_STRIP. If this is not a concern, feel free to skip this step. The below instructions will build a set of binary RPM packages:
                               
                              cd /home/OFS/linux-dfl\nmake INSTALL_MOD_STRIP=1 binrpm-pkg\n
                               
                              By default, a directory is created in your home directory called rpmbuild. This directory will house all the kernel packages which have been built. You need to navigate to the newly built kernel packages and install them. The following files were generated using the build command executed in the previous step:
                               
                              cd ~/rpmbuild/RPMS/x86_64\nls\nkernel-6.1.41_dfl.x86_64.rpm  kernel-headers-6.1.41_dfl.x86_64.rpm\nsudo dnf localinstall kernel*.rpm\n
                               
                              7. The system will need to be rebooted in order for changes to take effect. After a reboot, select the newly built kernel as the boot target. This can be done pre-boot using the command grub2-reboot, which removes the requirement for user intervention. After boot, verify that the currently running kernel matches expectation.
                               
                              uname -r\n6.1.41-dfl\n
                               
                              8. Verify the DFL drivers have been successfully installed by reading version information directly from /lib/modules. Recall that the name of the kernel built as a part of this section is 6.1.41-dfl. If the user set a different name for their kernel, change this path as needed:
                               
                              cd /usr/lib/modules/6.1.41-dfl/kernel/drivers/fpga\nls\ndfl-afu.ko     dfl-fme.ko      dfl-fme-region.ko  dfl.ko             dfl-pci.ko      fpga-mgr.ko     intel-m10-bmc-sec-update.ko\ndfl-fme-br.ko  dfl-fme-mgr.ko  dfl-hssi.ko        dfl-n3000-nios.ko  fpga-bridge.ko  fpga-region.ko\n
                               
                              If an OFS device that is compatible with these drivers is installed on the server, you can double check the driver versions by listing the currently loaded kernel modules with lsmod:
                               
                              lsmod | grep dfl\nuio_dfl                20480  0\ndfl_emif               16384  0\nuio                    20480  1 uio_dfl\nptp_dfl_tod            16384  0\ndfl_intel_s10_iopll    20480  0\n8250_dfl               20480  0\ndfl_fme_region         20480  0\ndfl_fme_br             16384  0\ndfl_fme_mgr            20480  2\ndfl_fme                49152  0\ndfl_afu                36864  0\ndfl_pci                20480  0\ndfl                    40960  11 dfl_pci,uio_dfl,dfl_fme,intel_m10_bmc_pmci,dfl_fme_br,8250_dfl,qsfp_mem,ptp_dfl_tod,dfl_afu,dfl_intel_s10_iopll,dfl_emif\nfpga_region            20480  3 dfl_fme_region,dfl_fme,dfl\nfpga_bridge            20480  4 dfl_fme_region,fpga_region,dfl_fme,dfl_fme_br\nfpga_mgr               20480  4 dfl_fme_region,fpga_region,dfl_fme_mgr,dfl_fme\n
                               
                              9. Two kernel parameters must be added to the boot command line for the newly installed kernel. First, open the file grub:
                               
                              sudo vim /etc/default/grub\n
                               
                              10.  In the variable GRUB_CMDLINE_LINUX add the following parameters in bold: GRUB_CMDLINE_LINUX=\"crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\".
                               
                              Note: If you wish to instead set hugepages on a per session basis, you can perform the following steps. These settings will be lost on reboot.
                               
                              mkdir -p /mnt/huge \nmount -t hugetlbfs nodev /mnt/huge \necho 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages \necho 2048 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages \n
                               
                              11.  Save your edits, then apply them to the GRUB2 configuration file.
                               
                              sudo grub2-mkconfig\n
                               
                              12.  Warm reboot. Your kernel parameter changes should have taken affect.
                               
                              cat /proc/cmdline\nBOOT_IMAGE=(hd1,gpt2)/vmlinuz-6.1.41-dfl root=/dev/mapper/cl-root ro crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200 rhgb quiet\n
                               
                              A list of all DFL drivers and their purpose is maintained on the DFL Wiki.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#33-installing-the-ofs-dfl-kernel-drivers-from-pre-built-packages","title":"3.3 Installing the OFS DFL Kernel Drivers from Pre-Built Packages","text":"
                              To use the pre-built Linux DFL packages, you first need to download the files from the OFS 2023.3 Release Page. You can choose to either install using the SRC or RPM packages.
                               
                              tar xf kernel-6.1.41_dfl-1.x86_64-version.tar.gz\n\nsudo dnf localinstall kernel-6.1.41_dfl_*.x86_64.rpm \\\nkernel-devel-6.1.41_dfl_*.x86_64.rpm \\\nkernel-headers-6.1.41_dfl_*.x86_64.rpm\n\n### OR\n\nsudo dnf localinstall kernel-6.1.41_dfl_*.src.rpm\n
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#40-opae-software-development-kit","title":"4.0 OPAE Software Development Kit","text":"
                              The OPAE SDK software stack sits in user space on top of the OFS kernel drivers. It is a common software infrastructure layer that simplifies and streamlines integration of programmable accelerators such as FPGAs into software applications and environments. OPAE consists of a set of drivers, user-space libraries, and tools to discover, enumerate, share, query, access, manipulate, and reconfigure programmable accelerators. OPAE is designed to support a layered, common programming model across different platforms and devices. To learn more about OPAE, its documentation, code samples, an explanation of the available tools, and an overview of the software architecture, visit opae.github.io.
                               
                              The OPAE SDK source code is contained within a single GitHub repository hosted at the OPAE Github. This repository is open source and does not require any permissions to access.
                               
                              You can choose to install the OPAE SDK by either using pre-built binaries created for the BKC OS, or by building them on your local server. If you decide to use the pre-built packages available on the OFS 2023.3 Release Page, skip to section 4.3 Installing the OPAE SDK with Pre-built Packages. Regardless of your choice you will need to follow the steps in this section to prepare your server for installation.
                               
                              You may also choose to use the supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3 Release Page.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#41-opae-sdk-installation-environment-setup","title":"4.1 OPAE SDK Installation Environment Setup","text":"
                              This installation process assumes you have access to an internet connection in order to pull specific GitHub repositories, and to satisfy package dependencies.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-6-opae-package-description","title":"Table 6: OPAE Package Description","text":"Package Name Description opae OPAE SDK is a collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. It provides a library implementing the OPAE C API for presenting a streamlined and easy-to-use interface for software applications to discover, access, and manage FPGA devices and accelerators using the OPAE software stack. opae-debuginfo This package provides debug information for package opae. Debug information is useful when developing applications that use this package or when debugging this package. opae-debugsource This package provides debug sources for package opae. Debug sources are useful when developing applications that use this package or when debugging this package. opae-devel OPAE headers, tools, sample source, and documentation opae-devel-debuginfo This package provides debug information for package opae-devel. Debug information is useful when developing applications that use this package or when debugging this package. opae-tools This package contains OPAE base tools binaries opae-extra-tools Additional OPAE tools opae-extra-tools-debuginfo This package provides debug information for package opae-extra-tools. Debug information is useful when developing applications that use this package or when debugging this package. 
                                 
                              1.  
                                Remove any currently installed OPAE packages.
                                 
                                sudo dnf remove opae*\n
                                 
                                2.  
                              2.  
                                Initialize an empty git repository and clone the tagged OPAE SDK source code.
                                 
                                cd /home/OFS/\ngit init\ngit clone https://github.com/OFS/opae-sdk opae-sdk\ncd /home/OFS/opae-sdk\ngit checkout tags/2.10.0-1\n
                                 
                                3.  
                              3.  
                                Verify that the correct tag/branch have been checkout out.
                                 
                                git describe --tags\n2.10.0-1\n
                                 
                                4.  
                              4.  
                                Update your system, then set up a temporary podman container to build OPAE, which will allow you to customize the python installation without affecting system packages.
                                 
                                sudo subscription-manager release --set=8.6\nsudo dnf update\n\ncd /home/OFS\npodman pull registry.access.redhat.com/ubi8:8.6\npodman run -ti -v \"$PWD\":/src:Z -w /src registry.access.redhat.com/ubi8:8.6\n\n# Everything after runs within container:\n\n# Enable EPEL\ndnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n\ndnf install --enablerepo=codeready-builder-for-rhel-8-x86_64-rpms -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel make numactl-devel\n\npip3 install --upgrade --prefix=/usr pip setuptools pybind11\n\n./opae-sdk/packaging/opae/rpm/create unrestricted\n\nexit\n
                                 
                                The following packages will be built in the same directory as create:
                                 
                                5.  
                              5.  
                                Install the packages you just created.
                                 
                                cd /home/OFS/opae-sdk/packaging/opae/rpm\nrm -rf opae-2.10.0-1.el8.src.rpm \nsudo dnf localinstall -y opae*.rpm\n
                                 
                                6.  
                              6.  
                                Check that all packages have been installed and match expectation:
                                 
                                rpm -qa | grep opae\nopae-2.10.0-1.el8.x86_64.rpm\nopae-debuginfo-2.10.0-1.el8.x86_64.rpm\nopae-debugsource-2.10.0-1.el8.x86_64.rpm\nopae-devel-2.10.0-1.el8.x86_64.rpm\nopae-devel-debuginfo-2.10.0-1.el8.x86_64.rpm\nopae-extra-tools-2.10.0-1.el8.x86_64.rpm\nopae-extra-tools-debuginfo-2.10.0-1.el8.x86_64.rpm\n
                                 
                                7.  
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#43-installing-the-opae-sdk-with-pre-built-packages","title":"4.3 Installing the OPAE SDK with Pre-Built Packages","text":"
                              You can skip the entire build process and use a set of pre-built binaries supplied by Intel. Visit the OFS 2023.3 Release Page and navigate to the bottom of the page, under the Assets tab you will see a file named opae-2.10.0-1.x86_64-*_*.tar.gz. Download this package and extract its contents:
                               
                              tar xf opae-2.10.0-1.x86_64-<<date>>_<<build>>.tar.gz\n
                               
                              For a fast installation you can delete the source RPM as it isn't necessary, and install all remaining OPAE RPMs:
                               
                              rm opae-*.src.rpm\nsudo dnf localinstall opae*.rpm\n
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#44-opae-tools-overview","title":"4.4 OPAE Tools Overview","text":"
                              The following section offers a brief introduction including expected output values for the utilities included with OPAE. A full explanation of each command with a description of its syntax is available in the opae-sdk GitHub repo.
                               
                              A list of all tools included in the OPAE SDK release can be found on the OPAE FPGA Tools tab of ofs.github.io.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#441-board-management-with-fpgainfo","title":"4.4.1 Board Management with fpgainfo","text":"
                              The fpgainfo utility displays FPGA information derived from sysfs files. As this release targets a development kit platform, it does not have access to an on-board BMC. Subcommands that target specific BMC functionality (such as reading temeletry) are not supported for this release.
                               
                              Displays FPGA information derived from sysfs files. The command argument is one of the following: errors, power, temp, port, fme, bmc, phy or mac, security. Some commands may also have other arguments or options that control their behavior.
                               
                              For systems with multiple FPGA devices, you can specify the BDF to limit the output to the FPGA resource with the corresponding PCIe configuration. If not specified, information displays for all resources for the given command.
                               
                              Note: Your Bitstream ID and PR Interface Id may not match the below examples.
                               
                              The following examples walk through sample outputs generated by fpgainfo. As the I-Series Development Kit does not contain a traditional BMC as used by other OFS products, those lines in fpgainfo's output will not return valid objects. The subcommand fpgainfo bmc will likewise fail to report telemetry data.
                               
                              Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : TBD\nBitstream Version                : 5.0.1\nPr Interface Id                  : TBD\nBoot Page                        : N/A\n
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#443-updating-with-fpgasupdate","title":"4.4.3 Updating with fpgasupdate","text":"
                              The fpgasupdate tool is used to program AFU workloads into an open slot in a FIM. The fpgasupdate tool only accepts images that have been formatted using PACsign.
                               
                              As the I-Series Development Kit does not contain a traditional BMC, you do not have access to a factory, user1, and user2 programmed image for both the FIM and BMC FW and RTL. Only the programming of a GBS workload is supported for this release.
                               
                              The process of programming a SOF with a new FIM version is shown in section 1.5 Upgrading the I-Series Development Kit FIM via JTAG
                               
                              sudo fpgasupdate ofs_pr_afu.gbs   <PCI ADDRESS>\n[2022-04-14 16:42:31.58] [WARNING ] Update starting. Please do not interrupt.                                           \n[2022-04-14 16:42:31.58] [INFO    ] updating from file ofs_pr_afu.gbs with size 19928064               \n[2022-04-14 16:42:31.60] [INFO    ] waiting for idle                                                                 \n[2022-04-14 16:42:31.60] [INFO    ] preparing image file                                                                \n[2022-04-14 16:42:38.61] [INFO    ] writing image file                                                                 \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [19928064/19928064 bytes][Elapsed Time: 0:00:16.01]                                       \n[2022-04-14 16:42:54.63] [INFO    ] programming image file                                                              \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588][Elapsed Time: 0:06:16.40]                                                                 \n[2022-04-14 16:49:11.03] [INFO    ] update of 0000:b1:00.0 complete                                                     \n[2022-04-14 16:49:11.03] [INFO    ] Secure update OK                                                                   \n[2022-04-14 16:49:11.03] [INFO    ] Total time: 0:06:39.45\n
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#444-verify-fme-interrupts-with-hello_events","title":"4.4.4 Verify FME Interrupts with hello_events","text":"
                              The hello_events utility is used to verify FME interrupts. This tool injects FME errors and waits for error interrupts, then clears the errors.
                               
                              Sample output from sudo hello_events.
                               
                              sudo hello_events\nWaiting for interrupts now...\ninjecting error\nFME Interrupt occurred\nSuccessfully tested Register/Unregister for FME events!\nclearing error\n
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#445-host-exercisor-modules","title":"4.4.5 Host Exercisor Modules","text":"
                              The reference FIM and unchanged FIM compilations contain Host Exerciser Modules (HEMs). These are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. There are three HEMs present in the Intel OFS Reference FIM - HE-LPBK, HE-MEM, and HE-HSSI. These exercisers are tied to three different VFs that must be enabled before they can be used. Execution of these exercisers requires you bind specific VF endpoint to vfio-pci. The host-side software looks for these endpoints to grab the correct FPGA resource.
                               
                              Refer to the Intel FPGA Interface Manager Technical Reference Manual for Intel Agilex PCIe Attach: Open FPGA Stack for a full description of these modules.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-7-module-pfvf-mappings","title":"Table 7: Module PF/VF Mappings","text":"Module PF/VF ST2MM PF0 HE-MEM PF0-VF0 HE-HSSI PF0-VF1 HE-MEM_TG PF0-VF2 HE-LB Stub PF1-VF0 HE-LB PF2 VirtIO LB Stub PF3 HPS Copy Engine PF4"},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#4451-he-mem-he-lb","title":"4.4.5.1 HE-MEM / HE-LB","text":"
                              The host exerciser used to exercise and characterize the various host-FPGA interactions eg. MMIO, Data transfer from host to FPGA , PR, host to FPGA memory etc. Host Exerciser Loopback (HE-LBK) AFU can move data between host memory and FPGA.
                               
                              HE-LBK supports: - Latency (AFU to Host memory read) - MMIO latency (Write+Read) - MMIO BW (64B MMIO writes) - BW (Read/Write, Read only, Wr only)
                               
                              Host Exerciser Loopback Memory (HE-MEM) AFU is used to exercise use of FPGA connected DDR, data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host.
                               
                              HE-LB is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth. HE-MEM is used to exercise use of FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host. HE-MEM uses external DDR memory (i.e. EMIF) to store data. It has a customized version of the AVMM interface to communicate with the EMIF memory controller. Both exercisers rely on the user-space tool host_exerciser. When using the I-Series Development Kit SmartNIC Platform, optimal performance requires the exercisers be run at 400 MHz.
                               
                              Execution of these exercisers requires you to bind specific VF endpoint to vfio-pci. The following commands will bind the correct endpoint for a device with B/D/F 0000:b1:00.0 and run through a basic loopback test.
                               
                              Note: While running the opae.io init command listed below, the command has failed if no output is present after completion. Double check that Intel VT-D and IOMMU have been enabled in the kernel as discussed in section 3.0 OFS DFL Kernel Drivers.
                               
                              sudo pci_device  0000:b1:00.0 vf 3\n\nsudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci                                                             \nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci \niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188                                                                  \nAssigning /dev/vfio/188 to user                                                                 \nChanging permissions for /dev/vfio/188 to rw-rw----\n\n\nsudo host_exerciser --clock-mhz 400 lpbk\n    starting test run, count of 1\nAPI version: 1\nAFU clock from command line: 400 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Number of clocks: 5224\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1022\n    Bandwidth: 5.018 GB/s\n    Test lpbk(1): PASS\n
                               
                              The following example will run a loopback throughput test using one cache line per request.
                               
                              sudo pci_device  0000:b1:00.0 vf 3\n\nsudo opae.io init -d 0000:b1:00.2 user:user\n\nsudo host_exerciser --clock-mhz 400 --mode trput --cls cl_1 lpbk\n    starting test run, count of 1\nAPI version: 1\nAFU clock from command line: 400 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 512\n    Host Exerciser numWrites: 513\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Number of clocks: 3517\n    Total number of Reads sent: 512\n    Total number of Writes sent: 512\n    Bandwidth: 7.454 GB/s\n    Test lpbk(1): PASS\n
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#4452-traffic-generator-afu-test-application","title":"4.4.5.2 Traffic Generator AFU Test Application","text":"
                              Beginning in OPAE version 2.0.11-1+ the TG AFU has an OPAE application to access & exercise traffic, targeting a specific bank. The supported arguments for test configuration are:
                               
                                 
                              • Number of test loops: --loops
                                •  
                              • Number of read transfers per test loop: -r,--read
                                •  
                              • Number of write transfers per test loop: -w,--write
                                •  
                              • Burst size of each transfer: -b,--bls
                                •  
                              • Address stride between each transfer: --stride
                                •  
                              • Target memory TG: -m,--mem-channel
                                •  
                               
                              Below are some example commands for how to execute the test application. To run the preconfigured write/read traffic test on channel 0:
                               
                              mem_tg tg_test\n
                               
                              Target channel 1 with a 1MB single-word write only test for 1000 iterations
                               
                              mem_tg --loops 1000 -r 0 -w 2000 -m 1 tg_test\n
                               
                              Target channel 2 with 4MB write/read test of max burst length for 10 iterations
                               
                              mem_tg --loops 10 -r 8 -w 8 --bls 255 -m 2 tg_test\n
                               
                              sudo mem_tg --loops 1000 -r 2000 -w 2000 --stride 2 --bls 2  -m 1 tg_test\n[2022-07-15 00:13:16.349] [tg_test] [info] starting test run, count of 1\nMemory channel clock frequency unknown. Assuming 300 MHz.\nTG PASS\nMem Clock Cycles: 17565035\nWrite BW: 4.37232 GB/s\nRead BW: 4.37232 GB/s\n
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#4453-he-hssi","title":"4.4.5.3 HE-HSSI","text":"
                              HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic.
                               
                              The hssi application provides a means of interacting with the 10G and with the 100G HSSI AFUs. In both 10G and 100G operating modes, the application initializes the AFU, completes the desired transfer as described by the mode- specific options, and displays the ethernet statistics by invoking ethtool --statistics INTERFACE.
                               
                              sudo fpgainfo phy b1:00.0\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** PHY ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x501020241BF165B\nBitstream Version                : TBD\nPr Interface Id                  : TBD\nQSFP0                            : Connected\nQSFP1                            : Connected\n//****** HSSI information ******//\nHSSI version                     : 2.0\nNumber of ports                  : 8\nPort0                            :25GbE        UP\nPort1                            :25GbE        UP\nPort2                            :25GbE        UP\nPort3                            :25GbE        DOWN\nPort4                            :25GbE        UP\nPort5                            :25GbE        UP\nPort6                            :25GbE        DOWN\nPort7                            :25GbE        DOWN\n
                               
                              The following example walks through the process of binding the VF corresponding with the HE-HSSI exerciser to vfio-pci, sending traffic, and verifying that traffic was received.
                              "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-8-accelerator-pfvf-and-guid-mappings","title":"Table 8: Accelerator PF/VF and GUID Mappings","text":"Component VF Accelerator GUID I Series Dev Kit base PF XXXX:XX:XX.0 N/A VirtIO Stub XXXX:XX:XX.1 3e7b60a0-df2d-4850-aa31-f54a3e403501 HE-MEM Stub XXXX:XX:XX.2 56e203e9-864f-49a7-b94b-12284c31e02b Copy Engine XXXX:XX:XX.4 44bfc10d-b42a-44e5-bd42-57dc93ea7f91 HE-MEM XXXX:XX:XX.5 8568ab4e-6ba5-4616-bb65-2a578330a8eb HE-HSSI XXXX:XX:XX.6 823c334c-98bf-11ea-bb37-0242ac130002 MEM-TG XXXX:XX:XX.7 4dadea34-2c78-48cb-a3dc-5b831f5cecbb 
                                 
                              1.  
                                Create 3 VFs in the PR region.
                                 
                                sudo pci_device b1:00.0 vf 3 \n
                                 
                                2.  
                              2.  
                                Verify all 3 VFs were created.
                                 
                                lspci -s b1:00 \nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01) \nb1:00.1 Processing accelerators: Intel Corporation Device bcce \nb1:00.2 Processing accelerators: Intel Corporation Device bcce \nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device \nb1:00.4 Processing accelerators: Intel Corporation Device bcce \nb1:00.5 Processing accelerators: Intel Corporation Device bccf \nb1:00.6 Processing accelerators: Intel Corporation Device bccf \nb1:00.7 Processing accelerators: Intel Corporation Device bccf \n
                                 
                                3.  
                              3.  
                                Bind all the PF/VF endpoints to the vfio-pci driver.
                                 
                                sudo opae.io init -d 0000:b1:00.1 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 187\nAssigning /dev/vfio/187 to user\nChanging permissions for /dev/vfio/187 to rw-rw----\n\nsudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188\nAssigning /dev/vfio/188 to user\nChanging permissions for /dev/vfio/188 to rw-rw----\n\n...\n\nsudo opae.io init -d 0000:b1:00.7 user:user\nBinding (0x8086,0xbccf) at 0000:b1:00.7 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.7 is 319\nAssigning /dev/vfio/319 to user\nChanging permissions for /dev/vfio/319 to rw-rw----\n
                                 
                                4.  
                              4.  
                                Check that the accelerators are present using fpgainfo. Note your port configuration may differ from the below.
                                 
                                sudo fpgainfo port \n//****** PORT ******//\nObject Id                        : 0xEC00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0xE0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.7\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 4dadea34-2c78-48cb-a3dc-5b831f5cecbb\n//****** PORT ******//\nObject Id                        : 0xC0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.6\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 823c334c-98bf-11ea-bb37-0242ac130002\n//****** PORT ******//\nObject Id                        : 0xA0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.5\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n//****** PORT ******//\nObject Id                        : 0x80B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.4\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 44bfc10d-b42a-44e5-bd42-57dc93ea7f91\n//****** PORT ******//\nObject Id                        : 0x40B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n//****** PORT ******//\nObject Id                        : 0x20B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 3e7b60a0-df2d-4850-aa31-f54a3e403501\n
                                 
                                5.  
                              5.  
                                Check Ethernet PHY settings with fpgainfo.
                                 
                                sudo fpgainfo phy -B 0xb1 \n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : TBD\nBitstream Version                : 5.0.1\nPr Interface Id                  : TBD\n//****** HSSI information ******//\nHSSI version                     : 1.0\nNumber of ports                  : 8\nPort0                            :25GbE        DOWN\nPort1                            :25GbE        DOWN\nPort2                            :25GbE        DOWN\nPort3                            :25GbE        DOWN\nPort4                            :25GbE        DOWN\nPort5                            :25GbE        DOWN\nPort6                            :25GbE        DOWN\nPort7                            :25GbE        DOWN\n
                                 
                                6.  
                              6.  
                                Set loopback mode.
                                 
                                sudo hssiloopback --loopback enable  --pcie-address 0000:b1:00.0 \nargs Namespace(loopback='enable', pcie_address='0000:b1:00.0', port=0)\nsbdf: 0000:b1:00.0\nFPGA dev: {'segment': 0, 'bus': 177, 'dev': 0, 'func': 0, 'path': '/sys/class/fpga_region/region0', 'pcie_address': '0000:b1:00.0'}\nargs.hssi_grps{0: ['dfl_dev.6', ['/sys/bus/pci/devices/0000:b1:00.0/fpga_region/region0/dfl-fme.0/dfl_dev.6/uio/uio0']]}\nfpga uio dev:dfl_dev.6\n\n--------HSSI INFO START-------\nDFH                     :0x3000000010002015\nHSSI ID                 :0x15\nDFHv                    :0.5\nguidl                   :0x99a078ad18418b9d\nguidh                   :0x4118a7cbd9db4a9b\nHSSI version            :1.0\nFirmware Version        :1\nHSSI num ports          :8\nPort0                   :25GbE\nPort1                   :25GbE\nPort2                   :25GbE\nPort3                   :25GbE\nPort4                   :25GbE\nPort5                   :25GbE\nPort6                   :25GbE\nPort7                   :25GbE\n--------HSSI INFO END-------\n\nhssi loopback enabled to port0\n
                                 
                                7.  
                              7.  
                                Send traffic through the 10G AFU.
                                 
                                sudo hssi --pci-address b1:00.6 hssi_10g --num-packets 100       \n10G loopback test\n  port: 0\n  eth_loopback: on\n  he_loopback: none\n  num_packets: 100\n  packet_length: 64\n  src_address: 11:22:33:44:55:66\n    (bits): 0x665544332211\n  dest_address: 77:88:99:aa:bb:cc\n    (bits): 0xccbbaa998877\n  random_length: fixed\n  random_payload: incremental\n  rnd_seed0: 5eed0000\n  rnd_seed1: 5eed0001\n  rnd_seed2: 25eed\n  eth:\n\nNo eth interface, so not honoring --eth-loopback.\n0x40000           ETH_AFU_DFH: 0x1000010000001000\n0x40008          ETH_AFU_ID_L: 0xbb370242ac130002\n0x40010          ETH_AFU_ID_H: 0x823c334c98bf11ea\n0x40030      TRAFFIC_CTRL_CMD: 0x0000000000000000\n0x40038     TRAFFIC_CTRL_DATA: 0x0000000100000000\n0x40040 TRAFFIC_CTRL_PORT_SEL: 0x0000000000000000\n0x40048        AFU_SCRATCHPAD: 0x0000000045324511\n\n0x3c00         number_packets: 0x00000064\n0x3c01          random_length: 0x00000000\n0x3c02         random_payload: 0x00000000\n0x3c03                  start: 0x00000000\n0x3c04                   stop: 0x00000000\n0x3c05           source_addr0: 0x44332211\n0x3c06           source_addr1: 0x00006655\n0x3c07             dest_addr0: 0xaa998877\n0x3c08             dest_addr1: 0x0000ccbb\n0x3c09        packet_tx_count: 0x00000064\n0x3c0a              rnd_seed0: 0x5eed0000\n0x3c0b              rnd_seed1: 0x5eed0001\n0x3c0c              rnd_seed2: 0x00025eed\n0x3c0d             pkt_length: 0x00000040\n0x3cf4          tx_end_tstamp: 0x000003d2\n0x3d00                num_pkt: 0xffffffff\n0x3d01               pkt_good: 0x00000064\n0x3d02                pkt_bad: 0x00000000\n0x3d07            avst_rx_err: 0x00000000\n0x3d0b          rx_sta_tstamp: 0x00000103\n0x3d0c          rx_end_tstamp: 0x0000053b\n0x3e00               mac_loop: 0x00000000\n\nHSSI performance:\n        Selected clock frequency : 402.832 MHz\n        Latency minimum : 642.948 ns\n        Latency maximum : 896.155 ns\n        Achieved Tx throughput : 18.4528 GB/s\n        Achieved Rx throughput : 16.7101 GB/s\n\nNo eth interface, so not showing stats.\n
                                 
                                8.  
                               
                              The hssi_loopback utility works in conjunction with a packet generator accelerator function unit (AFU) to test high-speed serial interface (HSSI) cards. hssi_loopback tests both external and internal loopbacks.
                               
                              The hssistats tool provides the MAC statistics.
                              "},{"location":"hw/n6001/","title":"Index","text":"
                              Repository folder for Agilex OFS PCIe Attach collateral targeting Intel FPGA SmartNIC N6001-PL.
                              "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/","title":"FPGA Interface Manager Developer Guide for Open FPGA Stack: Intel\u00ae FPGA SmartNIC N6001-PL PCIe Attach","text":"
                              Last updated: February 03, 2024 
                              "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1-introduction","title":"1. Introduction","text":""},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#11-about-this-document","title":"1.1. About This Document","text":"
                              This document serves as a guide for OFS Agilex PCIe Attach developers targeting the Intel\u00ae FPGA SmartNIC N6001-PL. The following topics are covered in this guide:
                               
                                 
                              • Compiling the OFS Agilex PCIe Attach FIM design
                                •  
                              • Simulating the OFS Agilex PCIe Attach design
                                •  
                              • Customizing the OFS Agilex PCIe Attach FIM design
                                •  
                              • Configuring the FPGA with an OFS Agilex PCIe Attach FIM design
                                •  
                               
                              The FIM Development Walkthroughs Table lists all of the walkthroughs provided in this guide. These walkthroughs provide step-by-step instructions for performing different FIM Development tasks.
                               
                              Table: FIM Development Walkthroughs
                               Walkthrough Name Category Install Quartus Prime Pro Software Setup Install Git Large File Storage Extension Setup Clone FIM Repository Setup Set Development Environment Variables Setup Set Up Development Environment Setup Compile OFS FIM Compilation Manually Generate OFS Out-Of-Tree PR FIM Compilation Change the Compilation Seed Compilation Run Individual Unit Level Simulation Simulation Run Regression Unit Level Simulation Simulation Add a new module to the OFS FIM Customization Modify and run unit tests for a FIM that has a new module Customization Modify and run UVM tests for a FIM that has a new module Customization Hardware test a FIM that has a new module Customization Debug the FIM with Signal Tap Customization Compile the FIM in preparation for designing your AFU Customization Resize the Partial Reconfiguration Region Customization Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS Customization Modify PCIe Sub-System and PF/VF MUX Configuration Using IP Presets Customization Create a Minimal FIM Customization Migrate to a Different Agilex Device Number Customization Modify the Memory Sub-System Using IP Presets With OFSS Customization Modify the Ethernet Sub-System Channels With Pre-Made HSSI OFSS Customization Add Channels to the Ethernet Sub-System Channels With Custom HSSI OFSS Customization Modify the Ethernet Sub-System With Pre-Made HSSI OFSS Plus Additional Modifications Customization Modify the Ethernet Sub-System Without HSSI OFSS Customization Remove the HPS Customization Set up JTAG Configuration Program the FPGA via JTAG Configuration Program the FPGA via RSU Configuration"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#111-knowledge-pre-requisites","title":"1.1.1 Knowledge Pre-Requisites","text":"
                              It is recommended that you have the following knowledge and skills before using this developer guide.
                               
                                 
                              • Basic understanding of OFS and the difference between OFS designs. Refer to the OFS Welcome Page.
                                •  
                              • Review the release notes for the Intel Agilex 7 PCIe Attach Reference Shells, with careful consideration of the Known Issues.
                                •  
                              • Review of Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL).
                                •  
                              • FPGA compilation flows using Intel\u00ae Quartus\u00ae Prime Pro Edition.
                                •  
                              • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
                                •  
                              • RTL (System Verilog) and coding practices to create synthesized logic.
                                •  
                              • RTL simulation tools.
                                •  
                              • Intel\u00ae Quartus\u00ae Prime Pro Edition Signal Tap Logic Analyzer tool software.
                                •  
                              "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#12-fim-development-theory","title":"1.2. FIM Development Theory","text":"
                              This section will help you understand how the OFS Agilex PCIe Attach FIM can be developed to fit your design goals.
                               
                              The Default FIM Features section provides general information about the default features of the OFS Agilex PCIe Attach FIM so you can become familiar with the default design. For more detailed information about the FIM architecture, refer to the [OFS Agilex PCIe Attach FIM Technical Reference Manual].
                               
                              The Customization Options section then gives suggestions of how this default design can be customized. Step-by-step walkthroughs for many of the suggested customizations are later described in the FIM Customization section.
                               
                              FIM development for a new acceleration card generally consists of the following steps:
                               
                                 
                              1. Install OFS and familiarize yourself with provided scripts and source code
                                2.  
                              2. Develop high level design with your specific functionality
                                   
                                1. Determine requirements and key performance metrics
                                  2.  
                                2. Select IP cores
                                  3.  
                                3. Select FPGA device
                                  4.  
                                4. Develop software memory map
                                  5.  
                                 
                                3.  
                              3. Select and implement FIM Physical interfaces including:
                                   
                                1. External clock sources and creation of internal PLL clocks
                                  2.  
                                2. General I/O
                                  3.  
                                3. Ethernet modules
                                  4.  
                                4. External memories
                                  5.  
                                5. FPGA programming methodology
                                  6.  
                                 
                                4.  
                              4. Develop device physical implementation
                                   
                                1. FPGA device pin assignment
                                  2.  
                                2. Create logic lock regions
                                  3.  
                                3. Create of timing constraints
                                  4.  
                                4. Create Intel Quartus Prime Pro FIM test project and validate:
                                     
                                  1. Placement
                                    2.  
                                  2. Timing constraints
                                    3.  
                                  3. Build script process
                                    4.  
                                  4. Review test FIM FPGA resource usage
                                    5.  
                                   
                                  5.  
                                 
                                5.  
                              5. Select FIM to AFU interfaces and development of PIM
                                6.  
                              6. Implement FIM design
                                   
                                1. Develop RTL
                                  2.  
                                2. Instantiate IPs
                                  3.  
                                3. Develop test AFU to validate FIM
                                  4.  
                                4. Develop unit and device level simulation
                                  5.  
                                5. Develop timing constraints and build scripts
                                  6.  
                                6. Perform timing closure and build validation
                                  7.  
                                 
                                7.  
                              7. Create FIM documentation to support AFU development and synthesis
                                8.  
                              8. Software Device Feature discovery
                                9.  
                              9. Integrate, validate, and debug hardware/software
                                10.  
                              10. Prepare for high volume production
                                11.  
                              "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#121-default-fim-features","title":"1.2.1 Default FIM Features","text":""},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1211-top-level","title":"1.2.1.1 Top Level","text":"
                              Figure: OFS Agilex PCIe Attach n6001 FIM Top-Level Diagram
                               
                              "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1212-interfaces","title":"1.2.1.2 Interfaces","text":"
                              The key interfaces in the OFS Agilex PCIe Attach design are listed in the Release Capabilities Table. It describes the capabilities of the n6001 hardware as well as the capabilities of the default OFS Agilex PCIe Attach design targeting the n6001.
                               
                              Table: Release Capabilities
                               Interface n6001 Hardware Capabilities OFS Agilex PCIe Attach Default Design Implementation Host Interface PCIe Gen4x16 PCIe Gen4x16 Network Interface 2 x QSFP-28/56 cages 2x4x25GbE | 2x2x100GbE | 2x4x10GbE External Memory 5xDDR4 DIMMs sockets - 40-bits (1 available for HPS) 4xDDR4 - 2400MHz - 4GB (1Gb x 32) - 32-bits - No ECC1xDDR4 - 2400MHz - 1GB (256Mb x 32 with 256 Mb x8 ECC) - 40-bits - With ECC - For HPS"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1213-subsystems","title":"1.2.1.3 Subsystems","text":"
                              The FIM Subsystems Table  describes the Platform Designer IP subsystems used in the OFS Agilex PCIe Attach n6001 FIM.
                               
                              Table: FIM Subsystems
                               Subsystem User Guide Document ID PCIe Subsystem PCIe Subsystem Intel FPGA IP User Guide for Intel Agilex OFS N/A Memory Subsystem Memory Subsystem Intel FPGA IP User Guide for Intel Agilex OFS 686148[1] Ethernet Subsystem Ethernet Subsystem Intel FPGA IP User Guide 773413[1] 
                              [1] You must log in to myIntel and request entitled access.
                              "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1214-host-exercisers","title":"1.2.1.4 Host Exercisers","text":"
                              The default AFU workload in the OFS Agilex PCIe Attach n6001 FIM contains several modules called Host Exercisers which are used to exercise the interfaces on the board. The Host Exerciser Descriptions Table describes these modules.
                               
                              Table: Host Exerciser Descriptions
                               Name Acronym Description OPAE Command Host Exerciser Loopback HE-LB Used to exercise and characterize host to FPGA data transfer. host_exerciser Host Exerciser Memory HE_MEM Used to exercise and characterize host to Memory data transfer. host_exerciser Host Exerciser Memory Traffic Generator HE_MEM_TG Used to exercise and test available memory channels with a configurable traffic pattern. mem_tg Host Exerciser High Speed Serial Interface HE-HSSI Used to exercise and characterize HSSI interfaces. hssi 
                              The host exercisers can be removed from the design at compile-time using command line arguments for the build script.
                              "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1215-module-access-via-apfbpf","title":"1.2.1.5 Module Access via APF/BPF","text":"
                              The OFS Agilex PCIe Attach n6001 FIM uses AXI4-Lite interconnect logic named the AFU Peripheral Fabric (APF) and Board Peripheral Fabric (BPF) to access the registers of the various modules in the design. The APF/BPF modules define master/slave interactions, namely between the host software and AFU and board peripherals. The APF Address Map Table describes the address mapping of the APF, followed by the BPF Address Map Table which describes the address mapping of the BPF.
                               
                              Table: APF Address Map
                               Address Size (Bytes) Feature 0x00000\u20130x3FFFF 256K Board Peripherals (See BPF Address Map table) 0x40000 \u2013 0x4FFFF 64K ST2MM 0x50000 \u2013 0x5FFFF 64K Reserved 0x60000 \u2013 0x60FFF 4K UART 0x61000 \u2013 0x6FFFF 4K Reserved 0x70000 \u2013 0x7FFFF 56K PR Gasket:4K= PR Gasket DFH, control and status4K= Port DFH4K=User Clock52K=Remote STP 0x80000 \u2013 0x80FFF 4K AFU Error Reporting 
                              Table: BPF Address Mapping
                               Address Size (Bytes) Feature 0x00000 - 0x0FFFF 64K FME 0x10000 - 0x10FFF 4K PCIe 0x11000 - 0x11FFF 4K Reserved 0x12000 - 0x12FFF 4K QSFP0 0x13000 - 0x13FFF 4K QSFP1 0x14000 - 0x14FFF 4K HSSI 0x15000 - 0x15FFF 4K EMIF 0x20000 - 0x3FFFF 128K PMCI Controller"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#122-customization-options","title":"1.2.2 Customization Options","text":"
                              OFS is designed to be easily customizable to meet your design needs. The OFS FIM Customization Examples Table lists the general user flows for OFS Agilex PCIe Attach n6001 FIM development, along with example customizations for each user flow, plus links to step-by-step walkthroughs where available.
                               
                              Table: OFS FIM Customization Examples
                               Walkthrough Name Add a new module to the OFS FIM Modify and run unit tests for a FIM that has a new module Modify and run UVM tests for a FIM that has a new module Hardware test a FIM that has a new module Debug the FIM with Signal Tap Compile the FIM in preparation for designing your AFU Resize the Partial Reconfiguration Region Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS Modify PCIe Sub-System and PF/VF MUX Configuration Using IP Presets Create a Minimal FIM Migrate to a Different Agilex Device Number Modify the Memory Sub-System Using IP Presets With OFSS Modify the Ethernet Sub-System Channels With Pre-Made HSSI OFSS Add Channels to the Ethernet Sub-System Channels With Custom HSSI OFSS Modify the Ethernet Sub-System With Pre-Made HSSI OFSS Plus Additional Modifications Modify the Ethernet Sub-System Without HSSI OFSS Remove the HPS"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#13-development-environment","title":"1.3 Development Environment","text":"
                              This section describes the components required for OFS FIM development, and provides a walkthrough for setting up the environment on your development machine.
                               
                              Note that your development machine may be different than your deployment machine where the FPGA acceleration card is installed. FPGA development work and deployment work can be performed either on the same machine, or on different machines as desired. Please see the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up the environment for deployment machines.
                              "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#131-development-tools","title":"1.3.1 Development Tools","text":"
                              The Development Environment Table describes the Best Known Configuration (BKC) for the tools that are required for OFS FIM development.
                               
                              Table: Development Environment BKC
                               Component Version Installation Walkthrough Operating System RedHatEnterprise Linux\u00ae (RHEL) 8.6 N/A Intel Quartus Prime Software Quartus Prime Pro Version 23.3 for Linux + Patches 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe) Section 1.3.1.1 Python 3.6.8 or later N/A GCC 7.4.0 or later N/A cmake 3.15 or later N/A git with git-lfs 1.8.3.1 or later Section 1.3.1.2 FIM Source Files ofs-2023.3-2 Section 1.3.2.1"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1311-walkthrough-install-quartus-prime-pro-software","title":"1.3.1.1 Walkthrough: Install Quartus Prime Pro Software","text":"
                              Intel Quartus Prime Pro Version 23.3 is verified to work with the latest OFS release ofs-2023.3-2.  However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.
                               
                              Use RedHatEnterprise Linux\u00ae (RHEL) 8.6 for compatibility with your development flow and also testing your FIM design in your platform. 
                               
                              Prior to installing Quartus:
                               
                                 
                              1.  
                                Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.
                                 
                                   
                                • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                                  •  
                                • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.
                                  •  
                                 
                                2.  
                              2.  
                                Perform the following steps to satisfy the required dependencies.
                                 
                                $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                                 
                                Apply the following configurations.
                                 
                                $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n
                                 
                                3.  
                              3.  
                                Create the default installation path: /intelFPGA_pro/, where  is the default path of the Linux workstation, or as set by the system administrator and  is your Quartus version number. 
                                The installation path must satisfy the following requirements:
                                 
                                   
                                • Contain only alphanumeric characters
                                  •  
                                • No special characters or symbols, such as !$%@^&*<>,
                                  •  
                                • Only English characters
                                  •  
                                • No spaces
                                  •  
                                 
                              4.  
                                Download your required Quartus Prime Pro Linux version here.
                                 
                                5.  
                              5.  
                                Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe).
                                 
                                6.  
                              6.  
                                After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                                 
                                export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                                 
                                For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:
                                 
                                export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n
                                 
                                7.  
                              7.  
                                Verify, Quartus is discoverable by opening a new shell:
                                 
                                $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                                 
                                8. "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1312-walkthrough-install-git-large-file-storage-extension","title":"1.3.1.2 Walkthrough: Install Git Large File Storage Extension","text":"
                                To install the Git Large File Storage (LFS) extension, execute the following commands:
                                 
                                   
                                1. Obtain Git LFS package     
                                  curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\n
                                  2.  
                                2. Install Git LFS package     
                                  sudo dnf install git-lfs\n
                                  3.  
                                3. Install Git LFS     
                                  git lfs install\n
                                  4.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#132-fim-source-files","title":"1.3.2 FIM Source Files","text":"
                                The source files for the OFS Agilex PCIe Attach FIM are provided in the following repository: https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2.
                                 
                                Some essential directories in the repository are described as follows:
                                 
                                ofs-agx7-pcie-attach\n|  syn                      // Contains files related to synthesis\n|  |  board                     // Contains synthesis files for several cards, including the n6001 \n|  |  |  n6001                  // Contains synthesis files for n6001\n|  |  |  |  setup                       // Contains setup files, including pin constraints and location constraints\n|  |  |  |  syn_top                     // Contains Quartus project files\n|  verification             // Contains files for UVM testing\n|  ipss                     // Contains files for IP Sub-Systems\n|  |  qsfp                      // Contains source files for QSFP Sub-System\n|  |  hssi                      // Contains source files for HSSI Sub-System\n|  |  pmci                      // Contains source files for PMCI Sub-System (not used in F-Tile FIM)\n|  |  pcie                      // Contains source files for PCIe Sub-System\n|  |  mem                       // Contains source files for Memory Sub-System\n|  sim                      // Contains simulation files\n|  |  unit_test                 // Contains files for all unit tests\n|  |  |  scripts                    // Contains script to run regression unit tests\n|  license                  // Contains Quartus patch\n|  ofs-common               // Contains files which are common across OFS platforms\n|  |  verification              // Contains common UVM files\n|  |  scripts                   // Contains common scripts\n|  |  |  common\n|  |  |  |  syn                         // Contains common scripts for synthesis, including build script\n|  |  |  |  sim                         // Contains common scripts for simulation\n|  |  tools                     // Contains common tools files\n|  |  |  mk_csr_module              // Contains common files for CSR modules\n|  |  |  fabric_generation          // Contains common files for APF/BPF fabric generation\n|  |  |  ofss_config                // Contains common files for OFSS configuration tool\n|  |  |  |  ip_params                   // Contains default IP parameters for certain Sub-Systems when using OFSS\n|  |  src                       // Contains common source files, including host exercisers\n|  tools                    //\n|  |  ofss_config               // Contains top level OFSS files for each pre-made board configuration\n|  |  |  hssi                       // Contains OFSS files for Ethernet-SS configuraiton\n|  |  |  memory                     // Contains OFSS files for Memory-SS configuration\n|  |  |  pcie                       // Contains OFSS files for PCIe-SS configuration\n|  |  |  iopll                      // Contains OFSS files for IOPLL configuration\n|  src                      // Contains source files for Agilex PCIe Attach FIM\n|  |  pd_qsys                   // Contains source files related to APF/BPF fabric\n|  |  includes                  // Contains source file header files\n|  |  top                       // Contains top-level source files, including design top module\n|  |  afu_top                   // Contains top-level source files for AFU\n
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1321-walkthrough-clone-fim-repository","title":"1.3.2.1 Walkthrough: Clone FIM Repository","text":"
                                Perform the following steps to clone the OFS Agilex PCIe Attach FIM Repository:
                                 
                                   
                                1. Create a new directory to use as a clean starting point to store the retrieved files.     
                                  mkdir OFS_BUILD_ROOT\ncd OFS_BUILD_ROOT\nexport OFS_BUILD_ROOT=$PWD\n
                                  2.  
                                2. Clone GitHub repository using the HTTPS git method     
                                  git clone --recurse-submodules https://github.com/OFS/ofs-agx7-pcie-attach.git\n
                                  3.  
                                3. Check out the correct tag of the repository     
                                  cd ofs-agx7-pcie-attach\ngit checkout --recurse-submodules tags/ofs-2023.3-2\n
                                  4.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#133-environment-variables","title":"1.3.3 Environment Variables","text":"
                                The OFS FIM compilation and simulation scripts require certain environment variables be set prior to execution.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1331-walkthrough-set-development-environment-variables","title":"1.3.3.1 Walkthrough: Set Development Environment Variables","text":"
                                Perform the following steps to set the required environment variables. These environment variables must be set prior to simulation or compilation tasks so it is recommended that you create a script to set these variables.
                                 
                                   
                                1.  
                                  Navigate to the top level directory of the cloned OFS FIM repository.
                                   
                                  cd ofs-agx7-pcie-attach\n
                                   
                                  2.  
                                2.  
                                  Set project variables   
                                  # Set OFS Root Directory - e.g. this is the top level directory of the cloned OFS FIM repository\nexport OFS_ROOTDIR=$PWD\n
                                   
                                  3.  
                                3.  
                                  Set variables based on your development environment   
                                  # Set proxies if required for your server\nexport http_proxy=<YOUR_HTTP_PROXY>\nexport https_proxy=<YOUR_HTTPS_PROXY>\nexport ftp_proxy=<YOUR_FTP_PROXY>\nexport socks_proxy=<YOUR_SOCKS_PROXY>\nexport no_proxy=<YOUR_NO_PROXY>\n\n# Set Quartus license path\nexport LM_LICENSE_FILE=<YOUR_LM_LICENSE_FILE>\n\n# Set Synopsys License path (if using Synopsys for simulation)\nexport DW_LICENSE_FILE=<YOUR_DW_LICENSE_FILE>\nexport SNPSLMD_LICENSE_FILE=<YOUR_SNPSLMD_LICENSE_FILE>\n\n# Set Quartus Installation Directory - e.g. $QUARTUS_ROOTDIR/bin contains Quartus executables\nexport QUARTUS_ROOTDIR=<YOUR_QUARTUS_INSTALLATION_DIRECTORY>\n\n# Set the Tools Directory - e.g. $TOOLS_LOCATION contains the 'synopsys' directory if you are using Synopsys. Refer to the $VCS_HOME variable for an example.\nexport TOOLS_LOCATION=<YOUR_TOOLS_LOCATION>\n
                                   
                                  4.  
                                4.  
                                  Set generic environment variables
                                   
                                  # Set Work directory \nexport WORKDIR=$OFS_ROOTDIR\n\n# Set Quartus Tools variables\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport QUARTUS_VER_AC=$QUARTUS_ROOTDIR\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport IMPORT_IP_ROOTDIR=$IP_ROOTDIR\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\n\n# Set Verification Tools variables (if running simulations)\nexport DESIGNWARE_HOME=$TOOLS_LOCATION/synopsys/vip_common/vip_Q-2020.03A\nexport UVM_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport MTI_HOME=$QUARTUS_ROOTDIR/../questa_fse\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n\n# Set PATH to include compilation and simulation tools\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/../qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/bin:$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$DESIGNWARE_HOME/bin:$VCS_HOME/bin:$PATH\n
                                   
                                  5.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#134-walkthrough-set-up-development-environment","title":"1.3.4 Walkthrough: Set Up Development Environment","text":"
                                This walkthrough guides you through the process of setting up your development environment in preparation for FIM development. This flow only needs to be done once on your development machine.
                                 
                                   
                                1.  
                                  Ensure that Quartus Prime Pro Version 23.3 for Linux with Intel Agilex FPGA device support is installed on your development machine. Refer to the Install Quartus Prime Pro Software section for step-by-step installation instructions.
                                   
                                     
                                  1. Verify version number
                                    quartus_sh --version\n
                                     
                                    Example Output:
                                     
                                    Quartus Prime Shell\nVersion 23.3 Build 94 06/14/2023 SC Pro Edition\nCopyright (C) 2023  Intel Corporation. All rights reserved.\n
                                     
                                    2.  
                                   
                                  2.  
                                2.  
                                  Ensure that all support tools are installed on your development machine, and that they meet the version requirements.
                                   
                                     
                                  1.  
                                    Python 3.6.8 or later
                                     
                                       
                                    1.  
                                      Verify version number
                                       
                                      python --version\n
                                       
                                      Example Output:
                                       
                                      Python 3.6.8\n
                                       
                                      2.  
                                     
                                    2.  
                                  2.  
                                    GCC 7.4.0 or later
                                     
                                       
                                    1.  
                                      Verify version number
                                       
                                      gcc --version\n
                                       
                                      Example output:
                                       
                                      gcc (GCC) 7.4.0\n
                                       
                                      2.  
                                     
                                    3.  
                                  3.  
                                    cmake 3.15 or later
                                     
                                       
                                    1.  
                                      Verify version number
                                       
                                      cmake --version\n
                                       
                                      Example output:
                                       
                                      cmake version 3.15\n
                                       
                                      2.  
                                     
                                    4.  
                                  4.  
                                    git with git-lfs 1.8.3.1 or later. Refer to the Install Git Large File Storage Extension section for step-by-step instructions on installing the Git Large File Storage (LFS) extension.
                                     
                                       
                                    1.  
                                      Verify version number
                                       
                                      git --version\n
                                       
                                      Example output:
                                       
                                      git version 1.8.3.1\n
                                       
                                      2.  
                                     
                                    5.  
                                   
                                  3.  
                                3.  
                                  Clone the ofs-agx7-pcie-attach repository. Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  4.  
                                4.  
                                  Install UART IP license patch .02.
                                   
                                     
                                  1.  
                                    Navigate to the license directory
                                     
                                    cd $IOFS_BUILD_ROOT/license\n
                                     
                                    2.  
                                  2.  
                                    Install Patch 0.02
                                     
                                    sudo ./quartus-0.0-0.02iofs-linux.run\n
                                     
                                    3.  
                                   
                                  5.  
                                5.  
                                  Install Quartus Patches 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe). All required patches are provided in the Assets of the OFS FIM Release: https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2
                                   
                                  6.  
                                6.  
                                  Verify that patches have been installed correctly. They should be listed in the output of the following command.
                                   
                                  quartus_sh --version\n
                                   
                                  7.  
                                7.  
                                  Set required environment variables. Refer to the [Set Environment Variables] section for step-by-step instructions.
                                   
                                  8.  
                                 
                                This concludes the walkthrough for setting up your development environment. At this point you are ready to begin FIM development.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2-fim-compilation","title":"2. FIM Compilation","text":"
                                This section describes the process of compiling OFS FIM designs using the provided build scripts. It contains two main sections:
                                 
                                   
                                • Compilation Theory - Describes the theory behind FIM compilation
                                  •  
                                • Compilation Flows - Describes the process of compiling a FIM
                                  •  
                                 
                                The walkthroughs provided in this section are:
                                 
                                   
                                • Compile OFS FIM
                                  •  
                                • Manually Generate OFS Out-Of-Tree PR FIM
                                  •  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#21-compilation-theory","title":"2.1 Compilation Theory","text":"
                                This section describes the theory behind FIM compilation.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#211-fim-build-script","title":"2.1.1 FIM Build Script","text":"
                                The OFS Common Repository contains a script named build_top.sh which is used to build OFS FIM designs and generate output files that can be programmed to the board. After cloning the OFS FIM repository (with the ofs-common repository included), the build script can be found in the following location:
                                 
                                $OFS_ROOTDIR/ofs-common/scripts/common/syn/build_top.sh\n
                                 
                                The usage of the build_top.sh script is as follows:
                                 
                                build_top.sh [-k] [-p] [-e] [--stage=<action>] [--ofss=<ip_config>] <build_target>[:<fim_options>] [<work_dir_name>]\n
                                 Field Options Description Requirement -k None Keep. Preserves and rebuilds within an existing work tree instead of overwriting it. Optional -p None When set, and if the FIM supports partial reconfiguration, a PR template tree is generated at the end of the FIM build. The PR template tree is located in the top of the work directory but is relocatable and uses only relative paths. See $OFS_ROOTDIR/syn/common/scripts generate_pr_release.sh for details. Optional -e None Run only Quartus analysis and elaboration. It completes the setup stage, passes -end synthesis to the Quartus compilation flow and exits without running the finish stage. Optional --stage all | setup | compile | finish Controls which portion of the OFS build is run.\u00a0\u00a0- all: Run all build stages (default)\u00a0\u00a0- setup: Initialize a project in the work directory\u00a0\u00a0- compile: Run the Quartus compilation flow on a project that was already initialized with setup\u00a0\u00a0- finish: Complete OFS post-compilation tasks, such as generating flash images and, if -p is set, generating a release. Optional --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. This parameter is consumed during the setup stage and IP is updated only inside the work tree. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. Optional <build_target> n6001 Specifies which board is being targeted. Required <fim_options> flat | null_he_lb | null_he_hssi | null_he_mem | null_he_mem_tg Used to change how the FIM is built.\u00a0\u00a0- flat - Compiles a flat design (no PR assignments). This is useful for bringing up the design on a new board without dealing with PR complexity.\u00a0\u00a0- null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null.\u00a0\u00a0- null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null.\u00a0\u00a0- null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null.\u00a0\u00a0- null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null. More than one FIM option may be passed included in the <fim_options> list by concatenating them separated by commas. For example: <build_target>:flat,null_he_lb,null_he_hssi Optional <work_dir_name> String Specifies the name of the work directory in which the FIM will be built. If not specified, the default target is $OFS_ROOTDIR/work Optional 
                                Note: The he_null is a minimal block with CSRs that responds to PCIe MMIO requests in order to keep PCIe alive.
                                 
                                The build script copies source files from the existing cloned repository into the specified work directory, which are then used for compilation. As such, any changes made in the base source files will be included in all subsequent builds, unless the -k option is used, in which case an existing work directories files are used as-is. Likewise, any changes made in a work directory is only applied to that work directory, and will not be updated in the base repository by default.
                                 
                                Refer to Compile OFS FIM which provides step-by-step instructions for running the build_top.sh script with some of the different available options.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#212-ofss-file-usage","title":"2.1.2 OFSS File Usage","text":"
                                The OFS FIM build script can use OFSS files to easily customize design IP prior to compilation using preset configurations. The OFSS files specify certain parameters for different IPs. Using OFSS is provided as a convenience feature for building FIMs. The Provided OFSS Files table below describes the pre-made OFSS files for the n6001 that can be found in the $OFS_ROOTDIR/tools/ofss_config directory. Only the OFSS files listed in this table are compatible with the n6001 In order to compile an n6001 FIM, you must supply OFSS files corresponding to each IP that is present in your design.
                                 
                                Table: Provided OFSS Files
                                 OFSS File Name Location Type Description n6001.ofss $OFS_ROOTDIR/tools/ofss_config Top Includes the following OFSS files: \u00a0\u00a0\u2022 n6001_base.ofss \u00a0\u00a0\u2022 pcie_host.ofss \u00a0\u00a0\u2022 iopll.ofss \u00a0\u00a0\u2022 memory.ofss n6001_2pf.ofss $OFS_ROOTDIR/tools/ofss_config Top Includes the following OFSS files: \u00a0\u00a0\u2022 n6001_base.ofss \u00a0\u00a0\u2022 pcie_2pf.ofss \u00a0\u00a0\u2022 iopll.ofss \u00a0\u00a0\u2022 memory.ofss n6001_1pf_1vf.ofss $OFS_ROOTDIR/tools/ofss_config Top Includes the following OFSS files: \u00a0\u00a0\u2022 n6001_base.ofss \u00a0\u00a0\u2022 pcie_1pf_1vf.ofss \u00a0\u00a0\u2022 iopll.ofss \u00a0\u00a0\u2022 memory.ofss n6001_base.ofss $OFS_ROOTDIR/tools/ofss_config ofs Defines certain attributes of the design, including the platform name, device family, fim type, part number, and device ID. pcie_host.ofss $OFS_ROOTDIR/tools/ofss_config/pcie pcie Defines the PCIe Subsystem with the following configuration:\u00a0\u00a0\u2022 PF0 (3 VFs)\u00a0\u00a0\u2022 PF1 (0 VFs)\u00a0\u00a0\u2022 PF2 (0 VFs)\u00a0\u00a0\u2022 PF3 (0 VFs)\u00a0\u00a0\u2022 PF4 (0 VFs) pcie_2pf.ofss $OFS_ROOTDIR/tools/ofss_config/pcie pcie Defines the PCIe Subsystem with the following configuration:\u00a0\u00a0\u2022 PF0 (0 VFs)\u00a0\u00a0\u2022 PF1 (0 VFs) pcie_1pf_1vf.ofss $OFS_ROOTDIR/tools/ofss_config/pcie pcie Defines the PCIe Subsystem with the following configuration:\u00a0\u00a0\u2022 PF0 (1 VF) iopll.ofss $OFS_ROOTDIR/tools/ofss_config/iopll iopll Sets the IOPLL frequency to 470 MHz memory.ofss $OFS_ROOTDIR/tools/ofss_config/memory memory Defines the memory IP preset file to be used during the build as n6001 hssi_8x25.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP configuration to be 8x25 GbE hssi_8x10.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP configuration to be 8x10 GbE hssi_2x100.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP configuration to be 2x100 GbE 
                                There can typically be three sections contained within an OFSS file.
                                 
                                   
                                •  
                                  [include]
                                   
                                     
                                  • This section of an OFSS file contains elements separated by a newline, where each element is the path to an OFSS file that is to be included for configuration by the OFSS Configuration Tool. Ensure that any environment variables (e.g. $OFS_ROOTDIR) is set correctly. The OFSS Config tool uses breadth first search to include all of the specified OFSS files; the ordering of OFSS files does not matter
                                    •  
                                   
                                  •  
                                 
                                   
                                •  
                                  [ip]
                                   
                                     
                                  • This section of an OFSS file contains a key value pair that allows the OFSS Config tool to determine which IP configuration is being passed in. The currently supported values of IP are ofs, iopll, pcie, memory, and hssi.
                                    •  
                                   
                                  •  
                                 
                                   
                                •  
                                  [settings]
                                   
                                     
                                  • This section of an OFSS file contains IP specific settings. Refer to an existing IP OFSS file to see what IP settings are set. For the IP type `ofss``, the settings will be information of the OFS device (platform, family, fim, part #, device_id)
                                    •  
                                   
                                  •  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2121-platform-ofss-file","title":"2.1.2.1 Platform OFSS File","text":"
                                The <platform>.ofss file (e.g. $OFS_ROOTDIR/tools/ofss_config/n6001.ofss) is the platform level OFSS wrapper file. This is typically the OFSS file that is provided to the build script. It only contains an include section which lists all other OFSS files that are to be used when the <platform>.ofss file is passed to the build script.
                                 
                                The generic structure of a <platform>.ofss file is as follows:
                                 
                                [include]\n<PATH_TO_PLATFORM_BASE_OFSS_FILE>\n<PATH_TO_PCIE_OFSS_FILE>\n<PATH_TO_IOPLL_OFSS_FILE>\n<PATH_TO_MEMORY_OFSS_FILE>\n<PATH_TO_HSSI_OFSS_FILE>\n
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2122-ofs-ip-ofss-file","title":"2.1.2.2 OFS IP OFSS File","text":"
                                An OFSS file with IP type ofs (e.g. $OFS_ROOTDIR/tools/ofss_config/n6001_base.ofss) contains board specific information for the target board.
                                 
                                Currently supported configuration options for an OFSS file with IP type ofs are described in the OFS IP OFSS File Options table.
                                 
                                Table: OFS IP OFSS File Options
                                 Section Parameter n6001 Default Value [ip] type ofs [settings] platform N6001 family agilex fim base_x16 part AGFB014R24A2E2V device_id 6001"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2123-pcie-ip-ofss-file","title":"2.1.2.3 PCIe IP OFSS File","text":"
                                An OFSS file with IP type pcie (e.g. $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss) is used to configure the PCIe-SS in the FIM.
                                 
                                The PCIe OFSS file has a special section type ([pf*]) which is used to define physical functions (PFs) in the FIM. Each PF has a dedicated section, where the * character is replaced with the PF number. For example, [pf0], [pf1], etc. For reference FIM configurations, you must have at least 1 PF with 1VF, or 2PFs. This is because the PR region cannot be left unconnected. PFs must be consecutive. The PFVF Limitations table describes the supported number of PFs and VFs.
                                 
                                Table: PF/VF Limitations
                                 Parameter Value Min # of PFs 1 (on PF0) Max # of PFs 8 Min # of VFs 0 (if 2PFs are used) Max # of VFs 2000 distributed across all PFs 
                                Currently supported configuration options for an OFSS file with IP type pcie are described in the PCIe IP OFSS File Options table.
                                 
                                Table: PCIe IP OFSS File Options
                                 Section Parameter Options Default Description [ip] type pcie N/A Specifies that this OFSS file configures the PCIe-SS [settings] output_name pcie_ss N/A Specifies the output name of the PCIe-SS IP preset String N/A OPTIONAL - Specifies the name of a PCIe-SS IP presets file to use when building the FIM. When used, a presets file will take priority over any other parameters set in this OFSS file. [pf*] num_vfs Integer 0 Specifies the number of Virtual Functions in the current PF bar0_address_width Integer 12 bar4_address_width Integer 14 vf_bar0_address_width Integer 12 ats_cap_enable 0 | 1 0 vf_ats_cap_enable 0 | 1 0 prs_ext_cap_enable 0 | 1 0 pasid_cap_enable 0 | 1 0 pci_type0_vendor_id 32'h Value 0x00008086 pci_type0_device_id 32'h Value 0x0000bcce revision_id 32'h Value 0x00000001 class_code 32'h Value 0x00120000 subsys_vendor_id 32'h Value 0x00008086 subsys_dev_id 32'h Value 0x00001771 sriov_vf_device_id 32'h Value 0x0000bccf exvf_subsysid 32'h Value 0x00001771 
                                The default values for all PCIe-SS parameters (that are not defined in the PCIe IP OFSS file) are defined in $OFS_ROOTDIR/ofs-common/tools/ofss_config/ip_params/pcie_ss_component_parameters.py. When using a PCIe IP OFSS file during compilation, the PCIe-SS IP that is used will be defined based on the values in the PCIe IP OFSS file plus the parameters defined in pcie_ss_component_parameters.py.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2124-iopll-ip-ofss-file","title":"2.1.2.4 IOPLL IP OFSS File","text":"
                                An OFSS file with IP type iopll (e.g. $OFS_ROOTDIR/tools/ofss_config/iopll/iopll.ofss) is used to configure the IOPLL in the FIM.
                                 
                                The IOPLL OFSS file has a special section type ([p_clk]) which is used to define the IOPLL clock frequency.
                                 
                                Currently supported configuration options for an OFSS file with IP type iopll are described in the IOPLL OFSS File Options table.
                                 
                                Table: IOPLL OFSS File Options
                                 Section Parameter Options Description [ip] type iopll Specifies that this OFSS file configures the IOPLL [settings] output_name sys_pll Specifies the output name of the IOPLL. instance_name iopll_0 Specifies the instance name of the IOPLL. [p_clk] freq Integer: 250 - 470 Specifies the IOPLL clock frequency in MHz. 
                                Note: The following frequencies have been tested on reference boards: 350MHz, 400MHz, 470MHz.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2125-memory-ip-ofss-file","title":"2.1.2.5 Memory IP OFSS File","text":"
                                An OFSS file with IP type memory (e.g. $OFS_ROOTDIR/tools/ofss_config/memory/memory.ofss) is used to configure the Memory-SS in the FIM.
                                 
                                The Memory OFSS file specifies a preset value, which selects a presets file (.qprs) to configure the Memory-SS.
                                 
                                Currently supported configuration options for an OFSS file with IP type memory are described in the Memory OFSS File Options table.
                                 
                                Table: Memory OFSS File Options
                                 Section Parameter Options Description [ip] type memory Specifies that this OFSS file configures the Memory-SS [settings] output_name mem_ss_fm Specifies the output name of the Memory-SS. preset n6001 | String[1] Specifies the name of the .qprs presets file that will be used to build the Memory-SS. 
                                [1] You may generate your own .qprs presets file with a unique name using Quartus. 
                                 
                                Memory-SS presets files are stored in the $OFS_ROOTDIR/ipss/mem/qip/presets directory.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2126-hssi-ip-ofss-file","title":"2.1.2.6 HSSI IP OFSS File","text":"
                                An OFSS file with IP type hssi (e.g. $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_8x25.ofss) is used to configure the Ethernet-SS in the FIM.
                                 
                                Currently supported configuration options for an OFSS file with IP type hssi are described in the HSSI OFSS File Options table.
                                 
                                Table: HSSI OFSS File Options
                                 Section Parameter Options Description [ip] type hssi Specifies that this OFSS file configures the Ethernet-SS [settings] output_name hssi_ss Specifies the output name of the Ethernet-SS num_channels Integer Specifies the number of channels. data_rate 10GbE | 25GbE | 100GCAUI-4 Specifies the data rate preset None | String[1] OPTIONAL - Selects the platform whose preset .qprs file will be used to build the Ethernet-SS. When used, this will overwrite the other settings in this OFSS file. 
                                [1] You may generate your own .qprs presets file with a unique name using Quartus. 
                                 
                                Ethernet-SS presets are stored in  $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss/presets directory.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#213-ofs-build-script-outputs","title":"2.1.3 OFS Build Script Outputs","text":"
                                The output files resulting from running the the OFS FIM build_top.sh build script are copied to a single directory during the finish stage of the build script. The path for this directory is: $OFS_ROOTDIR/<WORK_DIRECTORY>/syn/board/n6001/syn_top/output_files.
                                 
                                The output files include programmable images and compilation reports. The OFS Build Script Output Descriptions table describes the images that are generated by the build script.
                                 
                                Table: OFS Build Script Output Descriptions
                                 File Description ofs_top[_hps].bin This is an intermediate, raw binary file. This intermediate raw binary file is produced by taking the Quartus generated .sof file, and converting it to *.pof using quartus_pfg, then converting the *.pof to *.hexout using quartus_cpf, and finally converting the *.hexout to *.bin using objcopy. Depending on whether the FPGA design contains an HPS block, a different file will be generated. **ofs_top.bin* - Raw binary image of the FPGA generated if there is no HPS present in the design. ofs_top_hps.bin - Raw binary image of the FPGA generated if there is an HPS present in the design. ofs_top_page1.bin This is the binary of the Factory Image and is the input to PACSign utility to generate ofs_top_page1_unsigned.bin binary image file. This image will carry binary content for the HPS if it is included in the SOF image. ofs_top_page0_factory.bin This is an input file to PACSign to generate ofs_top_page0_unsigned_factory.bin. ofs_top_page0_unsigned_factory.bin This is the unsigned PACSign output generated for the Factory Image. ofs_top_page1_user1.bin This is an input file to PACSign to generate ofs_top_page1_unsigned_user1.bin. This file is created by taking the ofs_top_[hps].bin file and assigning the User1 or appending factory block information. ofs_top_page1_unsigned_user1.bin This is the unsigned FPGA binary image generated by the PACSign utility for the User1 Image. This file is used to load the FPGA flash User1 Image using the fpgasupdate tool. ofs_top_page2_user2.bin This is an input file to PACSign to generate ofs_top_page2_unsigned_user2.bin. This file is created by taking the ofs_top_[hps].bin file and assigning the User2 or appending factory block information. ofs_top_page2_unsigned_user2.bin This is the unsigned FPGA binary image generated by the PACSign utility for the User2 Image. This file is used to load the FPGA flash User2 Image using the fpgasupdate tool. ofs_top_hps.sof If your design contains an Intel\u00ae Agilex\u00ae 7 FPGA Hard Processor System, then the build assembly process combines the FPGA ofs_top.sof programming file with u-boot-spl-dtb.hex to produce this file."},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#22-compilation-flows","title":"2.2 Compilation Flows","text":"
                                This section provides information for using the build script to generate different FIM types. Walkthroughs are provided for each compilation flow. These walkthroughs require that the development environment has been set up as described in the Set Up Development Environment section.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#221-flat-fim","title":"2.2.1 Flat FIM","text":"
                                A flat FIM is compiled such that there is no partial reconfiguration region, and the entire design is built as a flat design. This is useful for compiling new designs without worrying about the complexity introduced by partial reconfiguration. The flat compile removes the PR region and PR IP; thus, you cannot use the -p build flag when using the flat compile setting. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#222-in-tree-pr-fim","title":"2.2.2 In-Tree PR FIM","text":"
                                An In-Tree PR FIM is the default compilation if no compile flags or compile settings are used. This flow will compile the design with the partial reconfiguration region, but it will not create a relocatable PR directory tree to aid in AFU development. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#223-out-of-tree-pr-fim","title":"2.2.3 Out-of-Tree PR FIM","text":"
                                An Out-of-Tree PR FIM will compile the design with the partial reconfiguration region, and will create a relocatable PR directory tree to aid in AFU workload development. This is especially useful if you are developing a FIM to be used by another team developing AFU workloads. This is the recommended build flow in most cases. There are two ways to create the relocatable PR directory tree:
                                 
                                   
                                • Run the FIM build script with the -p option. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                                  •  
                                • Run the generate_pr_release.sh script after running the FIM build script. Refer to the Walkthrough: Manually Generate OFS Out-Of-Tree PR FIM Section step-by-step instructions for this flow.
                                  •  
                                 
                                In both cases, the generate_pr_release.sh is run to create the relocatable build tree. This script is located at $OFS_ROOTDIR/ofs-common/scripts/common/syn/generate_pr_release.sh. Usage for this script is as follows:
                                 
                                generate_pr_release.sh -t <PATH_OF_RELOCATABLE_PR_TREE> <BOARD_TARGET> <WORK_DIRECTORY>\n
                                 
                                The Generate PR Release Script Options table describes the options for the generate_pr_release.sh script.
                                 
                                Table: Generate PR Release Script Options
                                 Parameter Options Description <PATH_OF_RELOCATABLE_PR_TREE> String Specifies the location of the relocatable PR directory tree to be created. <BOARD_TARGET> n6001 Specifies the name of the board target. <WORK_DIRECTORY> String Specifies the existing work directory from which the relocatable PR directory tree will be created from. 
                                After generating the relocatable build tree, it is located in the $OFS_ROOTDIR/<WORK_DIRECTORY>/pr_build_template directory (or the directory you specified if generated separately). The contents of this directory have the following structure:
                                 
                                \u251c\u2500\u2500 bin\n\u251c\u2500\u2500 \u251c\u2500\u2500 afu_synth\n\u251c\u2500\u2500 \u251c\u2500\u2500 qar_gen\n\u251c\u2500\u2500 \u251c\u2500\u2500 update_pim\n\u251c\u2500\u2500 \u251c\u2500\u2500 run.sh\n\u251c\u2500\u2500 \u251c\u2500\u2500 build_env_config\n\u251c\u2500\u2500 README\n\u251c\u2500\u2500 hw\n\u251c\u2500\u2500 \u251c\u2500\u2500 lib\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 build\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 fme-ifc-id.txt\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 platform\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 fme-platform-class.txt\n\u251c\u2500\u2500 \u251c\u2500\u2500 blue_bits\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 ofs_top_hps.sof\n\u2514\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 ofs_top.sof\n
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#223-he_null-fim","title":"2.2.3 HE_NULL FIM","text":"
                                An HE_NULL FIM refers to a design with one, some, or all of the Host Exercisers replaced by he_null blocks. The he_null is a minimal block with CSRs that responds to PCIe MMIO requests in order to keep PCIe alive. You may use any of the build flows (flat, in-tree, out-of-tree) with the HE_NULL compile options. The HE_NULL compile options are as follows:
                                 
                                   
                                • null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null
                                  •  
                                • null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null
                                  •  
                                • null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null
                                  •  
                                • null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null
                                  •  
                                 
                                The Compile OFS FIM section gives step-by-step instructions for this flow.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#225-walkthrough-compile-ofs-fim","title":"2.2.5 Walkthrough: Compile OFS FIM","text":"
                                Perform the following steps to compile the OFS Agilex PCIe Attach FIM for n6001:
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Navigate to the root directory.
                                   
                                  cd $OFS_ROOTDIR\n
                                   
                                  4.  
                                4.  
                                  Run the build_top.sh script with the desired compile options. Some examples are provided:
                                   
                                     
                                  •  
                                    Default FIM
                                     
                                    ./ofs-common/scripts/common/syn/build_top.sh n6001 work_n6001\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    Flat FIM using OFSS
                                     
                                    ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001:flat work_n6001_flat\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    In-Tree PR FIM using OFSS
                                     
                                    ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001 work_n6001_in_tree_pr\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    Out-of-Tree PR FIM using OFSS
                                     
                                    ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001 work_n6001_oot_pr\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    HE_NULL Flat FIM using OFSS
                                     
                                    ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001:flat,null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_n6001_flat\n
                                     
                                    •  
                                   
                                  5.  
                                5.  
                                  Once the build script is complete, the build summary should report that the build is complete and passes timing. For example:
                                   
                                  ***********************************\n***\n***        OFS_PROJECT: n6001\n***        OFS_BOARD: n6001\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 6\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                   
                                  6.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#226-walkthrough-manually-generate-ofs-out-of-tree-pr-fim","title":"2.2.6 Walkthrough: Manually Generate OFS Out-Of-Tree PR FIM","text":"
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Navigate to the root directory.
                                   
                                  cd $OFS_ROOTDIR\n
                                   
                                  4.  
                                4.  
                                  Run the build_top.sh script with the desired compile options using the n6001 OFSS presets. In order to create the relocatable PR tree, you may not compile with the flat option. For example:
                                   
                                  ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001 work_n6001\n
                                   
                                  5.  
                                5.  
                                  Run the generate_pr_release.sh script to create the relocatable PR tree.
                                   
                                  ./ofs-common/scripts/common/syn/generate_pr_release.sh -t work_n6001/pr_build_template n6001 work_n6001\n
                                   
                                  6.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#227-compilation-seed","title":"2.2.7 Compilation Seed","text":"
                                You may change the seed which is used by the build script during Quartus compilation to change the starting point of the fitter. Trying different seeds is useful when your design is failing timing by a small amount.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2271-walkthrough-change-the-compilation-seed","title":"2.2.7.1 Walkthrough: Change the Compilation Seed","text":"
                                Perform the following steps to change the compilation seed for the FIM build.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Edit the SEED assignment in the $OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_top.qsf file to your desired seed value. The value can be any non-negative integer value.
                                   
                                  set_global_assignment -name SEED 1\n
                                   
                                  4.  
                                4.  
                                  Build the FIM. Refer to the Compile OFS FIM section for instructions.
                                   
                                  5.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#3-fim-simulation","title":"3. FIM Simulation","text":"
                                Unit level simulation of key components in the FIM is provided for verification of the following areas:
                                 
                                   
                                • Ethernet
                                  •  
                                • PCIe
                                  •  
                                • External Memory
                                  •  
                                • Core FIM
                                  •  
                                 
                                The Unit Level simulations work with Synopsys VCS/VCSMX or Mentor Graphics Questasim simulators. The scripts to run each unit level simulation are located in $OFS_ROOTDIR/sim/unit_test. Each unit test directory contains a README which describes the test in detail.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#31-simulation-file-generation","title":"3.1 Simulation File Generation","text":"
                                The simulation files must be generated prior to running unit level simulations. The script to generate simulation files is in the following location:
                                 
                                $OFS_ROOTDIR/ofs-common/scripts/common/sim/gen_sim_files.sh\n
                                 
                                The usage of the gen_sim_files.sh script is as follows:
                                 
                                gen_sim_files.sh [--ofss=<ip_config>] <build_target>[:<fim_options>] [<device>] [<family>]\n
                                 
                                The Gen Sim Files Script Options table describes the options for the gen_sim_files.sh script.
                                 
                                Table: Gen Sim Files Script Options
                                 Field Options Description Requirement --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. Platform Dependent[1] <build_target> n6001 Specifies which board is being targeted. Required <fim_options> null_he_lb | null_he_hssi | null_he_mem | null_he_mem_tg Used to change how the FIM is built.\u00a0\u00a0- null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null.\u00a0\u00a0- null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null.\u00a0\u00a0- null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null.\u00a0\u00a0- null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null. More than one FIM option may be passed included in the <fim_options> list by concatenating them separated by commas. For example: <build_target>:null_he_lb,null_he_hssi Optional <device> string Specifies the device ID for the target FPGA. If not specified, the default device is parsed from the QSF file for the project. Optional <family> string Specifies the family for the target FPGA. If not specified, the default family is parsed from the QSF file for the project. Optional 
                                [1] Using OFSS is required for the F-Tile Development Kit.
                                 
                                Refer to the Run Individual Unit Level Simulation section for an example of the simulation files generation flow.
                                 
                                When running regression tests, you may use the -g command line argument to generate simulation files; refer to the Run Regression Unit Level Simulation section for step-by-step instructions.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#32-individual-unit-tests","title":"3.2 Individual Unit Tests","text":"
                                Each unit test may be run individually using the run_sim.sh script located in the following directory:
                                 
                                $OFS_ROOTDIR/ofs-common/scripts/common/sim/run_sim.sh\n
                                 
                                The usage for the run_sim.sh script is as follows:
                                 
                                sh run_sim.sh TEST=<test> [VCSMX=<0|1> | MSIM=<0|1>]\n
                                 
                                The Run Sim Script Options table describes the options for the run_sim.sh script.
                                 
                                Table: Run Sim Script Options
                                 Field Options Description TEST String Specify the name of the test to run, e.g. dfh_walker VCSMX 0 | 1 When set, the VCSMX simulator will be used MSIM 0 | 1 When set, the QuestaSim simulator will be used 
                                Note: The default simulator is VCS if neither VCSMX nor MSIM are set.
                                 
                                The log for a unit test is stored in a transcript file in the simulation directory of the test that was run.
                                 
                                $OFS_ROOTDIR/sim/unit_test/<TEST_NAME>/<SIMULATOR>/transcript\n
                                 
                                For example, the log for the DFH walker test using VCSMX would be found at:
                                 
                                $OFS_ROOTDIR/sim/unit_test/dfh_walker/sim_vcsmx/transcript\n
                                 
                                The simulation waveform database is saved as vcdplus.vpd for post simulation review.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#321-walkthrough-run-individual-unit-level-simulation","title":"3.2.1 Walkthrough: Run Individual Unit Level Simulation","text":"
                                Perform the following steps to run an individual unit test.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Generate the simulation files for the n6001
                                   
                                  cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/n6001.ofss n6001\n
                                   
                                  4.  
                                4.  
                                  Navigate to the common simulation directory   
                                  cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n
                                   
                                  5.  
                                5.  
                                  Run the desired unit test using your desired simulator
                                   
                                     
                                  •  
                                    Using VCS
                                     
                                    sh run_sim.sh TEST=<test_name>\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    Using VCSMX
                                     
                                    sh run_sim.sh TEST=<test_name> VCSMX=1\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    Using QuestaSim
                                     
                                    sh run_sim.sh TEST=<test_name> MSIM=1\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    For example, to run the DFH walker test using VCSMX:
                                     
                                    sh run_sim.sh TEST=dfh_walker VCSMX=1\n
                                     
                                    •  
                                   
                                  6.  
                                6.  
                                  Once the test is complete, check the output for the simulation results. Review the log for detailed test results.
                                   
                                  Test status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/ofs-agx7-pcie-attach/sim/unit_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time         356233750000\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 356233750000 fs\nCPU Time:     57.730 seconds;       Data structure size:  47.2Mb\nTue Sep  5 09:44:19 2023\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                                   
                                  7.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#33-regression-unit-tests","title":"3.3 Regression Unit Tests","text":"
                                You may use the regression script regress_run.py to run some or all of the unit tests available with a single command. The regression script is in the following location:
                                 
                                $OFS_ROOTDIR/sim/unit_test/scripts/regress_run.py\n
                                 
                                The usage of the regression script is as follows:
                                 
                                regress_run.py [-h] [-l] [-n <num_procs>] [-k <test_package>] [-s <simulator>] [-g] [--ofss <ip_config>] [-b <board_name>] [-e]\n
                                 
                                The Regression Unit Test Script Options table describes the options for the regress_run.py script.
                                 
                                Table: Regression Unit Test Script Options
                                 Field Options Description -h | --help N/A Show the help message and exit -l | --local N/A Run regression locally -n | --n_procs Integer Maximum number of processes/tests to run in parallel when run locally. This has no effect on farm runs. -k | --pack all | fme | he | hssi | list | mem | pmci Test package to run during regression. The \"list\" option will look for a text file named \"list.txt\" in the \"unit_test\" directory for a text list of tests to run (top directory names). The default test package is all. -s | --sim vcs | vcsmx | msim Specifies the simulator used for the regression tests. The default simulator is vcs -g | --gen_sim_files N/A Generate simulation files. This only needs to be done once per repo update. This is the equivalent of running the gen_sim_files.sh script. -o | --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. -b | --board_name n6001 Specifies the board target -e | --email_list String Specifies email list to send results to multiple recipients 
                                The log for each unit test that is run by the regression script is stored in a transcript file in the simulation directory of the test that was run.
                                 
                                $OFS_ROOTDIR/sim/unit_test/<TEST_NAME>/<SIMULATOR>/transcript\n
                                 
                                For example, the log for the DFH walker test using VCSMX would be found at:
                                 
                                $OFS_ROOTDIR/sim/unit_test/dfh_walker/sim_vcsmx/transcript\n
                                 
                                The simulation waveform database is saved as vcdplus.vpd for post simulation review.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#331-walkthrough-run-regression-unit-level-simulation","title":"3.3.1 Walkthrough: Run Regression Unit Level Simulation","text":"
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Run regression test with the your desired options. For example, to simulate with the options to generate simulation files, run locally, use 8 processes, run all tests, use VCS simulator, and target the n6001:
                                   
                                  cd $OFS_ROOTDIR/sim/unit_test/scripts\n\npython regress_run.py --ofss $OFS_ROOTDIR/tools/ofss_config/n6001.ofss -g -l -n 8 -k all -s vcs -b n6001\n
                                   
                                  4.  
                                4.  
                                  Once all tests are complete, check that the tests have passed.
                                   
                                  2024-02-01 11:58:56,220: Passing Unit Tests:37/37 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n  2024-02-01 11:58:56,220:    bfm_test:............................. PASS -- Time Elapsed:0:06:02.414524\n  2024-02-01 11:58:56,220:    csr_test:............................. PASS -- Time Elapsed:0:06:15.337707\n  2024-02-01 11:58:56,220:    dfh_walker:........................... PASS -- Time Elapsed:0:05:58.861082\n  2024-02-01 11:58:56,220:    flr:.................................. PASS -- Time Elapsed:0:05:56.581355\n  2024-02-01 11:58:56,220:    fme_csr_access:....................... PASS -- Time Elapsed:0:00:53.277935\n  2024-02-01 11:58:56,220:    fme_csr_directed:..................... PASS -- Time Elapsed:0:01:01.278407\n  2024-02-01 11:58:56,220:    he_lb_test:........................... PASS -- Time Elapsed:0:06:49.305322\n  2024-02-01 11:58:56,220:    he_mem_lb_test:....................... PASS -- Time Elapsed:0:42:19.990309\n  2024-02-01 11:58:56,220:    he_null_test:......................... PASS -- Time Elapsed:0:49:40.190675\n  2024-02-01 11:58:56,220:    hssi_csr_test:........................ PASS -- Time Elapsed:0:51:48.399285\n  2024-02-01 11:58:56,220:    hssi_kpi_test:........................ PASS -- Time Elapsed:2:14:31.673783\n  2024-02-01 11:58:56,220:    hssi_test:............................ PASS -- Time Elapsed:2:09:01.947835\n  2024-02-01 11:58:56,220:    indirect_csr:......................... PASS -- Time Elapsed:0:05:20.803518\n  2024-02-01 11:58:56,220:    mem_ss_csr_test:...................... PASS -- Time Elapsed:0:29:34.454916\n  2024-02-01 11:58:56,220:    mem_ss_rst_test:...................... PASS -- Time Elapsed:1:00:13.989947\n  2024-02-01 11:58:56,220:    mem_tg_test:.......................... PASS -- Time Elapsed:0:38:11.820365\n  2024-02-01 11:58:56,220:    pcie_csr_test:........................ PASS -- Time Elapsed:0:05:52.164841\n  2024-02-01 11:58:56,220:    pf_vf_access_test:.................... PASS -- Time Elapsed:0:05:50.003577\n  2024-02-01 11:58:56,220:    pmci_csr_test:........................ PASS -- Time Elapsed:0:06:00.823947\n  2024-02-01 11:58:56,220:    pmci_mailbox_test:.................... PASS -- Time Elapsed:0:07:33.270359\n  2024-02-01 11:58:56,220:    pmci_multi_master_test:............... PASS -- Time Elapsed:0:31:19.277990\n  2024-02-01 11:58:56,220:    pmci_qsfp_telemetry_test:............. PASS -- Time Elapsed:0:31:43.263409\n  2024-02-01 11:58:56,220:    pmci_rd_default_value_test:........... PASS -- Time Elapsed:0:06:12.833577\n  2024-02-01 11:58:56,220:    pmci_ro_mailbox_test:................. PASS -- Time Elapsed:0:09:07.148307\n  2024-02-01 11:58:56,220:    pmci_vdm_b2b_drop_err_scenario_test:.. PASS -- Time Elapsed:0:15:30.884401\n  2024-02-01 11:58:56,220:    pmci_vdm_len_err_scenario_test:....... PASS -- Time Elapsed:0:21:48.295012\n  2024-02-01 11:58:56,220:    pmci_vdm_mctp_mmio_b2b_test:.......... PASS -- Time Elapsed:0:07:40.951160\n  2024-02-01 11:58:56,220:    pmci_vdm_multipkt_error_scenario_test: PASS -- Time Elapsed:1:54:24.778252\n  2024-02-01 11:58:56,220:    pmci_vdm_multipkt_tlp_err_test:....... PASS -- Time Elapsed:0:28:16.872490\n  2024-02-01 11:58:56,220:    pmci_vdm_tlp_error_scenario_test:..... PASS -- Time Elapsed:0:28:45.735583\n  2024-02-01 11:58:56,221:    pmci_vdm_tx_rx_all_random_lpbk_test:.. PASS -- Time Elapsed:0:18:10.509131\n  2024-02-01 11:58:56,221:    pmci_vdm_tx_rx_all_toggle_test:....... PASS -- Time Elapsed:0:14:59.887002\n  2024-02-01 11:58:56,221:    pmci_vdm_tx_rx_lpbk_test:............. PASS -- Time Elapsed:0:15:07.519350\n  2024-02-01 11:58:56,221:    port_gasket_test:..................... PASS -- Time Elapsed:0:05:54.059223\n  2024-02-01 11:58:56,221:    qsfp_test:............................ PASS -- Time Elapsed:0:06:08.632942\n  2024-02-01 11:58:56,221:    remote_stp_test:...................... PASS -- Time Elapsed:0:05:58.309660\n  2024-02-01 11:58:56,221:    uart_csr:............................. PASS -- Time Elapsed:0:06:19.757272\n  2024-02-01 11:58:56,221: Failing Unit Tests: 0/37 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n  2024-02-01 11:58:56,221: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n  2024-02-01 11:58:56,221:       Number of Unit test results captured: 37\n  2024-02-01 11:58:56,221:       Number of Unit test results passing.: 37\n  2024-02-01 11:58:56,221:       Number of Unit test results failing.:  0\n  2024-02-01 11:58:56,221:     End Unit regression running at date/time................: 2024-02-01 11:58:56.221252\n  2024-02-01 11:58:56,221:     Elapsed time for Unit regression run....................: 3:19:03.056457\n
                                   
                                  5.  
                                 
                                --->
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#4-fim-customization","title":"4. FIM Customization","text":"
                                This section describes how to perform specific customizations of the FIM, and provides step-by-step walkthroughs for these customizations. Each walkthrough can be done independently. These walkthroughs require a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment. The FIM Customization Walkthroughs table lists the walkthroughs that are provided in this section. Some walkthroughs include steps for testing on hardware. Testing on hardware requires that you have a deployment environment set up. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                 
                                Table: OFS FIM Customization Examples
                                 Walkthrough Name Add a new module to the OFS FIM Modify and run unit tests for a FIM that has a new module Modify and run UVM tests for a FIM that has a new module Hardware test a FIM that has a new module Debug the FIM with Signal Tap Compile the FIM in preparation for designing your AFU Resize the Partial Reconfiguration Region Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS Modify PCIe Sub-System and PF/VF MUX Configuration Using IP Presets Create a Minimal FIM Migrate to a Different Agilex Device Number Modify the Memory Sub-System Using IP Presets With OFSS Modify the Ethernet Sub-System Channels With Pre-Made HSSI OFSS Add Channels to the Ethernet Sub-System Channels With Custom HSSI OFSS Modify the Ethernet Sub-System With Pre-Made HSSI OFSS Plus Additional Modifications Modify the Ethernet Sub-System Without HSSI OFSS Remove the HPS"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#41-adding-a-new-module-to-the-fim","title":"4.1 Adding a new module to the FIM","text":"
                                This section provides a information for adding a custom module to the FIM, simulating the new design, compiling the new design, implementing and testing the new design on hardware, and debugging the new design on hardware.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#411-hello-fim-theory-of-operation","title":"4.1.1 Hello FIM Theory of Operation","text":"
                                If you intend to add a new module to the FIM area, then you will need to inform the host software of the new module. The FIM exposes its functionalities to host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). This set of CSR registers and their operation is described in FIM MMIO Regions.
                                 
                                See FPGA Device Feature List (DFL) Framework Overview for a description of the software process to read and process the linked list of Device Feature Header (DFH) CSRs within a FPGA.
                                 
                                This example will add a hello_fim module to the design. The Hello FIM example adds a simple DFH register and 64bit scratchpad register connected to the Board Peripheral Fabric (BPF) that can be accessed by the Host. You can use this example as the basis for adding a new feature to your FIM.
                                 
                                The Hello FIM design can be verified by Unit Level simulation, Universal Verification Methodology (UVM) simulation, and running in hardware on the n6001 card. The process for these are described in this section.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#4111-hello-fim-board-peripheral-fabric-bpf","title":"4.1.1.1 Hello FIM Board Peripheral Fabric (BPF)","text":"
                                The Hello FIM module will be connected to the Board Peripheral Fabric (BPF), and will be connected such that it can be mastered by the Host. The BPF is an interconnect generated by Platform Designer. The Hello FIM BPF Interface Diagram figure shows the APF/BPF Master/Slave interactions, as well as the added Hello FIM module.
                                 
                                Figure: Hello FIM BPF Interface Diagram
                                 
                                 
                                The BPF fabric is defined in $OFS_ROOTDIR/src/pd_qsys/fabric/bpf.txt.
                                 
                                We will add the Hello FIM module to an un-used address space in the MMIO region. The Hello FIM MMIO Address Layout table below shows the MMIO region for the Host with the Hello FIM module added at base address 0x16000.
                                 
                                Table: Hello FIM MMIO Address Layout
                                 Offset Feature CSR set 0x00000 FME AFU 0x10000 PCIe Interface 0x12000 QSFP Controller 0 0x13000 QSFP Controller 1 0x14000 E-Tile Ethernet  Interface 0x15000 EMIF 0x16000 Hello FIM 0x20000 PMCI Controller 0x40000 ST2MM (Streaming to Memory-Mapped) 0x60000 VUART 0x70000 PR Control & Status (Port Gasket) 0x71000 Port CSRs (Port Gasket) 0x72000 User Clock (Port Gasket) 0x74000 Remote SignalTap (Port Gasket) 0x80000 AFU Errors (AFU Interface Handler)"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#4112-hello-fim-csr","title":"4.1.1.2 Hello FIM CSR","text":"
                                The Hello FIM CSR will consist of the three registers shown in the Hello FIM CSR table below. The DFH and Hello FIM ID registers are read-only. The Scratchpad register supports read and write accesses.
                                 
                                Table: Hello FIM CSR
                                 Offset Attribute Description Default Value 0x016000 RO DFH(Device Feature Headers) register 0x30000006a0000100 0x016030 RW Scrachpad register 0x0 0x016038 RO Hello FIM ID register 0x6626070150000034"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#412-walkthrough-add-a-new-module-to-the-ofs-fim","title":"4.1.2 Walkthrough: Add a new module to the OFS FIM","text":"
                                Perform the following steps to add a new module to the OFS FIM that can be accessed by the Host.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Make hello_fim source directory
                                   
                                  mkdir $OFS_ROOTDIR/src/hello_fim\n
                                   
                                  4.  
                                4.  
                                  Create hello_fim_top.sv file.
                                   
                                  touch $OFS_ROOTDIR/src/hello_fim/hello_fim_top.sv\n
                                   
                                  Copy the following code into hello_fim_top.sv:
                                   
                                  // ***************************************************************************\n//                               INTEL CONFIDENTIAL\n//\n//        Copyright (C) 2023 Intel Corporation All Rights Reserved.\n//\n// The source code contained or described herein and all  documents related to\n// the  source  code  (\"Material\")  are  owned  by  Intel  Corporation  or its\n// suppliers  or  licensors.    Title  to  the  Material  remains  with  Intel\n// Corporation or  its suppliers  and licensors.  The Material  contains trade\n// secrets  and  proprietary  and  confidential  information  of  Intel or its\n// suppliers and licensors.  The Material is protected  by worldwide copyright\n// and trade secret laws and treaty provisions. No part of the Material may be\n// used,   copied,   reproduced,   modified,   published,   uploaded,  posted,\n// transmitted,  distributed,  or  disclosed  in any way without Intel's prior\n// express written permission.\n//\n// No license under any patent,  copyright, trade secret or other intellectual\n// property  right  is  granted  to  or  conferred  upon  you by disclosure or\n// delivery  of  the  Materials, either expressly, by implication, inducement,\n// estoppel or otherwise.  Any license under such intellectual property rights\n// must be express and approved by Intel in writing.\n//\n// You will not, and will not allow any third party to modify, adapt, enhance, \n// disassemble, decompile, reverse engineer, change or create derivative works \n// from the Software except and only to the extent as specifically required by \n// mandatory applicable laws or any applicable third party license terms \n// accompanying the Software.\n//\n// -----------------------------------------------------------------------------\n// Engineer     : \n// Create Date  : September 2023\n// Module Name  : hello_fim_top.sv\n// Project      : OFS\n// -----------------------------------------------------------------------------\n//\n// Description: \n// This is a simple module that implements DFH registers and \n// AVMM address decoding logic.\n\nmodule hello_fim_top  #(\n   parameter ADDR_WIDTH  = 12, \n   parameter DATA_WIDTH = 64, \n   parameter bit [11:0] FEAT_ID = 12'h001,\n   parameter bit [3:0]  FEAT_VER = 4'h1,\n   parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n   parameter bit END_OF_LIST = 1'b0\n)(\n   input  logic    clk,\n   input  logic    reset,\n// -----------------------------------------------------------\n//  AXI4LITE Interface\n// -----------------------------------------------------------\n   ofs_fim_axi_lite_if.slave   csr_lite_if\n);\n\nimport ofs_fim_cfg_pkg::*;\nimport ofs_csr_pkg::*;\n\n//-------------------------------------\n// Signals\n//-------------------------------------\n   logic [ADDR_WIDTH-1:0]              csr_waddr;\n   logic [DATA_WIDTH-1:0]              csr_wdata;\n   logic [DATA_WIDTH/8-1:0]            csr_wstrb;\n   logic                               csr_write;\n   logic                               csr_slv_wready;\n   csr_access_type_t                   csr_write_type;\n\n   logic [ADDR_WIDTH-1:0]              csr_raddr;\n   logic                               csr_read;\n   logic                               csr_read_32b;\n   logic [DATA_WIDTH-1:0]              csr_readdata;\n   logic                               csr_readdata_valid;\n   logic [ADDR_WIDTH-1:0]              csr_addr;\n\n   logic [63:0]                        com_csr_writedata;\n   logic                               com_csr_read;\n   logic                               com_csr_write;\n   logic [63:0]                        com_csr_readdata;\n   logic                               com_csr_readdatavalid;\n   logic [5:0]                         com_csr_address;\n\n// AXI-M CSR interfaces\nofs_fim_axi_mmio_if #(\n   .AWID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .AWADDR_WIDTH (ADDR_WIDTH),\n   .WDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH),\n   .ARID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .ARADDR_WIDTH (ADDR_WIDTH),\n   .RDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH)\n) csr_if();\n\n// AXI4-lite to AXI-M adapter\naxi_lite2mmio axi_lite2mmio (\n   .clk       (clk),\n   .rst_n     (~reset),\n   .lite_if   (csr_lite_if),\n   .mmio_if   (csr_if)\n);\n\n//---------------------------------\n// Map AXI write/read request to CSR write/read,\n// and send the write/read response back\n//---------------------------------\nofs_fim_axi_csr_slave #(\n   .ADDR_WIDTH (ADDR_WIDTH),\n   .USE_SLV_READY (1'b1)\n\n   ) csr_slave (\n   .csr_if             (csr_if),\n\n   .csr_write          (csr_write),\n   .csr_waddr          (csr_waddr),\n   .csr_write_type     (csr_write_type),\n   .csr_wdata          (csr_wdata),\n   .csr_wstrb          (csr_wstrb),\n   .csr_slv_wready     (csr_slv_wready),\n   .csr_read           (csr_read),\n   .csr_raddr          (csr_raddr),\n   .csr_read_32b       (csr_read_32b),\n   .csr_readdata       (csr_readdata),\n   .csr_readdata_valid (csr_readdata_valid)\n);\n\n// Address mapping\nassign csr_addr             = csr_write ? csr_waddr : csr_raddr;\nassign com_csr_address      = csr_addr[5:0];  // byte address\nassign csr_slv_wready       = 1'b1 ;\n// Write data mapping\nassign com_csr_writedata    = csr_wdata;\n\n// Read-Write mapping\nalways_comb\nbegin\n   com_csr_read             = 1'b0;\n   com_csr_write            = 1'b0;\n   casez (csr_addr[11:6])\n      6'h00 : begin // Common CSR\n         com_csr_read       = csr_read;\n         com_csr_write      = csr_write;\n      end   \n      default: begin\n         com_csr_read       = 1'b0;\n         com_csr_write      = 1'b0;\n      end\n   endcase\nend\n\n// Read data mapping\nalways_comb begin\n   if (com_csr_readdatavalid) begin\n      csr_readdata       = com_csr_readdata;\n      csr_readdata_valid = 1'b1;\n   end\n   else begin\n      csr_readdata       = '0;\n      csr_readdata_valid = 1'b0;\n   end\nend\n\nhello_fim_com  #(\n   .FEAT_ID          (FEAT_ID),\n   .FEAT_VER         (FEAT_VER),\n   .NEXT_DFH_OFFSET  (NEXT_DFH_OFFSET),\n   .END_OF_LIST      (END_OF_LIST)\n) hello_fim_com_inst (\n   .clk                   (clk                     ),\n   .reset                 (reset                   ),\n   .writedata             (com_csr_writedata       ),\n   .read                  (com_csr_read            ),\n   .write                 (com_csr_write           ),\n   .byteenable            (4'hF                    ),\n   .readdata              (com_csr_readdata        ),\n   .readdatavalid         (com_csr_readdatavalid   ),\n   .address               (com_csr_address         )\n   );\nendmodule\n
                                   
                                  5.  
                                5.  
                                  Create hello_fim_com.sv file.
                                   
                                  touch $OFS_ROOTDIR/src/hello_fim/hello_fim_com.sv\n
                                   
                                  Copy the following code to hello_fim_com.sv:
                                   
                                  module hello_fim_com #(\n  parameter bit [11:0] FEAT_ID = 12'h001,\n  parameter bit [3:0]  FEAT_VER = 4'h1,\n  parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n  parameter bit END_OF_LIST = 1'b0\n)(\ninput clk,\ninput reset,\ninput [63:0] writedata,\ninput read,\ninput write,\ninput [3:0] byteenable,\noutput reg [63:0] readdata,\noutput reg readdatavalid,\ninput [5:0] address\n);\n\nwire reset_n = !reset;  \nreg [63:0] rdata_comb;\nreg [63:0] scratch_reg;\n\nalways @(negedge reset_n ,posedge clk)  \n   if (!reset_n) readdata[63:0] <= 64'h0; else readdata[63:0] <= rdata_comb[63:0];\n\nalways @(negedge reset_n , posedge clk)\n   if (!reset_n) readdatavalid <= 1'b0; else readdatavalid <= read;\n\nwire wr = write;\nwire re = read;\nwire [5:0] addr = address[5:0];\nwire [63:0] din  = writedata [63:0];\nwire wr_scratch_reg = wr & (addr[5:0]  == 6'h30)? byteenable[0]:1'b0;\n\n// 64 bit scratch register\nalways @( negedge  reset_n,  posedge clk)\n   if (!reset_n)  begin\n      scratch_reg <= 64'h0;\n   end\n   else begin\n   if (wr_scratch_reg) begin \n      scratch_reg <=  din;  \n   end\nend\n\nalways @ (*)\nbegin\nrdata_comb = 64'h0000000000000000;\n   if(re) begin\n      case (addr)  \n        6'h00 : begin\n                rdata_comb [11:0]   = FEAT_ID ;  // dfh_feature_id  is reserved or a constant value, a read access gives the reset value\n                rdata_comb [15:12]  = FEAT_VER ;  // dfh_feature_rev    is reserved or a constant value, a read access gives the reset value\n                rdata_comb [39:16]  = NEXT_DFH_OFFSET ;  // dfh_dfh_ofst is reserved or a constant value, a read access gives the reset value\n                rdata_comb [40]     = END_OF_LIST ;        //dfh_end_of_list\n                rdata_comb [59:40]  = 20'h00000 ;  // dfh_rsvd1     is reserved or a constant value, a read access gives the reset value\n                rdata_comb [63:60]  = 4'h3 ;  // dfh_feat_type  is reserved or a constant value, a read access gives the reset value\n        end\n        6'h30 : begin\n                rdata_comb [63:0]   = scratch_reg; \n        end\n        6'h38 : begin\n                rdata_comb [63:0]       = 64'h6626_0701_5000_0034;\n        end\n        default : begin\n                rdata_comb = 64'h0000000000000000;\n        end\n      endcase\n   end\nend\n\nendmodule\n
                                   
                                  6.  
                                6.  
                                  Create hello_fim_design_files.tcl file.
                                   
                                  touch $OFS_ROOTDIR/src/hello_fim/hello_fim_design_files.tcl\n
                                   
                                  Copy the following code into hello_fim_design_files.tcl
                                   
                                  # Copyright 2023 Intel Corporation.\n#\n# THIS SOFTWARE MAY CONTAIN PREPRODUCTION CODE AND IS PROVIDED BY THE\n# COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED\n# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE\n# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR\n# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\n# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR\n# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,\n# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE\n# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,\n# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# Hello FIM Files\n#--------------------\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_com.sv\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_top.sv\n
                                   
                                  7.  
                                7.  
                                  Modify $OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_top.qsf to include Hello FIM module
                                   
                                  ######################################################\n# Verilog Macros\n######################################################\n.....\nset_global_assignment -name VERILOG_MACRO \"INCLUDE_HELLO_FIM\"     # Includes Hello FIM\n
                                   
                                  8.  
                                8.  
                                  Modify $OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_top_sources.tcl to include Hello FIM design files
                                   
                                  ############################################\n# Design Files\n############################################\n...\n# Subsystems\n...\nset_global_assignment -name SOURCE_TCL_SCRIPT_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_design_files.tclset_global_assignment -name SOURCE_TCL_SCRIPT_FILE ../setup/hello_fim_design_files.tcl\n
                                   
                                  9.  
                                9.  
                                  Modify $OFS_ROOTDIR/src/pd_qsys/fabric/fabric_design_files.tcl to include BPF Hello FIM Slave IP.
                                   
                                  #--------------------\n# BPF\n#--------------------\n...\nset_global_assignment -name IP_FILE   $::env(BUILD_ROOT_REL)/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip\n
                                   
                                  10.  
                                10.  
                                  Modify $OFS_ROOTDIR/src/includes/fabric_width_pkg.sv to add Hello FIM slave information and update EMIF slave next offset.
                                   
                                  localparam bpf_hello_fim_slv_baseaddress = 'h16000;    // New\nlocalparam bpf_hello_fim_slv_address_width = 12;       // New\nlocalparam bpf_emif_slv_next_dfh_offset = 'h1000;      // Old value: 'hB000\nlocalparam bpf_hello_fim_slv_next_dfh_offset = 'hA000; // New\nlocalparam bpf_hello_fim_slv_eol = 'b0;                // New\n
                                   
                                  11.  
                                11.  
                                  Modify $OFS_ROOTDIR/src/top/top.sv
                                   
                                     
                                  1.  
                                    Add bpf_hello_fim_slv_if to AXI interfaces
                                     
                                    // AXI4-lite interfaces\n...\nofs_fim_axi_lite_if #(.AWADDR_WIDTH(fabric_width_pkg::bpf_hello_fim_slv_address_width), .ARADDR_WIDTH(fabric_width_pkg::bpf_hello_fim_slv_address_width)) bpf_hello_fim_slv_if();\n
                                     
                                    2.  
                                  2.  
                                    Add Hello FIM instantiation
                                     
                                    //*******************************\n// Hello FIM Subsystem\n//*******************************\n\n`ifdef INCLUDE_HELLO_FIM\nhello_fim_top #(\n   .ADDR_WIDTH       (fabric_width_pkg::bpf_hello_fim_slv_address_width),\n   .DATA_WIDTH       (64),\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset),\n   .END_OF_LIST      (fabric_width_pkg::bpf_hello_fim_slv_eol)\n) hello_fim_top_inst (\n    .clk (clk_csr),\n    .reset(~rst_n_csr),\n    .csr_lite_if    (bpf_hello_fim_slv_if)\n);\n`else\ndummy_csr #(\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset),\n   .END_OF_LIST      (fabric_width_pkg::bpf_hello_fim_slv_eol)\n) hello_fim_dummy (\n   .clk         (clk_csr),\n   .rst_n       (rst_n_csr),\n   .csr_lite_if (bpf_hello_fim_slv_if)\n);\n\n`endif\n
                                     
                                    3.  
                                  3.  
                                    Add interfaces for Hello FIM slv to bpf instantiation
                                     
                                    bpf bpf (\n...\n  .bpf_hello_fim_slv_awaddr   (bpf_hello_fim_slv_if.awaddr  ),\n  .bpf_hello_fim_slv_awprot   (bpf_hello_fim_slv_if.awprot  ),\n  .bpf_hello_fim_slv_awvalid  (bpf_hello_fim_slv_if.awvalid ),\n  .bpf_hello_fim_slv_awready  (bpf_hello_fim_slv_if.awready ),\n  .bpf_hello_fim_slv_wdata    (bpf_hello_fim_slv_if.wdata   ),\n  .bpf_hello_fim_slv_wstrb    (bpf_hello_fim_slv_if.wstrb   ),\n  .bpf_hello_fim_slv_wvalid   (bpf_hello_fim_slv_if.wvalid  ),\n  .bpf_hello_fim_slv_wready   (bpf_hello_fim_slv_if.wready  ),\n  .bpf_hello_fim_slv_bresp    (bpf_hello_fim_slv_if.bresp   ),\n  .bpf_hello_fim_slv_bvalid   (bpf_hello_fim_slv_if.bvalid  ),\n  .bpf_hello_fim_slv_bready   (bpf_hello_fim_slv_if.bready  ),\n  .bpf_hello_fim_slv_araddr   (bpf_hello_fim_slv_if.araddr  ),\n  .bpf_hello_fim_slv_arprot   (bpf_hello_fim_slv_if.arprot  ),\n  .bpf_hello_fim_slv_arvalid  (bpf_hello_fim_slv_if.arvalid ),\n  .bpf_hello_fim_slv_arready  (bpf_hello_fim_slv_if.arready ),\n  .bpf_hello_fim_slv_rdata    (bpf_hello_fim_slv_if.rdata   ),\n  .bpf_hello_fim_slv_rresp    (bpf_hello_fim_slv_if.rresp   ),\n  .bpf_hello_fim_slv_rvalid   (bpf_hello_fim_slv_if.rvalid  ),\n  .bpf_hello_fim_slv_rready   (bpf_hello_fim_slv_if.rready  ),\n...\n);\n
                                     
                                    4.  
                                   
                                  12.  
                                12.  
                                  Modify $OFS_ROOTDIR/src/pd_qsys/fabric/bpf.txt to add the hello_fim module as a slave to the apf.
                                   
                                  # NAME   FABRIC      BASEADDRESS    ADDRESS_WIDTH SLAVES\napf         mst     n/a             18            fme,pcie,pmci,qsfp0,qsfp1,emif,hssi,hello_fim\n...\nhello_fim   slv     0x16000         12            n/a\n
                                   
                                  13.  
                                13.  
                                  Execute helper script to generate BPF design files
                                   
                                  cd $OFS_ROOTDIR/src/pd_qsys/fabric/\n\nsh gen_fabrics.sh\n
                                   
                                  14.  
                                14.  
                                  Once the script completes, the following new IP is created: $OFS_ROOTDIR/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip.
                                   
                                  15.  
                                15.  
                                  [OPTIONAL] You may verify the BPF changes have been made correctly by opening bpf.qsys to analyze the BPF.
                                   
                                  cd $OFS_ROOTDIR/src/pd_qsys/fabric\n\nqsys-edit bpf.qsys --quartus-project=$OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_top.qpf\n
                                   
                                  Find the bpf_hello_fim_slv instance:
                                   
                                   
                                  16.  
                                16.  
                                  Compile the Hello FIM design
                                   
                                  cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001 work_n6001_hello_fim\n
                                   
                                  17.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#413-walkthrough-modify-and-run-unit-tests-for-a-fim-that-has-a-new-module","title":"4.1.3 Walkthrough: Modify and run unit tests for a FIM that has a new module","text":"
                                Perform the following steps to modify the unit test files to support a FIM that has had a new module added to it.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                   
                                • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design in order to simulate.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Modify $OFS_ROOTDIR/sim/unit_test/dfh_walker/test_csr_defs.sv
                                   
                                     
                                  1.  
                                    Add HELLO_FIM_IDX entry to t_dfh_idx enumeration.
                                     
                                    ...\ntypedef enum {\n    FME_DFH_IDX,\n    THERM_MNGM_DFH_IDX,\n    GLBL_PERF_DFH_IDX,\n    GLBL_ERROR_DFH_IDX,\n    QSFP0_DFH_IDX,\n    QSFP1_DFH_IDX,\n    HSSI_DFH_IDX,\n    EMIF_DFH_IDX,\n    HELLO_FIM_DFH_IDX,  // New\n    PMCI_DFH_IDX,\n    ST2MM_DFH_IDX,\n    VUART_DFH_IDX,\n    PG_PR_DFH_IDX,\n    PG_PORT_DFH_IDX,\n    PG_USER_CLK_DFH_IDX,\n    PG_REMOTE_STP_DFH_IDX,\n    AFU_ERR_DFH_IDX,\n    MAX_DFH_IDX\n} t_dfh_idx;\n...\n
                                     
                                    2.  
                                  2.  
                                    Add HELLO_FIM_DFH to get_dfh_names function.
                                     
                                    ...\nfunction automatic dfh_name[MAX_DFH_IDX-1:0] get_dfh_names();\n...\n  dfh_names[PMCI_DFH_IDX]        = \"PMCI_DFH\";\n  dfh_names[HELLO_FIM_DFH_IDX]   = \"HELLO_FIM_DFH\";  // New\n  dfh_names[ST2MM_DFH_IDX]       = \"ST2MM_DFH\";\n...\nreturn dfh_names;\n...\n
                                     
                                    3.  
                                  3.  
                                    Add expected DFH value for Hello FIM to the get_dfh_values function.
                                     
                                    ...\nfunction automatic [MAX_DFH_IDX-1:0][63:0] get_dfh_values();\n...\n  dfh_values[PMCI_DFH_IDX]       = 64'h3_00000_xxxxxx_1012;\n  dfh_values[PMCI_DFH_IDX][39:16] = fabric_width_pkg::bpf_pmci_slv_next_dfh_offset;\n\n  dfh_values[HELLO_FIM_DFH_IDX]  = 64'h3_00000_xxxxxx_0100;  // New\n  dfh_values[HELLO_FIM_DFH_IDX][39:16] = fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset; // New\n\n  dfh_values[ST2MM_DFH_IDX]      = 64'h3_00000_xxxxxx_0014;\n  dfh_values[ST2MM_DFH_IDX][39:16] = fabric_width_pkg::apf_st2mm_slv_next_dfh_offset;\n...\nreturn dfh_values;\n...\n
                                     
                                    4.  
                                   
                                  3.  
                                3.  
                                  Generate simulation files
                                   
                                  cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/n6001.ofss n6001\n
                                   
                                  4.  
                                4.  
                                  Run DFH Walker Simulation   
                                  cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\nsh run_sim.sh TEST=dfh_walker\n
                                   
                                  5.  
                                5.  
                                  Verify that the test passes, and that the output shows the Hello FIM in the DFH sequence
                                   
                                  ********************************************\n Running TEST(0) : test_dfh_walking\n********************************************\n...\n\nREAD64: address=0x00015000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x3000000010001009\n\nEMIF_DFH\n   Address   (0x15000)\n   DFH value (0x3000000010001009)\n\nREAD64: address=0x00016000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x30000000a0000100\n\nHELLO_FIM_DFH\n   Address   (0x16000)\n   DFH value (0x30000000a0000100)\n\nREAD64: address=0x00020000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x3000000200001012\n\nPMCI_DFH\n   Address   (0x20000)\n   DFH value (0x3000000200001012)\n\n...\n\nTest status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/ofs-agx7-pcie-attach/sim/unit_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time         356791250000\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 356791250000 fs\nCPU Time:     61.560 seconds;       Data structure size:  47.4Mb\nTue Aug 15 16:29:45 2023\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                                   
                                  6.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#414-walkthrough-modify-and-run-uvm-tests-for-a-fim-that-has-a-new-module","title":"4.1.4 Walkthrough: Modify and run UVM tests for a FIM that has a new module","text":"
                                Perform the following steps to modify the UVM simulation files to support the Hello FIM design.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                   
                                • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design in order to simulate.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Modify $OFS_ROOTDIR/verification/tests/sequences/dfh_walking_seq.svh
                                   
                                     
                                  1.  
                                    Modify the dfh_offset_array to insert the Hello FIM.
                                     
                                    dfh_offset_array = new[16];\ndfh_offset_array[ 0] = tb_cfg0.PF0_BAR0;                    // FME_DFH                0x8000_0000\ndfh_offset_array[ 1] = dfh_offset_array[ 0] + 64'h0_1000;   // THERM_MNGM_DFH         0x8000_1000\ndfh_offset_array[ 2] = dfh_offset_array[ 1] + 64'h0_2000;   // GLBL_PERF_DFH          0x8000_3000\ndfh_offset_array[ 3] = dfh_offset_array[ 2] + 64'h0_1000;   // GLBL_ERROR_DFH         0x8000_4000\ndfh_offset_array[ 4] = dfh_offset_array[ 3] + 64'h0_E000;   // QSFP0_DFH              0x8001_2000\ndfh_offset_array[ 5] = dfh_offset_array[ 4] + 64'h0_1000;   // QSFP1_DFH              0x8001_3000\ndfh_offset_array[ 6] = dfh_offset_array[ 5] + 64'h0_1000;   // HSSI_DFH               0x8001_4000\ndfh_offset_array[ 7] = dfh_offset_array[ 6] + 64'h0_1000;   // EMIF_DFH               0x8001_5000\ndfh_offset_array[ 8] = dfh_offset_array[ 7] + 64'h0_1000;   // HELLO_FIM_DFH          0x8001_6000\ndfh_offset_array[ 9] = dfh_offset_array[ 8] + 64'h6_a000;   // PMCI_DFH               0x8008_0000\ndfh_offset_array[ 10] = dfh_offset_array[ 9] + 64'h8_0000;  // ST2MM_DFH              0x8010_0000\ndfh_offset_array[ 11] = dfh_offset_array[10] + 64'h3_0000;  // PG_PR_DFH_IDX          0x8013_0000\ndfh_offset_array[ 12] = dfh_offset_array[11] + 64'h0_1000;  // PG_PORT_DFH_IDX        0x8013_1000\ndfh_offset_array[ 13] = dfh_offset_array[12] + 64'h0_1000;  // PG_USER_CLK_DFH_IDX    0x8013_2000\ndfh_offset_array[ 14] = dfh_offset_array[13] + 64'h0_1000;  // PG_REMOTE_STP_DFH_IDX  0x8013_3000\ndfh_offset_array[ 15] = dfh_offset_array[14] + 64'h0_D000;  // PG_AFU_ERR_DFH_IDX     0x8014_0000\n
                                     
                                    2.  
                                   
                                  2.  
                                2.  
                                  Modify `$OFS_ROOTDIR/verification/tests/sequences/mmio_seq.svh``
                                   
                                     
                                  1.  
                                    Add test code related to the Hello FIM. This code will verify the scratchpad register at 0x16030 and read only the register at 0x16038.
                                     
                                    // HELLO_FIM_Scratchpad 64 bit access\n`uvm_info(get_name(), $psprintf(\"////Accessing PF0 HELLO_FIM_Scratchpad Register %0h+'h16030////\", tb_cfg0.PF0_BAR0), UVM_LOW)\n\nassert(std::randomize(wdata));\naddr = tb_cfg0.PF0_BAR0+'h1_6000+'h30;\n\nmmio_write64(.addr_(addr), .data_(wdata));\nmmio_read64 (.addr_(addr), .data_(rdata));\n\nif(wdata !== rdata)\n    `uvm_error(get_name(), $psprintf(\"Data mismatch 64! Addr = %0h, Exp = %0h, Act = %0h\", addr, wdata, rdata))\nelse\n    `uvm_info(get_name(), $psprintf(\"Data match 64! addr = %0h, data = %0h\", addr, rdata), UVM_LOW)\n\naddr = tb_cfg0.PF0_BAR0+'h1_6000+'h38;\nwdata = 64'h6626_0701_5000_0034;\nmmio_read64 (.addr_(addr), .data_(rdata));\nif(wdata !== rdata)\n    `uvm_error(get_name(), $psprintf(\"Data mismatch 64! Addr = %0h, Exp = %0h, Act = %0h\", addr, wdata, rdata))\nelse\n    `uvm_info(get_name(), $psprintf(\"Data match 64! addr = %0h, data = %0h\", addr, rdata), UVM_LOW)\n
                                     
                                    Note: uvm_info and uvm_error statements will put a message into log file.
                                     
                                    2.  
                                   
                                  3.  
                                3.  
                                  Modify $OFS_ROOTDIR/verification/scripts/Makefile_VCS.mk
                                   
                                     
                                  1.  
                                    Add INCLUDE_HELLO_FIM define option to enable Hello FIM on UVM
                                     
                                    VLOG_OPT += +define+INCLUDE_HELLO_FIM\n
                                     
                                    2.  
                                   
                                  4.  
                                4.  
                                  Re-generate the UVM files
                                   
                                     
                                  1.  
                                    Navigate to the verification scripts directory
                                     
                                    cd $VERDIR/scripts\n
                                     
                                    2.  
                                  2.  
                                    Clean the output of previous builds
                                     
                                    gmake -f Makefile_VCS.mk clean\n
                                     
                                    3.  
                                  3.  
                                    Compile the IP files
                                     
                                    gmake -f Makefile_VCS.mk cmplib_adp\n
                                     
                                    4.  
                                  4.  
                                    Build the RTL and Test Benches
                                     
                                    gmake -f Makefile_VCS.mk build_adp DUMP=1 \n
                                     
                                    5.  
                                   
                                  5.  
                                5.  
                                  Run the UVM DFH Walker Simulation 
                                   
                                     
                                  1.  
                                    Run the DFH Walker simulation
                                     
                                    cd $VERDIR/scripts\ngmake -f Makefile_VCS.mk run TESTNAME=dfh_walking_test DUMP=1\n
                                     
                                    2.  
                                  2.  
                                    The output logs are stored in the $VERDIR/sim/dfh_walking_test directory. The main files to note are described in Table 5-3:
                                     
                                    Table 5-3 UVM Output Logs
                                     File Name Description runsim.log A log file of UVM trans.log A log file of transactions on PCIe bus inter.vpd A waveform for VCS 
                                    3.  
                                  3.  
                                    Run the following command to quickly verify- that the Hello FIM module was successfully accessed. In the example below, the message DFH offset Match! Exp = 80016000 Act = 80016000 shows that the Hello FIM module was successfully accessed.
                                     
                                    cd $VERDIR/sim/dfh_walking_test\ncat runsim.log | grep \"DFH offset\"\n
                                     
                                    Expected output:
                                     
                                    UVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 111950000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp = 80000000 Act = 80000000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 112586000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80001000 Act = 80001000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 113222000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80003000 Act = 80003000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 113858000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80004000 Act = 80004000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 114494000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80012000 Act = 80012000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 115147000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80013000 Act = 80013000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 115801000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80014000 Act = 80014000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 116628000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80015000 Act = 80015000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 117283000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80016000 Act = 80016000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 117928000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80080000 Act = 80080000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 118594000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80100000 Act = 80100000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 119248000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80130000 Act = 80130000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 119854000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80131000 Act = 80131000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 120460000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80132000 Act = 80132000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 121065000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80133000 Act = 80133000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 121672000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80140000 Act = 80140000\n
                                     
                                    4.  
                                   
                                  6.  
                                6.  
                                  Run the UVM MMIO Simulation 
                                   
                                     
                                  1.  
                                    Run the MMIO test
                                     
                                    cd $VERDIR/scripts\ngmake -f Makefile_VCS.mk run TESTNAME=mmio_test DUMP=1\n
                                     
                                    2.  
                                  2.  
                                    Run the following commands to show the result of the scratchpad register and Hello FIM ID register. You can see the \"Data match\" message indicating that the registers are successfuly verified.
                                     
                                    cd $VERDIR/sim/mmio_test\ncat runsim.log | grep \"Data\" | grep 1603\n
                                     
                                    Expected output:
                                     
                                    UVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/mmio_seq.svh(68) @ 115466000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] Data match 64! addr = 80016030, data = 880312f9558c00e1\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/mmio_seq.svh(76) @ 116112000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] Data match 64! addr = 80016038, data = 6626070150000034\n
                                     
                                    3.  
                                   
                                  7.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#415-walkthrough-hardware-test-a-fim-that-has-a-new-module","title":"4.1.5 Walkthrough: Hardware test a FIM that has a new module","text":"
                                Perform the following steps to program and hardware test a FIM that has had a new module added to it.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                  •  
                                 
                                   
                                • This walkthrough uses a FIM design that has been generated with a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for generating a Hello FIM design.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  [OPTIONAL] In the work directory where the FIM was compiled, determine the PR Interface ID of your design. You can use this value at the end of the walkthrough to verify that the design has been configured to the FPGA.
                                   
                                  cd $OFS_ROOTDIR/<work_directory>/syn/board/n6001/syn_top/\n\ncat fme-ifc-id.txt\n
                                   
                                  Example output:
                                   
                                  1d6beb4e-86d7-5442-a763-043701fb75b7\n
                                   
                                  2.  
                                2.  
                                  Switch to your deployment environment.
                                   
                                  3.  
                                3.  
                                  Program the FPGA with the Hello FIM image. Refer to the Program the FPGA via RSU Section for step-by-step programming instructions.
                                   
                                  4.  
                                4.  
                                  Run fpgainfo to determine the PCIe B:D.F of your board, and to verify the PR Interface ID matches the ID you found in Step 1.
                                   
                                  fpgainfo fme\n
                                   
                                  Example output:
                                   
                                  Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                   
                                  5.  
                                5.  
                                  Initialize opae.io
                                   
                                  sudo opae.io init -d <B:D.F>\n
                                   
                                  For example:
                                   
                                  sudo opae.io init -d 98:00.0\n
                                   
                                  6.  
                                6.  
                                  Run DFH walker. Note the value read back from offset 0x16000 indicates the DFH ID is 0x100 which matches the Hello FIM module.
                                   
                                  sudo opae.io walk -d <B:D.F>\n
                                   
                                  For example:
                                   
                                  sudo opae.io walk -d 98:00.0\n
                                   
                                  Example output:
                                   
                                  ...\noffset: 0x15000, value: 0x3000000010001009\n    dfh: id = 0x9, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x16000, value: 0x30000000a0000100\n    dfh: id = 0x100, rev = 0x0, next = 0xa000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x20000, value: 0x3000000200001012\n    dfh: id = 0x12, rev = 0x1, next = 0x20000, eol = 0x0, reserved = 0x0, feature_type = 0x3\n...\n
                                   
                                  7.  
                                7.  
                                  Read all of the registers in the Hello FIM module
                                   
                                     
                                  1.  
                                    Read the DFH Register
                                     
                                    opae.io -d 98:00.0 -r 0 peek 0x16000\n
                                     
                                    Example Output:
                                     
                                    0x30000006a0000100\n
                                     
                                    2.  
                                  2.  
                                    Read the Scratchpad Register
                                     
                                    opae.io -d 98:00.0 -r 0 peek 0x16030\n
                                     
                                    Example Output:
                                     
                                    0x0\n
                                     
                                    3.  
                                  3.  
                                    Read the ID Register
                                     
                                    opae.io -d 98:00.0 -r 0 peek 0x16038\n
                                     
                                    Example Output:
                                     
                                    0x6626070150000034\n
                                     
                                    4.  
                                   
                                  8.  
                                8.  
                                  Verify the scratchpad register at 0x16030 by writing and reading back from it.
                                   
                                     
                                  1.  
                                    Write to Scratchpad register
                                     
                                    opae.io -d 0000:15:00.0 -r 0 poke 0x16030 0x123456789abcdef\n
                                     
                                    2.  
                                  2.  
                                    Read from Scratchpad register
                                     
                                    opae.io -d 15:00.0 -r 0 peek 0x16030\n
                                     
                                    Expected output:
                                     
                                    0x123456789abcdef\n
                                     
                                    3.  
                                  3.  
                                    Write to Scratchpad register
                                     
                                    opae.io -d 15:00.0 -r 0 poke 0x16030 0xfedcba9876543210\n
                                     
                                    4.  
                                  4.  
                                    Read from Scratchpad register
                                     
                                    opae.io -d 15:00.0 -r 0 peek 0x16030\n
                                     
                                    Expected output:
                                     
                                    0xfedcba9876543210\n
                                     
                                    5.  
                                   
                                  9.  
                                9.  
                                  Release the opae.io tool
                                   
                                  opae.io release -d 15:00.0\n
                                   
                                  10.  
                                10.  
                                  Confirm the driver has been set back to dfl-pci
                                   
                                  opae.io ls\n
                                   
                                  Example output:
                                   
                                  [0000:98:00.0] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                                   
                                  11.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#416-walkthrough-debug-the-fim-with-signal-tap","title":"4.1.6 Walkthrough: Debug the FIM with Signal Tap","text":"
                                The following steps guide you through the process of adding a Signal Tap instance to your design. The added Signal Tap instance provides hardware to capture the desired internal signals and connect the stored trace information via JTAG. Please be aware that the added Signal Tap hardware will consume FPGA resources and may require additional floorplanning steps to accommodate these resources. Some areas of the FIM use logic lock regions and these regions may need to be re-sized.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                   
                                • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                  •  
                                 
                                   
                                • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design.
                                  •  
                                 
                                Perform the following steps in your development environment:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Synthesize the design using the -e build script option. You may skip this step if you are using a pre-synthesized design.
                                   
                                  cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -e --ofss tools/ofss_config/n6001.ofss n6001 work_hello_fim_with_stp\n
                                   
                                  4.  
                                4.  
                                  Open the design in Quartus. The Compilation Dashboard should show that the Analysis & Synthesis step has completed.
                                   
                                  quartus $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/n6001/syn_top/ofs_top.qpf\n
                                   
                                   
                                  5.  
                                5.  
                                  Open Tools -> Signal Tap Logic Analyzer
                                   
                                   
                                     
                                  1.  
                                    Select the Default template and click Create
                                     
                                    2.  
                                  2.  
                                    Assign the clock for sampling the Signal Tap instrumented signals of interest. Note that the clock selected should correspond to the signals you want to view for best trace fidelity. Different clocks can be used, however, there maybe issues with trace inaccuracy due to sampling time differences. This example instruments the hello_fim_top module previously intetegrated into the FIM. If unfamiliar with code, it is helpful to use the Quartus Project Navigator to find the block of interest and open the design instance for review.
                                     
                                       
                                    1.  
                                      In the Signal Configuration -> Clock box of the Signal Tap Logic Analyzer window, click the \"...\" button
                                       
                                       
                                      2.  
                                    2.  
                                      In the Node Finder tool that opens, type hello_fim_top_inst|clock into the Named field, then click Search. Select the clk signal from the Matching Nodes box and click the \">\" button to move it to the Nodes Found box. Click OK to close the Node Finder dialog.
                                       
                                       
                                      3.  
                                     
                                    3.  
                                  3.  
                                    Update the sample depth and other Signal Tap settings as needed for your debugging criteria.
                                     
                                     
                                    4.  
                                  4.  
                                    In the Signal Tap GUI add the nodes to be instrumented by double-clicking on the \"Double-click to add nodes\" legend.
                                     
                                     
                                    5.  
                                  5.  
                                    This brings up the Node Finder to add the signals to be traced. In this example we will monitor the memory mapped interface to the Hello FIM. Select the signals that appear from the  search patterns hello_fim_top_inst|reset and hello_fim_top_inst|csr_lite_if\\*. Click Insert and close the Node Finder dialog.
                                     
                                     
                                    6.  
                                  6.  
                                    To provide a unique name for your Signal Tap instance, select \"auto_signaltap_0\", right-click, and select Rename Instance (F2). Provide a descriptive name for your instance, for example, stp_for_hello_fim.
                                     
                                     
                                    7.  
                                  7.  
                                    Save the newly created Signal Tap file, in the $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/n6001/syn_top/ directory, and give it the same name as the instance. Ensure that the Add file to current project checkbox is ticked.
                                     
                                     
                                    8.  
                                  8.  
                                    In the dialog that pops up, click \"Yes\" to add the newly created Signal Tap file to the project settings files.
                                     
                                     
                                    This will automatically add the following lines to $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/n6001/syn_top/ofs_top.qsf:
                                     
                                    set_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE stp_for_hello_fim.stp\nset_global_assignment -name SIGNALTAP_FILE stp_for_hello_fim.stp\n
                                     
                                    9.  
                                   
                                  6.  
                                6.  
                                  Close all Quartus GUIs.
                                   
                                  7.  
                                7.  
                                  Compile the project with the Signal Tap file added to the project. Use the -k switch to perform the compilation using the files in a specified working directory and not the original ones from the cloned repository. 
                                   
                                  ./ofs-common/scripts/common/syn/build_top.sh -p -k --ofss tools/ofss_config/n6001.ofss n6001 work_hello_fim_with_stp\n
                                   
                                  Alternatively, you can copy the ofs_top.qsf and stp_for_hello_fim.stp files from the Hello FIM with STP work directory to replace the original files in the cloned OFS repository. In this scenario, all further FIM compilation projects will include the Signal Tap instance integrated into the design. Execute the following commands for this alternative flow:
                                   
                                  Copy the modified file work_hello_fim_with_stp/syn/board/n6001/syn_top/ofs_top.qsf to the source OFS repository, into syn/board/n6001/syn_top/.
                                   
                                  cd $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/n6001/syn_top\n\ncp ofs_top.qsf $OFS_ROOTDIR/syn/board/n6001/syn_top\n\ncp stp_for_hello_fim.stp $OFS_ROOTDIR/syn/board/n6001/syn_top\n
                                   
                                  Compile the FIM to create a new work directory.
                                   
                                  cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001 work_hello_fim_with_stp_src_repo\n
                                   
                                  8.  
                                8.  
                                  Ensure that the compile completes successfully and meets timing:
                                   
                                  ***********************************\n***\n***        OFS_PROJECT: n6001\n***        OFS_BOARD: n6001\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 6\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                   
                                  9.  
                                9.  
                                  Set up a JTAG connection to the n6001. Refer to Set up JTAG section for step-by-step instructions.
                                   
                                  10.  
                                10.  
                                  Copy the ofs_top.sof and stp_for_hello_fim.stp files to the machine which is connected to the n6001 via JTAG.
                                   
                                  11.  
                                11.  
                                  From the JTAG connected machine, program the $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/n6001/syn_top/output_files/ofs_top.sof image to the n6001 FPGA. Refer to the Program the FPGA via JTAG section for step-by-step programming instructions.
                                   
                                  12.  
                                12.  
                                  Open the Quartus Signal Tap GUI
                                   
                                  $QUARTUS_ROOTDIR/bin/quartus_stpw\n
                                   
                                  13.  
                                13.  
                                  In the Signal Tap Logic Analyzer window, select File -> Open, and choose the stp_for_hello_fim.stp file.
                                   
                                   
                                  14.  
                                14.  
                                  In the right pane of the Signal Tap GUI, in the Hardware: selection box select the cable for the n6001. In the Device: selection box select the Agilex device.
                                   
                                   
                                  15.  
                                15.  
                                  If the Agilex Device is not displayed in the Device: list, click the 'Scan Chain' button to re-scan the JTAG device chain.
                                   
                                  16.  
                                16.  
                                  Create the trigger conditions. In this example, we will capture data on a rising edge of the Read Address Valid signal.
                                   
                                   
                                  17.  
                                17.  
                                  Start analysis by selecting the 'stp_for_hello_fim' instance and pressing 'F5' or clicking the Run Analysis icon in the toolbar. You should see a green message indicating the Acquisition is in progress. Then, move to the Data Tab to observe the signals captured.
                                   
                                   
                                  18.  
                                18.  
                                  To generate traffic in the csr_lite_if signals of the Hello FIM module, walk the DFH list or peek/poke the Hello FIM registers.
                                   
                                  opae.io init -d 0000:98:00.0\nopae.io walk -d 0000:98:00.0\nopae.io release -d 0000:98:00.0\n
                                   
                                  The signals should be captured on the rising edge of arvalid in this example. Zoom in to get a better view of the signals.
                                   
                                   
                                  19.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#42-preparing-fim-for-afu-development","title":"4.2 Preparing FIM for AFU Development","text":"
                                To save area, the default Host Excercisers in the FIM can be replaced by a \"he_null\" blocks. There are a few things to note:
                                 
                                   
                                • \"he_null\" is a minimal block with registers that responds to PCIe MMIO request. MMIO responses are required to keep PCIe alive (end points enabled in PCIe-SS needs service downstream requests).
                                  •  
                                • If an exerciser with other I/O connections such as \"he_mem\" or \"he_hssi\" is replaced, then then those I/O ports are simply tied off.
                                  •  
                                • The options supported are null_he_lb, null_he_hssi, null_he_mem and null_he_mem_tg. Any combination, order or all can be enabled at the same time. 
                                  •  
                                • Finer grain control is provided for you to, for example, turn off only the exercisers in the Static Region in order to save area.
                                  •  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#421-walkthrough-compile-the-fim-in-preparation-for-designing-your-afu","title":"4.2.1 Walkthrough: Compile the FIM in preparation for designing your AFU","text":"
                                Perform the following steps to compile a FIM for where the exercisers are removed and replaced with an he_null module while keeping the PF/VF multiplexor connections.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the [Walkthrough: Set Up Development Environment] Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the FIM repository (or use an existing cloned repository). Refer to the [Walkthrough: Clone FIM Repository] section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the [Walkthrough: Set Development Environment Variables] section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Compile the FIM with the HE_NULL compile options
                                   
                                  cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_n6001\n
                                   
                                  4.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#43-partial-reconfiguration-region","title":"4.3 Partial Reconfiguration Region","text":"
                                To take advantage of the available resources in the Intel\u00ae Agilex\u00ae 7 FPGA for an AFU design, you can adjust the size of the AFU PR partition. An example reason for the changing the size of PR region is if you add more logic to the FIM region, then you may need to adjust the PR region to fit the additional logic into the static region.  Similarly, if you reduce logic in the FIM region, then you can adjust the PR region to provide more logic resources for the AFU.
                                 
                                After the compilation of the FIM, the resources usage broken down by partitions as reported in the following two files 
                                $OFS_ROOTDIR/<WORDK_DIRECTORY>/syn/board/n6001/syn_top/output_files/ofs_top.fit.place.rpt\n$OFS_ROOTDIR/<WORDK_DIRECTORY>/syn/board/n6001/syn_top/output_files/ofs_top.fit.rpt\n
                                 
                                The next is a report of the resources usage by partitions defined for the FIM. 
                                 
                                 
                                In this case, the default size for the afu_top|port_gasket|pr_slot|afu_main PR partition is large enough to hold the logic of the default AFU, which is mainly occupied by the Host Exercisers. However, larger designs might require additional resources.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#431-walkthrough-resize-the-partial-reconfiguration-region","title":"4.3.1 Walkthrough: Resize the Partial Reconfiguration Region","text":"
                                Perform the following steps to customize the resources allocated to the AFU in the PR regions:
                                 
                                   
                                1.  
                                  The OFS flow provides the TCL file $OFS_ROOTDIR/syn/board/n6001/setup/pr_assignments.tcl which defines the PR partition where the AFU is allocated.
                                   
                                   
                                  2.  
                                2.  
                                  Use Quartus Chip Planner to identify the locations of the resources available within the Intel\u00ae Agilex\u00ae 7 FPGA chip for placement and routing your AFU. You need to identify a pair of coordinates, the origin (X0, Y0) and top right corner (X1, Y1) of the new or existing rectangles to modify as shown in the following image. 
                                   
                                   
                                  3.  
                                 
                                The coordinates of the top right corner of the lock regions are computed indirectly based on the Width and Height, as follows.
                                 
                                X1 = X0 + Width \nY1 = Y0 + Height\n
                                 
                                   
                                1.  
                                  Make changes to the pr_assignments.tcl based on your findings in Quartus Chip Planner. You can modify the size and location of existing lock regions or add new ones and assign them to the AFU PR partition.
                                   
                                  2.  
                                2.  
                                  Recompile your FIM and create the PR relocatable build tree using the following commands.
                                   
                                  cd $OFS_ROOTDIR    \nofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001 work_n6001_resize_pr\n
                                   
                                  3.  
                                3.  
                                  Analyze the resource utilization report per partition produced after recompiling the FIM.
                                   
                                  4.  
                                4.  
                                  Perform further modification to the PR regions until the results are satisfactory. Make sure timing constraints are met.
                                   
                                  5.  
                                 
                                For more information on how to optimize the floor plan of your Partial Reconfiguration design refer to the following online documentation.
                                 
                                   
                                • Analyzing and Optimizing the Design Floorplan
                                  •  
                                 
                                   
                                • Partial Reconfiguration Design Flow - Step 3 - Floorplan the Design
                                  •  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#44-pcie-configuration","title":"4.4 PCIe Configuration","text":"
                                The PCIe sub-system IP and PF/VF MUX can be modified either using the OFSS flow or the IP Presets flow. The OFSS flow supports a subset of all available PCIe Sub-system settings, while the IP Preset flow can make any available PCIe Sub-system settings change. With PCIe-SS modifcations related to the PFs and VFs, the PF/VF MUX logic is automatically configured based on the PCIe-SS configuration. The sections below describe each flow.
                                 
                                   
                                • PCIe Configuration Using OFSS
                                  •  
                                • [PCIe Configuration Using IP Presets]
                                  •  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#441-pfvf-mux-configuration","title":"4.4.1 PF/VF MUX Configuration","text":"
                                The default PF/VF MUX configuration for OFS PCIe Attach FIM for the n6001 can support up to 8 PFs and 2000 VFs distributed accross all PFs.
                                 
                                For reference FIM configurations, you must have at least 1 PF with 1VF, or 2PFs. This is because the PR region cannot be left unconnected. PFs must be consecutive. The PFVF Limitations table describes the supported number of PFs and VFs.
                                 
                                Table: PF/VF Limitations
                                 Parameter Value Min # of PFs 1 (on PF0) Max # of PFs 8 Min # of VFs 1 on PF0 Max # of VFs 2000 distributed across all PFs 
                                New PF or VF instances will automatically have a null_afu module instantiated and connected to the new PF or VF.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#442-pcie-ss-configuration-registers","title":"4.4.2 PCIe-SS Configuration Registers","text":"
                                The PCIe-SS configuration registers contain the Vendor, Device and Subsystem Vendor ID registers which are used in PCIe add-in cards to uniquely identify the card for assignment to software drivers. OFS has these registers set with Intel values for out of the box usage. If you are using OFS for a PCIe add in card that your company manufactures, then update the PCIe Subsytem Subsystem ID and Vendor ID registers as described below and change OPAE provided software code to properly operate with your company's register values.
                                 
                                The Vendor ID is assigned to your company by the PCI-SIG (Special Interest Group). The PCI-SIG is the only body officially licensed to give out IDs. You must be a member of PCI-SIG to request your own ID. Information about joining PCI-SIG is available here: PCI-SIG. You select the Subsystem Device ID for your PCIe add in card.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#443-pcie-configuration-using-ofss","title":"4.4.3 PCIe Configuration Using OFSS","text":"
                                The general flow for using OFSS to modify the PCIe Sub-system and PF/VF MUX is as follows:
                                 
                                   
                                1. Create or modify a PCIe OFSS file with the desired PCIe configuration. 
                                  2.  
                                2. Call this PCIe OFSS file when running the FIM build script.
                                  3.  
                                 
                                The PCIe IP OFSS File Options table lists all of the configuration options supported by the OFSS flow. Any other customizations to the PCIe sub-system must be done with the IP Presets Flow.
                                 
                                Table: PCIe IP OFSS File Options
                                 Section Parameter Options Default Description [ip] type pcie N/A Specifies that this OFSS file configures the PCIe-SS [settings] output_name pcie_ss N/A Specifies the output name of the PCIe-SS IP preset String N/A OPTIONAL - Specifies the name of a PCIe-SS IP presets file to use when building the FIM. When used, a presets file will take priority over any other parameters set in this OFSS file. [pf*] num_vfs Integer 0 Specifies the number of Virtual Functions in the current PF bar0_address_width Integer 12 bar4_address_width Integer 14 vf_bar0_address_width Integer 12 ats_cap_enable 0 | 1 0 vf_ats_cap_enable 0 | 1 0 prs_ext_cap_enable 0 | 1 0 pasid_cap_enable 0 | 1 0 pci_type0_vendor_id 32'h Value 0x00008086 pci_type0_device_id 32'h Value 0x0000bcce revision_id 32'h Value 0x00000001 class_code 32'h Value 0x00120000 subsys_vendor_id 32'h Value 0x00008086 subsys_dev_id 32'h Value 0x00001771 sriov_vf_device_id 32'h Value 0x0000bccf exvf_subsysid 32'h Value 0x00001771"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#4431-walkthrough-modify-the-pcie-sub-system-and-pfvf-mux-configuration-using-ofss","title":"4.4.3.1 Walkthrough: Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS","text":"
                                Perform the following steps to modify the PF/VF MUX configuration.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                   
                                • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  View the default OFS PCIe Attach FIM for the n6001 PF/VF configuration in the the $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss file.
                                   
                                  [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n\n[pf0]\nnum_vfs = 3\nbar0_address_width = 20\nvf_bar0_address_width = 20\n\n[pf1]\n\n[pf2]\nbar0_address_width = 18\n\n[pf3]\n\n[pf4]\n
                                   
                                  4.  
                                4.  
                                  Create a new PCIe OFSS file from the existing pcie_host.ofss file
                                   
                                  cp $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_pfvf_mod.ofss\n
                                   
                                  5.  
                                5.  
                                  Configure the new pcie_pfvf_mod.ofss with your desired settings. In this example we will add PF5 with 2 VFs.
                                   
                                  [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n\n[pf0]\nnum_vfs = 3\nbar0_address_width = 20\nvf_bar0_address_width = 20\n\n[pf1]\n\n[pf2]\nbar0_address_width = 18\n\n[pf3]\n\n[pf4]\n\n[pf5]\nnum_vfs = 2\n
                                   
                                  6.  
                                6.  
                                  Edit the $OFS_ROOTDIR/tools/ofss_config/n6001.ofss file to use the new PCIe configuration file pcie_pfvf_mod.ofss
                                   
                                  [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/n6001_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_pfvf_mod.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory.ofss\n
                                   
                                  7.  
                                7.  
                                  Compile the FIM. 
                                   
                                  cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001 work_n6001_pfvf_mod\n
                                   
                                  8.  
                                8.  
                                  Copy the resulting $OFS_ROOTDIR/work_n6001_pfvf_mod/syn/board/n6001/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                                   
                                  9.  
                                9.  
                                  Switch to your deployment environment.
                                   
                                  10.  
                                10.  
                                  Program the .bin image to the n6001 FPGA. Refer to the Program the FPGA via RSU Section for step-by-step programming instructions.
                                   
                                  11.  
                                11.  
                                  Verify the number of VFs on the newly added PF8. In this example, we defined 2 VFs on PF5 in Step 5.
                                   
                                  sudo lspci -vvv -s 98:00.5 | grep VF\n
                                   
                                  Example output:
                                   
                                  Initial VFs: 2, Total VFs: 2, Number of VFs: 0, Function Dependency Link: 05\nVF offset: 4, stride: 1, Device ID: bccf\nVF Migration: offset: 00000000, BIR: 0\n
                                   
                                  12.  
                                12.  
                                  Verify communication with the newly added PF5. New PF/VF are seamlessly connected to their own CSR stub, which can be read at DFH Offset 0x0. You can bind to the function and perform opae.io peek commands to read from the stub CSR. Similarly, perform opae.io poke commands to write into the stub CSRs. Use this mechanism to verify that the new PF/VF Mux configuration allows to write and read back values from the stub CSRs. 
                                   
                                  13.  
                                 
                                The GUID for every new PF/VF CSR stub is the same.
                                 
                                * NULL_GUID_L           = 64'haa31f54a3e403501\n* NULL_GUID_H           = 64'h3e7b60a0df2d4850\n
                                 
                                1. Initialize the driver on PF5
                                 
                                ```bash\nsudo opae.io init -d 98:00.5\n```\n
                                 
                                2. Read the GUID for the PF5 CSR stub.
                                 
                                ```bash\nsudo opae.io -d 98:00.5 -r 0 peek 0x8\nsudo opae.io -d 98:00.5 -r 0 peek 0x10\n```\n\nExample output:\n\n```bash\n0xaa31f54a3e403501\n0x3e7b60a0df2d4850\n```\n
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#444-pcie-sub-system-configuration-using-ip-presets","title":"4.4.4 PCIe Sub-System configuration Using IP Presets","text":"
                                The general flow for using IP Presets to modify he PCIe Sub-system is as follows:
                                 
                                   
                                1. [OPTIONAL] Create a work directory using OFSS files if you wish to use OFSS configuration settings as a starting point.
                                  2.  
                                2. Open the PCIe-SS IP and make desired modifications.
                                  3.  
                                3. Create an IP Presets file.
                                  4.  
                                4. Create an PCIe OFSS file that uses the IP Presets file.
                                  5.  
                                5. Build the FIM with the PCIe OFSS file from Step 4.
                                  6.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#4441-walkthrough-modify-pcie-sub-system-and-pfvf-mux-configuration-using-ip-presets","title":"4.4.4.1 Walkthrough: Modify PCIe Sub-System and PF/VF MUX Configuration Using IP Presets","text":"
                                Perform the following steps to use an IP preset file to configure the PCIe Sub-system and PF/VF MUX. In this example, we will change the Revision ID of PF0. While this modification can be done with the OFSS flow, this walkthrough is intended to show the procedure for making any PCIe configuration change using IP presets.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment to build the FIM. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                   
                                • This walkthrough requires an OFS Agilex PCIe Attach deployment environment to test the design. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  [OPTIONAL] Run the setup stage of the build script using your desired OFSS configration to create a working directory for the n6001 design.
                                   
                                  ./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/n6001.ofss n6001 work_n6001\n
                                   
                                  4.  
                                4.  
                                  Open the PCIe-SS in the work directory using Quartus Parameter Editor. If you performed Step 3, open the PCIe-SS IP from the work directory; otherwise, open the PCIe-SS IP from the source files.
                                   
                                  qsys-edit $OFS_ROOTDIR/work_n6001/ipss/pcie/qip/pcie_ss.ip\n
                                   
                                  5.  
                                5.  
                                  Modify the settings as desired. In this example we are changing the Device ID to 0xbccf and the Revision ID to 0x2. In the IP Parameter Editor, scroll down and expand the PCIe Interfaces Ports Settings -> Port 0 -> PCIe0 Device Identification Registers -> PCIe0 PF0 IDs tab and make these changes.
                                   
                                   
                                  6.  
                                6.  
                                  Once you are satisfied with your modifcations, create a new IP Preset file.
                                   
                                     
                                  1.  
                                    click New... in the Presets window.
                                     
                                     
                                    2.  
                                  2.  
                                    In the New Preset window, set a unique Name for the preset; for example, n6001-rev2.
                                     
                                     
                                    3.  
                                  3.  
                                    Click the ... button to set the save location for the IP Preset file to $OFS_ROOTDIR/ipss/pcie/presets. Set the File Name to match the name selected in Step 9. Click OK.
                                     
                                     
                                    4.  
                                  4.  
                                    In the New Preset window, click Save. Click No when prompted to add the file to the IP search path.
                                     
                                     
                                    5.  
                                   
                                  7.  
                                7.  
                                  Close IP Parameter Editor without saving or generating HDL.
                                   
                                  8.  
                                8.  
                                  Create a new PCIe OFSS file in the $OFS_ROOTDIR/tools/ofss_config/pcie directory. For example:
                                   
                                  touch $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host_mod_preset.ofss\n
                                   
                                  Insert the following into the OFSS file to specify the IP Preset file created in Step 6. 
                                   
                                  [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\npreset = n6001-rev2\n
                                   
                                  9.  
                                9.  
                                  Edit the $OFS_ROOTDIR/tools/ofss_config/n6001.ofss file to call new OFSS file created in Step 10.
                                   
                                  [include]  \"$OFS_ROOTDIR\"/tools/ofss_config/n6001_base.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host_mod_preset.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory.ofss
                                   
                                  10.  
                                10.  
                                  Compile the design with the modified n6001.ofss file.
                                   
                                  cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss,tools/ofss_config/hssi/hssi_8x25.ofss n6001 work_n6001_pcie_mod\n
                                   
                                  11.  
                                11.  
                                  Copy the resulting $OFS_ROOTDIR/work_n6001_pcie_mod/syn/board/n6001/syn_top/output_files/ofs_top_hps.sof image to your deployment environmenment for JTAG programming, or copy a bin file (e.g. ofs_top_page1_unsigned_user1.bin) for RSU programming.
                                   
                                  Note: OPAE FPGA management commands require recognition of the FPGA PCIe Device ID for control.  If there is a problem between OPAE management recognition of FPGA PCIe values, then control of the card will be lost.  For this reason, you are strongly encouraged to program the FPGA via JTAG to load the test FPGA image.  If there is a problem with the SOF image working with your host software that is updated for the new PCIe settings, then you can load a known good SOF file to recover.  Once you sure that both the software and FPGA work properly, you can load the FPGA into FPGA flash using the OPAE command fpgasupdate.
                                   
                                  12.  
                                12.  
                                  Switch to your deployment environment.
                                   
                                  13.  
                                13.  
                                  Program the image to the n6001 FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step JTAG programming instructions, or the Program the FPGA via RSU Section for step-by-step RSU programming instructions.
                                   
                                  14.  
                                14.  
                                  Use lspci to verify that the PCIe changes have been implemented.
                                   
                                  lspci -nvmms 98:00.0\n
                                   
                                  Example output:
                                   
                                  Slot:   98:00.0\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nPhySlot:        1-1\nRev:    02\nNUMANode:       1\nIOMMUGroup:     8\n
                                   
                                  15.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#45-minimal-fim","title":"4.5 Minimal FIM","text":"
                                In a minimal FIM, the exercisers and Ethernet subsystem are removed from the design, and the AFU PR area is expanded to make use of the available area previously used by the removed components. This minimal FIM is useful for HDL applications.
                                 
                                There are two types of provided minimal FIMs:
                                 
                                   
                                • 2PF: this minimal FIM has two physical functions, with the APF/BPF on PF0, and the AFU PR region on PF1.
                                   
                                  •  
                                 
                                   
                                • 1PF/1VF: This minmal FIM has a single physical function, with the APF/BPF on PF0, and the AFU PR region on PF0-VF0.
                                   
                                  •  
                                 
                                The FIM source repository contains OFSS file that can be used to build the two different types of minimal FIM.
                                 
                                   
                                • $OFS_ROOTDIR/tools/ofss_config/n6001_2pf.ofss
                                  •  
                                • $OFS_ROOTDIR/tools/ofss_config/n6001_1pf_1vf.ofss
                                  •  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#451-walkthrough-create-a-minimal-fim","title":"4.5.1 Walkthrough: Create a Minimal FIM","text":"
                                Perform the following steps to create a Minimal FIM.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                   
                                • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  The OFS FIM repo provides a PR assignments TCL file which optimizes the PR region for the minimal FIM. Copy the minimal PR assignments TCL file into the pr_assignments.tcl file location for use in the FIM build process.
                                   
                                     
                                  1.  
                                    Rename the current pr_assignments.tcl file to pr_assignments_base.tcl for future use
                                     
                                    mv $OFS_ROOTDIR/syn/board/n6001/setup/pr_assignments.tcl $OFS_ROOTDIR/syn/board/n6001/setup/pr_assignments_base.tcl\n
                                     
                                    2.  
                                  2.  
                                    Copy the pr_assignments_slim.tcl file to pr_assignments.tcl to be used in the current build
                                     
                                    cp $OFS_ROOTDIR/syn/board/n6001/setup/pr_assignments_slim.tcl $OFS_ROOTDIR/syn/board/n6001/setup/pr_assignments.tcl\n
                                     
                                    3.  
                                   
                                  4.  
                                4.  
                                  Compile the FIM with Null HEs compile option, the No HSSI compile option, and 1PF/1VF configuration OFSS file.
                                   
                                  cd $OFS_ROOTDIR\n
                                   
                                     
                                  •  
                                    For 2PF Minimal FIM
                                     
                                    ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001_2pf.ofss n6001:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_n6001_minimal_fim\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    For 1PF/1VF Minimal FIM
                                     
                                    ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001_1pf_1vf.ofss n6001:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_n6001_minimal_fim\n
                                     
                                    •  
                                   
                                  5.  
                                5.  
                                  Review the $OFS_ROOTDIR/work_n6001_minimal_fim/syn/board/n6001/syn_top/output_files/ofs_top.fit.rpt utilization report to see the utilization statistics for the Minimal fim. Refer to [Appendix A] Table A-4 for the expected utilization for this Minimal FIM.
                                   
                                  6.  
                                6.  
                                  Copy the resulting $OFS_ROOTDIR/work_n6001_minimal_fim/syn/board/n6001/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                                   
                                  7.  
                                7.  
                                  Switch to your deployment environment, if different than your development environment.
                                   
                                  8.  
                                8.  
                                  Program the .bin image to the n6001 FPGA. Refer to the Program the FPGA via RSU Section for step-by-step programming instructions.
                                   
                                  9.  
                                9.  
                                  Verify the minimal FIM design on the board.
                                   
                                     
                                  •  
                                    For 2PF Minimal FIM:
                                     
                                       
                                    1. Use opae.io ls to verify that there are two PFs
                                      2.  
                                     
                                    sudo opae.io ls\n
                                     
                                    Example output:
                                     
                                    [0000:98:00.1] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n[0000:98:00.0] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                                     
                                       
                                    1.  
                                      Use fpgainfo port to verify the ports.
                                       
                                      sudo fpgainfo port\n
                                       
                                      Example output:
                                       
                                      //****** PORT ******//\nInterface                        : DFL\nObject Id                        : 0xEE00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nInterface                        : UIO\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 00000000-0000-0000-0000-000000000000\n
                                       
                                      2.  
                                    2.  
                                      Bind the VFIO driver on PF1.
                                       
                                      sudo opae.io init -d 98:00.1\n
                                       
                                      Example output:
                                       
                                      Unbinding (0x8086,0xbcce) at 0000:98:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:98:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:98:00.1 is 15\n
                                       
                                      3.  
                                    3.  
                                      Use fpgainfo port to verify the ports. There should now be a port entry for PF1. The Accelerator GUID for PF1 should be as shown below, which is the GUID for the HE-MEM.
                                       
                                        //****** PORT ******//\n  Interface                        : DFL\n  Object Id                        : 0xEE00000\n  PCIe s:b:d.f                     : 0000:98:00.0\n  Vendor Id                        : 0x8086\n  Device Id                        : 0xBCCE\n  SubVendor Id                     : 0x8086\n  SubDevice Id                     : 0x1771\n  Socket Id                        : 0x00\n  //****** PORT ******//\n  Interface                        : VFIO\n  Object Id                        : 0x2098000000000000\n  PCIe s:b:d.f                     : 0000:98:00.1\n  Vendor Id                        : 0x8086\n  Device Id                        : 0xBCCE\n  SubVendor Id                     : 0x8086\n  SubDevice Id                     : 0x1771\n  Socket Id                        : 0x01\n  Accelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n  //****** PORT ******//\n  Interface                        : UIO\n  Object Id                        : 0xED00000\n  PCIe s:b:d.f                     : 0000:98:00.0\n  Vendor Id                        : 0x8086\n  Device Id                        : 0xBCCE\n  SubVendor Id                     : 0x8086\n  SubDevice Id                     : 0x1771\n  Socket Id                        : 0x01\n  Accelerator GUID                 : 00000000-0000-0000-0000-000000000000\n
                                       
                                      4.  
                                     
                                    •  
                                   
                                     
                                  •  
                                    For 1PF/1VF Minimal FIM:
                                     
                                       
                                    1. Use opae.io ls to verify that there is one PF.
                                      2.  
                                     
                                    sudo opae.io ls\n
                                     
                                    Example output:
                                     
                                    [0000:98:00.0] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                                     
                                       
                                    1.  
                                      Use lspci to verify that there is 1 VF on PF0.
                                       
                                      sudo lspci -vvv -s 98:00.0 | grep VF\n
                                       
                                      Example output:
                                       
                                      Initial VFs: 1, Total VFs: 1, Number of VFs: 0, Function Dependency Link: 00\nVF offset: 1, stride: 1, Device ID: bcce\nVF Migration: offset: 00000000, BIR: 0\n
                                       
                                      2.  
                                    2.  
                                      Use fpgainfo port to verify the ports.
                                       
                                      sudo fpgainfo port\n
                                       
                                      Example output:
                                       
                                      //****** PORT ******//\nInterface                        : DFL\nObject Id                        : 0xEE00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nInterface                        : UIO\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 00000000-0000-0000-0000-000000000000\n
                                       
                                      3.  
                                    3.  
                                      Initialize one virtual function
                                       
                                      sudo pci_device 0000:98:00.0 vf 1\n
                                       
                                      4.  
                                    4.  
                                      Use opae.io ls to verify the VF has been initialized.
                                       
                                      sudo opae.io ls\n
                                       
                                      Example output:
                                       
                                      [0000:98:00.0] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n[0000:98:00.1] (0x8086:0xbccf 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                                       
                                      5.  
                                    5.  
                                      Bind the VFIO driver on VF0
                                       
                                      sudo opae.io init -d 0000:98:00.1\n
                                       
                                      6.  
                                    6.  
                                      Use fpgainfo port to verify the ports. There should now be a port entry for VF0. The Accelerator GUID for VF0 should be as shown below, which is the GUID for the HE-MEM.
                                       
                                      sudo fpgainfo port\n
                                       
                                      Example output:
                                       
                                      //****** PORT ******//\nInterface                        : DFL\nObject Id                        : 0xEE00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nInterface                        : VFIO\nObject Id                        : 0x2098000000000000\nPCIe s:b:d.f                     : 0000:98:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n//****** PORT ******//\nInterface                        : UIO\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 00000000-0000-0000-0000-000000000000\n
                                       
                                      7.  
                                     
                                    •  
                                   
                                  10.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#46-migrate-to-a-different-agilex-device-number","title":"4.6 Migrate to a Different Agilex Device Number","text":"
                                The following instructions enable a user to change the device part number of the Intel\u00ae FPGA SmartNIC N6001-PL. Please be aware that this release works with Intel\u00ae Agilex\u00ae 7 FPGA devices that have P tile for PCIe and E tile for Ethernet.  Other tiles will take further work.
                                 
                                You may wish to change the device part number for the following reasons
                                 
                                   
                                1. Migrate to same device package but with a different density
                                  2.  
                                2. Migrate to a different package and with a different or same density
                                  3.  
                                 
                                The default device for the Intel\u00ae FPGA SmartNIC N6001-PL is AGFB014R24A2E2V
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#461-walkthrough-migrate-to-a-different-agilex-device-number","title":"4.6.1 Walkthrough: Migrate to a Different Agilex Device Number","text":"
                                Perform the following steps to migrate your design to target a different Agilex device using the OFSS build flow. In this example we will change the device from the default AGFB014R24A2E2V to a new device AGFB022R25A2E2V.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Modify the part field in the $OFS_ROOTDIR/tools/ofss_config/n6001_base.ofss file to use AGFB022R25A2E2V. This is only necessary if you are using the OFSS flow.
                                   
                                  [ip]\ntype = ofs\n\n[settings]\nplatform = n6001\nfamily = agilex\nfim = base_x16\npart = AGFB022R25A2E2V\ndevice_id = 6001\n
                                   
                                  4.  
                                4.  
                                  Modify the DEVICE field in the $OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_top.qsf file.
                                   
                                  ############################################################################################\n# FPGA Device\n############################################################################################\n\nset_global_assignment -name FAMILY Agilex\nset_global_assignment -name DEVICE AGFB022R25A2E2V\n
                                   
                                  5.  
                                5.  
                                  Modify the DEVICE field in the $OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_pr_afu.qsf file.
                                   
                                  ############################################################################################\n# FPGA Device\n############################################################################################\n\nset_global_assignment -name FAMILY Agilex\nset_global_assignment -name DEVICE AGFB022R25A2E2V\n
                                   
                                  6.  
                                6.  
                                  Modify the DEVICE field in te $OFS_ROOTDIR/ipss/pmci/pmci_ss.qsf file.
                                   
                                  set_global_assignment -name DEVICE AGFB022R25A2E2V\n
                                   
                                  7.  
                                7.  
                                  If you are changing to a device with a different package, you must change the pin assignments in the location files. If you would like Quartus to attempt to pin out the design automatically, you may remove all pin assignments instead. Typically you will be required to set certain pins manually in order to guide Quartus for a successful compile (e.g. transceiver reference clocks). In this example we will start by commenting out all of the pin constraints in the following files and attempt to let Quartus pin out the design as much as possibe:
                                   
                                  $OFS_ROOTDIR/syn/board/n6001/setup/emif_loc.tcl\n$OFS_ROOTDIR/syn/board/n6001/setup/hps_loc.tcl\n$OFS_ROOTDIR/syn/board/n6001/setup/pmci_loc.tcl\n$OFS_ROOTDIR/syn/board/n6001/setup/top_loc.tcl\n
                                   
                                  For example:
                                   
                                  #set_location_assignment PIN_CU26 -to hssi_rec_clk[0]\n
                                   
                                  8.  
                                8.  
                                  Identify the pins you wish to assign prior to compiling. In this example, we will re-pin some of the reference clocks to help guide the fitter. Refer to the Pin-Out Files for Intel FPGAs for the pin list of your device. In this example, the Migration Re-Pin Mapping table below shows the pins we will re-pin in the constraints files.
                                   
                                  Table: Migration Re-Pin Mapping
                                   Pin Name FIM Signal Name AGF 014 R24A Pin # AGF 022 R25A Pin # REFCLK_GXER9A_CH0p cr3_cpri_reflclk_clk[0] PIN_AT13 PIN_CE18 REFCLK_GXER9A_CH0n \"cr3_cpri_reflclk_clk[0](n)\" PIN_AP13 PIN_CA18 REFCLK_GXER9A_CH1p cr3_cpri_refclk_clk[1] PIN_AR14 PIN_CC19 REFCLK_GXER9A_CH1n \"cr3_cpri_refclk_clk[1](n)\" PIN_AN14 PIN_BW19 REFCLK_GXER9A_CH2p cr3_cpri_refclk_clk[2] PIN_AJ12 PIN_BL17 REFCLK_GXER9A_CH2n \"cr3_cpri_refclk_clk[2](n)\" PIN_AH11 PIN_BJ15 REFCLK_GXER9A_CH3p qsfp_ref_clk PIN_AK13 PIN_BN18 REFCLK_GXER9A_CH3n \"qsfp_ref_clk(n)\" PIN_AH13 PIN_BJ18 REFCLK_GXER9A_CH4p cr3_cpri_reflclk_clk_184_32m PIN_AJ14 PIN_BL19 REFCLK_GXER9A_CH4n \"cr3_cpri_reflclk_clk_184_32m(n)\" PIN_AL14 PIN_BR19 REFCLK_GXER9A_CH5p cr3_cpri_reflclk_clk_153_6m PIN_AR16 PIN_CC21 REFCLK_GXER9A_CH5n \"cr3_cpri_reflclk_clk_153_6m(n)\" PIN_AN16 PIN_BW21 REFCLK_GXPL10A_CH0n \"PCIE_REFCLK0(n)\" PIN_AH49 PIN_DD56 REFCLK_GXPL10A_CH0p PCIE_REFCLK0 PIN_AJ48 PIN_DF57 REFCLK_GXPL10A_CH2n \"PCIE_REFCLK1(n)\" PIN_AD49 PIN_CT56 REFCLK_GXPL10A_CH2p PCIE_REFCLK1 PIN_AE48 PIN_CV57 
                                  9.  
                                9.  
                                  Re-pin the reference clocks defined in $OFS_ROOTDIR/syn/board/n6001/setup/top_loc.tcl
                                   
                                  set_location_assignment PIN_BN18 -to qsfp_ref_clk\nset_location_assignment PIN_BJ18 -to \"qsfp_ref_clk(n)\"\nset_location_assignment PIN_CC19 -to cr3_cpri_refclk_clk[1]\nset_location_assignment PIN_BW19 -to \"cr3_cpri_refclk_clk[1](n)\"\nset_location_assignment PIN_BL17 -to cr3_cpri_refclk_clk[2]\nset_location_assignment PIN_BJ15 -to \"cr3_cpri_refclk_clk[2](n)\"\nset_location_assignment PIN_CE18 -to cr3_cpri_reflclk_clk[0]\nset_location_assignment PIN_CA18 -to \"cr3_cpri_reflclk_clk[0](n)\"\nset_location_assignment PIN_BL19 -to cr3_cpri_reflclk_clk_184_32m\nset_location_assignment PIN_BR19 -to \"cr3_cpri_reflclk_clk_184_32m(n)\"\nset_location_assignment PIN_CC21 -to cr3_cpri_reflclk_clk_153_6m\nset_location_assignment PIN_BW21 -to \"cr3_cpri_reflclk_clk_153_6m(n)\"\n\nset_location_assignment PIN_DD56 -to \"PCIE_REFCLK0(n)\"\nset_location_assignment PIN_DF57 -to PCIE_REFCLK0\nset_location_assignment PIN_CT56 -to \"PCIE_REFCLK1(n)\"\nset_location_assignment PIN_CV57 -to PCIE_REFCLK1\n
                                   
                                  10.  
                                10.  
                                  Un-comment the instance assignemnts for the transceiver reference clocks defined in $OFS_ROOTDIR/syn/board/n6001/setup/top_loc.tcl.
                                   
                                  set_instance_assignment -name IO_STANDARD \"DIFFERENTIAL LVPECL\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=disable_3p3v_tol\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=156250000\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=TRUE\" -to qsfp_ref_clk\nset_instance_assignment -name IO_STANDARD \"DIFFERENTIAL LVPECL\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=disable_3p3v_tol\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=184320000\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=FALSE\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name IO_STANDARD \"DIFFERENTIAL LVPECL\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=disable_3p3v_tol\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=153600000\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=FALSE\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name IO_STANDARD \"DIFFERENTIAL LVPECL\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=disable_3p3v_tol\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=245760000\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=FALSE\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name IO_STANDARD \"DIFFERENTIAL LVPECL\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=enable_3p3v_tol\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=184320000\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=FALSE\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name IO_STANDARD \"DIFFERENTIAL LVPECL\" -to cr3_cpri_reflclk_clk_153_6m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to cr3_cpri_reflclk_clk_153_6m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=enable_3p3v_tol\" -to cr3_cpri_reflclk_clk_153_6m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to cr3_cpri_reflclk_clk_153_6m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=153600000\" -to cr3_cpri_reflclk_clk_153_6m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to cr3_cpri_reflclk_clk_153_6m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=FALSE\" -to cr3_cpri_reflclk_clk_153_6m\n
                                   
                                  11.  
                                11.  
                                  Compile a flat design. It is recommended to compile a flat design first before incorporating a PR region in the design. This reduces design complexity while you determine the correct pinout for your design.
                                   
                                  cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001:flat work_n6001_migrate_device_ofss\n
                                   
                                  12.  
                                12.  
                                  The compile should succeed. If the compile fails with errors relating to the pinout, review the error messages and modify the pinout.
                                   
                                  ***********************************\n***\n***        OFS_PROJECT: n6001\n***        OFS_BOARD: n6001\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 3\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                   
                                  13.  
                                13.  
                                  After a successful compile, to preserve pin assignemnts youmust hard code the new pin asigments back to the constraints files.
                                   
                                  $OFS_ROOTDIR/syn/board/n6001/setup/emif_loc.tcl\n$OFS_ROOTDIR/syn/board/n6001/setup/hps_loc.tcl\n$OFS_ROOTDIR/syn/board/n6001/setup/pmci_loc.tcl\n$OFS_ROOTDIR/syn/board/n6001/setup/top_loc.tcl\n
                                   
                                  14.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#47-modify-the-memory-sub-system","title":"4.7 Modify the Memory Sub-System","text":"
                                OFS allows modifications on the Memory Sub-System in the FIM. This section provides examples walkthroughs for modifiying the Memory-SS.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#471-walkthrough-modify-the-memory-sub-system-using-ip-presets-with-ofss","title":"4.7.1 Walkthrough: Modify the Memory Sub-System Using IP Presets With OFSS","text":"
                                This walkthrough will go through the flow of modifying the Memory-SS in the OFS FIM. In this example, we will enable ECC on memory channels 0-3. Note that routes for the ECC pins on Channels 0 and 1 are not physiclly present on standard n6001 board hardware; the purpose of this walkthrough is only to show an example of how to make modifications to the IP.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Open the Memory Subsystem mem_ss.ip in IP Parameter Editor
                                   
                                  qsys-edit $OFS_ROOTDIR/ipss/mem/qip/mem_ss/mem_ss.ip\n
                                   
                                  4.  
                                4.  
                                  The Memory Subsystem IP will open in IP Parameter Editor. Click Dive Into Packaged Subsystem.
                                   
                                   
                                  5.  
                                5.  
                                  The Platform Designer mem_ss view opens. All of the EMIFs are shown in the Filter window.
                                   
                                   
                                  6.  
                                6.  
                                  Click each EMIF 0 through 3 and perform the following actions.
                                   
                                     
                                  1.  
                                    In the Parameters window, click the Memory tab and change the DQ width to 40.
                                     
                                     
                                    2.  
                                  2.  
                                    In the Parameters window, click the Controller tab.
                                     
                                     
                                    3.  
                                  3.  
                                    Scroll down and check the box for Enable Error Detection and Correction Logic with ECC. 
                                     
                                     
                                    4.  
                                   
                                  7.  
                                7.  
                                  Once Step 6 has been done for each EMIF 0-3, click File -> Save. Close the Platform Designer window.
                                   
                                  8.  
                                8.  
                                  In the IP Parameter Editor Presets window, click New to create an IP Presets file.
                                   
                                   
                                  9.  
                                9.  
                                  In the New Preset window, set the Name for the preset. In this case we will name it n6001-ecc.
                                   
                                   
                                  10.  
                                10.  
                                  Click the ... button to select the location for the Preset file.
                                   
                                  11.  
                                11.  
                                  In the Save As window, change the save location to $OFS_ROOTDIR/ipss/mem/qip/presets and change the File Name to n6001-ecc.qprs. Click OK.
                                   
                                   
                                  12.  
                                12.  
                                  Click Save in the New Preset window. Click No when prompted to add the file to the IP search path.
                                   
                                   
                                  13.  
                                13.  
                                  Close the IP Parameter Editor. You do not need to generate or save the IP.
                                   
                                  14.  
                                14.  
                                  Edit the $OFS_ROOTDIR/syn/board/n6001/setup/emif_loc.tcl file to add pin assignments for the new signals supporting ECC on Channels 0-3. Note that routes for the ECC pins on Channels 0 and 1 are not physiclly present on a standard n6001 board.
                                   
                                  # CH0 DQS4 (ECC)\nset_location_assignment PIN_CG48  -to ddr4_mem[0].dbi_n[4]\nset_location_assignment PIN_CF47  -to ddr4_mem[0].dqs_n[4]\nset_location_assignment PIN_CH47  -to ddr4_mem[0].dqs[4]\nset_location_assignment PIN_CE50  -to ddr4_mem[0].dq[32]\nset_location_assignment PIN_CG50  -to ddr4_mem[0].dq[33]\nset_location_assignment PIN_CF49  -to ddr4_mem[0].dq[34]\nset_location_assignment PIN_CH49  -to ddr4_mem[0].dq[35]\nset_location_assignment PIN_CE46  -to ddr4_mem[0].dq[36]\nset_location_assignment PIN_CG46  -to ddr4_mem[0].dq[37]\nset_location_assignment PIN_CF45  -to ddr4_mem[0].dq[38]\nset_location_assignment PIN_CH45  -to ddr4_mem[0].dq[39]\n\n# CH1 DQS4 (ECC)\nset_location_assignment PIN_DC34  -to ddr4_mem[1].dbi_n[4]\nset_location_assignment PIN_CY33  -to ddr4_mem[1].dqs_n[4]\nset_location_assignment PIN_DB33  -to ddr4_mem[1].dqs[4]\nset_location_assignment PIN_DA36  -to ddr4_mem[1].dq[32]\nset_location_assignment PIN_DC36  -to ddr4_mem[1].dq[33]\nset_location_assignment PIN_CY35  -to ddr4_mem[1].dq[34]\nset_location_assignment PIN_DB35  -to ddr4_mem[1].dq[35]\nset_location_assignment PIN_DA32  -to ddr4_mem[1].dq[36]\nset_location_assignment PIN_DC32  -to ddr4_mem[1].dq[37]\nset_location_assignment PIN_CY31  -to ddr4_mem[1].dq[38]\nset_location_assignment PIN_DB31  -to ddr4_mem[1].dq[39]\n\n\n# CH2 DQS4 (ECC)\nset_location_assignment PIN_G36  -to ddr4_mem[2].dbi_n[4]\nset_location_assignment PIN_H35  -to ddr4_mem[2].dqs_n[4]\nset_location_assignment PIN_F35  -to ddr4_mem[2].dqs[4]\nset_location_assignment PIN_G38  -to ddr4_mem[2].dq[32]\nset_location_assignment PIN_J38  -to ddr4_mem[2].dq[33]\nset_location_assignment PIN_H33  -to ddr4_mem[2].dq[34]\nset_location_assignment PIN_J34  -to ddr4_mem[2].dq[35]\nset_location_assignment PIN_F33  -to ddr4_mem[2].dq[36]\nset_location_assignment PIN_H37  -to ddr4_mem[2].dq[37]\nset_location_assignment PIN_F37  -to ddr4_mem[2].dq[38]\nset_location_assignment PIN_G34  -to ddr4_mem[2].dq[39]\n\n# CH3 DQS4 (ECC)\nset_location_assignment PIN_L50 -to ddr4_mem[3].dbi_n[4]\nset_location_assignment PIN_P49 -to ddr4_mem[3].dqs_n[4]\nset_location_assignment PIN_M49 -to ddr4_mem[3].dqs[4]\nset_location_assignment PIN_M51 -to ddr4_mem[3].dq[32]\nset_location_assignment PIN_N48 -to ddr4_mem[3].dq[33]\nset_location_assignment PIN_M47 -to ddr4_mem[3].dq[34]\nset_location_assignment PIN_L48 -to ddr4_mem[3].dq[35]\nset_location_assignment PIN_P47 -to ddr4_mem[3].dq[36]\nset_location_assignment PIN_P51 -to ddr4_mem[3].dq[37]\nset_location_assignment PIN_N52 -to ddr4_mem[3].dq[38]\nset_location_assignment PIN_L52 -to ddr4_mem[3].dq[39]\n
                                   
                                  15.  
                                15.  
                                  Edit the $OFS_ROOTDIR/tools/ofss_config/memory/memory.ofss file to use the n6001-ecc preset that was generated previously.
                                   
                                  [ip]\ntype = memory\n\n[settings]\noutput_name = mem_ss_fm\npreset = n6001-ecc\n
                                   
                                  16.  
                                16.  
                                  Compile the design with the n6001.ofss file, which will use the modified memory.ofss file.
                                   
                                  ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss,tools/ofss_config/hssi/hssi_8x25.ofss n6001 work_n6001_mem_ecc_preset\n
                                   
                                  17.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#48-modify-the-ethernet-sub-system","title":"4.8 Modify the Ethernet Sub-System","text":"
                                This section describes the flows for modifying the Ethernet Sub-System. There are three flows you may use to make modifications.
                                 
                                   
                                • Modify the Ethernet Sub-System with OFSS supported changes only. These modifications are supported natively by the build script, and are made at run-time of the build script. This flow is useful for users who only need to leverage natively supported HSSI OFSS settings.
                                  •  
                                • Modify the Ethernet Sub-System with OFSS supported changes, then make additional custom modifications not covered by OFSS. These modifications will be captured in a presets file which can be used in future compiles. This flow is useful for users who whish to leverage pre-made HSSI OFSS settings, but make additional modifications not natively supported by HSSI OFSS.
                                  •  
                                • Modify the Ethernet Sub-System without HSSI OFSS. These modification will be made directly in the source files.
                                  •  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#481-walkthrough-modify-the-ethernet-sub-system-channels-with-pre-made-hssi-ofss","title":"4.8.1 Walkthrough: Modify the Ethernet Sub-System Channels With Pre-Made HSSI OFSS","text":"
                                This walkthrough describes how to use OFSS to configure the Ethernet-SS. Refer to section HSSI IP OFSS File for detailed information about modifications supported by Ethernet-SS OFSS files. This walkthrough is useful for users who only need to leverage the pre-made, natively supported HSSI OFSS settings.
                                 
                                Pre-Requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Edit the $OFS_ROTDIR/tools/ofss_config/n6001.ofss file to use the desired Ethernet-SS OFSS configuration. The pre-provided OFSS configurations are as follows:
                                   
                                     
                                  •  
                                    To select 2x4x25GbE configuration, add the following line
                                     
                                    \"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_8x25.ofss\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    To select 2x4x10GbE configuration, add the following line
                                     
                                    \"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_8x10.ofss\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    To select 2x1x100GbE configuration, add the following line
                                     
                                    \"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_2x100.ofss\n
                                     
                                    •  
                                   
                                  4.  
                                4.  
                                  Compile the FIM using the n6001.ofss file.
                                   
                                  cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001 work_n6001\n
                                   
                                  5.  
                                5.  
                                  The resulting FIM will contain the Ethernet-SS configuration specified in Step 3. The Ethernet-SS IP in the resulting work directory shows the parameter settings that are used.
                                   
                                  6.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#482-walkthrough-add-channels-to-the-ethernet-sub-system-channels-with-custom-hssi-ofss","title":"4.8.2 Walkthrough: Add Channels to the Ethernet Sub-System Channels With Custom HSSI OFSS","text":"
                                This walkthrough describes how to create an use a custom OFSS file to add channels to the Ethernet-SS and compile a design with a 3x4x10GbE Ethernet-SS configuration. This walkthrough is useful for users who wish to leverage the natively supported HSSI OFSS settings.
                                 
                                Pre-Requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Create a new HSSI OFSS file $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_12x10.ofss with the following contents.
                                   
                                  [ip]\ntype = hssi\n\n[settings]\noutput_name = hssi_ss\nnum_channels = 12\ndata_rate = 10GbE\n
                                   
                                  4.  
                                4.  
                                  Edit the $OFS_ROOTDIR/tools/ofss_config/n6001.ofss file to use the new HSSI OFSS file generated in Step 3.
                                   
                                  [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/n6001_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_12x10.ofss\n
                                   
                                  5.  
                                5.  
                                  Identify the which channels will be added. You may use the E-Tile Channel Placement Tool to aid in your design. In this example we will add the 4 new 10GbE channels to Channels 8-11.
                                   
                                   
                                  6.  
                                6.  
                                  Based on your channel selection, identify which pins will be used. Refer to the Pin-Out Files for Intel FPGAs determine the required pins for your device. In this example we are targeting the AGFB014R24A2E2V device. Set the pin assignments in the $OFS_ROOTDIR/syn/board/n6001/setup/top_loc.tcl file.
                                   
                                  set_location_assignment PIN_AV7  -to qsfp_serial[2].rx_p[0]\nset_location_assignment PIN_AW10 -to qsfp_serial[2].rx_p[1]\nset_location_assignment PIN_BB7  -to qsfp_serial[2].rx_p[2]\nset_location_assignment PIN_BC10 -to qsfp_serial[2].rx_p[3]\n\nset_location_assignment PIN_AV1 -to qsfp_serial[2].tx_p[0]\nset_location_assignment PIN_AW4 -to qsfp_serial[2].tx_p[1]\nset_location_assignment PIN_BB1 -to qsfp_serial[2].tx_p[2]\nset_location_assignment PIN_BC4 -to qsfp_serial[2].tx_p[3]\n
                                   
                                  7.  
                                7.  
                                  Edit the NUM_QSFP_PORTS value in the $OFS_ROOTDIR/ipss/hssi/rtl/inc/ofs_fim_eth_plat_if_pkg.sv file to 3.
                                   
                                  localparam NUM_QSFP_PORTS       = 3; // QSFP cage on board\n
                                   
                                  8.  
                                8.  
                                  Compile the design. It is recommended to compile a flat design first before incorporating a PR region in the design. This reduces design complexity while you determine the correct pinout for your design.
                                   
                                  cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001:flat work_n6001_12x10\n
                                   
                                  9.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#483-walkthrough-modify-the-ethernet-sub-system-with-pre-made-hssi-ofss-plus-additional-modifications","title":"4.8.3 Walkthrough: Modify the Ethernet Sub-System With Pre-Made HSSI OFSS Plus Additional Modifications","text":"
                                This walkthrough describes how to use OFSS to first modify the Ethernet-SS, then make additional modifications on top. Refer to section HSSI IP OFSS File for detailed information about modifications supported by Ethernet-SS OFSS files. This flow is useful for users who whish to leverage pre-made OFSS settings, but make additional modifications not natively supported by OFSS.
                                 
                                Pre-Requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Edit the $OFS_ROTDIR/tools/ofss_config/n6001.ofss file to use the desired Ethernet-SS OFSS configuration starting point.
                                   
                                     
                                  •  
                                    To select 2x4x25GbE configuration, add the following line
                                     
                                    \"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_8x25.ofss\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    To select 2x4x10GbE configuration, add the following line
                                     
                                    \"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_8x10.ofss\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    To select 2x1x100GbE configuration, add the following line
                                     
                                    \"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_2x100.ofss\n
                                     
                                    •  
                                   
                                  4.  
                                4.  
                                  Run the setup stage of the build script with the OFSS file to create a work directory which contains the Ethernet-SS IP configuration specified in Step 3.
                                   
                                  cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/n6001.ofss n6001 work_n6001\n
                                   
                                  5.  
                                5.  
                                  Open the Ethernet-SS IP in Quartus Parameter Editor. The IP settings will match te configuration of the OFSS file defined in Step 3. Make any additional modifications in the Parameter Editor.
                                   
                                  qsys-edit $OFS_ROOTDIR/work_n6001/ipss/hssi/qip/hssi_ss/hssi_ss.ip\n
                                   
                                  6.  
                                6.  
                                  Once you are satisfied with your changes, click the New... button in the Presets pane of IP Parameter Editor.
                                   
                                   
                                  7.  
                                7.  
                                  In the New Preset window, create a unique Name. In this example the name is n6001-hssi-presets.
                                   
                                   
                                  8.  
                                8.  
                                  Click the ... button to select where to save the preset file. Give it a name, and save it to $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss/presets
                                   
                                   
                                  9.  
                                9.  
                                  Click Save in the New Preset window. Click No when prompted to add the file to the IP search path.
                                   
                                  10.  
                                10.  
                                  Close out of all Quartus GUIs. You do not need to save or compile the IP.
                                   
                                  11.  
                                11.  
                                  Create a new HSSI OFSS file in the $OFS_ROOTDIR/tools/ofss_config/hssi directory named hssi_preset_n6001.ofss with the following contents. Note that the num_channels and data_rate settings will be overwritten by the contents of the preset file. The preset setting must match the name you selected in Step 7.
                                   
                                  [ip]\ntype = hssi\n\n[settings]\noutput_name = hssi_ss\nnum_channels = 8\ndata_rate = 25GbE\npreset = n6001-hssi-presets\n
                                   
                                  12.  
                                12.  
                                  Edit the $OFS_ROOTDIR/tools/ofss_config/n6001.ofss file to use the new HSSI OFSS file created in Step 10.
                                   
                                  [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/n6001_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_preset_n6001.ofss\n
                                   
                                  13.  
                                13.  
                                  Compile the design using the n6001 OFSS file. It is recommended to compile a flat design first before incorporating a PR region in the design. This reduces design complexity while you modify the FIM.
                                   
                                  ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001:flat work_n6001_hssi_preset\n
                                   
                                  14.  
                                14.  
                                  The resulting FIM will contain the Ethernet-SS configuration specified by the presets file. The Ethernet-SS IP in the resulting work directory shows the parameter settings that are used.
                                   
                                  15.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#484-walkthrough-modify-the-ethernet-sub-system-without-hssi-ofss","title":"4.8.4 Walkthrough: Modify the Ethernet Sub-System Without HSSI OFSS","text":"
                                This walkthrough describes how to modify the Ethernet-SS wihout using OFSS. This flow will edit the Ethernet-SS IP source directly. This walkthrough is useful for users who wish to make all Ethernet-SS modifications manually, without leveraging HSSI OFSS.
                                 
                                Pre-Requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Open the Ethernet-SS IP in Quartus Parameter Editor. Make your modifications in the Parameter Editor.
                                   
                                  qsys-edit $OFS_ROOTDIR/work_n6001/ipss/hssi/qip/hssi_ss/hssi_ss.ip\n
                                   
                                  4.  
                                4.  
                                  Once you are satisfied with your changes, click the Generate HDL. Save the design if prompted.
                                   
                                  5.  
                                5.  
                                  Compile the design.
                                   
                                     
                                  •  
                                    If you are not using any other OFSS files in your compilation flow, use the following command to compile. It is recommended to compile a flat design before incorporating a PR region in the design. This reduces design complexity while you modify the FIM.
                                     
                                    ./ofs-common/scripts/common/syn/build_top.sh n6001:flat work_n6001\n
                                     
                                    •  
                                   
                                     
                                  • If you are using OFSS files for other IP in the design, ensure that the top level OFSS file (e.g. $OFS_ROOTDIR/tools/ofss_config/n6001.ofss) does not specify an HSSI OFSS file. Then use the following command to compile. It is recommended to compile a flat design first before incorporating a PR region in the design. This reduces design complexity while you modify the FIM.
                                    •  
                                   
                                  ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001:flat work_n6001\n
                                   
                                  6.  
                                6.  
                                  The resulting FIM will contain the Ethernet-SS configuration contained in the hssi_ss.ip source file.
                                   
                                  7.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#49-modifying-the-hps","title":"4.9 Modifying the HPS","text":"
                                This section describes ways to modify the HPS.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#491-walkthrough-remove-the-hps","title":"4.9.1 Walkthrough: Remove the HPS","text":"
                                Perform the following steps to remove the HPS from the FIM design.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                   
                                  2.  
                                2.  
                                  Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                   
                                  3.  
                                3.  
                                  Create a Memory Sub-system IP presets file with the connection to the HPS removed.
                                   
                                     
                                  1.  
                                    Open the the Memory Sub-System IP
                                     
                                    qsys-edit $OFS_ROOTDIR/ipss/mem/qip/mem_ss/mem_ss.ip\n
                                     
                                    2.  
                                  2.  
                                    In the IP Parameter Editor window that opens, remove the entries corresponding to the HPS (row #4) in the Memory Interfaces and Application Interfaces sections. To do this, click the row to be removed, then click the minus (-) button.
                                     
                                     
                                    3.  
                                  3.  
                                    In the Presets pane, click New... to create a new IP presets file.
                                     
                                     
                                    4.  
                                  4.  
                                    In the New Preset window, create a unique preset name. For example, n6001-mem-no-hps.
                                     
                                     
                                    5.  
                                  5.  
                                    Click the ... button to select the save location of the IP presets file. In the Save As window, set the Look In field to the memory IP presets directory $OFS_ROOTDIR/ipss/mem/qip/presets. Set the File Name field to match the name selected in Step 4. Click OK.
                                     
                                     
                                    6.  
                                  6.  
                                    Click Save in the New Preset window. Click No when prompted to add the file to the IP search path.
                                     
                                     
                                    7.  
                                  7.  
                                    Close IP Parameter Editor without saving or generating HDL.
                                     
                                    8.  
                                   
                                  4.  
                                4.  
                                  Edit the Memory OFSS file $OFS_ROOTDIR/tools/ofss_config/memory/memory.ofss to use the IP presets file generated in Step 4.
                                   
                                  [ip]\ntype = memory\n\n[settings]\noutput_name = mem_ss\npreset = n6001-mem-no-hps\n
                                   
                                  5.  
                                5.  
                                  Edit $OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_top.qsf to comment out the INCLUDE_HPS and INCLUDE_UART macro definitions.
                                   
                                  #set_global_assignment -name VERILOG_MACRO \"INCLUDE_HPS\"\n#set_global_assignment -name VERILOG_MACRO \"INCLUDE_UART\"\n
                                   
                                  6.  
                                6.  
                                  Build the FIM.
                                   
                                  ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss,tools/ofss_config/hssi/hssi_8x25.ofss n6001 work_n6001_no_hps\n
                                   
                                  7.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#5-fpga-configuration","title":"5. FPGA Configuration","text":"
                                Configuring the Agilex FPGA on the n6001 can be done by Remote System Update (RSU) using OPAE commands, or by programming a SOF image to the FPGA via JTAG using Quartus Programer.
                                 
                                Programming via RSU will program the flash device on the board for non-volatile image updates. Programming via JTAG will configure the FPGA for volatile image updates.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#51-walkthrough-set-up-jtag","title":"5.1 Walkthrough: Set up JTAG","text":"
                                Perform the following steps to set up a JTAG connection to the Intel\u00ae FPGA SmartNIC N6001-PL.
                                 
                                Pre-requisites:
                                 
                                   
                                • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                  •  
                                 
                                   
                                • This walkthrough requires a workstation with Quartus Prime Pro Version 23.3 tools installed, specifically the jtagconfig tool.
                                  •  
                                 
                                   
                                • This walkthrough requires an Intel FPGA Download Cable II.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Set the board switches to dynamically select either the Intel\u00ae Agilex\u00ae 7 FPGA or MAX\u00ae 10 device on the JTAG chain.
                                   
                                     
                                  1. Set SW1.1=ON as shown in the next image. The switches are located at the back of the Intel\u00ae FPGA SmartNIC N6001-PL.
                                    2.  
                                   
                                   
                                  2.  
                                2.  
                                  The Intel\u00ae FPGA SmartNIC N6001-PL has a 10 pin JTAG header on the top side of the board. Connect an Intel\u00ae FPGA Download II Cable to the JTAG header of the Intel\u00ae FPGA SmartNIC N6001-PL as shown in picture below. This picture shows the Intel\u00ae FPGA SmartNIC N6001-PL card installed in the middle bay, top slot of a SuperMicro\u00ae SYS-220HE-FTNR server where the lower slot does not have card installed allowing the Intel\u00ae Download II cables to pass through removed the slot access. 
                                   
                                   
                                  Note: If using the Intel FGPA download Cable on Linux, add the udev rule as described in Intel FPGA Download Cable Driver for Linux.
                                   
                                  3.  
                                3.  
                                  Set the JTAG chain to select the Intel\u00ae Agilex\u00ae 7 FPGA as the target by writing to the JTAG enable register in the BMC (Register 0x378). This is done via PMCI registers 0x2040C and 0x20400.
                                   
                                  Note: The commands below are targeted to a board with PCIe B:D.F of 98:00.0. Use the correct PCIe B:D.F of your card.
                                   
                                  sudo opae.io init -d 0000:98:00.0 $USER\nsudo opae.io -d 0000:98:00.0 -r 0 poke 0x2040c 0x100000000\nsudo opae.io -d 0000:98:00.0 -r 0 poke 0x20400 0x37800000002\nsudo opae.io release -d 0000:98:00.0\n
                                   
                                  Note: To later re-direct the JTAG back to the MAX 10 device, execute the following commands.
                                   
                                  sudo opae.io init -d 0000:b1:00.0 $USER\nsudo opae.io -d 0000:b1:00.0 -r 0 poke 0x2040c 0x000000000\nsudo opae.io -d 0000:b1:00.0 -r 0 poke 0x20400 0x37800000002\nsudo opae.io release -d 0000:b1:00.0\n
                                   
                                  Optionally, rather than dynamically commanding Intel\u00ae Agilex\u00ae 7 FPGA/MAX10 selection with the PMCI register settings, you can fix the selection with the following switch settings shown in the table below:
                                   SW1.1 SW2 JTAG Target OFF OFF Intel\u00ae Agilex\u00ae 7 FPGA OFF ON MAX\u00ae 10 FPGA ON X Intel\u00ae Agilex\u00ae 7 FPGA if BMC register 0x378=0x1 ON X MAX\u00ae 10 FPGA if BMC register 0x378=0x0 
                                  4.  
                                4.  
                                  Use the jtagconfig tool to check that the JTAG chain contains the AGFB014R24A2E2V device.
                                   
                                  <QUARTUS_INSTALL_DIR>/23.3/quartus/bin/jtagconfig\n
                                   
                                  Example expected output:
                                   
                                  TBD\n
                                   
                                  5.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#52-walkthrough-program-the-fpga-via-jtag","title":"5.2 Walkthrough: Program the FPGA via JTAG","text":"
                                This walkthrough describes the steps to program the Agilex FPGA on the Intel\u00ae FPGA SmartNIC N6001-PL with a SOF image via JTAG.
                                 
                                Pre-Requisites:
                                 
                                   
                                • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                  •  
                                 
                                   
                                • This walkthrough requires a SOF image which will be programmed to the Agilex FPGA. Refer to the Compile OFS FIM Section for step-by-step instructions for generating a SOF image.
                                  •  
                                 
                                   
                                • This walkthrough requires a JTAG connection to the n6001. Refer to the Set up JTAG section for step-by-step instructions.
                                  •  
                                 
                                   
                                • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel\u00ae FPGA SmartNIC N6001-PL is connected via JTAG.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Start in your deployment environment.
                                   
                                  2.  
                                2.  
                                  If the card is already programmed with an OFS enabled design, determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                                   
                                  fpgainfo fme\n
                                   
                                  Example output:
                                   
                                  Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                   
                                  3.  
                                3.  
                                  Remove the card from PCIe prior to programming. This will disable AER for the PCIe root port to prevent a surprise link-down event during programming.
                                   
                                  sudo pci_device b1:00.0 unplug\n
                                   
                                  4.  
                                4.  
                                  Switch to the machine with JTAG connection to the n6001, if different than your deployment machine.
                                   
                                  5.  
                                5.  
                                  Open the Quartus programmer GUI
                                   
                                  quartus_pgmw\n
                                   
                                   
                                  6.  
                                6.  
                                  Click Hardware Setup to open the Hardware Setup dialog window.
                                   
                                     
                                  1.  
                                    In the Currently selected hardware field select the n6001.
                                     
                                    2.  
                                  2.  
                                    In the Hardware frequency field enter 16000000 Hz
                                     
                                     
                                    3.  
                                  3.  
                                    Click Close
                                     
                                    4.  
                                   
                                  7.  
                                7.  
                                  In the Quartus Prime Programmer window, click Auto Detect.
                                   
                                  8.  
                                8.  
                                  If prompted, select the AGFB014R24A2E2V device. The JTAG chain should show the divice.
                                   
                                   
                                  9.  
                                9.  
                                  Right click the AGFB014R24A2E2V row and selct Change File.
                                   
                                   
                                  10.  
                                10.  
                                  In the Select New Programming File window that opens, select the .sof image you wish to program and click Open.
                                   
                                  11.  
                                11.  
                                  Check the Program/Configure box for the AGFB014R24A2E2V row, then click Start. Wait for the Progress bar to show 100% (Success).
                                   
                                  12.  
                                12.  
                                  Close the Quartus Programmer GUI. You can answer 'No' if a dialog pops up asking to save the 'Chain.cdf' file
                                   
                                  13.  
                                13.  
                                  Switch to the deployment environment, if different than the JTAG connected machine.
                                   
                                  14.  
                                14.  
                                  Replug the board PCIe
                                   
                                  sudo pci_device b1:00.0 plug\n
                                   
                                  15.  
                                15.  
                                  Run fpgainfo fme to verify communication with the board, and to check the PR Interface ID.
                                   
                                  Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                   
                                  16.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#53-remote-system-update","title":"5.3 Remote System Update","text":"
                                The OPAE fpgasupdate tool can be used to update the Intel Max10 Board Management Controller (BMC) image and firmware (FW), root entry hash, and FPGA Static Region (SR) and user image (PR). The fpgasupdate tool only accepts images that have been formatted using PACsign. If a root entry hash has been programmed onto the board, then you must also sign the image using the correct keys. Refer to the Security User Guide: Intel Open FPGA Stack for information on created signed images and on programming and managing the root entry hash.  
                                 
                                The Intel\u00ae FPGA SmartNIC N6001-PL ships with a factory, user1, and user2 programmed image for both the FIM and BMC FW and RTL on all cards. The platform ships with a single FIM image that can be programmed into either user1 or user2, depending in the image selected.
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#531-walkthrough-program-the-fpga-via-rsu","title":"5.3.1 Walkthrough: Program the FPGA via RSU","text":"
                                This walkthrough describes the steps to program the Agilex FPGA on the Intel\u00ae FPGA SmartNIC N6001-PL with a BIN image via JTAG.
                                 
                                Pre-Requisites:
                                 
                                   
                                • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                  •  
                                 
                                   
                                • This walkthrough requires a BIN image which will be programmed to the Agilex FPGA. Refer to the Compile OFS FIM Section for step-by-step instructions for generating a BIN image. The image used for programming must be formatted with PACsign before programming. This is done automatically by the build script.
                                  •  
                                 
                                   
                                • This walkthrough requires a JTAG connection to the n6001. Refer to the Set up JTAG section for step-by-step instructions.
                                  •  
                                 
                                   
                                • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel\u00ae FPGA SmartNIC N6001-PL is connected via JTAG.
                                  •  
                                 
                                Steps:
                                 
                                   
                                1.  
                                  Start in your deployment environment.
                                   
                                  2.  
                                2.  
                                  Determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                                   
                                  fpgainfo fme\n
                                   
                                  Example output:
                                   
                                  Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                   
                                  3.  
                                3.  
                                  Use the OPAE fpgasupdate command to program a PACsign signed image to flash. The flash slot which will be programmed is determined by the PACsign header.
                                   
                                  sudo fpgasupdate <IMAGE> <PCIe B:D.F>\n
                                   
                                     
                                  •  
                                    Example: update User Image 1 in flash
                                     
                                    sudo fpgasupdate ofs_top_page1_unsigned_user1.bin 98:00.0\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    Example: update User Image 2 in flash
                                     
                                    sudo fpgasupdate ofs_top_page2_unsigned_user2.bin 98:00.0\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    Example: update Factory Image in flash
                                     
                                    sudo fpgasupdate ofs_top_page0_unsigned_factory.bin 98:00.0\n
                                     
                                    •  
                                   
                                  4.  
                                4.  
                                  Use the OPAE rsu command to reconfigure the FPGA with the new image. You may select which image to configure from (User 1, User 2, Factory).
                                   
                                  sudo rsu fpga --page <PAGE> <PCIe B:D.F>\n
                                   
                                     
                                  •  
                                    Example: configure FPGA with User 1 Image
                                     
                                    sudo rsu fpga --page user1 98:00.0\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    Example: configure FPGA with User 2 Image
                                     
                                    sudo rsu fpga --page user2 98:00.0\n
                                     
                                    •  
                                   
                                     
                                  •  
                                    Example: configure FPGA with Factory Image
                                     
                                    sudo rsu fpga --page factory 98:00.0\n
                                     
                                    •  
                                   
                                  5.  
                                "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#appendix","title":"Appendix","text":""},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#appendix-a-resource-utilization-tables","title":"Appendix A: Resource Utilization Tables","text":"
                                Table A-1 Default Out-of-Tree FIM Resource Utilization
                                 Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % top 181,018.30 37.15 784 11.03 afu_top 104,994.20 21.55 287 4.04 pcie_wrapper 36,565.00 7.51 195 2.74 hssi_wrapper 20,132.10 4.13 173 2.43 mem_ss_top 9,092.80 1.87 76 1.07 pmci_wrapper 4,269.30 0.88 26 0.37 alt_sld_fab_0 2,726.90 0.56 13 0.18 bpf 1,364.60 0.28 0 0.00 qsfp_top 620.10 0.13 4 0.06 fme_top 615.30 0.13 6 0.08 qsfp_top 614.00 0.13 4 0.06 rst_ctrl 17.90 0.00 0 0.00 sys_pll 0.50 0.00 0 0.00 hps_ss 0.00 0.00 0 0.00 
                                Table A-2 Minimal FIM Resource Utilization
                                 Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 103192.5 21.18 410 5.77 afu_top 45848.7 9.41 150 2.11 auto_fab_0 1770.1 0.36 9 0.13 bpf 1265.8 0.26 0 0.0 fme_top 662.4 0.14 6 0.08 hps_ss 0.0 0.0 0 0.0 hssi_dummy_csr 675.5 0.14 0 0.0 mem_ss_top 8723.9 1.79 60 0.84 pcie_wrapper 38361.7 7.87 159 2.24 pmci_wrapper 4509.9 0.93 26 0.37 qsfp0_dummy_csr 672.2 0.14 0 0.0 qsfp1_dummy_csr 681.7 0.14 0 0.0 rst_ctrl 18.0 0.0 0 0.0 sys_pll 0.5 0.0 0 0.0"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#appendix-b-glossary","title":"Appendix B: Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                 
                                OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/","title":"Hard Processor System Software Developer Guide: OFS for Intel Agilex FPGAs Targeting Intel\u00ae N6000/1-PL FPGA SmartNIC Platform","text":"
                                Quartus Prime Pro Version: 23.1
                                 
                                Last updated: Last updated: February 03, 2024 
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#glossary","title":"Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel Max10 or Intel Cyclone10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implemented on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Management Component Transport Protocol MCTP A standardized model for communication with management controllers. Defines the transport protocol carrying PLDM messages through the BMC. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE-SDK The OPAE-SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Level Data Model PLDM A specification for reporting telemetry data to the host, such as board temperature, voltage, and current. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#file-types","title":"File Types","text":"Extension Description ITS File (*.its) The image source file which describes the contents of the image and defines various properties used during boot. Actual contents to be included in the image (kernel, ramdisk, etc.) are specified in the image source file as paths to the appropriate data files. ITB File (*.itb) Produced as output from mkimage, using an image source file. Contains all the referenced data (kernel, ramdisk, SSBL, etc.) and other information needed by UBoot to handle the image properly. This image is transferred to the target and booted. DTB File (*.dtb) The Device Tree Blob is loaded into memory by U-Boot during the boot process, and a pointer to it is shared with the kernel. This file describes the system's hardware layout to the kernel. FIT Image (*.fit) Format used for uImage payloads developed by U-Boot. On aarch64 the kernel must be in image format and needs a device tree to boot. SPL (*.spl) The Secondary Program Loader is a small binary which is embedded in a FIM SOF and loaded into board DDR4 RAM when the FPGA is initially configured. This file is responsible for loading U-Boot into system RAM."},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#10-introduction","title":"1.0 Introduction","text":"
                                The Open FPGA Stack (OFS) is a modular collection of hardware platform components, open source upstreamed software, and broad ecosystem support that enables an efficient path to develop a custom FPGA platform. OFS Provides a framework of FPGA synthesizable code, a simulation environment and synthesis/simulation scripts.  The updated OFS architecture for Intel Agilex FPGA devices improves upon the modularity, configurability and scalability of the first release of the OFS architecture while maintaining compatibility with the original design.  The primary components of the FPGA Interface Manager or shell of the reference design are:
                                 
                                   
                                • PCIe Subsystem
                                  •  
                                • HSSI Subsystem
                                  •  
                                • Memory Subsystem
                                  •  
                                • Hard Processor System (HPS)
                                  •  
                                • Reset Controller
                                  •  
                                • FPGA Management Engine (FME)
                                  •  
                                • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                                  •  
                                • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                                  •  
                                • SPI Interface to BMC controller
                                  •  
                                 
                                The Intel\u00ae N6000-PL and N6001-PL FPGA SmartNIC Platforms are acceleration cards that use the OFS infrastructure. The key difference between these two platforms is:
                                 
                                   
                                • Intel\u00ae N6000-PL SmartNIC Platform has a bifurcated PCIe bus with Gen4x8 interfacing to the the Intel Agilex FPGA and Gen4x8 interfacing to an Intel E810 SmartNIC.  This platform is targeted specifically for VRAN, UPF and vCSR applications.  The FPGA designs targeting these vertical market applications were generated using the OFS infrastructure.
                                  •  
                                • Intel\u00ae N6001-PL SmartNIC Platform has a Gen4x16 interface directly to the Intel Agilex FPGa and is not populated with an Intel E810 SmartNIC.  This platform is the reference platform for the OFS reference designs for Intel Agilex FPGA.  
                                  •  
                                 
                                Note: throughout this document \"Intel N6000/1-PL FPGA SmartNIC Platform\" denotes both cards. This document describes the software package that runs on the Hard Processor System (HPS) which is a key component within both platforms.
                                 
                                The Intel N6000/1-PL FPGA SmartNIC Platform has a customized build script that can be used to both set up a development environment and build the essential pieces of the HPS software image. This script, meta-bake.py, has its own dedicated Section 3.1 Building U-Boot which can be used to quickly get started with the HPS build flow. It is recommended you use this build script to construct the first and second stage bootloader files, as it will handle all of the setup and patching required to build out your complete Yocto image. You can familiarize yourself with the contents of this package in its public GitHub repository located at https://github.com/OPAE/meta-opae-fpga/tree/main/tools/meta-bake. All other information included for individual components is included for learning purposes only and is not meant as a recipe for image creation.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#11-reference-documents","title":"1.1 Reference Documents","text":"
                                This document pulls much of its information from related Agilex FPGA documentation hosted on intel.com. Reading these references is not required for initial platform bring up, but will serve to further your knowledge of the FPGA SoC boot and configuration process.
                                 
                                Table 1. Reference Documents
                                 Document Title Intel\u00ae Agilex\u2122 Hard Processor System Technical Reference Manual Intel\u00ae Agilex\u2122 SoC FPGA Boot User Guide Intel\u00ae Agilex\u2122 Configuration User Guide"},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#12-reference-images","title":"1.2 Reference Images","text":"
                                Intel has provided a set of two pre-compiled ITB images that can be used for exploration and evaluation of the HPS bring-up flow. These images contain the complete SSBL package specific to the board and can be copied to the N6000/1-PL SmartNIC Platform with an HPS enabled FIM loaded. Refer to Section 4.1 Example Boot for an example on how to use the built-in copy engine IP in tandem with the host-side cpeng software to transfer an SSBL.
                                 
                                The package is found on the official OFS 2023.3-2 Release on GitHub. Two ITB artifacts are included at the bottom of the page under Assets - one with the Vendor Authorized Boot (VAB) certificate included, and one without. Which you choose to load depends on whether the currently loaded FIM requires VAB authentication. Section 4.3 Example Boot contains instructions on the boot flow using these files for platform bring up.
                                 
                                The default username for these two images is root and the password is empty. A good place to start after loading the ITB is to set up SSH for file transfers and the remote console, as seen in 8.0 Connecting remotely to the HPS using ssh.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#20-architecture-overview","title":"2.0 Architecture Overview","text":"
                                The OFS architecture is classified into:
                                 
                                   
                                • 1. Host Interface Adapters (PCIe)
                                  •  
                                • 2. Low Performance Peripherals
                                  •  
                                     
                                  • 2.1. Slow speed peripherals (example: JTAG, I2C, SMBus, and so on)
                                    •  
                                  • 2.2. Management peripherals (example: FPGA FME)
                                    •  
                                   
                                • 3. High Performance Peripherals
                                  •  
                                     
                                  • 3.1. Memory peripherals
                                    •  
                                  • 3.2. Acceleration Function Units (AFUs)
                                    •  
                                  • 3.3. HPS Peripheral
                                    •  
                                   
                                • 4. Fabrics
                                  •  
                                     
                                  • 4.1. Peripheral Fabric (multi drop)
                                    •  
                                  • 4.2. AFU Streaming fabric (point to point)
                                    •  
                                   
                                 
                                The HPS is connected to the AFU and implements all the board specific flows that customers require to begin the application development using the HPS such as host communication, firmware load and update, integration with OFS, and memory. The HPS implements a basic Hello World software application and is intended as a starting point for customers to begin development with HPS.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#21-hps-peripherals","title":"2.1 HPS Peripherals","text":"
                                Figure 1  Intel Agilex FPGA HPS Peripherals
                                 
                                 
                                The Intel Agilex\u2122 SoC integrates a full-featured Arm\u00ae Cortex-A53\u00ae MPCore Processor.
                                 
                                The Cortex-A53 MPCore supports high-performance applications and provides the capability for secure processing and virtualization.
                                 
                                Each CPU in the processor has the following features:
                                 
                                   
                                • Support for 32-bit and 64-bit instruction sets.
                                  •  
                                • To pipeline with symmetric dual issue of most instructions.
                                  •  
                                • Arm NEON\u00ae Single Instruction Multiple Data (SIMD) co-processor with a Floating-Point Unit (FPU)
                                  •  
                                • Single and double-precision IEEE-754 floating point math support
                                  •  
                                • Integer and polynomial math support.
                                  •  
                                • Symmetric Multiprocessing (SMP) and Asymmetric Multiprocessing (AMP) modes.
                                  •  
                                • Armv8 Cryptography Extension.
                                  •  
                                • Level 1 (L1) cache:
                                  •  
                                • 32 KB two-way set associative instruction cache.
                                  •  
                                • Single Error Detect (SED) and parity checking support for L1 instruction cache.
                                  •  
                                • 32 KB four-way set associative data cache.
                                  •  
                                • Error checking and correction (ECC), Single Error Correct, Double Error Detect (SECDED) protection for L1 data cache.
                                  •  
                                • Memory Management Unit (MMU) that communicates with the System MMU (SMMU).
                                  •  
                                • Generic timer.
                                  •  
                                • Governor module that controls clock and reset.
                                  •  
                                • Debug modules:
                                  •  
                                • Performance Monitor Unit.
                                  •  
                                • Embedded Trace Macrocell (ETMv4).
                                  •  
                                • Arm CoreSight\u00ae cross trigger interface, the four CPUs share a 1 MB L2 cache with ECC, SECDED protection.
                                  •  
                                 
                                A Snoop Control Unit (SCU) maintains coherency between the CPUs and communicates with the system Cache Coherency Unit (CCU). At a system level, the Cortex-A53 MPCore interfaces to a Generic Interrupt Controller (GIC), CCU, and System Memory Management Unit (SMMU).
                                 
                                Beyond the Arm Cortex-A53 MPCore Processor, the HPS integrates a variety of useful peripherals for use in your design, such as Ethernet, USB, Memory Controller, on-chip RAM, SPI, UART and more. Refer to the Intel\u00ae Agilex\u2122 Hard Processor System Technical Reference Manual for more information.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#22-zarlink-device","title":"2.2 Zarlink Device","text":"
                                The Microchip\u00ae Zarlink device ZL30793 is used for time synchronization. It acts as the protocol engine that drives IEEE 1588-2008 PTP protocol. The Zarlink device is connected to the HPS side and the programming interface is SPI. The FPGA bitstream containing the HPS has the First Stage Bootloader (FSBL) only. This enable commands to be given from a terminal program connected through UART.
                                 
                                The software in HPS can access the Clock generator through SPI to enable write and read operations controlled by the terminal program. It can also read the status of the hold over and Loss of Lock signals and control the LED.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#23-copy-engine","title":"2.3 Copy Engine","text":"
                                A host with OPAE SDK and Linux DFL installed will provide the hps OPAE command with related options to transfer images from host to the HPS image. The module in the OFS FIM and HPS software that performs this transfer is called the Copy Engine (CPE), which is included by default within the HPS image.
                                 
                                Refer to the Getting Started Guide: \u00ae Open FPGA Stack for Intel Agilex FPGAs for platform and software installation instructions.
                                 
                                The CPE software is patched into Linux on the HPS in Yocto through the meta-intel-fpga-refdes layer. This service is daemonized and requires systemd in order to operate. This service will communicate with the HPS IP integrated in the FIM in order to coordinate and monitor file transfers from the host CPE software to DDR connected the HPS. The CPE HPS-side software takes advantage of the built-in I/O lightweight kernel module to communicate with the FIM's HPS IP. It can restart the transfer if the initial transfer of the image is not successful. The CPE can also serve as reference on how to integrate your own systemd service in the Linux build running on the HPS.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#24-boot-flow","title":"2.4 Boot Flow","text":"
                                The boot flow for the Agilex OFS design for the Intel N6000/1-PL FPGA SmartNIC Platform is an FPGA-first boot flow, meaning the Intel Agilex Secure Device Manager (SDM) configures the FPGA first and then boots the HPS. Alternatively, you can boot the HPS first and then configure the FPGA core as part of the Second-Stage Boot Loader (SSBL) or after the Operating System (OS) boots. HPS-first boot is not covered in this document, but for more information please refer to the Intel\u00ae Agilex\u2122 SoC FPGA Boot User Guide.
                                 
                                For the FPGA-first boot flow supported by the Intel Agilex OFS FIM, the First Stage Bootloader is part of FPGA bitstream. The available Secure Device Manager (SDM) in the FPGA initially configures the FPGA core and periphery in this mode. The first stage bootloader is produced as a part of a U-Boot build and cna be manually inserted into a Quartus generated SOF file as shown in step 7 of Section 9.2 Configuring the HPS.
                                 
                                After completion, the HPS boots. All the I/O, including the HPS-allocated I/O, are configured, and brought out of tri-state. If the HPS is not booted:
                                 
                                   
                                • The HPS is held in reset
                                  •  
                                • HPS-dedicated I/O are held in reset
                                  •  
                                • HPS-allocated I/O are driven with reset values from the HPS.
                                  •  
                                • If the FPGA is configured before the HPS boots, then the boot flow looks as shown in the example figure below.
                                  •  
                                 
                                Figure 2. Typical FPGA First Configuration Steps
                                 
                                 
                                The flow includes the Time from Power-on-Reset (TPOR) to boot completion (TBoot_Complete).
                                 
                                Table 2. FPGA Configuration First Stages
                                 Time Boot Stage Device State TPOR to T1 POR Power-on reset T1 to T2 SDM: Boot ROM 1. SDM samples the MSEL pins to determine the configuration scheme and boot source. 2. SDM establishes the device security level based on eFuse values. 3. SDM initializes the device by reading the configuration firmware (initial part of the bitstream) from the boot source. 4. SDM authenticates and decrypts the configuration firmware (this process occurs as necessary throughout the configuration). 5. SDM starts executing the configuration firmware. T2 to T3 SDM: Configuration Firmware 1. SDM I/O are enabled. 2. SDM configures the FPGA I/O and core (full configuration) and enables the rest of your configured SDM I/O. 3. SDM loads the FSBL from the bitstream into HPS on-chip RAM. 4. SDM enables HPS SDRAM I/O and optionally enables HPS debug. 5. FPGA is in user mode. 6. HPS is released from reset. CPU1-CPU3 are in a wait-for-interrupt (WFI) state. T3 to T4 First-Stage Boot Loader (FSBL) 1. HPS verifies the FPGA is in user mode. 2. The FSBL initializes the HPS, including the SDRAM. 3. The user application through the host must request the copy engine using the OPAE command hps to transfer the itb image (SSBL +Linux) to the HPS DRAM. 4. HPS peripheral I/O pin mux and buffers are configured. Clocks, resets, and bridges are also configured. 5. HPS I/O peripherals are available. T4 to T5 Second-Stage Boot Loader (SSBL) 1. HPS bootstrap completes. 2. OS is loaded into SDRAM. T5 to TBoot\\Complete Operating System (OS) The OS boots and applications are scheduled for runtime launch. 
                                When using the Pre-boot Execution Environment (PXE) boot mechanism, you must use an option ROM. OFS FIM does not have PXE boot implemented in the HPS.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#25-authorization","title":"2.5 Authorization","text":"
                                The HPS FSBL is part of the static region (SR) FPGA bitstream. Intel provides the capability to sign the FPGA bitstream binaries so that they can be authenticated when remotely updated and when configuring the FPGA. Signing of the SR bitstream is a two-stage process where you must sign with:
                                 
                                1. `quartus_sign` tool\n2. OPAE `PACSign` tool\n
                                 
                                Signing with PACSign ensures the security of the BMC RSU update process to the flash, and requires a compatible binary file. Quartus signing provides ensures security when the bitstream is configured through the SDM into the Intel Agilex FPGA device using Vendor Authorized Boot.
                                 
                                Vendor Authorized Bootloader (VAB) considers the SDM as a trusted root entity such that when firmware is authenticated and booted and is running on the SDM with dedicated crypto HW IP blocks, it is considered a trusted entity. As such it is trusted to perform the authentication and authorization steps for subsequent bitstreams.
                                 
                                Each subsequent loaded object after the SDM boot firmware does not need to re-implement the authentication and authorization functions. The authentication and authorization functions are centralized. Arm Trusted Firmware (ATF) is used to make a trusted execution environment (TEE) in the HPS. The source code for both Arm Trusted firmware and the First Stage Boot Loader (FSBL) is provided in the GitHub.
                                 
                                The SSBL + Linux is a part of an itb file and may also be signed with Quartus_sign and PACSign for authentication. This process is demonstrated in Section 9.2 Configuring the HPS.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#30-environment-setup","title":"3.0 Environment Setup","text":""},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#31-building-u-boot","title":"3.1 Building U-Boot","text":"
                                When creating a development environment using meta-bake.py both U-Boot and the patches required to work with the Intel N6000/1-PL FPGA SmartNIC Platform are located at /meta-opae-fpga/tools/meta-bake/build/agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/build/socfpga_agilex_n6000_defconfig. To review the required patches applied to U-Boot, navigate to /meta-opae-fpga/tools/meta-bake/build/agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/git/patches. From there, using git commands such as git status and git branch will show changes to the build environment. 
                                 
                                Currently the meta-bake build flow requires a specific environment and software dependencies. Refer to section 6.1 Running meta-bake.py for more information. 
                                 
                                Invoke the meta-bake.py build script to build your entire image, including U-Boot.
                                 
                                $ cd /meta-opae-fpga/tools/meta-bake\n$ ./meta-bake.py --conf n6000/layers.yaml builddir\n
                                 
                                This build process is highly system dependent and can take upwards of 1 hour to complete. Make sure you have at least 200 GB of free space on the system before running a build.
                                 
                                To build U-Boot manually after execution of meta-bake.py navigate to /meta-bake/build/agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-srcfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/build/socfpga_agilex_n6000_defconfig and run make. After running meta-bake.py, you can rebuild U-Boot to incorporate any changes you have made. Navigate to the U-Boot directory at /meta-opae-fpga/tools/meta-bake/build/agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/build/socfpga_agilex_n6000_defconfig and run the following commands to rebuild.
                                 
                                $ wget https://developer.arm.com/-/media/Files/downloads/gnu-a/10.2-2020.11/binrel/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz\n$ tar xf gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz\n$ rm gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz\n$ export CROSS_COMPILE=`pwd`/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-\n$ export ARCH=arm64\n$ make -j `nproc`\n
                                 
                                This recompile will result in a new ITB SSBL which may be loaded on an Intel FPGA SmartNIC N6000/1 platform. Several components of the ITB image are present under the U-Boot directory but are not rebuilt as a part of this flow. These files will need to be replaced before rebuilding U-Boot for changes to take affect.
                                 
                                U-Boot comes with its own dumpimage tool, which can be used to identify an image and extract and identify its contents. This tool is built by default under /u-boot-socfpga/tools, and in the meta-bake.py environment setup under /meta-opae-fpga/tools/meta-bake/build/agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/build/socfpga_agilex_n6000_defconfig/tools. This tool can also be used to extract specific components of the ITB file.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#32-yocto","title":"3.2 Yocto","text":"
                                Yocto is an open source toolkit used to create Linux distributions and commonly used for creating Linux images and bootloaders for embedded environments. A Yocto build environment is made up of one or more layers, with each layer consisting of recipes that get processed to build/install components in a layer. The workhorse of a Yocto build is the program called bitbake. This program processes the recipes to compile and build packages and images for the target platform. For SoC platforms, like the HPS, the ARM cross compiler is required.
                                 
                                The build script used for the Agilex SoC GSRD, create-linux-distro-release, is a bash script that automates the build of Linux images of different types (gsrd, nand, etc.) that are compatible with a target FPGA platform (agilex, stratix10, etc.). This script has been ported to Python 3 and modified to build an environment for the Intel FPGA SmartNIC N6000/1 platform, named meta-bake.py. This script pulls in the necessary patches and additional changes needed to support the platform.
                                 
                                In general, meta-bake.py pulls Yocto layers/recipes from public repositories, configures a Yocto build environment, and builds an image for a supported FPGA platform. The Yocto layer is always the first to be built, and includes the bitbake utility. The following table lists remote repositories hosting Yocto meta data source used by meta-bake.py and create-linux-distro as well as source code used for building binaries that make up the Linux image (kernel and rootfs).
                                 
                                Note: Not all repositories can be viewed in a web browser. All can be cloned using git.
                                 Repository Description https://git.yoctoproject.org/git/poky Base build tools and meta data layers https://git.openembedded.org/meta-openembedded Layers for OE-core packages https://git.yoctoproject.org/git/meta-intel-fpga Meta data layers for Intel FPGA SoC platforms https://github.com/altera-opensource/meta-intel-fpga-refdes BSP layer for Intel SoC FPGA GSRD https://github.com/altera-opensource/linux-socfpga Linux kernel source repository for socfpga https://github.com/altera-opensource/u-boot-socfpga U-Boot bootloader source repository for socfpga https://github.com/altera-opensource/arm-trusted-firmware Source for ATF 
                                Recipes in the meta-intel-fpga-refdes layer mostly inherit from and extend recipes in other layers. The following table lists the new or modified recipes (in meta-intel-fpga-refdes) necessary to support an Intel FPGA SmartNIC N6000/1 HPS boot image.
                                 Component Recipe Description Linux Kernel recipes-kernel/linux/linux-socfpga-lts_5.10.bbappend Recipe to append the GSRC SoC FPGA device tree to the Yocto build U-Boot recipes-bsp/u-boot/u-boot-socfpga_v2021.07.bbappend Recipe to fetch and build socfpga U-Boot. Modified to support N6000/1 in U-Boot. This also creates a shell script, *mkuboot-fit.sh. copy-engine recipes-bsp/copy-engine/copy-engine-0.1.bb New recipe to build copy-engine daemon in rootfs. N6000/1 Image recipes-images/poky/n6000-image-minimal.bb New recipe to create the N6000/1 image with copy-engine and linuxptp packages installed. 
                                mkuboot-fit.sh is meant to be called after a Yocto build to create the U-Boot FIT image for N6000/1, and is called automatically by meta-bake.py. This is a workaround for the Yocto build order which builds the bootloader (U-Boot) before building the Linux image rootfs. Because the rootfs is part of the U-Boot FIT image, the rootfs must be built before building the bootloader. The result of calling this script is copying the rootfs (as a .cpio file) to the U-Boot source file tree and calling make in the U-Boot build tree. When called again with the rootfs present, the resulting image will contain the rootfs. This is performed automatically as a part of the meta-bake.py build flow.
                                 
                                See here for more information regarding Yocto. Several reference designs found in rocketboards.org use Yocto for building the Linux image and/or bootloader. For the N6000/1 image and boot flow, the Yocto build script for the Agilex SoC Golden System Reference Design has been adapted to automate building the boot loader, Linux Image, and filesystem needed to support N6000/1 devices.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#321-customizing-the-yocto-image","title":"3.2.1 Customizing the Yocto Image","text":"
                                The following is a list of customizations made for building Yocto to run on the Intel FPGA SmartNIC N6000/1-PL platform.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#3211-extending-the-u-boot-recipe","title":"3.2.1.1 Extending the U-Boot recipe","text":"
                                A recipe extension file (recipes-bsb/u-boot/u-boot-socfpga_v2021.07.bbappend) has been added to the meta-intel-fpga-refdes layer which accomplishes the following:
                                 
                                   
                                • Adds patches using Yocto's patching mechanism
                                  •  
                                • Introduces a new U-Boot config, socfpga_agilex_n6000_defconfig, and associates it with a keyword, agilex-n6000, that can be referenced in Yocto configuration files. These patches are necessary until those changes are merged into the public u-boot-socfpga repository. This config works for both Smartnic Platforms.
                                  •  
                                • Creates mkuboot-fit.sh script file with variables for U-Boot source and build directories that will get expanded to the actual paths that Yocto uses for fetching/building U-Boot. Along with this recipe file, relevant patch files have been added. Once the changes are in the U-Boot repository, the patches and any references to them must be removed.
                                  •  
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#3212-patching-the-linux-kernel","title":"3.2.1.2 Patching The Linux Kernel","text":"
                                The kernel extension recipe, meta-intel-fpga-refdes/recipes-kernel/linux/linux-socfpga-lts_5.10.bbappend, in the meta-intel-fpga-refdes layer, has been modified to add a patch file using Yocto's patching mechanism. This patch file adds the device tree for N6000/1 and is only necessary until this change is merged into the public linux-socfpga repository.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#3213-adding-custom-user-space-software","title":"3.2.1.3 Adding Custom User Space Software","text":"
                                A new recipe, meta-intel-fpga-refdes/recipes-bsp/copy-engine-0.1.bb and relevant source files, have been added to the meta-intel-fpga-refdes layer. This recipe includes instructions for building the copy-engine program as well as installing it as a systemd service. Yocto will build this into an RPM package that gets installed into any image that includes it in the IMAGE_INSTALL variable. This recipe may be used as a guide for installing additional user space software.
                                 
                                You may also create a new Hello World application and add it to the Yocto build as shown below.
                                 
                                   
                                1. Generate a BSD 3-Clause License and create an MD5 hash of it.
                                  2.  
                                 
                                Copyright (c) 2023, User's Name \nAll rights reserved. \n\nRedistribution and use in source and binary forms, with or without \nmodification, are permitted provided that the following conditions are met: \n\n * Redistributions of source code must retain the above copyright notice, \n   this list of conditions and the following disclaimer. \n * Redistributions in binary form must reproduce the above copyright \n   notice, this list of conditions and the following disclaimer in the \n   documentation and/or other materials provided with the distribution. \n * Neither the name of Company Name nor the names of its contributors may \n   be used to endorse or promote products derived from this software \n   without specific prior written permission. \n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" \nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE \nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE \nARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE \nLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR \nCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF \nSUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS \nINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN \nCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) \nARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE \nPOSSIBILITY OF SUCH DAMAGE. \n
                                 
                                After license creation you will need to create an MD5 Hash of the clause. On Linux you can either pipe the raw text into echo \"text\" | md5sum, or create a new file and point to it md5sum licensefile.txt.
                                 
                                   
                                1. Create a BB recipe file, following the same directory structure as other Yocto recipes.
                                  2.  
                                 
                                $ cd /meta-opae-fpga/tools/meta-bake/build/meta-intel-fpga-refdes\n$ mkdir -p recipe-example/helloworld && cd recipe-example/helloworld\n
                                 
                                Create recipe file helloworld.bb in directory helloworld.
                                 
                                SUMMARY = \"Example hello world\" DESCRIPTION = \"helloworld in HPS\" \n\nAUTHOR = \"Your Name <your.email@address.com>\" \n\nLICENSE = \"BSD-3-Clause\" \n\nLIC_FILES_CHKSUM = \"file://${COMMON_LICENSE_DIR}/BSD-3-Clause;md5=<Your MD5 Hash>\" \n\ninherit systemd pkgconfig \n\nSRC_URI = \"file://helloworld.c\" \n\nS = \"${WORKDIR}\" \n\ndo_compile() { \n        ${CC} ${CFLAGS} ${LDFLAGS} helloworld.c -o helloworld \n}\n\ndo_install() { \n        install -d ${D}${bindir} \n        install -m 0755 helloworld ${D}${bindir} \n} \n
                                 
                                   
                                1. Create source file helloworld.c in the same helloworld directory.
                                  2.  
                                 
                                #include <stdio.h> \n\nvoid main void() \n{ \n    Printf(\u201c\\nHello World\\n\u201d) \n} \n
                                 
                                   
                                1. Re-run ./meta-bake.py --conf n6000/layers.yaml <Build Directory>. This will a new programmable SSBL that contains your Hello World program. Program the resulting ITB file as shown in Section 4.3 Example Boot and verify the application has been included in your build.
                                  2.  
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#3214-adding-kernel-driver-software","title":"3.2.1.4 Adding Kernel Driver Software","text":"
                                New recipes for custom kenel modules can be created at /build/meta-intel-fpga-refdes/recipes-kernel/linux/, and instructed to include custom module code. These can be patched in, included as a part of a new branch, or included by default if upstreamed. For more information visit the YoctoProject's Linux Kernel Development Manual. An example file from N6000/1 that can be used as an example is /build/meta-intel-fpga-refdes/recipes-kernel/linux/linux-socfpga-lts_5.10.bbappend.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#3215-creating-an-image-type","title":"3.2.1.5 Creating an Image Type","text":"
                                A new recipe, meta-intel-fpga-refdes/recipes-images/poky/n6000-image-minimal.bb, has been added that includes directives to install the copy-engine package (built in this layer) as well as the linuxptp package (available in other layers). In addition to including these packages, this recipe includes a rootfs post processing command that removes the Linux kernel image files from the rootfs. This is done because the Linux kernel is part of the U-Boot FIT image and therefore not used from the rootfs. Removing this redundant file reduces the final U-Boot FIT image by about 30Kb. This recipe may be modified or used as a guide to add additional user space software.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#3216-testing-and-debugging","title":"3.2.1.6 Testing and Debugging","text":"
                                As mentioned previously, the script will erase source files every time it is executed. This means that any changes made locally will be lost when the script is run again after making these changes. The example below shows how to test local changes without executing the script again.
                                 
                                $ cd build\n$ source poky/oe-init-build-env agilex-gsrd-rootfs/\n$ bitbake n6000-image-minimal\n$ ./agilex-n6000-rootfs/tmp/deploy/images/agilex/mkuboot-fit.sh\n
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#40-booting-the-hps","title":"4.0 Booting the HPS","text":""},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#41-ofs-fim-boot-overview","title":"4.1 OFS FIM Boot Overview","text":"
                                This implementation of an FPGA First boot flow requires that the FSBL poll on a given register before continuing to boot the HPS. Once this register indicates it is (copy engine) ready, the FSBL loads a monolithic U-Boot FIT image at a given offset 0x02000000.
                                 
                                This image is made up of the following components:
                                 
                                   
                                • U-Boot bootloader also referred to as second stage bootloader
                                  •  
                                • Linux Kernel image
                                  •  
                                • Root filesystem (rootfs) consisting of kernel modules as well as user space software.
                                  •  
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#42-booting-ofs-fim-for-intel-agilex-fpga","title":"4.2 Booting OFS FIM for Intel Agilex FPGA","text":"
                                As mentioned before, the Intel N6000/1-PL FPGA SmartNIC boot flow is an FPGA-first boot flow which requires that the Intel Agilex FPGA to be configured with the necessary components (SPL/FSBL, copyengine) in order for the HPS to boot.
                                 
                                SD/eMMC is not supported for FSBL for HPS first
                                 
                                   
                                • First Stage Bootloader (FSBL): u-boot-spl-dtb.hex is embedded into FPGA image
                                  •  
                                • Monolithic FIT Image Downloaded from Host, u-boot.itb contains the following components
                                  1) Second Stage Bootloader (SSBL): U-Boot + U-Boot Device Tree
                                   
                                  2) Linux kernel, Image, + Linux Device Tree
                                   
                                  3) Linux RAM based Root File System
                                   
                                  •  
                                 
                                   
                                • First Stage Bootloader (FSBL) is U-boot-spl
                                  •  
                                • U-boot-spl is built when U-Boot is built
                                  •  
                                • Artifact is u-boot-spl-dtb.hex
                                     
                                  • The user has to check into build location : ofs-n6001/syn/setup/vab_sw/u-boot-spl-dtb.hex
                                    •  
                                  • Then run the command
                                    •  
                                  • quartus/pfg -c -o hps/path=u-boot-spl-dtb.hex orig.sof orig/fsbl.sof
                                    •  
                                   
                                  •  
                                 
                                Things to consider while developing your ITB image:
                                 
                                - The size of the u-boot.itb matters.\n- FIT is downloaded to [0x2000000](https://github.com/altera-opensource/u-boot-socfpga/blob/541b6afcb183ddb350ad367c9b63cc6db94c1f6e/configs/socfpga_agilex_n6010_defconfig#L4)\n- Linux Device Tree and RootFS are unpacked to high memory\n- Linux is unpacked to an address specified in the FIT, [0xb600000](https://github.com/altera-opensource/u-boot-socfpga/blob/541b6afcb183ddb350ad367c9b63cc6db94c1f6e/arch/arm/dts/socfpga_agilex_n6010-u-boot.dtsi#L4)\n- If size of u-boot.itb is greater than 0xb600000 \u2013 0x2000000, then FIT will be corrupted mid extraction, resulting in unpredictable kernel crashes.\n
                                 
                                This example assumes the following preconditions have been met prior to booting HPS:
                                 
                                1) A SOF file synthesized with the SPL (u-boot-spl-dtb.hex).
                                 
                                2) Copy engine IP with relevant registers accessible to host and HPS.
                                 
                                Once the host FPGA boots with the required bitstream, the SPL in the HPS begins polling a register in the copy engine. One way to get an indication that the HPS is ready to continue past the SPL is to use a terminal emulator on a host with a serial cable connected to the FPGA's UART port. To transfer the U-Boot FIT image, use the hps cpeng subcommand from the host. Note, the hps program can be installed as part of installing the OPAE SDK and Linux DFL suite of packages.
                                 
                                hps command details are located in Section 5.0 HPS Command Usage.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#43-example-boot","title":"4.3 Example Boot","text":"
                                This example assumes the following preconditions have been met prior to booting HPS:
                                 
                                   
                                • A SOF file synthesized with the SPL (u-boot-spl-dtb.hex).
                                  •  
                                • Copy engine IP with relevant registers accessible to host and HPS.
                                  •  
                                 
                                Once the host FPGA boots with the required bitstream, the SPL in the HPS will begin polling a register in the copy engine. One way to get an indication that the HPS is ready to continue past the SPL is to use a terminal emulator on a host with a serial cable connected to the FPGA's UART port.
                                 
                                To transfer the U-Boot FIT image, use the hps program with cpeng subcommand from the host. Note, the hps program is installed as part of installing the OPAE SDK suite of packages. See here for information on running the hps program. The following example assumes your N6000/1 board is at PCIe BDF 0000:b1:00.0.
                                 
                                # Bind vfio-pci driver to Copy Engine PCIe Physical Function\n$ sudo opae.io init -d b1:00.4 root\n# Load the HPS SSBL\n$ hps cpeng -f u-boot.itb\n[2021-09-25 01:59:25.538] [cpeng] [info] starting copy of file:u-boot.itb, size: 116725656, chunk size: 4096\n[2021-09-25 01:59:29.935] [cpeng] [info] last chunk 1944, aligned 2048\n[2021-09-25 01:59:29.935] [cpeng] [info] transferred file in 28498 chunk(s)\n[2021-09-25 01:59:29.935] [cpeng] [info] waiting for ssbl verify...\n[2021-09-25 01:59:33.848] [cpeng] [info] ssbl verified\n[2021-09-25 01:59:33.848] [cpeng] [info] waiting for kernel verify...\n[2021-09-25 01:59:39.626] [cpeng] [info] kernel verified\n
                                 
                                This will transfer the U-Boot FIT image via the copy engine IP to the HPS DDR and then signal completion of the transfer to the copy engine. Once the copy engine completes the actual transfer, it will write to the register the HPS SPL is polling on allowing the SPL to load the U-Boot bootloader which will in turn boot into the Linux image embedded in the U-Boot FIT image. If a terminal emulator is connected to the UART as described above, a user can observe U-Boot and Linux running on the HPS.
                                 
                                   
                                1. Validate the HPS SSBL has been loaded by checking for its heartbeat.
                                  2.  
                                 
                                $ hps heartbeat\n[2021-09-25 01:59:42.722] [heartbeat] [info] heartbeat value: 0x30015\n[2021-09-25 01:59:43.722] [heartbeat] [info] heartbeat value: 0x40015\n[2021-09-25 01:59:44.722] [heartbeat] [info] heartbeat value: 0x50015\n[2021-09-25 01:59:45.723] [heartbeat] [info] heartbeat value: 0x60015\n[2021-09-25 01:59:46.723] [heartbeat] [info] heartbeat value: 0x70015\n
                                 
                                   
                                1. Login to HPS as user, root, with no password over serial connection. This process is covered in 8.0 Connecting remotely to the HPS using ssh.
                                  2.  
                                 
                                agilex login: root\nroot@agilex:~# ls\nroot@agilex:~# ls /\nbin dev home lib mnt root sbin sys usr\nboot etc init media proc run srv tmp var\nroot@agilex:~#\n
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#50-hps-command-usage","title":"5.0 HPS Command Usage","text":""},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#51-synopsis","title":"5.1 Synopsis","text":"
                                hps OPTIONS SUBCOMMAND SUBCOMMAND\\_OPTIONS
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#52-description","title":"5.2 Description","text":"
                                hps is an application to aid in the development, deployment, and debugging of an HPS (hard processor system) on an Intel Agilex device using OFS. The current version of the hps program assumes an AFU (accelerator functional unit) is configured into the FPGA that can be discovered/accessed through an OPAE library and used for communicating with the HPS. When invoked with one of its subcommands, hps will enumerate the AFU for that subcommand before executing it.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#53-options","title":"5.3 Options","text":"
                                -h,--help\n\nPrint this help message and exit\n\n-p,--pci-address address\n\nUse address in the filter for locating the AFU where address must be in\n\nthe following format: [domain]\\bus\\:\\device\\.\\function\\\n\n-l,--log-level \\level\\\n\nstdout logging level. Must be one of:\n\n{trace,debug,info,warning,error,critical,off}\n\nDefault is info.\n\n-s,--shared\n\nopen in shared mode, default is off\n\n-t,--timeout timeout\n\nProgram timeout in milliseconds. Default is 60000 ms.
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#54-subcommands","title":"5.4 Subcommands","text":""},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#541-cpeng","title":"5.4.1 cpeng","text":"
                                The copy engine command is used to program copy engine AFU registers to copy an image file from the host into the FPGA DDR. When the HPS boots, the first stage boot loader loads an image from a specific offset in DDR that will be used to transition into the second stage boot loader and subsequently boot into the embedded Linux that is also part of this image.
                                 cpeng  options description -h,--help Print this help message and exit -f,--filename filename Path to image file to copy. Default is u-boot.itb -d,--destination offset DDR Offset. Default is 0x2000000. -t,--timeout cpeng timeout Timeout of cpeng command in microseconds. Default is 1 sec (1000000 usec). -r,--data-request-limit size Can be 64, 128, 512, or 1024 and represents the PCIe request size in bytes that the copy engine IP will use. This is encoded to 0, 1, 2, or 3 and written to the copy engine DATA\\REQUEST\\LIMIT register. Default is 512. -c,--chunk size Split the copy into chunks of size size. 0 indicates no chunks. Chunk sizes must be aligned with data request limit. Default is 4096. --soft-reset Issue a soft reset only. --skip-ssbl-verify Do not wait for ssbl verify. --skip-kernel-verify Do now wait for kernel verify."},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#542-heartbeat","title":"5.4.2 heartbeat","text":"
                                This subcommand reads the value in the HPS2HOST register to check for a hearbeat. This compares the value to previous value read and determines the HPS is alive if the value is incrementing. This relies on the hps running the hello-cpeng program in heartbeat mode which will increment the upper 16 bits in the HPS2HOST register. Please see a typical sequence of using the rsu and hps commands as below for a device with BDF 15:00:0
                                 
                                rsu fpga -p user1 15:00.0\nsudo opae.io release -d 15:00.0\nsudo opae.io init -d 15:00.4 root:root\nhps cpeng -f u-boot-userkey-vab.itb\ntimeout 5 hps heartbeat\nsudo opae.io release -d 15:00.0\nhps cpeng -f u-boot-userkey-vab.itb\n
                                 
                                The above command will transfer the U-Boot FIT image via the copy engine IP to the HPS DDR and then signal completion of the transfer to the copy engine. After the copy engine completes the actual transfer, it writes to the register the HPS SPL is polling on allowing the SPL to load the U-Boot bootloader which in turn boots into the Linux image embedded in the U-Boot FIT image. If a terminal emulator is connected to the UART as described above, a user can observe U-Boot and Linux running on the HPS.
                                 
                                First FSBL is loaded and executed by FPGA configuration. Then Board/Server gets powered on. FPGA Configuration is done via JTAG followed by a reboot
                                 
                                The FSBL will send the following output the serial port: 
                                U-Boot SPL 2021.07-00312-g32c0556437 (Sep 17 2021 - 08:42:45 -0700)\nReset state: Cold\nMPU 1200000 kHz\nL4 Main 400000 kHz\nL4 sys free 100000 kHz\nL4 MP 200000 kHz\nL4 SP 100000 kHz\nSDMMC 50000 kHz\nDDR: Warning: DRAM size from device tree (1024 MiB)\nmismatch with hardware (2048 MiB).\nDDR: 1024 MiB\nSDRAM-ECC: Initialized success with 239 ms\nwaiting for host to copy image\n
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#60-meta-bakepy","title":"6.0 meta-bake.py","text":"
                                A script called meta-bake.py has been added to allow for more control of configuration/customization of recipes and their dependencies. This script separates the data from the logic by requiring that data be expressed in a yaml configuration file. This file contains the following confiration data:
                                 
                                   
                                • machine - The FPGA/SoC platform to build for, choices are agilex, stratix10, arria10, cyclone5
                                  •  
                                • image - The image type to build, choices are gsrd, nand, pcie, pr, qsqpi, sgmii, tse, n6000
                                  •  
                                • target - The build target to build. This is typically a Yocto image to build.
                                  •  
                                • fit - Make a monolothic FIT image after the Yocto build. This will use U-Boot source and binaries as well as the rootfs made for the image.
                                  •  
                                • repos - A list of repositories to pull for Yocto recipes. This information is made up of:
                                     
                                  • name - The project name (this is also the directory where source is clone to)
                                    •  
                                  • url - The URL to pull the source from
                                    •  
                                  • branch - The branch to checkout
                                    •  
                                  • add_layers - Can be either True or a list of sub-directories to add as layers in bblayers.conf
                                    •  
                                  • patch - Path to a file to use to patch the source code
                                    •  
                                  • keep - When set to true, this will leave the source tree untouched on subsequent runs
                                    •  
                                   
                                  •  
                                • upstream_versions - Dependencies/versions to use for either Linux kernel, U-Boot, and/or ATF. This information is made up of:
                                     
                                  • name - Project name
                                    •  
                                  • version - version to configure recipes that use it
                                    •  
                                  • branch - branch to use, will use git hash in recipe
                                    •  
                                  • url - URL to pull the source from
                                    •  
                                  • disabled - when set to True, this project will be ignored
                                    •  
                                   
                                  •  
                                • local - Used to configure local.conf used by Yocto/bitbake build. This information is made up of:
                                     
                                  • remove - List of keys to remove from local.conf
                                    •  
                                  • values - Dictionary of key/value pairs to use to insert into local.conf. Any existing key/value pairs will be overwritten.
                                    •  
                                   
                                  •  
                                "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#61-running-meta-bakepy","title":"6.1 Running meta-bake.py","text":"
                                To create an U-Boot fit and spl image for N6000/1 platforms, run the following command after meeting these setup conditions:
                                 
                                   
                                • Host PC with Ubuntu 20.04 LTS
                                     
                                  • ARM cross compiler
                                       
                                    • set CROSS_COMPILE environment variable to: /bin/aarch64-linux-gnu-     - e.x. export CROSS_COMPILE=aarch64-linux-gnu- 
                                    • set ARCH to: arm64     - e.x. export ARCH=arm64
                                      •  
                                    • At least 100 Gb of disk space
                                      •  
                                    • Tested on OFS 3.0.0
                                      •  
                                      This script does not require any bare-metal accesses to perform its build and can be run from a VM with no alterations. Ensure that the Ubuntu 20.04 Guest VM you create has enough to space to perform the entire build (recommend at least 200 GiB total space), as does the drive it is stored on. You will need to configure proxy access in the VM if required for your environment. You can use any VM technology; having ssh enabled in the VM will allow you to more easily transfer the completed build files back to the host system but is not required.
                                       
                                      Package dependencies to build Yocto on each supported OS can be found on the Yocto Quick Start page.
                                       
                                      wget https://developer.arm.com/-/media/Files/downloads/gnu-a/10.2-2020.11/binrel/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz\ntar xf gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz\nrm gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz\nexport CROSS_COMPILE=`pwd`/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-\nexport ARCH=arm64\n./meta-bake.py build\n
                                       
                                      After running this build, the images you need to boot the HPS are located under build/agilex-n6000-images. Follow the steps in Section 4.3 Example Boot to finish bringing up your board.
                                       
                                      This script will do the following:
                                       
                                         
                                      • Parse layers.yaml for configuration to use for build
                                        •  
                                      • Download recipe repositories (including poky) listed in repos secion of layers.yaml
                                        •  
                                      • Apply refdes-n6000 patch to meta-intel-fpga-refdes source tree
                                        •  
                                      • Configure Yocto build in build directory
                                        •  
                                      • Source build/poky/oe-init-build-env passing in agilex-n6000-rootfs. This will initialize conf files.
                                        •  
                                      • Configure build/agilex-n6000-rootfs/conf/local.conf using values in local section of layers.yaml          * Note: IMAGE_FSTYPES is configured to include cpio
                                        •  
                                      • Configure build/agilex-n6000-rootfs/conf/bblayers.conf using layer specification in repos section of layers.yaml
                                        •  
                                      • Run Yocto build for target listed in layers.conf
                                        •  
                                      • Call bitbake n6000-image-minimal
                                        •  
                                      • Get environment variables to locate rootfs cpio file as well as U-Boot source and build directories
                                        •  
                                      • Copy rootfs created by Yocto build for U-Boot
                                        •  
                                      • Copy rootfs cpio file (n6000-image-minimal-agilex*.rootfs.cpio) to U-Boot build directory for selected configuration (socfpga_agilex_n6000_defconfig)
                                        •  
                                      • Call U-Boot build in directory for selected configuration
                                        •  
                                      • Copy FIT image (u-boot.itb) to images directory, build/agilex-n6000-images
                                        •  
                                      • Many important images are copied to build/agilex-n6000-images, which may be useful if using VAB
                                        •  
                                      "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#62-required-changes","title":"6.2 Required Changes","text":"
                                      The patch file applied on top of the meta-intel-fpga-refdes repository introduces patches to:
                                       
                                         
                                      • Add patch files so that Yocto can modify Linux kernel to add configuration for creating a device tree binary (DTB) compatible with N6000/1
                                        •  
                                      • Add patch files so that Yocto can modify the bootloader in U-Boot to support booting with the assistance of the copy engine IP
                                        •  
                                      • Modify rootfs to include copy-engine daemon as well as other packages that can be useful for debug
                                        •  
                                       
                                      These changes may eventually be merged into upstream repositories for linux-socfpga, u-boot-socfpga, and meta-intel-fpga-refdes. Once all changes make their way into the repositories for the aforementioned projects, it will no longer be necessary to apply patches.
                                      "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#63-manual-build","title":"6.3 Manual Build","text":"
                                      One may use meta-bake.py to only pull down required repositories and configure a Yocto build environment by using the --skip-build command line argument. To initiate a build after this, source poky/oe-init-build-env passing in a directory as the only argument. This will set up the user's environment to be able to run bitbake. To build the Yocto image, run bitbake n6000-image-minimal. This will build all the components necessary to build a FIT image. Once the build is complete, U-Boot make system may be used to make the FIT. The U-Boot build directory for the selected configuration can be found in the Yocto build environment directory at: 
                                      $ cd tmp/work/agilex-poky-linux/u-boot-socfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/build/socfpga_agilex_n6000_defconfig\n
                                       Once in this directory, ensure that the necessary files are present in here in order to assemble the FIT image (u-boot.itb) 
                                      $ cp ../../../../../../deploy/images/agilex/n6000-image-minimal-agilex.cpio rootfs.cpio\n$ ls Image linux.dtb rootfs.cpio\nImage  linux.dtb  rootfs.cpio\n$ make\n
                                      "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#64-manual-vab-signing","title":"6.4 Manual VAB Signing","text":"
                                         
                                      • By default, meta-bake.py will sign and certify the proper files for use with VAB. Below is an example on how to perform the manual VAB Signing Process.
                                        •  
                                       
                                      Make sure Quartus already installed and its tools added to environment. Example PATH=$PATH:/home/intelFPGA\\pro/21.3/quartus/bin/
                                       
                                      $ cd HPS_VAB\n$ quartus_sign --family=agilex --operation=make_private_pem --curve=secp384r1 --no_passphrase userkey_root_private.pem\n$ quartus_sign --family=agilex --operation=make_public_pem userkey_root_private.pem userkey_root_public.pem\n$ quartus_sign --family=agilex --operation=make_rootuserkey_root_public.pem userkey_root_public.qky\n$ chmod +x fcs_prepare\n$ ./fcs_prepare --hps_cert bl31.bin -v\n$ quartus_sign --family=agilex --operation=SIGN --qky=userkey_root_public.qky --pem=userkey_root_private.pem unsigned_cert.ccert signed_cert_bl31.bin.ccert\n\n# ATF Sign\n\n$ ./fcs_prepare --finish signed_cert_bl31.bin.ccert --imagefile bl31.bin\n$ mv hps_image_signed.vab signed-bl31.bin\n$ rm unsigned_cert.ccert\n\n# u-boot-nodtb\n\n$ ./fcs_prepare --hps_cert u-boot-nodtb.bin -v\n\n#signed_u-boot-nodtb.bin.ccert\n\n$ quartus_sign --family=agilex --operation=SIGN --qky=userkey_root_public.qky --pem=userkey_root_private.pem unsigned_cert.ccert signed_u-boot-nodtb.bin.ccert\n\n# u-boot-nodtb.bin Sign\n\n$ ./fcs_prepare --finish signed_u-boot-nodtb.bin.ccert --imagefile u-boot-nodtb.bin\n$ mv hps_image_signed.vab signed-u-boot-nodtb.bin\n$ rm unsigned\\_cert.ccert\n\n# u-boot.dtb\n\n$ ./fcs_prepare --hps_cert u-boot.dtb -v\n\n#signed_u-boot.dtb.ccert\n\n$ quartus_sign --family=agilex --operation=SIGN --qky=userkey_root_public.qky --pem=userkey_root_private.pem unsigned_cert.ccert signed_u-boot.dtb.ccert\n\n# u-boot.dtb Sign\n\n$ ./fcs_prepare --finish signed_u-boot.dtb.ccert --imagefile u-boot.dtb\n$ mv hps_image_signed.vab signed-u-boot.dtb\n$ rm unsigned_cert.ccert\n\n# Image\n\n$ ./fcs_prepare --hps/cert Image -v\n\n#signed_Image.ccert\n\n$ quartus_sign --family=agilex --operation=SIGN --qky=userkey_root_public.qky --pem=userkey_root_private.pem unsigned_cert.ccert signed_Image.ccert\n\n# Image Sign\n\n$ ./fcs_prepare --finish signed_Image.ccert --imagefile Image\n$ mv hps_image_signed.vab signed-Image\n$ rm unsigned_cert.ccert\n\n# linux.dtb\n\n$ ./fcs_prepare --hps_cert linux.dtb -v\n\n#signed_linux.dtb.ccert\n\n$ quartus_sign --family=agilex --operation=SIGN --qky=userkey_root_public.qky --pem=userkey_root_private.pem unsigned_cert.ccert signed_linux.dtb.ccert\n\n# linux.dtb Sign\n\n$ ./fcs_prepare --finish signed_linux.dtb.ccert --imagefile linux.dtb\n$ mv hps_image_signed.vab signed-linux.dtb\n$ rm unsigned_cert.ccert\n\n# rootfs.cpio\n\n$ ./fcs_prepare --hps_cert rootfs.cpio -v\n\n#signed_rootfs.cpio.ccert\n\n$ quartus_sign --family=agilex --operation=SIGN --qky=userkey_root_public.qky --pem=userkey_root_private.pem unsigned_cert.ccert signed_rootfs.cpio.ccert\n\n# rootfs.cpio\n\n$ ./fcs_prepare --finish signed_rootfs.cpio.ccert --imagefile rootfs.cpio\n$ mv hps_image_signed.vab signed-rootfs.cpio\n$ rm unsigned_cert.ccert\n
                                       
                                      Copy the following files to u-boot-socfpga folder:
                                       
                                      #Copy the image back to uboot folder\n$ cp signed-bl31.bin ../u-boot-socfpga/\n$ cp signed-u-boot-nodtb.bin ../u-boot-socfpga/\n$ cp signed-u-boot.dtb ../u-boot-socfpga/\n$ cp signed-Image ../u-boot-socfpga/\n$ cp signed-linux.dtb ../u-boot-socfpga/\n$ cp signed-root\n$ fs.cpio ../u-boot-socfpga/\n
                                       Recompile the U-Boot
                                       
                                      $ git clone https://github.com/altera-opensource/u-boot-socfpga\n$ cd u-boot-socfpga\n$ export CROSS\\COMPILE=aarch64-none-linux-gnu-; export ARCH=arm\n$ make socfpga/agilex/n6000/vab/defconfig\n$ make -j 24\n$ cd ..\n
                                       
                                      Figure 3.1 N6000/1 Configs
                                       
                                       
                                      If you not see the defconfig desired, please checkout the correct branch version. Example config shown above is socfpga_v2021.10.
                                       
                                      If the memory device tree it mismatches with your hardware (figure below), change the memory device tree at u-boot-socfpga/arch/arm/dts/socfpga_agilex_n6000-u-boot.dtsi
                                       
                                      To make it 2GB, change as
                                       
                                      memory {\n\n\\* 2GB \\*\n\nreg = <0 0x00000000 0 0x40000000>,<0 0x00000000 0 0x40000000>;\n\n};\n
                                       Figure 3.2 Device tree mismatches example
                                       
                                       
                                      Refer to 6. Host Side Startup
                                       
                                      $ sudo opae.io init -d 4b:00.4 root:root\n$ hps cpeng -f u-boot.itb\n$ timeout 5 hps heartbeat\n
                                       The error happen (Figure below) when the Images do not sign with VAB.
                                       
                                      Figure 3.3 VAB certificate error example
                                       
                                      "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#70-debugging","title":"7.0 Debugging","text":"
                                      Debugging the HPS from the host side is standard HPS debugging. The primary debug tool is UART on the HPS and Arm DS-5 debugger.
                                       
                                      A UART connection can be enabled on the board using the following procedure:
                                       
                                         
                                      1.  
                                        Connect the HPS Debug Card and HPS UART to the Intel N6000/1-PL FPGA SmartNIC Platform board
                                         
                                        2.  
                                      2.  
                                        Open Putty with the following setting 
                                            Port:COM4\n\n    Baudrate:115200\n\n    Data bits : 8\n\n    Stop bits : 1\n\n    Parity : None\n\n    Flow Control : None\n
                                         
                                        3.  
                                      3.  
                                        Reboot the Intel N6000/1-PL FPGA SmartNIC Platform board by typing reboot in the shell. You will be able to see the HPS UART traffic in the putty. If any issues are encountered in this step, check the HPS UART connection and the UART driver.
                                         
                                        4.  
                                      4.  
                                        Check the PCI bdf ( lspci | grep acc ) or fpgainfo fme at the shell prompt.
                                         
                                        5.  
                                      5.  
                                        Run the rsu and fpga\\reconfig scripts with respective arguments to print the logs.
                                         
                                        6.  
                                      "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#80-connecting-remotely-to-the-hps-using-ssh","title":"8.0 Connecting remotely to the HPS using ssh","text":"
                                      The HPS running on the Intel FPGA N6000/1-PL SmartNIC Platform can be remotely accessed to via the utility ssh, allowing the user to run commands and copy files. SSH must be run over a Point-To-Point Protocol daemon that has been included in the HPS software (as a part of the meta-openembedded layer, in the recipes-daemons/ippool recipe). In this example, the HPS is set up as a PPP Server, and the host OS is set up as a PPP Client. Serial communication between the host and HPS is accomplished via HPS UART1, which communicates through the FIM to the Soft UART on the FPGA, who in turn communicates with the host over PCIe.
                                       
                                      The following steps assume the SSBL has not yet been loaded onto the HPS. If it has, a cold boot will reset the system.
                                       
                                         
                                      1. The HPS Copy Engine Module is available for access on PF 4 via the PF/VF Mux on the FPGA. This port needs to be bound to driver vfio-pci (the following example assumes PCIE BDF 0000:b1:00.0). Substitute your device's BDF address and desired user/group access permissions into the following command.
                                        2.  
                                       
                                      $ sudo opae.io init -d 0000:b1:00.4 <USER>[:<GROUP>]\nUnbinding (0x8086,0xbcce) at 0000:b1:00.4 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.4 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.4 is 190\nAssigning /dev/vfio/190 to DCPsupport\nChanging permissions for /dev/vfio/190 to rw-rw----\n
                                       
                                         
                                      1. When an HPS enabled SOF or BIN with the FSBL is loaded onto the FPGA, a message will be displayed on the host OS (seen via dmesg) after boot once the serial port has been registered with the dfl-uart driver. The UART driver is included as a part of the linux-dfl driver package. An example output from dmesg is shown below (search dmesg using dmesg | grep dfl-uart):
                                        2.  
                                       
                                      [    7.343014] dfl-uart dfl_dev.7: serial8250_register_8250_port 2\n
                                       
                                      The device file that corresponds with serial UART port 2 is /dev/ttyS2 (format is /dev/ttyS<port number>). A serial communication program can be used to view the HPS boot in realtime, then log in and run commands when boot has completed. Minicom is the program that will be used in this example, although others will work. Install Minicom using DNF sudo dnf install minicom. 
                                       
                                         
                                      1. Minicom requires configuration changes before it can listen to the serial device. Using the built-in menu accessed by sudo minicom -s, ensure the information under \"Serial port setup\" matches the following, where the serial device corresponds with the serial port discussed previously:
                                        2.  
                                       
                                       +-----------------------------------------------------------------------+\n    | A -    Serial Device      : /dev/ttyS2                                |\n    |                                                                       |\n    | C -   Callin Program      :                                           |\n    | D -  Callout Program      :                                           |\n    | E -    Bps/Par/Bits       : 115200 8N1                                |\n    | F - Hardware Flow Control : Yes                                       |\n    | G - Software Flow Control : No                                        |\n    |                                                                       |\n    |    Change which setting?                                              |\n    +-----------------------------------------------------------------------+\n
                                       
                                         
                                      1.  
                                        Save and exit the configuration menu. Run Minicom using the command sudo minicom and keep the terminal open and connected.
                                         
                                        2.  
                                      2.  
                                        Load the SSBL onto the HPS using a second terminal. This requires a built ITB image.
                                         
                                        3.  
                                       
                                      $ hps cpeng -f u-boot.itb\n
                                       
                                         
                                      1. You should see the HPS boot sequence continue through your Minicom terminal. Once boot has completed, log in using the user root with an empty password.
                                        2.  
                                       
                                      ...\n...\n...\n[  OK  ] Finished Load/Save Random Seed.\n[  OK  ] Finished OpenSSH Key Generation.\n\nPoky (Yocto Project Reference Distro) 3.3.6 agilex ttyS0\n\nagilex login: root\nroot@agilex:~#\n
                                       
                                         
                                      1. Configure the running Yocto image on the HPS as a PPP server. Run the following command through Minicom on the HPS (connects address 192.168.250.2 on the HPS to 192.168.250.1 on the host):
                                        2.  
                                       
                                      root@agilex:~# pppd noauth passive 192.168.250.1:192.168.250.2\n[  410.465450] PPP generic driver version 2.4.2\n...\n
                                       
                                         
                                      1. Exit the Minicom program running on the host using ^A X. Execute the following command on the host to establish a PPP connection as the client (if not installed on the host, run sudo dnf install ppp):
                                        2.  
                                       
                                      $ sudo pppd ttyS2 115200 crtscts lock local noauth passive debug\n
                                       
                                         
                                      1. A new network interface device registered to ppp should be visible.
                                        2.  
                                       
                                      $ ip -4 addr\n...\n8: ppp0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UNKNOWN group default qlen 3\n    inet 192.168.250.2 peer 192.168.250.1/32 scope global ppp0\n       valid_lft forever preferred_lft forever\n
                                       
                                      With both the client and server communicating, ssh and scp can be used to run commands and transfer files using IPv4 address 192.168.250.1 on the host. An example operation run on the host OS is shown below:
                                       
                                      [user@localhost ]: scp file_package.tar.gz root@192.168.250.1\n
                                       
                                      Note: If you are developing software for the HPS and altering system settings, it is possible for ssh to prohibit a connection due to a false man-in-the-middle attack warning. The flag <ssh/scp> -o StrictHostKeyChecking=no can be used to ignore the warning.
                                      "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#90-example-design-enabling-emmc-i2c-and-uart-in-platform-design","title":"9.0 Example Design - Enabling eMMC, I2C and UART in Platform Design","text":"
                                      The following section will walk through the process by which eMMC, I2C, and UART can be added to the FIM and the HPS image. The goal of this section is to allow the HPS to configure eMMC memory on boot and uses WNC's FPGA SmartNIC Series as a reference.
                                      "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#91-configuring-the-fim","title":"9.1 Configuring the FIM","text":"
                                         
                                      1. Configure eMMC, I2C, and UAET in Platform Designer. ACtual pin assignments are determined by the WNC board schematic. In Quartus, navigate to the HPS Processor Subsystem Intel Agilex FPGA IP -> Pin Mux and Peripherals -> Advanced -> Advanced IP Placement.
                                        2.  
                                       
                                      Check your pin assigments for the eMMC, UART and I2C in the Pin Planner. If these assignments are not present, then they can be found at the following link. Based on the changes shown above, the UART pins are removed on HPS IO3 and IO4 what are mapped on AG6 and AB1.
                                       
                                       
                                         
                                      1. Click Apply Selections->Generate HDL
                                        2.  
                                       
                                       
                                         
                                      1. Check for instantiation in top.sv. Click Generate -> Show Instatiation Template.
                                        2.  
                                       
                                       
                                      The following image demonstrates eMMC and I2C properly instatiated in top.sv.
                                       
                                       
                                         
                                      1. Add the following to the hps_ss modules in top.sv.
                                        2.  
                                       
                                       
                                       
                                         
                                      1. Compile the design.
                                        2.  
                                       
                                      "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#92-configuring-the-hps","title":"9.2 Configuring the HPS","text":"
                                         
                                      1. Enable mmc and DesignWare Memory Card interface flags in U-Boot (.config). After building U-Boot with meta-bake.py, this file is located at /agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga//build/socfpga_agilex_n6000_defconfig/.config. 
                                        CONFIG_CMD_MMC=y //Enable mmc command tool in uboot \nCONFIG_MMC_DW=y // support DesignWare Memory Card Interface\n
                                         
                                           
                                        1. Enable and configure I2C and eMMC in U-Boot (socfpga_agilex_n6000.dts). After building U-Boot with meta-bake.py, this file is located at /agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga/git/arch/arm/dts/socfpga_agilex_n6000.dts. 
                                             
                                          1. Enable and configure I2C and eMMC in the Linux device tree (socfpga_agilex_n6000.dts). After building U-Boot with meta-bake.py, this file is located at /agilex-n6000-rootfs/tmp/work-shared/agilex/kernel-source/arch/arm64/boot/dts/intel/socfpga_agilex_n6000.dts. 
                                               
                                            1. Compile Linux by navigating to the directory agilex-n6000-rootfs/tmp/work/agilex-poky-linux/linux-socfpga-lts/5.10.60-lts+gitAUTOINC+c35d63f9c7-r0/linux-agilex-standard-build and running the following:
                                              2.  
                                             
                                            $ make -j `nproc` Image dtbs  \n
                                             
                                               
                                            1. Add the software utilitiesutil-linux-mkfs e2fsprogs.mke2fs e2fsprogs in to the Linux RootFS. Thes utilities will be used to create a filsystem (ext4, FAT32, etc.) and partition the eMMC. Make the following changes in meta-intel-fpga-refdes/recipes-images/poky/n6000-image-minimal.bb.
                                              2.  
                                             
                                               
                                            1.  
                                              As an output from the Linux compilation from step 4 you will produce the files Image and socfpga_agilex_n6000.dtb. Transfer both over to the socfpga_agilex_n6000_defconfig directory. Rename socfpga_agilex_n6000.dtb to linux.dtb. Compile U-Boot by running make in directory agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/build/socfpga_agilex_n6000_defconfig/. This compilation will produce both spl/u-boot-spl-dtb.hex and u-boot.itb. 
                                               
                                              2.  
                                            2.  
                                              Combines these files with your SOF FSBL bootloader (created in Section 9.1 Configuring the FIM).
                                               
                                              3.  
                                             
                                            $ quartus_pfg -c ofs_top.sof ofs_top_hps.sof -o hps_path=u-boot-spl-dtb.hex\n$ quartus_pfg -c  ofs_top_hps_pof_flash.pfg //to pof file and flash to qspi\n
                                             
                                            Program the FSBL enabled SOF onto the FPGA and warm reboot the server for changes to take affect. After reboot has completed use the following commands to program the SSBL.
                                             
                                            $ sudo opae.io release -d d8:00.0\n$ sudo opae.io init -d d8:00.4 root:root\n$ hps cpeng -f u-boot.itb\n
                                             
                                            During HPS boot you should see the following message if the eMMC has been properly configured.
                                             
                                            "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#93-emmc-testing-from-the-hps","title":"9.3 eMMC Testing from the HPS","text":"
                                            The following memory test was run from the U-Boot shell.
                                             
                                               
                                            1. Erase eMMC using a start block offset 0x400.
                                              2.  
                                             
                                            $ mmc erase 0x400 20\n
                                             
                                             
                                            Write test data (at 0x00A00000) into the eMMC offset 0x400.
                                             
                                            $ mmc write 0x00A00000 0x400 10\n
                                             
                                             
                                            Read test data (at 0x00A00040) back from eMMC offset 0x400.
                                             
                                            $ mmc read 0x00A00040 0x400 10\n
                                             
                                             
                                            Data comparison at memory offset 0x00A00000 and 0x00A00040. Data should match.
                                             
                                            $ cmp.l 0x00A00000 0x00A00040 10.\n
                                             
                                             
                                            The following memory test was run from Linux running on the HPS.
                                             
                                               
                                            1. Display the eMMC and its partitions.
                                              2.  
                                             
                                            $ fdisk -l\n
                                             
                                             
                                               
                                            1. Create a primary partition on the eMMC.
                                              2.  
                                             
                                             
                                               
                                            1. Verify the partition has been created.
                                              2.  
                                             
                                             
                                               
                                            1. Format the ext3 filesystem in the partition you just created (p1).
                                              2.  
                                             
                                             
                                               
                                            1. Create the directory mydata in /mnt. Mount the eMMC p1 partition to /mnt/mydata and verify the filsystem mount was successful.
                                              2.  
                                             
                                            $ mkdir -p /mnt/mydata\n$ mount /dev/mmcblk0p1 /mnt/mydata\n$ df\n
                                             
                                             
                                               
                                            1. Create a new text file and write some data in it - \"Hello World!\". After the device has been written to run sync, unmount eMMC p1 partition and verify the unmount was successful.
                                              2.  
                                             
                                            $ sync\n$ umount /dev/mmcblk0p1\n$ df\n
                                             
                                            "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#layersyaml-reference","title":"layers.yaml Reference","text":"
                                            machine: agilex\nimage: n6000\ntarget: n6000-image-minimal\nfit: true\nuboot-dtb: [u-boot-nodtb.bin, u-boot.dtb]\nlinux-binary: [bl31.bin, linux.dtb, rootfs.cpio, Image]\nroot-public-qky: userkey_root_public.qky\nroot-private-pem: userkey_root_private.pem\nroot-public-pem: userkey_root_public.pem\n\nrepos:\n  - name: poky\n    url: https://git.yoctoproject.org/git/poky.git\n    branch: hardknott\n  - name: meta-intel-fpga\n    url: https://git.yoctoproject.org/git/meta-intel-fpga.git\n    branch: hardknott\n    add_layers: true\n  - name: meta-intel-fpga-refdes\n    url: https://github.com/altera-opensource/meta-intel-fpga-refdes.git\n    #url__: https://github.com/intel-innersource/os.linux.yocto.reference-design.meta-intel-fpga-refdes.git\n    #url: git@github.com:intel-innersource/os.linux.yocto.reference-design.meta-intel-fpga-refdes.git\n    #branch: rrojo/n6000\n    branch: hardknott\n    patch: refdes-n6000-ppp.patch\n    keep: true\n    add_layers: true\n  - name: meta-openembedded\n    url: https://github.com/openembedded/meta-openembedded.git\n    branch: hardknott\n    add_layers:\n      - meta-oe\n      - meta-networking\n      - meta-python\n  - name: fcs_prepare\n    url: https://github.com/altera-opensource/fcs_apps.git\n    branch: fcs_prepare\n\ningredients:\n  linux:\n    name: linux-socfpga\n    version: '5.10.100'\n    branch: socfpga-5.10.100-lts\n    url: https://github.com/altera-opensource/linux-socfpga.git\n  uboot:\n    name: u-boot-socfpga\n    version: '2021.07'\n    branch: socfpga_v2021.07\n    url: https://github.com/altera-opensource/u-boot-socfpga.git\n  atf:\n    disabled: true\n    version: '2.4.1'\n    branch: socfpga_v2.4.1\n    url: https://github.com/altera-opensource/arm-trusted-firmware.git\nlocal:\n  remove:\n    - MACHINE\n    - UBOOT_CONFIG\n    - IMAGE\n    - SRC_URI\n  values:\n    MACHINE: $machine\n    DL_DIR: $build_dir/downloads\n    DISTRO_FEATURES_append: \" systemd\"\n    VIRTUAL-RUNTIME_init_manager: systemd\n    IMAGE_TYPE: $image\n    IMAGE_FSTYPES: \"+=cpio tar.gz\"\n    PREFERRED_PROVIDER_virtual/kernel: linux-socfpga-lts\n    PREFERRED_VERSION_linux-socfpga-lts: 5.10%\n    UBOOT_CONFIG: agilex-n6000\n    PREFERRED_PROVIDER_virtual/bootloader: u-boot-socfpga\n    PREFERRED_VERSION_u-boot-socfpga: v2021.07%\n
                                            "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#faqs","title":"FAQs","text":"
                                            Below are the Frequently Asked Questions:
                                             
                                               
                                            1.  
                                              How will you get the software stack for HPS (FSBL, U-Boot, Kernel)? Or will there be a package available to them on Git, Intel RDC?
                                               
                                              Answer : HPS software has been around for quite a long time. Support for the OFS and the N6000-PL FPGA SmartNIC Platform will be upstreamed and available from rocketboards.com, just like any other HPS based project.
                                               
                                              2.  
                                            2.  
                                              What are the recommended steps for building the binaries and where will those be located?
                                               
                                              Answer: There are many documents on building the binaries at rocketboards.com. Any reference binaries can be stored at rocketboards.com as well.
                                               
                                              3.  
                                            3.  
                                              What are the host side commands used to put the binaries to Copy Engine and from there to HPS?
                                               
                                              Answer: There is a single command, hps to download the single binary through the Copy Engine to the HPS.
                                               
                                              4.  
                                            4.  
                                              What are the host side commands used to reset the HPS from Host side?
                                               
                                              Answer: This functionality is planned to be added to the hps command.
                                               
                                              5.  
                                            5.  
                                              What is the procedure used to debug the HPS from Host side?
                                               
                                              Answer: Debugging the HPS from the host side is standard HPS debugging. The primary debug tool is UART on the HPS and the Arm DS debugger.
                                               
                                              6.  
                                            6.  
                                              Do we have performance metrics about HPS, like whether any bench marking information available with any sample application is available?
                                               
                                              Answer: Any performance metrics on the HPS would be available on rocketboards.com.
                                               
                                              7.  
                                            7.  
                                              What is the PXE boot flow and what is required to enable the same?
                                               
                                              Answer: On some configurations, HPS is treated as a fully-fledged SoC and can PXE boot itself. But you must add this functionality.
                                               
                                              8.  
                                            "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#further-links","title":"Further Links","text":"Description Link OFS Getting Started User Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs https://ofs.github.io/hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/ Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs https://ofs.github.io/hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/ Open FPGA Stack Technical Reference Manual for Intel Agilex FPGA PCIe Attach https://ofs.github.io/hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/ AFU Development Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs https://ofs.github.io/hw/N6001/dev_guides/afu_dev/ug_dev_afu_n6001/ Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs https://ofs.github.io/hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/ FPGA Device Feature List (DFL) Framework Overview https://github.com/OFS/linux-dfl/blob/fpga-ofs-dev/Documentation/fpga/dfl.rst#fpga-device-feature-list-dfl-framework-overview ofs-platform-afu-bbb https://github.com/OFS/ofs-platform-afu-bbb Connecting an AFU to a Platform using PIM https://github.com/OFS/ofs-platform-afu-bbb/blob/master/plat_if_develop/ofs_plat_if/docs/PIM_AFU_interface.md example AFUs https://github.com/OFS/examples-afu.git PIM Tutorial https://github.com/OFS/examples-afu/tree/main/tutorial Non-PIM AFU Development https://github.com/OFS/examples-afu/tree/main/tutorial Intel FPGA IP Subsystem for PCI Express IP User Guide https://github.com/OFS/ofs.github.io/docs/hw/common/user_guides/ug_qs_pcie_ss.pdf Memory Subsystem Intel FPGA IP User Guide https://github.com/OFS/ofs.github.io/docs/hw/common/user_guides/ug_qs_mem_ss.pdf OPAE.io https://opae.github.io/latest/docs/fpga_tools/opae.io/opae.io.html OPAE GitHub https://github.com/OFS/opae-sdk README_ofs_n6001_eval.txt https://github.com/OFS/ofs-n6001/blob/release/ofs-2023.1/eval_scripts/README_ofs_n6001_eval.txt FIM MMIO Regions https://ofs.github.io/hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#mmio_regions evaluation script https://github.com/OFS/ofs-n6001/tree/release/ofs-2023.1/eval_scripts OFS https://github.com/OFS OFS GitHub page https://ofs.github.io DFL Wiki https://github.com/OPAE/linux-dfl/wiki"},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                            Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                             
                                            OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                            "},{"location":"hw/n6001/doc_modules/Glossary/","title":"Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel Max10 or Intel Cyclone10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implemented on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Management Component Transport Protocol MCTP A standardized model for communication with management controllers. Defines the transport protocol carrying PLDM messages through the BMC. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE-SDK The OPAE-SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Level Data Model PLDM A specification for reporting telemetry data to the host, such as board temperature, voltage, and current. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/doc_modules/Notices_%26_Disclaimers/","title":"Notices & Disclaimers","text":""},{"location":"hw/n6001/doc_modules/Notices_%26_Disclaimers/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                            Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                             
                                            OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                            "},{"location":"hw/n6001/doc_modules/n6001_wt_program_fpga_via_jtag/","title":"N6001 wt program fpga via jtag","text":"
                                            This walkthrough describes the steps to program the Agilex FPGA on the Intel\u00ae FPGA SmartNIC N6001-PL with a SOF image via JTAG.
                                             
                                            Pre-Requisites:
                                             
                                               
                                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the [OFS Agilex PCIe Attach Getting Started Guide] for instructions on setting up a deployment environment.
                                              •  
                                             
                                               
                                            • This walkthrough requires a SOF image which will be programmed to the Agilex FPGA. Refer to the [Walkthrough: Compile OFS FIM] Section for step-by-step instructions for generating a SOF image.
                                              •  
                                             
                                               
                                            • This walkthrough requires a JTAG connection to the n6001. Refer to the [Walkthrough: Set up JTAG] section for step-by-step instructions.
                                              •  
                                             
                                               
                                            • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel\u00ae FPGA SmartNIC N6001-PL is connected via JTAG.
                                              •  
                                             
                                            Steps:
                                             
                                               
                                            1.  
                                              Start in your deployment environment.
                                               
                                              2.  
                                            2.  
                                              If the card is already programmed with an OFS enabled design, determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                                               
                                              fpgainfo fme\n
                                               
                                              Example output:
                                               
                                              Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                               
                                              3.  
                                            3.  
                                              Remove the card from PCIe prior to programming. This will disable AER for the PCIe root port to prevent a surprise link-down event during programming.
                                               
                                              sudo pci_device b1:00.0 unplug\n
                                               
                                              4.  
                                            4.  
                                              Switch to the machine with JTAG connection to the n6001, if different than your deployment machine.
                                               
                                              5.  
                                            5.  
                                              Open the Quartus programmer GUI
                                               
                                              quartus_pgmw\n
                                               
                                               
                                              6.  
                                            6.  
                                              Click Hardware Setup to open the Hardware Setup dialog window.
                                               
                                                 
                                              1.  
                                                In the Currently selected hardware field select the n6001.
                                                 
                                                2.  
                                              2.  
                                                In the Hardware frequency field enter 16000000 Hz
                                                 
                                                 
                                                3.  
                                              3.  
                                                Click Close
                                                 
                                                4.  
                                               
                                              7.  
                                            7.  
                                              In the Quartus Prime Programmer window, click Auto Detect.
                                               
                                              8.  
                                            8.  
                                              If prompted, select the AGFB014R24A2E2V device. The JTAG chain should show the divice.
                                               
                                               
                                              9.  
                                            9.  
                                              Right click the AGFB014R24A2E2V row and selct Change File.
                                               
                                               
                                              10.  
                                            10.  
                                              In the Select New Programming File window that opens, select the .sof image you wish to program and click Open.
                                               
                                              11.  
                                            11.  
                                              Check the Program/Configure box for the AGFB014R24A2E2V row, then click Start. Wait for the Progress bar to show 100% (Success).
                                               
                                              12.  
                                            12.  
                                              Close the Quartus Programmer GUI. You can answer 'No' if a dialog pops up asking to save the 'Chain.cdf' file
                                               
                                              13.  
                                            13.  
                                              Switch to the deployment environment, if different than the JTAG connected machine.
                                               
                                              14.  
                                            14.  
                                              Replug the board PCIe
                                               
                                              sudo pci_device b1:00.0 plug\n
                                               
                                              15.  
                                            15.  
                                              Run fpgainfo fme to verify communication with the board, and to check the PR Interface ID.
                                               
                                              Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                               
                                              16.  
                                            "},{"location":"hw/n6001/doc_modules/n6001_wt_program_fpga_via_rsu/","title":"N6001 wt program fpga via rsu","text":"
                                            This walkthrough describes the steps to program the Agilex FPGA on the Intel\u00ae FPGA SmartNIC N6001-PL with a BIN image via JTAG.
                                             
                                            Pre-Requisites:
                                             
                                               
                                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the [OFS Agilex PCIe Attach Getting Started Guide] for instructions on setting up a deployment environment.
                                              •  
                                             
                                               
                                            • This walkthrough requires a BIN image which will be programmed to the Agilex FPGA. Refer to the [Walkthrough: Compile OFS FIM] Section for step-by-step instructions for generating a BIN image. The image used for programming must be formatted with PACsign before programming. This is done automatically by the build script.
                                              •  
                                             
                                               
                                            • This walkthrough requires a JTAG connection to the n6001. Refer to the [Walkthrough: Set up JTAG] section for step-by-step instructions.
                                              •  
                                             
                                               
                                            • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel\u00ae FPGA SmartNIC N6001-PL is connected via JTAG.
                                              •  
                                             
                                            Steps:
                                             
                                               
                                            1.  
                                              Start in your deployment environment.
                                               
                                              2.  
                                            2.  
                                              Determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                                               
                                              fpgainfo fme\n
                                               
                                              Example output:
                                               
                                              Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                               
                                              3.  
                                            3.  
                                              Use the OPAE fpgasupdate command to program a PACsign signed image to flash. The flash slot which will be programmed is determined by the PACsign header.
                                               
                                              sudo fpgasupdate <IMAGE> <PCIe B:D.F>\n
                                               
                                                 
                                              •  
                                                Example: update User Image 1 in flash
                                                 
                                                sudo fpgasupdate ofs_top_page1_unsigned_user1.bin 98:00.0\n
                                                 
                                                •  
                                               
                                                 
                                              •  
                                                Example: update User Image 2 in flash
                                                 
                                                sudo fpgasupdate ofs_top_page2_unsigned_user2.bin 98:00.0\n
                                                 
                                                •  
                                               
                                                 
                                              •  
                                                Example: update Factory Image in flash
                                                 
                                                sudo fpgasupdate ofs_top_page0_unsigned_factory.bin 98:00.0\n
                                                 
                                                •  
                                               
                                              4.  
                                            4.  
                                              Use the OPAE rsu command to reconfigure the FPGA with the new image. You may select which image to configure from (User 1, User 2, Factory).
                                               
                                              sudo rsu fpga --page <PAGE> <PCIe B:D.F>\n
                                               
                                                 
                                              •  
                                                Example: configure FPGA with User 1 Image
                                                 
                                                sudo rsu fpga --page user1 98:00.0\n
                                                 
                                                •  
                                               
                                                 
                                              •  
                                                Example: configure FPGA with User 2 Image
                                                 
                                                sudo rsu fpga --page user2 98:00.0\n
                                                 
                                                •  
                                               
                                                 
                                              •  
                                                Example: configure FPGA with Factory Image
                                                 
                                                sudo rsu fpga --page factory 98:00.0\n
                                                 
                                                •  
                                               
                                              5.  
                                            "},{"location":"hw/n6001/doc_modules/n6001_wt_set_up_jtag/","title":"N6001 wt set up jtag","text":"
                                            Perform the following steps to set up a JTAG connection to the Intel\u00ae FPGA SmartNIC N6001-PL.
                                             
                                            Pre-requisites:
                                             
                                               
                                            • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the [OFS Agilex PCIe Attach Getting Started Guide] for instructions on setting up a deployment environment.
                                              •  
                                             
                                               
                                            • This walkthrough requires a workstation with Quartus Prime Pro Version 23.3 tools installed, specifically the jtagconfig tool.
                                              •  
                                             
                                               
                                            • This walkthrough requires an [Intel FPGA Download Cable II].
                                              •  
                                             
                                            Steps:
                                             
                                               
                                            1.  
                                              Set the board switches to dynamically select either the Intel\u00ae Agilex\u00ae 7 FPGA or MAX\u00ae 10 device on the JTAG chain.
                                               
                                                 
                                              1. Set SW1.1=ON as shown in the next image. The switches are located at the back of the Intel\u00ae FPGA SmartNIC N6001-PL.
                                                2.  
                                               
                                               
                                              2.  
                                            2.  
                                              The Intel\u00ae FPGA SmartNIC N6001-PL has a 10 pin JTAG header on the top side of the board. Connect an Intel\u00ae FPGA Download II Cable to the JTAG header of the Intel\u00ae FPGA SmartNIC N6001-PL as shown in picture below. This picture shows the Intel\u00ae FPGA SmartNIC N6001-PL card installed in the middle bay, top slot of a SuperMicro\u00ae SYS-220HE-FTNR server where the lower slot does not have card installed allowing the Intel\u00ae Download II cables to pass through removed the slot access. 
                                               
                                               
                                              Note: If using the Intel FGPA download Cable on Linux, add the udev rule as described in [Intel FPGA Download Cable Driver for Linux].
                                               
                                              3.  
                                            3.  
                                              Set the JTAG chain to select the Intel\u00ae Agilex\u00ae 7 FPGA as the target by writing to the JTAG enable register in the BMC (Register 0x378). This is done via PMCI registers 0x2040C and 0x20400.
                                               
                                              Note: The commands below are targeted to a board with PCIe B:D.F of 98:00.0. Use the correct PCIe B:D.F of your card.
                                               
                                              sudo opae.io init -d 0000:98:00.0 $USER\nsudo opae.io -d 0000:98:00.0 -r 0 poke 0x2040c 0x100000000\nsudo opae.io -d 0000:98:00.0 -r 0 poke 0x20400 0x37800000002\nsudo opae.io release -d 0000:98:00.0\n
                                               
                                              Note: To later re-direct the JTAG back to the MAX 10 device, execute the following commands.
                                               
                                              sudo opae.io init -d 0000:b1:00.0 $USER\nsudo opae.io -d 0000:b1:00.0 -r 0 poke 0x2040c 0x000000000\nsudo opae.io -d 0000:b1:00.0 -r 0 poke 0x20400 0x37800000002\nsudo opae.io release -d 0000:b1:00.0\n
                                               
                                              Optionally, rather than dynamically commanding Intel\u00ae Agilex\u00ae 7 FPGA/MAX10 selection with the PMCI register settings, you can fix the selection with the following switch settings shown in the table below:
                                               SW1.1 SW2 JTAG Target OFF OFF Intel\u00ae Agilex\u00ae 7 FPGA OFF ON MAX\u00ae 10 FPGA ON X Intel\u00ae Agilex\u00ae 7 FPGA if BMC register 0x378=0x1 ON X MAX\u00ae 10 FPGA if BMC register 0x378=0x0 
                                              4.  
                                            4.  
                                              Use the jtagconfig tool to check that the JTAG chain contains the AGFB014R24A2E2V device.
                                               
                                              <QUARTUS_INSTALL_DIR>/23.3/quartus/bin/jtagconfig\n
                                               
                                              Example expected output:
                                               
                                              TBD\n
                                               
                                              5.  
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/","title":"FPGA Interface Manager Technical Reference Manual for Intel Agilex PCIe Attach: Open FPGA Stack","text":""},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1-overview","title":"1 Overview","text":""},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#11-about-this-document","title":"1.1 About this Document","text":"
                                            This document describes the hardware architecture for the PCIe attach reference FIM of the Open FPGA Stack (OFS) targeting the Intel\u00ae Agilex\u00ae FPGA.  After reviewing this document you should understand the features and functions of the components that comprise the FPGA Interface Manager (FIM), also known as the \"shell.\"
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#12-glossary","title":"1.2 Glossary","text":"
                                            This table defines some of the common terms used when discussing OFS.
                                             
                                            Table 1-1 Glossary Table
                                             Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#13-introduction-to-open-fpga-stack","title":"1.3 Introduction to Open FPGA Stack","text":"
                                            The Open FPGA Stack (OFS) is a modular infrastructure of hardware platform components, open source upstreamed software, and broad ecosystem support that enables an efficient path to develop a custom FPGA platform.  OFS provides a framework of FPGA synthesizable code, simulation environment and synthesis/simulation scripts.  The key components of OFS include:
                                             
                                               
                                            • Target development platforms such as Intel-branded Programmable Acceleration Cards (PACs), Acceleration Development Platforms (ADPs) and third-party platforms.
                                              •  
                                             
                                               
                                            • Board Management Controller RTL and firmware that supports telemetry monitoring and capability for remote configuration updates.
                                              •  
                                             
                                               
                                            • Source accessible, modular FPGA Interface  manager (FIM) RTL with a UVM infrastructure unit tests that can be leveraged for your own custom FIM design. The FIM can be thought of as the FPGA shell that provides the I/O ring and timing closed management components for the FPGA.
                                              •  
                                             
                                               
                                            • Basic building blocks for interconnect and PF/VF translation and arbitration; Platform Interface Manager (PIM) which provides Avalon\u00ae bus compliant interfaces.
                                              •  
                                             
                                               
                                            • AFU examples both in the git repository and workload examples provided by 3rd party vendors.
                                              •  
                                             
                                               
                                            • A OneAPI acceleration support package (oneapi-asp) that provides a bridge layer that is used by OneAPI runtime to communicate with the kernel.
                                              •  
                                             
                                               
                                            • Unit level simulation test suite
                                              •  
                                             
                                               
                                            • System level simulation through a unified verification methodology (UVM)
                                              •  
                                             
                                               
                                            • OPAE software development kit (APIs, upstreamed Linux drivers and software tools)
                                              •  
                                             
                                               
                                            • Support for other frameworks to be built on top of the OPAE such as DPDK 
                                              •  
                                             
                                            These components are available under the https://github.com/OFS site.
                                             
                                            The OFS hardware repository supports hardware development and simulation.  Repositories for OFS high level design support and board management controller RTL and firmware source code are also provided.  These repositories can be found in the Intel Opensource Technology GitHub location, which requires entitlement access.  To request access, please contact your local Intel sales representative.
                                             
                                            Table 1-2 OFS Hardware Repositories
                                             Repository Contains ofs-n6001 Contains FIM or shell RTL, automated compilation scripts, and unit tests and UVM framework. oneapi-asp Contains the hardware and software components you need to develop your own OneAPI board support package ofs-platform-afu-bbb Contains the files and scripts to build the platform interface manager. ofs-examples-afu Contains AFU examples you can use. ofs-bmc Provides the OFS Board Management Controller RTL, firmware, scripts and collateral targeting the Intel\u00ae FPGA SmartNIC N6001-PL which can be leveraged for your own OFS design. 
                                            Table 1-3 OFS Software Repositories
                                             OPAE Git Repository Folder Contains opae-sdk Contains the files for building and installing OPAE SDK from source. linux-dfl Contains OFS Linux drivers that are being upstreamed to the Linux kernel. opae-sim Contains the files for an AFU developer to build the Accelerator Functional Unit Simulation Environment (ASE) for workload development. 
                                            Providing the hardware and software source code and supporting test frameworks in a GitHub repository allows you to customize your designs with the latest versions easily.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#14-ofs-features","title":"1.4 OFS Features","text":"
                                            The OFS architecture within the FPGA comprises two partitions:
                                             
                                               
                                            • FPGA Interface Manager (FIM)
                                              •  
                                             
                                               
                                            • Accelerator Functional Unit (AFU)
                                              •  
                                             
                                            The FIM or shell provides platform management functionality, clocks, resets and interface access to the host and peripheral features of the acceleration platform. The FIM architecture along with the supporting OPAE software supports features such as partial reconfiguration and virtualization. The FIM provides a standard Arm\u00ae AMBA\u00ae 4 AXI4 datapath interface. The FIM resides in the static region of the FPGA.
                                             
                                            The AFU partition is provided for custom acceleration workloads and may contain both static and partial reconfiguration regions.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#141-fpga-interface-manager-fim","title":"1.4.1 FPGA Interface Manager (FIM)","text":"
                                            The primary components of the FPGA Interface Manager or shell of this reference design are: 
                                             
                                               
                                            • PCIe Subsystem
                                              •  
                                            • HSSI Subsystem
                                              •  
                                            • Memory Subsystem
                                              •  
                                            • Hard Processor System
                                              •  
                                            • Reset Controller
                                              •  
                                            • FPGA Management Engine
                                              •  
                                            • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                                              •  
                                            • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                                              •  
                                            • Platform Management Controller Interface (PMCI) to the board management controller
                                              •  
                                             
                                            The AFU Region provides design space for custom workloads and contains both static and partial reconfiguration regions.  Partial reconfiguration allows you to update your specific logic blocks or entire workload while the rest of your static design is still in operation.
                                             
                                            Note that the BMC RTL and firmware that works with this OFS design provided in a separate entitled repository.  Please email ofs.github@intel.com if you would like to use our BMC code for your own design.
                                             
                                            Figure 1-2 OFS for OFS FIM for Agilex OFS Block Diagram
                                             
                                             
                                            The table provides an overview of the OFS features targeting the Intel\u00ae Agilex\u00ae 7 FPGA.  This reference FIM (shell) is a starting point for your custom FPGA design.  With this initial starting point, you can add or subtract interfaces or ports to different Agilex devices.
                                             
                                            Table 1-4 OFS FIM for Intel\u00ae Agilex\u00ae 7 FPGA Features
                                             Key Feature Description PCIe P-tile PCIe* Gen4x16 Virtualization Includes support for 5 physical functions/4 virtual functions with ability to expand to the P-tile PCIe Hard IP maximum (8PFs, 2K VFs per each Endpoint). Memory 5 DDR Channels:* One HPS DDR4 bank, x40 (x32 Data and x8 ECC), 1200 MHz, 1GB each* Four Fabric DDR4 banks, x32 (no ECC), 1200 MHz, 4GB Ethernet Eight Arm\u00ae AMBA\u00ae 4 AXI4-Stream channels of 25G Ethernet interfacing to an E-tile Ethernet Subsystem. Hard Processor System 64-bit quad core Arm\u00ae Cortex\u00ae-A53 MPCore with integrated peripherals. Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration)* Platform Controller Management Interface (PMCI) Module contained within the Agilex FPGA that interfaces through Avalon-Streaming x8 QuadSPI and SPI to a Board Management Controller Partial Reconfiguration Partial Reconfiguration region supported in hardware and software OneAPI OneAPI Acceleration Support Package (ASP) provided with compiled FIM to support OneAPI Runtime Software Support * Linux DFL drivers targeting OFS FIMs* OPAE Software Development Kit* OPAE Tools"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#subsystem-interfaces","title":"Subsystem Interfaces","text":"
                                            The PCIe, Memory and Ethernet interfaces in this design use a new flexible subsystem design that provides a standard Arm\u00ae AMBA\u00ae 4 AXI4 interface.  To access these Intel FPGA IP Subsystem documents. Please go to the links below: * Intel FPGA IP Subsystem for PCI Express IP User Guide * Memory Subsystem Intel FPGA IP User Guide * Ethernet Subsystem Intel FPGA IP User Guide (public document)
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#hard-processor-system-hps","title":"Hard Processor System (HPS)","text":"
                                            The HPS SoC contains a 64-bit quad core ARM\u00ae Cortex\u00ae-A53 MPCore with a variety of integrated modules such as on-chip RAM, Ethernet, USB, UARTs and SPI controllers and memory controllers.  For more information about the Intel Agilex HPS, please refer to the Intel Agilex Hard Processor System Technical Reference Manual.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#fpga-management-engine-fme","title":"FPGA Management Engine (FME)","text":"
                                            The FIM contains only one FME, regardless of the number of host interfaces to the FIM. The FME provides management features for the platform and controls reset and loading of the AFU into the partial reconfiguration region of the FPGA.
                                             
                                            Any feature, such as a memory interface or global error control that you want to control through FME, must expose its capability to host software drivers.  New features are exposed to the FME by adding a device feature header (DFH) register at the beginning of the feature's control status register (CSR) space. The FME CSR maps to physical function 0 (PF0) Base address register 0 (BAR0) so that software can access it through a single PCIe link.  For more information about DFHs, refer to the FPGA Device Feature List Framework Overview.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#streaming-datapath","title":"Streaming Datapath","text":"
                                            The FIM implements an AXI4-Stream bus protocol for data transfer in the FIM. AXI4-Stream channels send data packets to and from the host channel IP without data abstraction. Memory-mapped I/O (MMIO) CSR accesses are routed to the ST2MM module, which converts the AXI4-Stream to an AXI4 memory-mapped protocol.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#virtualization","title":"Virtualization","text":"
                                            This design supports virtualization by making use of the virtualization functionality in the PCIe Hard IP and mapping packets to the appropriate physical or virtual function through a PF/VF multiplexer.  
                                             
                                            This reference FIM example supports 5 PFs and 4 VFs; however, you may extend your configuration to whatever the PCIe Hard IP can support or your application requires.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#142-afu","title":"1.4.2 AFU","text":"
                                            An AFU is an acceleration workload that interfaces with the FIM. The AFU boundary in this design comprises both static and partial reconfiguration (PR) regions.  You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region.  A port gasket within the design provides all the PR specific modules and logic required for partial reconfiguration. Only one partial reconfiguration region is supported in this design.
                                             
                                            Similar to the FME, the port gasket exposes its capability to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space.  
                                             
                                            In this design, the AFU region is comprised of:
                                             
                                               
                                            • AFU Interface handler to verify transactions coming from AFU region.
                                              •  
                                            • PF/VF Mux to route transactions to and from corresponding AFU components: 
                                                 
                                              • ST2MM module.
                                                •  
                                              • Null Host exerciser (HE_NULL) stub.
                                                •  
                                              • PCIe loopback host exerciser (HE-LB).
                                                •  
                                              • HSSI host exerciser (HE-HSSI).
                                                •  
                                              • Memory Host Exerciser (HE-MEM).
                                                •  
                                              • Traffic Generator to memory (HE-MEM-TG).      Port Gasket (PRG). 
                                                •  
                                              • HPS Copy Engine.
                                                •  
                                               
                                              •  
                                            • Arm\u00ae AMBA\u00ae 4 AXI4 Streaming to Memory Map (ST2MM) Module that routes MMIO CSR accesses to FME and board peripherals.
                                              •  
                                            • Host exercisers to test PCIe, memory and HSSI interfaces (these can be removed from the AFU region after your FIM design is complete to provide more resource area for workloads)
                                              •  
                                            • HPS Copy Engine to copy second-stage bootloader and Linux OS image from Host DDR to HPS DDR.  
                                              •  
                                            • Port gasket and partial reconfiguration support.
                                              •  
                                            • Component for handling PLDM over MCTP over PCIe Vendor Defined Messages (VDM)
                                              •  
                                             
                                            For this design the PF/VF Mux provides the following mappings (found in src/afu_top/mux/top_cfg_pkg.sv):
                                             
                                            Table 1-5 PF/VF Mapping
                                             Module PF/VF Mapping AXI4 Stream to Memory Mapped Module (ST2MM) PF0 Memory Host Exerciser (HE_MEM) PF0-VF0 HSSI Host Exerciser (HE_HSSI) PF0-VF1 Memory Traffic Generator (HE_MEM_TG) PF0-VF2 Null Host exerciser (HE_NULL) stub PF1-VF0 PCIe Loopback (HE_LB) PF2 Null Host exerciser (HE_NULL) stub PF3 HPS Copy Engine Module PF4 
                                            Figure 1-3 AFU Diagram 
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#143-platform-interface-manager","title":"1.4.3 Platform Interface Manager","text":"
                                            The PIM provides a way to abstract the AXI4-Stream interface to the AFU by providing a library of shims that convert the host channel native packet into other protocols such as AXI4 memory-mapped, Avalon\u00ae streaming (Avalon-ST) or Avalon\u00ae memory-mapped (Avalon-MM). 
                                             
                                            The FPGA or AFU developer implements these interface abstractions in the AFU region (afu_main) of the design.  
                                             
                                            For more information, refer to [AFU Development Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs].
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#144-platform-feature-discovery","title":"1.4.4 Platform Feature Discovery","text":"
                                            This reference design comes with specific Intel FPGA drivers that are upstreamed to linux-dfl.  These drivers abstract the hardware and operating system specific details of the platform to the host. 
                                             
                                            The FIM implements a device feature header (DFH) at the beginning of each host-discoverable feature in a linked list format that allows an FPGA platform driver running on the host to discover FME, port, and AFU features. 
                                             
                                            You must implement a 64-bit DFH Device Feature Header register at the beginning (first 8B aligned address) of the feature CSR space for a new feature to be discovered or enumerated by a driver.  
                                             
                                            During host discovery, the driver traverses the DFH of the first feature from the first address on PF0 BAR0. Based on the information in the DFH, a driver can determine the CSR address range of the feature and other associated details. The end of the DFH contains a \"next DFH offset\" field that points the driver to the DFH of the next feature. 
                                             
                                            The software must continue traversing the linked list until it sees the EOL (End-Of-List) bit set to 1 in the \"next DFH offset\" field it inspects.  A 1 indicates this is the last feature in the feature set. The figure below gives a simple illustration of the feature discovery by traversing the DFH registers. This model is similar to how PCIe enumeration occurs.
                                             
                                            Figure 1-4 Device Feature Header Linked List Traversal
                                             
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#145-ofs-reference-design","title":"1.4.5 OFS Reference Design","text":"
                                            OFS provides FIM designs you can use as a starting point for your own custom design. These designs target a specific programmable acceleration card or development kit and exercise key FPGA device interfaces. 
                                             
                                            The Intel Agilex\u00ae code line for OFS targets the Intel N6001-PL FPGA SmartNIC Platform.  FIM designs are released to https://github.com/OFS/ofs-agx7-pcie-attach for evaluation and use. 
                                             
                                            In addition to the OFS FIM for Agilex that targets the Intel N6001-PL FPGA SmartNIC Platform, vertical market FIMs are available for the Intel N6000-PL SmartNIC Platform. 
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#146-fim-simulation","title":"1.4.6 FIM Simulation","text":"
                                            OFS provides unit tests and a UVM environment for the FIM and a framework for new feature verification. UVM provides a modular, reusable, and scalable testbench structure by providing an API framework that can be deployed across multiple projects. 
                                             
                                            The FIM testbench is UVM compliant and integrates third-party verification IPs from Synopsys that require a license. 
                                             
                                            Verification components include:
                                             
                                               
                                            • FIM monitor to detect correct design behavior
                                              •  
                                             
                                               
                                            • FIM assertions for signal level integrity testing
                                              •  
                                             
                                               
                                            • Arm\u00ae AMBA\u00ae 4 AXI4 scoreboards to check data integrity
                                              •  
                                             
                                               
                                            • FIM coverage to collect functional data
                                              •  
                                             
                                            The verification infrastructure can be found here for evaluation and use. Please refer to the [Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs] for more information.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#2-ofs-high-level-architecture","title":"2 OFS High Level Architecture","text":"
                                            OFS provides distinct data paths that simplify the design and integration process for adding or removing interface modules:
                                             
                                               
                                            • High Bandwidth data path for AFU-attached high-performance peripherals (HSSI, Memory, HPS, workload).
                                              •  
                                            • Low Bandwidth data path for OFS management and slow peripheral components (JTAG, I2C, SMBus).
                                              •  
                                            • AFU Peripheral Fabric (APF) to Board Peripheral Fabric (BPF) path to communicate with interface control and status registers (CSRs) and board components.
                                              •  
                                            • Peer-to-peer datapath between AFU components.
                                              •  
                                            • Peer-to-peer datapath between BPF components.
                                              •  
                                             
                                            Depending on your design goals, you can present peripherals to software as:
                                             
                                               
                                            • OFS managed peripherals with a device feature header that is part of a device feature list.
                                              •  
                                            • Native driver managed peripherals that are exposed through an independent physical function or virtual function.
                                              •  
                                             
                                            Figure 2-1 OFS Datapath Structure
                                             
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#3-pcie-subsystem","title":"3 PCIe Subsystem","text":"
                                            The FIM's PCIe Subsystem is a hierarchical design that targets the P-tile PCIe* hard IP and is configured to support Gen4 speeds and Arm\u00ae AMBA\u00ae 4 AXI4 Data Mover functional mode.  The IP supports SR-IOV and is configured to provide 5 PFs and 4 VFs.  Native PCIe TLP packets are sent through the PCIe using Arm\u00ae AMBA\u00ae 4 AXI4 Stream Protocol.  Before they reach the AFU, the packets go through an adapter in the subsystem that converts any headers to a data mover format.  Tag allocation and management for packets sent from the application to the host are done by the PF/VF Mux that is part of the AFU region.
                                             
                                            Figure 3-1 OFS FIM RX-TX Datapath
                                             
                                             
                                            Some key features of the PCIe interface are:
                                             Feature OFS for Intel Agilex FPGA PCIe Subsystem Configuration Mode PCIe Gen4x16 Port Mode Native Endpoint SR-IOV 5PFs, 4VFs MSI-X Support Yes Functional Mode Data Mover Profile Basic TLP Bypass No Header Packing Scheme Simple Data Width 512-bit (64-byte) AXI-ST Clock Frequency 400 MHz Tags Supported 128 Reordering No reordering of requests, no completion reordering Maximum Payload Size 512 Bytes Memory Requests Supported 1CL, 2CL, 4CL MMIO transaction Size 4B, 8B Control Shadow Interface Enabled Completion Timeout Interface Enabled 
                                            The PCIe PF, VF and Base Address Register (BAR) configuration can be modified in the PCIe Subsystem Platform Designer GUI interface.  The current implementation for the OFS FIM for Agilex FPGA is as follows:
                                             
                                            Table 3-1 Function and BAR Table for OFS for Agilex FPGA
                                             PF VF Feature BAR BAR Size PF0 - OFS Managed Peripherals (PCIe, Memory, Ethernet) BAR0 512 KB PF0 - AFU Peripherals BAR0 256 KB PF0 - Board Peripherals BAR0 256 KB PF0 - MSI-X BAR4 16 KB PF0 VF0 Memory Host Exerciser (HE-MEM) BAR0 256 KB PF0 VF1 HSSI Host Exerciser (HE-HSSI) BAR0 256 KB PF0 VF2 Memory Traffic Generator (HE-MEM-TG) BAR0 256 KB PF1 VF0 Null Host exerciser (HE_NULL) BAR0 4 KB PF2 PCIe Loopback (HE-LB) BAR0 256 KB PF3 Null Host exerciser (HE_NULL) BAR0 4 KB PF4 HPS Copy Engine BAR0 4 KB"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#31-pcie-subsystem-header-format","title":"3.1 PCIe Subsystem Header Format","text":"
                                            The first 32 bytes of the TLP from the PCIe subsystem denotes the PCIe header. There are two types of header format supported \u2013 Power User Mode header and Data Mover mode header. The tuser_vendor[0] bit on the AXI4-Stream channel indicates the header format of the TLP; tuser_vendor[0] =0 indicates Power User Mode header and tuser_vendor[0] =1 indicates Data Mover Mode header.
                                             
                                            The OFS FIM for Intel Agilex FPGA implements the Data Mover Functional Mode.  With this implementation, the application has the flexibility to use either mode for PCIe transaction, as shown in the following table. For more detailed information about the PCIe Subsystem, see the PCIe Subsystem Intel FPGA User Guide. 
                                             
                                            Table 3-2 PCIe Subsystem Header Format Support for OFS for Agilex FPGA
                                             Direction Type Power User Data Mover Host to Endpoint MWr, MRd Yes No Host to Endpoint CPL/CPLd Yes Yes Host to Endpoint Msg Yes No Endpoint to Host MWr, MRd Yes Yes Endpoint to Host Intr Yes (MWr) Yes Endpoint to Host CPL/CPLd Yes Yes Endpoint to Host Msg Yes Yes"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#311-power-user-header-format","title":"3.1.1 Power User Header Format","text":"
                                            The Power User Format provides user complete control over PCIe Hard IP. The user can implement functionality of interest with finer control over PCIe Transaction Layer Packet (TLP), credit handling and various mode provided by HIP. 
                                             
                                            The lower 16 bytes of the Power User Format are standard PCIe header as defined by PCIe specification, and the upper 16 bytes are specific to the PCIe Subsystem Intel FPGA IP. 
                                             
                                            Table 3-3 Power User Header Format
                                             
                                             
                                            The mapping of the PCIe defined header to the lower 16 bytes of the AXI4-Stream data channel is shown in the figure below. Each double word (DW) or 4 bytes in the PCIe header is mapped from big-endian to little-endian on AXI-S data channel.  
                                             
                                            Figure 3-2 Power User Header Mapping to Arm\u00ae AMBA\u00ae 4 AXI4 Channel
                                             
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#312-data-mover-header-format","title":"3.1.2 Data Mover Header Format","text":"
                                            The data mover mode allows high bandwidth data transfers to and from Host memory. It hides the complexity of handling PCIe TLPs. This format provides a simple interface for reading and writing to Host Memory. The data mover checks link partner credits before transmitting packets. It also provides MSI-X interrupt generation capability. 
                                             
                                            In Data Mover Mode, the lower 16 bytes are data mover specific and do not follow a PCIe standard header.  
                                             
                                            Table 3-4 Data Mover Header Format
                                             
                                             
                                            The mapping of the data mover header to the lower 16 bytes of the AXI-S data channel is shown below. Each byte in the data mover header is mapped directly to the AXI-S data channel.  
                                             
                                            Figure 3-3 Data Mover Header Mapping to Arm\u00ae AMBA\u00ae 4 AXI4 Channel
                                             
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#32-pcie-subsystem-interface-module","title":"3.2 PCIe Subsystem Interface Module","text":"
                                            The PCIe Subsystem Interface module (/ipss/pcie/rtl/pcie_ss_if.sv), provides the supporting interface between software and the PCIe subsystem.  
                                             
                                            The interface module provides the following:
                                             
                                               
                                            • Device Feature Header Registers
                                              •  
                                            • Control and Status Registers
                                              •  
                                            • Indirect access to PCIe subsystem CSR registers through a CSR mailbox in the PCIe Subsystem Interface.
                                              •  
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#33-data-mover-request-cycles","title":"3.3 Data Mover Request Cycles","text":"
                                            For Host read request cycles using the OFS FIM for Intel Agilex FPGA:
                                             
                                               
                                            • All requests in the RX direction will be MMIO. 
                                              •  
                                            • Requester ID from the request does get sent to the AFU. It is the AFU's responsibility to send back a completion to the host with the correct completer ID. 
                                              •  
                                            • Prefix is not supported. 
                                              •  
                                            • Memory Mapped (MM) Mode is not supported.
                                              •  
                                            • Slot Number is 0.
                                              •  
                                            • Base address is not sent to the AFU.
                                              •  
                                            • Local Address field is not used.
                                              •  
                                             
                                            For AFU/application request cycles using the OFS FIM for Intel Agilex FPGA:
                                             
                                               
                                            • All requests in the TX direction will be Memory Read/Write. 
                                              •  
                                            • The tag must be generated by the AFU/application.
                                              •  
                                            • Prefix is not supported. 
                                              •  
                                            • MM Mode is not supported.
                                              •  
                                            • Slot Number is 0 (non-0 only for switch)
                                              •  
                                            • VF Active, VF number and PF number are obtained from Data Mover Header Packet.
                                              •  
                                             
                                            Figure 3-4 Data Mover Request Cycles
                                             
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#34-data-mover-completion-cycles","title":"3.4 Data Mover Completion Cycles","text":"
                                            For Host completion cycles using the OFS FIM for Intel Agilex FPGA:
                                             
                                               
                                            • All completions in the RX direction will be Data Completions. 
                                              •  
                                            • Prefix is not supported.
                                              •  
                                            • MM Mode is not supported.
                                              •  
                                            • Slot Number is 0.
                                              •  
                                            • Data packet responses (for Memory Read requests from AFU) from the PCIe SS may come out of order when the size is >64B.
                                              •  
                                             
                                            For AFU/application completion cycles using the OFS FIM for Intel Agilex FPGA: * All requests in the TX direction will be Memory Read/Write.  * Requester ID is generated within the FIM. * That tag must be generated by the AFU/application. * Prefix is not supported.  * MM Mode is not supported. * Slot Number is 0. * VF Active, VF Number and PF number are obtained from the Data Mover Header Packet. 
                                             
                                            Figure 3-5 Data Mover Completion Cycles
                                             
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#4-platform-interface-manager","title":"4 Platform Interface Manager","text":"
                                            The FIM interfaces to an application in the AFU region through AXI4-Stream channels.  This format allows the AFU to access the host channel's raw interface without any translation. 
                                             
                                            As a FIM developer, you have the option to provide the raw data format associated with the host interface channel to the workload or AFU developer or you can provide an intermediate protocol using Platform Interface Manager Components or your own custom interface.
                                             
                                            If you expose the raw AXI4-Stream interface of the FIM, workload developers also have the option to convert to a desired protocol using the PIM resources as well.  
                                             
                                            Refer to the AFU Developer Guide and the FPGA Interface Manager Developer Guide for more information on using the PIM in your design.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#5-interconnect-fabric","title":"5 Interconnect Fabric","text":"
                                            There are three types of interconnect fabric in the OFS FIM design:  * AXI4-Stream PF/VF mux/demux fabric  * AFU Peripheral Fabric (APF)  * Board Peripheral Fabric (BPF)
                                             
                                            Figure 5-1 Interonnect Fabric Diagram
                                             
                                             
                                            TLP packets sent from upstream PCIe Subsystem on AXI4-Stream channel are demultiplexed in the AXI4-Stream PF/VF mux/demux fabric and routed to the respective PF/VF function based on the PF/VF information in the TLP header, such as vf_active or the PF/VF number. In the opposite direction, TLP packets from downstream PF/VF function are muxed in the fabric and sent to PCIe subsystem over AXI4-Stream channel.
                                             
                                            All host MMIO requests targeting PF0 BAR0 are routed to the ST2MM module. The ST2MM converts MMIO TLP packets into AXI-Lite memory requests and places the requests onto AFU Peripheral Fabric (APF). AFU peripherals, such as OFS managed AFU features and ST2MM, and Board Peripheral Fabric (BPF) are interconnected by APF. The BPF is the interconnect fabric one hierarchy below APF which connects all the board peripherals. Both APF and BPF allow multiple AXI4-Lite master and slave interconnect topology.
                                             
                                            If you are modifying the APF or BPF connections, you must use Platform Designer to generate the fabrics directly.  Please refer to the FPGA Interface Manager Developer Guide for directions on what files must be modified and how to generate the interconnect.
                                             
                                            For modifying the PF/VF mux you must update the tools/pfvf_config_tool/pcie_host.ofss file and run the ofs-fim-common/pfvf_config_tool/gen_ofs_settings.py script to initiate the PCIe SS and PF/VF mux parameters to be regenerated before running the FIm build script.  
                                             
                                            For details on these modifications, please refer to the [Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs].  
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#51-afu-peripheral-fabric-apf","title":"5.1   AFU Peripheral Fabric (APF)","text":"
                                            The AFU Peripheral Fabric (APF) is a 64-bit Arm AXI4-lite compliant interconnect fabric that connects AFU peripheral modules to board peripheral modules through the Board Peripheral Fabric (BPF). 
                                             
                                            The fabric is clocked by clk_csr and has a read allowance and write allowance of 1, i.e. only 1 active write/read is allowed in the fabric at any single time. 
                                             
                                            The address mapping for components interconnected by the APF is listed below. All components are mapped to PF0 BAR0 and implement AXI-lite slave interface. Note that none of the features in the APF mapping are designed to act as a master.
                                             
                                            Table 5-1 APF Address Mapping
                                             Address Size (Byte) Feature 0x00000\u20130x3FFFF 256K Board Peripherals(See BPF address mapping) AFU Peripherals 0x40000 \u2013 0x4FFFF 64K ST2MM 0x50000 \u2013 0x5FFFF 64K Reserved 0x60000 \u2013 0x60FFF 4K UART 0x61000 \u2013 0x6FFFF 4K Reserved 0x70000 \u2013 0x7FFFF 56K PR Gasket:4K= PR Gasket DFH, control and status4K= Port DFH4K=User Clock52K=Remote STP 0x80000 \u2013 0x80FFF 4K AFU Error Reporting"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#52-board-peripheral-fabric-bpf","title":"5.2   Board Peripheral Fabric (BPF)","text":"
                                            The Board Peripheral Fabric is the 64-bit AXI4-Lite compliant interconnect fabric that connects board peripheral modules to APF. The fabric is clocked by clk_csr and has a read allowance and write allowance of 1, i.e. only 1 active write/read is allowed in the fabric at any single time.
                                             
                                            The address mapping for components interconnected by BPF is listed below. All components are mapped to PF0 BAR0 and implement AXI4-lite slave interface. The Master column indicates if a component also implements AXI4-lite master interface which can send requests to the BPF.
                                             
                                            Table 5-2 BPF Address Mapping
                                             Address Size (Byte) Feature Master 0x00000 \u2013 0x0FFFF 64K FME (FME, Error, etc) No 0x10000 \u2013 0x10FFF 4K PCIe Interface No 0x11000 \u2013 0x11FFF 4K Reserved - 0x12000 \u2013 0x12FFF 4K QSFP Controller 0 No 0x13000 \u2013 0x13FFF 4K QSFP Controller 1 No 0x14000 \u2013 0x14FFF 4K Ethernet Subsystem No 0x20000 \u2013 0x3FFFF 128K PMCI Controller Yes"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#53-arm-amba-4-axi4-stream-pfvf-muxdemux","title":"5.3 Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF Mux/Demux","text":"
                                            The Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF Mux/Demux routes the PCIe TLP packets from the PCIe subsystem AXI4-Stream RX channel to downstream PF/VF based on the pf_num and vf_num information in the PCIe TLP header.
                                             
                                            The Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF mux arbitrates PCIe TLP packets from downstream PF/VF to the PCIe SS AXI-S TX channel. The PF/VF Mux/Demux is an M X N switch that allows any M port to target any N port, and any N port to target any M port, where M is the number of host/upstream ports, and N is the numbers functions/downstream ports.   
                                             
                                            The fpga top package file, /src/afu_top/mux/top_cfg_pkg.sv, contains the PF/VF parameters and mappings.
                                             
                                            Figure 5-2 PF/VF Mux
                                             
                                             
                                            The PF/VF mux integration is part of afu_top (/src/afu_top/mux/top_cfg_pkg.sv).  There are two independent TX PF/VF MUX trees, labeled \"A\" and \"B\".
                                             
                                            Both an A and a B port are passed to each AFU component with a unique PF/VF. You can design your AFU components to send all requests to the primary A port or partition requests across both A and B ports. A typical high-performance AFU sends read requests to the B port and everything else to the A port, giving the arbiter freedom to keep both the host TX and RX channels busy.
                                             
                                            In the reference FIM provided for Intel Agilex OFS, the A and B TX trees have been multiplexed down to a single channel for A and another for B. The A/B multiplexer merges them into a single TX stream that will be passed to the tag remapper.
                                             
                                            The tag remapper provides unique tags as required by the PCIe specification.  Tags are not provided by the PCIe Subsystem FPGA IP.  When creating your own AFU you can leverage this module to generate unique tags.
                                             
                                            Note that the primary PF/VF Mux A supports RX and TX ports.  For the secondary PF/VF Mux B only TX ports are supported and the RX input to the Mux is tied off.  
                                             
                                            The default mapping is shown below:
                                             
                                            Table 5-3 PF/VF Mapping
                                             Module PF/VF Mapping AXI4 Stream to Memory Mapped Module (ST2MM) PF0 Memory Host Exerciser (HE_MEM) PF0-VF0 HSSI Host Exerciser (HE_HSSI) PF0-VF1 Memory Traffic Generator (HE_MEM_TG) PF0-VF2 Virtio Loopback Stub (Virtio_LB) PF1-VF0 PCIe Loopback (HE_LB) PF2 Virtio Loopback Stub (Virtio_LB) PF3 HPS Copy Engine Module PF4 
                                            For information on how to modify the PF/VF mapping for your own design, refer to the [Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs].
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#54-afu-interface-handler","title":"5.4 AFU Interface Handler","text":"
                                            The AFU Interface Handler resides inline between the PCIe AXI4-Stream Adapter and the AXI4-Stream PF/VF Demux/Mux logic. Its main function is to provide: * Unique PCIe tags \u2013 Each PCIe transaction shares the 512 tags across all VFs in the AFU region * AFU error logging for all VFs in the AFU region
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#541-unified-tag-remapping","title":"5.4.1 Unified Tag Remapping","text":"
                                            When a FPGA function sends out a read cycle, it allocates a unique tag which is subsequently used to identify the read completion.  The tag is considered busy; it cannot be assigned to another read cycle until read completion.  While a tag may be unique within a unit, two different units could unknowingly send out two read cycles of the same tag.  The PCIe subsystem requires unique tags for all read cycles irrespective of their origins.  Therefore, a mechanism is needed to uniquify tag globally across different units.
                                             
                                            OFS contains a tag remapper (/ofs-fim-common/src/common/tag_remap/tag_remap.sv) that intercepts the read cycle, finds a globally unique tag, and replaces the original tag value.  It also restores the original tag value when returning completion to the read requester.  tag_remap is placed between the AXI4-Stream interface of the PCIE subsystem and the PF/VF Mux/Demux.
                                             
                                            The logic is described as follows:
                                             
                                               
                                            1. A sub-module (/ofs-fim-common/src/common/tag_remap/ofs_fim_tag_pool.sv) maintains a pool of available tags.
                                              2.  
                                            2. TX read requests are held until a tag is available from the pool by setting tvalid=0 to the host, and tready=0 to the PF/VF Mux/Demux.
                                              3.  
                                            3. When a TX read is dispatched, the tag is marked busy in the pool.
                                              4.  
                                            4. The original tag is stored in tag_reg, so it can be recovered when returning a completion to the unit/function.
                                              5.  
                                            5. Because completion to a read request can split into multiple smaller transfer sizes, responses are monitored and the final completion is detected using PCIe TLP rules.
                                              6.  
                                            6. Tags are released in the pool only when all requested data are transferred.
                                              7.  
                                            7. When the completion returns, the original tag is restored from tag_reg.
                                              8.  
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#542-afu-error-handling","title":"5.4.2 AFU Error Handling","text":"
                                            In this OFS design, the AFU Interface Handler handles error logging for all VFs in the AFU. Errors handled are as follows
                                             
                                            Table 5-4 AFU Error Descriptions
                                             Checker Field Description AFU protocol checker (PCIe TLP) Malformed TLP AFU PCIe TLP contains unsupported format type AFU protocol checker (PCIe TLP) MaxPayloadError AFU memory write payload size exceeds max_payload_length limit AFU protocol checker (PCIe TLP) MaxReadReqSizeError AFU memory read payload size exceeds max_read_request_size limit AFU protocol checker (PCIe TLP) MaxTagError AFU memory read request tag value exceeds the maximum supported tag count AFU protocol checker (PCIe TLP) UnalignedAddrErr The address field in AFU memory write/read request TLP is not DW-aligned. AFU protocol checker (PCIe TLP) UnexpMMIOResp AFU is sending a MMIO read response with no matching MMIO read request. AFU protocol checker (PCIe TLP) MMIOTimedOut AFU is not responding to a MMIO read request within the pre-defined response timeout period. AFU protocol checker (PCIe TLP) MMIODataPayloadOverrun The number of data payload sent by AFU for a MMIO response (cplD) is more than the data length specified in the response. AFU protocol checker (PCIe TLP) MMIOInsufficientData The number of data payload sent by AFU for a memory write request is more than the data length specified in the request. AFU protocol checker (PCIe TLP) TxMWrDataPayloadOverrun The number of data payload sent by AFU for a memory write request is less than the data length specified in the request. AFU protocol checker (PCIe TLP) TxMWrInsufficientData The number of data payload sent by AFU for a memory write request is less than the data length specified in the request. AFU Protocol Checker (Arm\u00ae AMBA\u00ae 4 AXI4) TxValidViolation Three checkers are implemented in the FIM to catch errors and protocol violations."},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#55-tlp-to-axi4-lite-memory-mapped-bridge-st2mm","title":"5.5 TLP to AXI4-Lite Memory Mapped Bridge (ST2MM)","text":"
                                            ST2MM implements the following key features:
                                             
                                               
                                            • Host MMIO bridge     * Maps MMIO TLP packets received from the PCIe Subsystem over streaming interface to AXI4-Lite memory-mapped request. The memory-mapped request is sent to AFU or Board peripherals over APF and BPF.     * Maps AXI4-lite MM response received from AFU or Board peripherals to TLP packets and send the packets over ST streaming channel to host HIA subsystem.
                                              •  
                                            • Sends MMIO response of all 0\u2019s for MMIO read to unused BAR region.
                                              •  
                                            • Interrupt     * Sends interrupt packets to the PCIe subsystem when interrupt requests are received from the peripherals. Interrupts can be requested by a peripheral through a memory write to interrupt CSR registers in the ST2MM.
                                              •  
                                             
                                            Figure 5-3 ST2MM Module
                                             
                                             
                                            ST2MM implements both AXI4-lite master and slave interfaces that are connected to the designated slave and master port on APF. Host memory requests are sent on the ST2MM master interface to AFP where the requests are routed to the targeted peripherals. 
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#6-mmio-regions","title":"6 MMIO Regions","text":"
                                            The FIM and AFU expose their functionalities to the host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). An MMIO region is an address space within a base address register (BAR) region to which features are memory mapped.  
                                             
                                            For example, when a feature is mapped to an MMIO region, the CSR registers of that feature are located within the address range of that region. There can be multiple MMIO regions within a BAR region. 
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#61-feature-region","title":"6.1 Feature Region","text":"
                                            A group of related CSRs can be categorized as a feature region. For example, a DMA engine has queue management function and quality of service (QoS) function; these are two different features of the DMA engine. A feature region is contained within a single PCIe BAR and cannot span across two BAR region boundaries. 
                                             
                                            A Device Feature Header (DFH) is a block of registers that mark the start of the feature region and sub-feature region, and you must place it at the first address of the region. Each DFH starts at 4KB boundary. A DFH register contains information that OPAE software requires to enumerate the feature. It also has an offset field that points to the next DFH in a feature list. OPAE software traverses the linked list of DFHs in each BAR region to discover all the features implemented on the platform. 
                                             
                                            The EOL field in a DFH Start marks the end of a DFH list and is only set in the DFH of the last feature in the feature list. The feature type field in the DFH is used to differentiate between the different types of feature region. Basic building blocks (BBB) and private features are always a child of an AFU or FPGA Interface Unit (FIU) and must be contained within an AFU or FIU, respectively.  
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#611-device-feature-header-dfh-structure","title":"6.1.1 Device Feature Header (DFH) Structure","text":"
                                            All DFHs must follow a specific structure to be compatible with OPAE software.  Note that only features you want to be exposed to the OPAE software must have a DFH.  For the latest information on DFH structure, please refer to https://github.com/OFS/linux-dfl/blob/fpga-ofs-dev/Documentation/fpga/dfl.rst#device-feature-list-dfl-overview.
                                             
                                            6.2 Control and Status Registers
                                             
                                            All the Control and Status Registers (CSRs) in the FIM are 64-bit registers with the following MMIO write, and MMIO read support.
                                             
                                            Table 6-5: CSR MMIO Read and Write Support
                                             Request Memory Attribute Payload size Memory Ordering MMIO Write UC 4B or 8B Strongly ordered MMIO Read UC 4B or 8B Strongly ordered 
                                            The FIM does not reorder the MMIO requests or responses. For MMIO writes, there is no reordering of requests in FIM, and uncacheable (UC) ordering rules are followed. Similarly, for MMIO reads, there is no re-ordering of requests or responses in the FIM. An AFU may opt to re-order the MMIO read responses but the FIM does not enforce read response ordering. 
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#621-software-access-to-registers","title":"6.2.1 Software Access to Registers","text":"
                                               
                                            • Software accesses 64-bit registers as aligned quadwords. For example, to modify a field (bit or byte) in a 64-bit register, the entire quadword is read, the appropriate field(s) are modified, and the entire quadword is written back.
                                              •  
                                            • When updating registers through multiple accesses (whether in software or due to hardware disassembly), certain registers may have specific requirements on how the accesses must be ordered for proper behavior. These are documented as part of the respective register descriptions.
                                              •  
                                            • For compatibility with future extensions or enhancements, software must assign the last read value to all \u201cReserved and Preserved\u201d (RsvdP) fields when written. In other words, any updates to a register must be read so that the appropriate merge between the RsvdP and updated fields occurs. Also, software must assign a value of zero for \u201cReserved and Zero\u201d (RsvdZ) fields when written.
                                              •  
                                            • PCIe locked operations to FPGA hardware registers are not supported. Software must not issue locked operations to access FPGA hardware registers.
                                              •  
                                             
                                            In the following two cases, the FIM terminates MMIO Read requests by sending a completion with the data (CplD) specified below: * MMIO Timeout:  This occurs when the AFU does not respond within a set timeout. The timeout value is currently configured to 512 pclks (clk_2x). In this case, the FIM returns all 1s.
                                             
                                               
                                            • Illegal MMIO Accesses: This occurs when the read is accessing undefined registers in the FIM or if an AFU access violation.  An example of an access violation is when a PF attempts to access the AFU when it is set to VF mode, or when a VF attempts to access the AFU when it is set to PF mode. In this case, the FME will returns all 0s.
                                              •  
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#622-register-attribute-definition","title":"6.2.2    Register Attribute Definition","text":"
                                            Table 6-6: OFS Register Attribute Definitions
                                             Attribute Expansion Description RW Read/Write This bit can be read or written by software. RO Read Only The bit is set by hardware only. Software can only read this bit. Writes do not have any effect. RW1C Read/ Write 1 to Clear Software can read or clear this bit. The software must write 1 to clear this bit. Writing zero to RW1C bit has no effect. Note that a multi-bit RW1C field may exist. In this case, all bits in the field are cleared if a 1 is written to any of the bits. RW1S Read/ Write 1 to Set Software can read this bit. Writing a 1 to the bit sets it to 1. Writing a 0 has no effect. It is not possible for software to set this bit to 0. The 1 to 0 transition can only be performed by HW. RW1CS Read/Write 1 to Clear Sticky Software can read and clear this bit. Writing a 1 to a bit clears it, while writing a 0 to a bit has no effect. This bit is only reinitialized to its default value by a power-on reset. RWD Read/Write Sticky across Hard Reset The bit can be read or written by SW. This bit is sticky or unchanged by any reset type, including Hard Reset. The bit gets cleared only with power on. *S Sticky across Soft Reset The bit will be sticky or unchanged by soft reset. These bits are only re-initialized to their default value by a power-on reset. *D Sticky across Hard Reset The bit is sticky or unchanged by or unchanged by any reset type, including hard reset. The bit gets cleared only with power on. Rsvd Reserved Reserved for future definitions. Currently don\u2019t care bits. RsvdP Reserved and Protected Reserved for future RW implementations. The software must preserve the value of this bit by read modify write. RsvdZ Reserved and Zero Reserved for future RW1C implementations. The software must write zero to this bit."},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#623-csr-offset-in-bars","title":"6.2.3 CSR Offset in BARs","text":"
                                            The table below captures the FIM and AFU features in the supported BAR regions. The highlighted offset indicates the first DFH in the DFH list of a BAR region where device driver starts the DFH traversal.
                                             
                                            Table 6-7: PF0 BAR0 Features
                                             Offset Feature CSR set 0x00000 FME 0x01000 Thermal Management 0x03000 Global Performance 0x04000 Global Error 0x10000 PCIe 0x12000 QSFP0 0x13000 QSFP1 0x20000 PMCI 0x40000 ST2MM 0x60000 Transceiver CSR Interface 0x61000 Virtual UART 0x62000 EMIF 0x70000 Port Gasket PR logic 0x71000 Port Gasket Port interface 0x73000 Port Gasket Remote Signal Tap 0x80000 AFU Interface Handler- AFU Errors 
                                            Table 6-8: PF0 BAR4 Features
                                             Offset Feature CSR set 0x03000 MSI-X Vector Tables"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#7-clocks","title":"7 Clocks","text":"
                                            The following table provides external clock sources which correspond to pins on the FPGA that drive blocks of internal logic or are used as a reference clock for internal PLLs.  The names in the table are used in the top.sv or are the logical clock names used by their respective functional blocks.  
                                             
                                            Table 7-1: External Clock Source
                                             Clock Frequency Description SYS_RefClk 100 MHz Reference clock to system IOPLL (sys_pll) which provides FIM system clocks. PCIE_REFCLK 100MHz PCIe reference clock 0 PCIE_REFCLK 100MHz PCIe reference clock 1 qsfp_ref_clk 156.25 MHz Ethernet Reference Clock ddr4_x32[0].ref_clk 150MHz Refclk for 32-bit EMIF channel 0 ddr4_x32[1].ref_clk 150 MHz Refclk for 32-bit EMIF channel 1 ddr4_x40[0].ref_clk 150 MHz Refclk for 40-bit EMIF channel 0 ddr4_x40[1].ref_clk 150 MHz Refclk for 40-bit EMIF channel 1 ddr4_hps.ref_clk 150 MHz Refclk for HPS EMIF sdm_config_clk 125 MHz Refclk for HPS EMIF hps_refclk 25 MHz Refclk for HPS EMIF altera_reserved_tck 10 MHz Refclk for HPS EMIF 
                                            Table 7-2: Internal Clocks
                                             
                                            Internal clock generated by the IOPLL as outclk outputs.
                                             outclk# Clock Frequency Description outclk0 clk_sys 470 MHz System clock. Primary clock for PCIe Datapath outclk1 clk_100m 100 MHz For RemoteSTP and user clock, or any logic that requires a 100 MHz clock. outclk2 clk_100m 175 MHz Drives AFU in PR slot outclk3 clk_ptp_slv 155.56 MHz Unused outclk4 clk_50m 50 MHz Drives Virtual UART"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#8-reset","title":"8 Reset","text":""},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#81-reset-signals","title":"8.1   Reset Signals","text":"
                                            The FIM system reset signals are driven according to their respective requirements derived from their use cases.  The behavioral code defining the reset operation is located in the file rst_ctrl.sv. The FIM System Reset Drivers table below provides a list of the input and feedback signals that compose the various resets.
                                             
                                            Table 8-1: FIM System Reset Drivers
                                             Reset Description nPERST pin Active low PCIe reset pin from the PCIe card edge that may be set by the host machine for cold or warm resets. nCONFIG pin Active low input to the FPGA that causes the FPGA to lose its configuration data, enter a reset state, and tri-state all I/O pins. Host software must reload the FPGA FIM after nCONFIG is activated. ninit_done Active low signal indicating FPGA core configuration is complete and FPGA has entered usermode.  This signal is provided by the configuration monitor block in rst_ctrl.sv. pcie_reset_status Active high reset status from PCIe hard IP. When driven high, this signal indicates that the PCIe IP core is not ready for usermode. pll_locked Active high SYS IOPLL locked signal pcie_cold_rst_ack_n Reset Acknowledge signal from the PCIe subsystem that lets the Reset Controller know that the Cold Reset sequence has been completed. pcie_warm_rst_ack_n Reset Acknowledge signal from the PCIe subsystem that lets the Reset Controller know that the Warm Reset sequence has been completed. 
                                            Upon power-on, the reset module in the FIM holds the FIM in reset until all the reset conditions are de-activated:
                                             
                                               
                                            • nPERST signal is asserted.
                                              •  
                                            • The INIT_DONE pin is driven high to indicate core configuration is complete.
                                              •  
                                            • The SYS IOPLL is locked.
                                              •  
                                            • The reset status from PCIe hard IP is de-asserted indicating the IP is ready for transfer. 
                                              •  
                                             
                                            The reset module places the FIM back into reset if any of these conditions becomes active again. The only way to invoke a system reset to the FIM after power-up is to de-assert the nPERST pin either by performing a warm reboot or through PCIe driver intervention. There are soft reset signals set aside to allow software to reset the Port, AFU and partial reconfiguration IP. 
                                             
                                            THe following table lists the reset outputs from the rst_ctrl.sv block.
                                             
                                            \u200b                                                      Table 8-2: FIM System Reset Outputs
                                             Reset Description rst_n_sys pin System general reset synchronous to clk_sys.  This is a warm reset of the FPGA logic.  Sticky bits in the FME and other CSR registers will not be reset by this signal. rst_n_100m pin System general reset synchronous to clk_100m. ninit_done Active low signal indicating FPGA core configuration is complete and FPGA has entered usermode.  This signal is provided by the configuration monitor block in rst_ctrl.sv. pwr_good_n Hardware reset conditioned by ninit_done and the pll_locked signal.  The signal is generally set when power has been cycled or a physical hardware fault has occurred, requiring a reset of the FPGA logic.   This signal is synchronous to clk_sys. pcie_cold_rst_ack_n Hardware reset conditioned by the pcie_reset_status which is a summary reset driven by the PCIe Hard IP logic tile on the FPGA die.  This signal is synchronous to clk_sys. pcie_warm_rst_ack_n Soft reset conditioned by nPERST when the pcie_reset_status is not asserted, meaning a warm reset is desired.  Cold reset sticky bits in the PCIe subsystem will not be reset by this signal."},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#9-interrupts","title":"9 Interrupts","text":"
                                            The OFS platform supports interrupts through MSI-X feature. The MSI-X Interrupt feature handles FME and AFU interrupts. FME interrupts are primarily used to notify the host of error events happened in the FIM. When any of the bit in the FME error status registers is set, an interrupt request is sent to the MSI-X module. There are FME error status registers for OFS for Agilex FPGA features.  An AFU sends interrupt to the MSI-X module in the PCIE SS on the AXI interrupt request channel. The MSI-X table entries and PBA vectors are implemented in the PCIE SS. The PCIE SS supports upto 4096 vectors in \"Static MSI-X mode.
                                             
                                            Please refer to the Intel FPGA IP Subsystem for PCI Express IP User Guide for more information.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#10-external-memory-interface-emif","title":"10 External Memory Interface (EMIF)","text":"
                                            There are 5 EMIF channels (5 DDR4 banks) on the Intel\u00ae FPGA SmartNIC N6001-PL.  Banks 3D and 2A are dedicated for HPS use, and Banks 3B, 3A, and 2B are connected to the HE-MEM exerciser module in AFU. The provided FIM reference design implements ECC for the HPS external memory bank while ECC is not implemented for the fabric external memory interfaces. The Intel\u00ae FPGA SmartNIC N6001-PL that this design targets supports ECC on all banks except 2B and the user may create a FIM design with ECC support.  Both memory subsystem and HE-MEM implement AXI-MM interface.
                                             
                                            Table 10-1 Memory Subsystem Configuration on the Intel\u00ae FPGA SmartNIC N6001-PL
                                             
                                            This table shows the capabilities of the memory populated on the Intel\u00ae FPGA SmartNIC N6001-PL for reference.
                                             FPGA I/O Bank Width ECC Width ECC Supported Speed Size 3D, HPS x32 x8 1GB 2400 Three 256Mx16 3B x32 x8 4GB 2400 Three 1024Mx16 3A x32 x8 4GB 2400 Three 1024Mx16 2B x32 No ECC 4GB 2400 Two 1024Mx16 2A, HPS x32 x8 1GB 2400 Two 1024Mx16 
                                            Figure 10-1: EMIF Interfaces
                                             
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#101-emif-csr","title":"10.1 EMIF CSR","text":"
                                            The CSR for the EMIF feature is memory mapped to the FME BAR region. The following table captures the EMIF CSR registers.
                                             
                                            Table 10-2: EMIF CSR Registers
                                             EMIF_DFH 0x62000 0x300000001000100F EMIF Management DFH FIELD NAME RANGE ACCESS DEFAULT DESCRIPTION FeatureType [63:60] RO 0x3 Feature Type = Private Feature Reserved40 [59:40] RsvdZ 0x0 Reserved NextDfhByteOffset [39:16] RO 0x1000 Next DFH Byte offset FeatureRev [15:12] RO 0x0 Feature Revision FeatureID [11:0] RO 0x9 Feature Id EMIF_STAT 0x62008 0x0000000000000000 EMIF Status FIELD NAME RANGE ACCESS DEFAULT DESCRIPTION Reserved [63:8] RsvdZ 0x0 Reserved CalFailure [7:4] RO 0x0 EMIF PHY Calibration Failure (1 bit per interface) CalSuccess [3:0] RO 0x0 EMIF PHY Calibration Successful (1 bit per interface) EMIF_CAPABILITY 0x62010 0x000000000000000F EMIF Capability Register FIELD NAME RANGE ACCESS DEFAULT DESCRIPTION Reserved [63:4] RsvdZ 0x0 Reserved EMIFCap [3:0] RO 0xF Attached Memory Capability (1 bit per interface) 
                                            \u200b           
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#11-ethernet-subsystem","title":"11 Ethernet Subsystem","text":""},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#111-ethernet-subsystem-overview","title":"11.1 Ethernet Subsystem Overview","text":"
                                            The Ethernet Subsystem (hssi_ss) provides portability to different Ethernet configurations across platforms and generations and reusability of the hardware framework and software stack. This reference FIM implements an E-tile Ethernet Subsystem IP in a 2x4x25GbE configuration and provides a Linux driver that can be leveraged for customization. Each group of 4x25GbE routes to a QSFP.
                                             
                                            For more information about how to reconfigure the Ethernet Subsystem please refer to the Ethernet Subsystem Intel FPGA IP.
                                             
                                            Table 11-1: Ethernet Configurations for example OFS FIMs for Agilex FPGAs
                                             Parameter Configuration for PCIe Attach Intel Agilex OFS (2x4x25GbE) IP file name hssi_ss_8x25g Number of ports enabled 8 Enabled ports Port 0-7 Port{x} profile 25GbE Port{x} subprofile MAC+PCS Port{x} RSFEC True Port{x} PTP False Enable AN/LT False Enable NPDME True Enable JTAG to Avalon master bridge False Enable Tx packet classifier NA PTP accuracy mode NA Enable iCAL and pCAL recipe at power True TX/RX maximum frame size 1518 Enforce maximum frame size False Link fault generation mode Bidirectional Stop TX traffic when link partner sends PAUSE Yes Bytes to remove from RX frames Remove CRC bytes Forward RX PAUSE requests False Use source address insertion False Enable TX/RX VLAN detection True Enable asynchronous adapter clocks False Enable preamble Passthrough False Enable asynchronous adapter clocks False Enable preamble Passthrough False Enable strict preamble check False Enable strict SFD check False Average IPG 12 Enable adaptation load soft IP True Additional IPG removed as per AM period 0 
                                            To determine which Transceiver Subsystem port maps to the QSFP A and B lanes, please refer to the following table:
                                             
                                            Table 11-2: Transceiver Subsystem Port Mapping 
                                             Port number Configuration for PCIe Attach Intel Agilex OFS (2x4x25GbE) 1 QSFP-A Lane-0 2 QSFP-A Lane-1 3 QSFP-A Lane-2 4 QSFP-A Lane-3 5 QSFP-B Lane-0 6 QSFP-B Lane-1 7 QSFP-B Lane-2 8 QSFP-B Lane-3 9 NA 10 NA 11 NA 12 NA 13 NA 14 NA 15 NA 16 NA 
                                            Figure 11-1: Transceiver Subsystem Block Diagram
                                             
                                             
                                            A host exerciser, named he-hssi, is provided in the pr_slot of the AFU partition. The Ethernet Subsystem interface to the AFU has an Arm\u00ae AMBA\u00ae 4 AXI4 data and sideband interface.  
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#112-parameterization-options","title":"11.2 Parameterization Options","text":"
                                            The Ethernet Subsystem features are highly parameterized to provide the various features/configurations required for the different FIMs. The macro defines can be added to the ofs_top.qsf file and are as follows:
                                             
                                               
                                            • ETH_100G: includes 100G related logic. Used to generate 2x1x100G.
                                              •  
                                            • ETH_10G:  includes 10G related logic. Used to generate 2x4x10G
                                              •  
                                            • ETH_25G: includes 25G related logic (Default behaviour even with no `DEFINES). Used to generate 2x4x25G
                                              •  
                                             
                                            Parameterization is also included in the ofs_fim_eth_plat_defines.svh package file:
                                             
                                            \u2022   INCLUDE_HSSI_PORT_{n}: These are available for every port. If defined for a particular port, that indicates that port is enabled. \u2022  Based on which INCLUDE_HSSI_PORT_{n} is defined in ofs_fim_eth_plat_defines_pkg.svh file, hssi_ss IP needs to be configured manually to have only those ports enabled. 
                                             
                                            Following are parameters defined in ofs_fim_eth_plat_if_pkg.sv: 
                                             
                                            \u2022   MAX_NUM_ETH_CHANNELS: This indicates how many maximum ethernet channels are supported for platfrom \u2022   NUM_QSFP_PORTS: Indicates number of QSFP cages are available on a platform \u2022   NUM_ETH_CHANNELS: Indicates number of channels present for a FIM, i.e. number for ports for Transceiver SS IP  \u2022   NUM_QSFP_LANES: Number of lanes per QSFP port \u2022   NUM_LANES: Number of lanes per HSSI port \u2022   ETH_PACKET_WIDTH: Datawidth of client side AXI-ST interface coming from HSSI SS IP. This is not an user configurabale parameter. User need update this value to reflect HSSI SS IP client side data width for selected configuration.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#113-ofs-ethernet-subsystem-interface-module","title":"11.3 OFS Ethernet Subsystem Interface Module","text":"
                                            A wrapper around the Transceiver Subsystem integrates the following features: * DFH registers  * Control & status registers  * Indirect access to Transceiver SS CSR registers via CSR mailbox in the HSSI SS interface 
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1131-ethernet-subsystem-control-and-status-register-csr-map","title":"11.3.1   Ethernet Subsystem Control and Status Register (CSR) Map","text":"
                                            The Ethernet Subsystem connects to the APF which is memory mapped to PF0 BAR0. The Ethernet Subsystem CSR space in the FIM consists of two parts:
                                             
                                               
                                            • Ethernet Subsystem CSRs assigned from offset 0x000 to 0x7FF
                                              •  
                                            • Additional CSRs are added to FIM at offset 0x800 to 0xFFF
                                              •  
                                             
                                            The PCIe subsystem uses AXI Memory Mapped accesses to read and write Ethernet Subsystem Control and Status Registers in the FIM. The Ethernet Subsystem CSR Map structure is designed to scale according to IP capabilities.
                                             
                                            The Ethernet Subsystem CSR Map can be found ipss/hssi/HSSI_SS_CSR.xls.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#114-ethernet-subsytem-software","title":"11.4 Ethernet Subsytem Software","text":"
                                            There are two pieces of software related to running the Ethernet Subsystem: The Linux* dfl network driver and the user space OPAE Tools.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1141-ethernet-subsystem-linux-driver","title":"11.4.1 Ethernet Subsystem Linux Driver","text":"
                                            The Ethernet subystem is exposed as a feature in the PCIe PF BAR0 region.  It has a Device Feature Header (DFH) specifying the interface. 
                                             
                                            The primary functionality of the driver is to interact with the ethernet MAC and PHY through an indirect register access mailbox implemented by the HSSI_RCFG_CMD0, HSSI_RCFG_DATA0 registers described above. To aid in RTL bringup, the driver provides a debugfs interface directly to the indirect register access mailbox. For each HSSI interface in the system there would be a directory with the following form containing two files, regaddr and regval: /sys/kernel/debug/dfl-fme.X.Y
                                             
                                            To read a register offset in the MAC/PHY write the offset to the regaddr file as a C hex string (e.g. 0x04) and then read the value as string out of regval file. To write a register offset in the MAC/PHY write the offset to the regaddr file as a C hex string (e.g. 0x04) and then write the value as a C hex string to regval file.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1142-ethernet-subsystem-opae-user-space-tool","title":"11.4.2   Ethernet Subsystem OPAE User Space Tool","text":"
                                            User space OPAE Tools that are part of OPAE SDK provide support for the Ethernet Subsystem.  You can use the --help option to print more information about each of these commands: * hssi: Provides a means of interacting with the 10G and 100G HSSI AFUs through the host exerciser. * hssiloopback: Enables and disables Ethernet loopback. * hssistats: Provides MAC statistics.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#12-partial-reconfiguration","title":"12 Partial Reconfiguration","text":"
                                            Partial Reconfiguration (PR) is an Intel FPGA technology that allows users to reconfigure parts of the FPGA device dynamically, while the remainder of the device continues to operate. In a non-partial reconfiguration flow, any change to the design requires full reprogramming of the entire configuration RAM (CRAM) arrays in the device. With partial reconfiguration, you can dynamically reprogram one or more CRAM frames. A partial reconfiguration design has a static region, and a PR regions, which can be modified to implement new logic. The portion of the CRAM on the chip to be reconfigured is contained within a PR region. For the PR flow, your design must be partitioned into static region and reconfigurable region. The static region is the area of your FPGA that is not reconfigured without reprogramming the entire FPGA. An area of the chip that you plan to partially reconfigure is a PR region. 
                                             
                                            The Port Gasket contains all the PR specific modules and logic, such as PR slot reset/freeze control, user clock, remote STP etc. For this reference example only one PR slot is supported.
                                             
                                            The Figure below shows the fundamental concepts required to support PR in OFS platform. There are 4 OFS management DFH \u2013 PR, Port, User Clock and Remote STP in Port Gasket that is exposed to OFS software. These platform capabilities are generally used in conjunction to Partial Reconfiguration. The Figure below also demonstrates the concepts of adding a new interface to the PR region. 
                                             
                                            Figure 12-1 Partial Reconfiguration Gasket
                                             
                                             
                                            The isolation logic is provided on the output signals of the PR region to ensure they don\u2019t glitch and affect the interfacing logic in the Static Region (SR). The isolation logic is controlled by the PR Freeze logic during PR operation.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#user-clock-support","title":"User Clock Support","text":"
                                            The reference platform provides two user configurable clock (uclk_usr, uclk_usr_div2) for use by the AFU. These clocks are generated by a reconfigurable IOPLL. The control and status of these clocks is expose through two pairs of command and status registers (USR_CLK_FREQ_CMD0 / USR_CLK_FREQ_STS0 & USR_CLK_FREQ_CMD1 / USR_CLK_FREQ_STS1). The first pair controls the fPLL reconfiguration functionality. The second pair controls a clock frequency measurement block. 
                                             
                                            The following are the default values of the userclk. 
                                             
                                            uclk_usr: 312.5 MHz 
                                             
                                            uclk_usr_div2: 156.2 MHz 
                                             
                                            Table 12-1 usr_clk_* Acceptable Programming Range
                                             Fabric Frequency Range uclk_usr (H) uclk_usr_div2 (L) Details 0-400 MHz 0-800 MHz 0-400 MHz Clocks set on 2x relationship for L<400 MHz 400-800 MHz 400-800 MHz 400-800 MHz Clks are equal for L>400 MHz 
                                            PLL Reconfiguration 
                                             
                                            The blue bit stream hardware exposes the low level IOPLL reconfiguration interfaces directly to software control. Through the USR_CLK_FREQ_CMD0 register software can select the reference clock, assert the PLL power down pin and issue reads and writes on the IOPLL Avalon-mm reconfiguration bus. Through the USR_CLK_FREQ_STS0 register software can query the IOPLL active reference clock, locked status and readdata returned from the IOPLL AVMM interface for read requests. 
                                             
                                            Clock Frequency Counter 
                                             
                                            The user clocks, generated by the reconfigurable IOPLL, are connected to a frequency measurement circuit. Software selects which of the two clocks to measure by setting the clock select bit in the USER_CLK_FREQ_CMD1 register. After 10ms software can read the frequency, in 10 KHz units, from the USER_CLK_FREQ_STS1 register. Reading this register before 10ms has passed will return undefined data but otherwise has no effect. 
                                             
                                            Configuring User Clocks 
                                             
                                            To program the user clock to the desired frequency, user will set the clock-frequency-low and clock-frequency-high fields in the PR AFU GBS .json file to the desired frequency value. During PR, SW will try to provide the closest possible frequency to the value specified in the .json file. 
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#13-host-exercisers","title":"13 Host Exercisers","text":"
                                            The Host Exerciser (HE) modules are responsible for generating traffic with the intent of exercising the specific interface they are designed to connect to. They are intended to test the interface in simulation and hardware, enable measurement of bandwidth and other performance KPIs and, in come cases, provide an example of data movement between interfaces (PCIe to DDR for e.g.) for adoption into a customer application. 
                                             
                                            ### 13.1 HE-LB and HE-MEM Host Exerciser
                                             
                                            The Host Exerciser Loopback module is a traffic generator that can be attached both to host memory, over PCIe (HE-LB), and to local memory, such as board-attached DDR (HE-MEM). The Host Exerciser (HE) is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth as well as demonstrating data path correctness. The FIM picks up the HE-LB module behind the PIM (Platform Interface Manager).  The PIM converts the AXI with TLP out of the PCIe SS to standard Arm\u00ae AMBA\u00ae 4 AXI4 (MM for completions and Lite for CSR) interfaces. The figure below shows how the HE-LB is instantiated in the FIM.
                                             
                                            Figure 13-1 HE-LB Dataflow Diagram 
                                             
                                            The exerciser has two primary modes: loopback, in which read data is fed back as writes, and read/write mode, in which reads and writes are generated independently. Furthermore, the host_exerciser software provided with OPAE that drives the RTL has two major modes: \"lpbk\" and \"mem\". These modes only select the UUID of the HE AFU, with lpbk selecting a version configured with no local memory (56e203e9-864f-49a7-b94b-12284c31e02b) and mem seeking a version with local memory (8568ab4e-6ba5-4616-bb65-2a578330a8eb). The behavior of each engine with and without local memory is described below.
                                             
                                            Figure 13-2 HE Engine Heirarchy 
                                             
                                            For more details of HE-LB and HE-MEM IP, please refer to ofs-fim-common/src/common/he_lb/README.md
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#132-hssi-host-exerciser-he-hssi","title":"13.2 HSSI Host Exerciser (HE-HSSI)","text":"
                                            HE-HSSI is an Ethernet AFU that handles client side ethernet traffic. The reference HE-HSSI has following features:
                                             
                                               
                                            • HE-HSSI provides an E-tile compatible interface with the Transceiver Subsystem.
                                              •  
                                            • Includes a traffic generator and checker or monitor
                                              •  
                                            • Provides pause signals to the Transceiver Subsystem for XON and XOFF generation
                                              •  
                                            • Generates traffic or incoming traffic that can be looped back into transmit path by enabling loopback mode, which will bypass traffic generator
                                              •  
                                            • At the HE-HSSI interface boundary the Ethernet data interface is AXI4-Stream with 64-bit data at eth_clk clock
                                              •  
                                             
                                               
                                            • The Traffic generator and checker modules have a 64-bit data interface at eth_clk clock.
                                              •  
                                            • The traffic generator supports the following modes:     * Fixed length or Random Length     * Incremental pattern or Random pattern
                                              •  
                                            • Traffic checker does a 32-bit crc check in 10/25G. In the 100G configuration, there is no data integrity check, only a packet counter. 
                                              •  
                                            • The CSR of this AFU is accessible through AXI4-Stream PCIe TLP interface
                                              •  
                                            • The PCIe TLP to CSR Interface Conversion module converts PCIe TLPs into simple CSR interface
                                              •  
                                            • The CSR space of the traffic generator and checker modules are accessed in an indirect way using mailbox registers
                                              •  
                                            • If used for more than one channel, each channel has a separate traffic generator and traffic checker with separate CSR space.
                                              •  
                                            • Reads and Writes to individual traffic controller CSR spaces can be done by selecting that particular channel using channel select register.
                                              •  
                                             
                                            The HE-HSSI Ethernet block diagram is below. 
                                             
                                            Figure 13-3: HE-HSSI Block Diagram Block Diagram
                                             
                                             
                                            The diagram below shows the path followed depending on if you enable loopback mode in HE-HSSI or not.  In loopback mode, traffic is looped back into the transmit path which will bypass traffic.  Alternatively, the traffic generates traffic.
                                             
                                            Figure 13-4: HE-HSSI Operation Mode Traffic Patterns
                                             
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1322-he-hssi-csr-map","title":"13.2.2 HE-HSSI CSR Map","text":"
                                            The reference HSSI AFU contains the following registers and a similar arrangement of register space can be implemented for other usecase specific HSSI AFUs. * AFU DFH Register: Device feature header for the AFU (AFU_DFH) * AFU ID Registers: 128-bit UUID for the AFU which occupies two 64-bit registers (AFU_ID_L, AFU_ID_H) * Mailbox Registers: Command and Data register for traffic controller register access. It follows the standard access method defined for OFS. Access method and implementation is same as Reconfiguration Interface defined for the HSSI FIM. (TRAFFIC_CTRL_CMD, TRAFFIC_CTRL_DATA) * Channel Select Register: Channel select register for traffic controller mailbox access. It is used in cases where more than one channel is in the AFU, else it defaults to zero, meaning channel-0 is selected. (TRAFFIC_CTRL_PORT_SEL) * Scratchpad Register: Scratchpad register for CSR access checking. (AFU_SCRATCHPAD) 
                                             
                                            The CSR excel for HE-HSSI module can be found at ofs-common/src/common/he_hssi/HE_HSSI_CSR.xls.
                                             
                                            13.3 HE-Null Overview
                                             
                                            This module is a simple stub that is used to replace various HE and other blocks in the FIM whenever they are bypassed using the qsf compiler directive such as null_he_lb, null_he_hssi, null_he_mem and null_he_mem_tg.  To find out more about these compiler directives, refer to the [Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs].
                                             
                                            Table 13-1  HE-Null DFH
                                             REGISTER NAME ADDRESS ACCESS DEFAULT DESCRIPTION DFH 0x0000 RO 0x0000_0000_1000_0000 DFH register GUID_L 0x0008 RO 0xaa31f54a3e403501 Lower 64b of GUID GUID_H 0x0010 RO 0x3e7b60a0df2d4850 Upper 64b of GUID SCRATCHPAD 0x0018 RW 0x0 Scratchpad 
                                            Figure 13-5: HE-Null Block Diagram
                                             
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#14-reliability-accessibility-serviceability-ras-and-error-handling","title":"14 Reliability, Accessibility, Serviceability (RAS) and Error Handling","text":"
                                               
                                            1. Downstream AFU checker: Identifies AFU violations.  For example, these checker flags violations of the interface specification. 
                                              2.  
                                            2. Upstream software or PCIe link checker: Identifies invalid traffic from PCIe that violates FIM or PCIe specifications. For example, this checker flags an application sending traffic if it violates the FIM specification or creates a PCIe link issue by causing completion timeout or malformed TLP. 
                                              3.  
                                            3. FIM - Checks for bugs in the FIM fabric.
                                              4.  
                                             
                                            Errors reported by the checker are logged in either the FME error registers or Port error registers, or both, as shown in the table below.  For more details on each of the registers, please refer to ofs-fim-common/src/common/fme/xls/n6000/FME_CSR.xls or the SystemVerilog file: ofs-common/src/common/fme/fme_csr.sv.
                                             
                                            Table 14-1: Error Registers
                                             MMIO Region Area Register Description FME CoreFIM FME_ERROR FME Error Status Register 0.  Registers parity errors, underflow or overflow errors and access mismatches. FME CoreFIM FME_ERROR0_MASK FME Error Mask Register 0.  Write a 0 to mask errors in the FME Error Status Register 0. FME External PCIE0_ERROR PCIe0 Error Status Register. FME External PCIE0_ERROR_MASK PCIe0 Error Mask Register 0.  Write a 0 to mask errors in the PCIe0 Error Status Register 0. FME CoreFIM FME_FIRST_ERROR First FME Error Register. FME CoreFIM FME_NEXT_ERROR FME Next Error Register. FME CoreFIM RAS_NOFAT_ERR_STAT Reliability/Accessibility/Serviceability (RAS) Non-Fatal Error Status Register. FME CoreFIM RAS_NOFAT_ERR_MASK RAS Non-Fatal Error Mask Register. Write 0 to mask error fields in RAS_NOFAT_ERR_STAT Register. FME CoreFIM RAS_CATFAT_ERR_STAT RAS Catastrophic and Fatal Errors Status Register. FME CoreFIM RAS_CATFAT_ERR_MASK RAS Catastrophic and Fatal Errors Mask Register. Write 0 to mask error fields in the RAS_CATFAT_ERR_STAT Register. FME CoreFIM RAS_ERROR_INJ RAS error Injection Register. PORT CoreFIM PORT_ERROR Port Error Status Register. PORT CoreFIM PORT_FIRST_ERROR Port First Error Register . PORT CoreFIM PORT_MALFORMED_REQ0 Port Malformed Request Register 0.  Provides malformed request header LSBs. PORT CoreFIM PORT_MALFORMED_REQ1 Port Malformed Request Register 1.  Provides malformed request header MSBs."},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#141-fme-errors","title":"14.1 FME Errors","text":""},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1411-fme_error0","title":"14.1.1   FME_ERROR0","text":"
                                            The FME_ERROR0 register flags CoreFIM FME errors in the Global Error (GLBL_ERROR) private feature. The error bits in this register are sticky bits. You can only clear these bits through software or through a system reset. Writing a 1 to the error field in the register clears the corresponding error bit. Writing a 1 to the corresponding bit in FME_ERROR0_MASK register masks the error.
                                             
                                            Table 14-2: FME Error Types
                                             Error Type Description Fabric errors FIFO underflow/overflow condition in CoreFIM. These errors only occur if you have introduced bugs into the FIM or during very rare single event upset (SEU) or SEU-like events. Invalid port access A port can either be mapped to a PF or a VF, and not both. The checker reads the AfuAccessCtrl field in the FME CSR PORT0_OFFSET register to determine the access type of the Port. If it finds a PF is trying to access a port that is mapped to a VF or vice-versa, an error will be reported. Invalid AFU access An AFU can either be mapped to a PF or a VF, and not both. The checker reads the AfuAccessCtrl field in the FME CSR PORT0_OFFSET register to determine the access type of the AFU associated with the Port. If it finds a PF is trying to access an AFU that is mapped to a VF or vice-versa, an error is reported and a fake response is sent back to the requester to avoid a completion timeout on the host."},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1412-pcie0_error","title":"14.1.2 PCIE0_ERROR","text":"
                                            The PCIe Avalon-ST to AXI4-Stream bridge monitors the PCIe link for errors and logs any such errors in the PCIE0_ERROR register (in PCIE0 feature region) and PCIE0_ERROR register in the GLBL_ERR private feature.    The error bits in the PCIE0_ERROR register are sticky bits that you can only clear through software or through a system reset. Writing a 1 to the error field in the register clears the corresponding error bit. Writing a 1 to the corresponding bit in PCIE0_ERROR0_MASK masks the error.  
                                             
                                            If you have other external FME features, you can add similar _ERROR registers to this space. Please refer to the following spreadsheet in the release branch found at: /ipss/pcie/rtl/PCIE_CSR.xls or the SystemVerilog file: ipss/pcie/rtl/pcie_csr.sv for more details on this register.  
                                            NOTE
                                             
                                            The PCIE0_ERROR register is located in both the Global Error external feature memory space and a separate PCIe external feature memory space.  OPAE software supports the PCIe external feature memory space beginning at offset 0x40000 for OFS EA and going forward.  PCIe registers beginning at 0x4000 in the Global Error external feature memory space is there for backward compatibility to the Intel FPGA PAC D5005 v2.0.1 Acceleration Stack.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1413-fme_first_error-fme_next_error","title":"14.1.3   FME_FIRST_ERROR, FME_NEXT_ERROR","text":"
                                            The FME_FIRST_ERROR register flags which of the FME error reporting registers, such as FME_ERROR0, PCIE0_ERROR0, has reported the first error occurrence. The error fields of the first error register are then continuously logged into the FME_FIRST_ERROR register until a system reset or software clears all the errors in that first error register. Likewise, the FME_NEXT_ERROR indicates which of the FME error reporting registers (except the first error register) has reported the next occurrence of error after the first error register. The error fields of the next error register are continuously logged into the FME_NEXT_ERROR register until a system reset or software clears all the errors in the second error register.
                                             
                                            Please refer to the file in the ofs-fim-common repository folder: src/common/fme/fme_csr.sv for individual register field descriptions or the SystemVerilog file src/common/fme/fme_csr.sv.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1414-reliability-accessibility-serviceability-ras-error-status","title":"14.1.4   Reliability, Accessibility, Serviceability (RAS) Error Status","text":"
                                            The RAS feature in CoreFIM labels errors as non-fatal, fatal or catastrophic based on their impact to the system.  * A non-fatal error usually originates from software or an AFU.  With a non-fatal error, the user application may still be able to recover from the error by performing a soft reset on the AFU, fixing the user application software if necessary, and clearing the error. On the other hand, a fatal or catastrophic error is non-recoverable and requires the platform to be reset. * Non-fatal errors are logged in the RAS_NOFAT_ERR_STAT register and fatal or catastrophic errors are logged in the RAS_CATFAT_ERR_STAT register.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#14141-non-fatal-errors","title":"14.1.4.1    Non-Fatal Errors","text":"
                                            The RAS_NOFAT_ERR_STAT is a read-only status register that is specifically added for the RAS feature. It logs the high-level status of non-fatal errors in the hardware.  Unlike the error bits in the PCIE0_ERROR and FME_ERROR0 registers which are RW1C (software can write a 1 to clear the error), the error bits in this register are read-only and can only be cleared by system reset. Software has an option to mask the error using RAS_NOFAT_ERR_MASK. Please refer to the following file in the ofs-fim-common resository: src/common/fme/xls/n6000/FME_CSR.xls for individual register field descriptions or the SystemVerilog file: src/common/fme/fme_csr.sv.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#14142-catastrophic-fatal-errors","title":"14.1.4.2    Catastrophic & Fatal Errors","text":"
                                            The RAS_CATFAT_ERR_STAT is a read-only status register that is specifically added for the RAS feature. It captures the high-level status of errors that can only be recovered with a system reset. Therefore, the error bits in the RAS_CATFAT_ERR_STAT register are read-only and can only be cleared by system reset or masked through RAS_CATFAT_ERR_MASK. Please refer to the following file in the ofs-fim-common resository: src/common/fme/xls/n6000/FME_CSR.xls for individual register field descriptions or the SystemVerilog file: src/common/fme/fme_csr.sv.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1415-ras-error-injection","title":"14.1.5   RAS Error Injection","text":"
                                            For software testing purposes, you can inject non-fatal, fatal and catastrophic errors into the platform through the RAS_ERROR_INJ register.  These errors are reported in the RAS_CATFAT_ERR_STAT and RAS_NOFAT_ERR_STAT registers. Please refer to the following file in the ofs-fim-common resository: src/common/fme/xls/n6000/FME_CSR.xls for individual register field descriptions or the SystemVerilog file: src/common/fme/fme_csr.sv.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1416-fme-error-interrupts","title":"14.1.6   FME Error Interrupts","text":"
                                            In an event of an FME error, the MSI-X module in the FIM generates an interrupt so the host can decide on the next course of action. The FIM does not stall upstream and downstream traffic after the FME error. However, a port error triggered by invalid request from a user AFU stalls all the traffic going from AFU to PCIe. The interrupt capability is discoverable by querying the NumbSuppInterrupt field of the PORT_CAPABILITY register in the Port private feature.  The MSI-X vector number is recorded in the InterruptVectorNumber field of the GLBL_ERROR_CAPABILITY register of the Global Error external feature.
                                             
                                            An FME error interrupt is generated in response to the FME_ERROR0, PCIE0_ERROR0, RAS_NOFAT_ERR_STAT or RAS_CATFAT_ERR_STAT registers recording a new, unmasked, error per the rules defined by CoreFIM interrupt feature.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1417-fme-error-handling","title":"14.1.7   FME Error Handling","text":"
                                            When the host receives an FME error interrupt, it must take the recommended actions described below to bring the system back to its normal operation.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#14171-catastrophicfatal-error","title":"14.1.7.1    Catastrophic/Fatal Error","text":"
                                            A system reset is mandatory for any catastrophic or fatal error.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#14172-non-fatal-error","title":"14.1.7.2    Non-Fatal Error","text":"
                                            When software receives a non-fatal error interrupt which does not require a system reset, it can take the following steps to clear the error after software handles the error: 1.  Set the *_ERROR_MASK register to all 1\u2019s to mask all errors 2. Clear the *_FIRST_ERROR register 3. Clear the *_ERROR register 4. Set *_ERROR_MASK register to all 0\u2019s to enable all errors
                                             
                                               
                                            • Result: The *_ERROR & *_FIRST_ERROR registers begin capturing new errors.
                                              •  
                                             
                                            NOTE
                                             
                                            A system reset can only clear RAS Error status registers.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#142-mmio-requests","title":"14.2 MMIO Requests","text":"
                                            The FIM is designed to gracefully handle MMIO request scenarios.  
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1421-unsupported-functions-and-bars","title":"14.2.1 Unsupported Functions and BARs","text":"
                                            The PCIe hard IP in the FIM guarantees that only TLP packets for the functions and BARs supported by the FIM (as configured in PCIe HIP IP instantiation) are sent to the FIM.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1422-mmio-request-decoding","title":"14.2.2 MMIO Request Decoding","text":"
                                            The packet router and memory decoder in the FIM ensure that only legal MMIO requests are forwarded to the targeted MMIO region. Full address and BAR decoding is done both in the packet router and the memory decoder to ensure the requests are forwarded to the designated CSR region as defined in the MMIO Regions chapter.  Any unsolicited/illegal MMIO request is dropped, and an error is reported back to host through the FME error register.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1423-unused-fmeport-csr-regions","title":"14.2.3 Unused FME/Port CSR Regions","text":"
                                            All the CSR slaves in FIM which are mapped to the FME or Port CSR regions must always respond to MMIO read requests targeting its associated CSR region. A CSR slave must return all 0s for MMIO reads to its unused CSR region such as a reserved space or a region where no CSR register is implemented for the address. The FIM ensures MMIO reads to FME or Port CSR regions that are not mapped to any CSR slave always gets a response of all 0s. The memory decoder module and fake responder module in the FIM provide this guaranteed response.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1424-unsupported-mmio-request","title":"14.2.4 Unsupported MMIO Request","text":"
                                            Any MMIO request targeting FME or Port CSR regions with a length or address alignment that are not  supported by the FIM is dropped, and an error is logged in PCIE0_ERROR register. The MMIO checker module in the FIM guarantees this response. When an unsupported MMIO read request to the FIM CSR region is detected, the FIM sends back a CPL (completion without data) with error status (UR) back to host.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1425-afu-access-violation","title":"14.2.5 AFU Access Violation","text":"
                                            AFU access violations refer to the scenarios where a PF is attempting to access the MMIO region of an AFU bound to a VF (virtualization enabled), or when a VF is trying to access the MMIO region of an AFU bound to a PF (virtualization disabled). When such a violation is detected, the FIM drops the request and logs the error in the FME_ERROR0 register. If the request is an MMIO read request, the FIM returns a fake response to the host.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1426-afu-mmio-response-timeout","title":"14.2.6    AFU MMIO Response Timeout","text":"
                                            An AFU MMIO Response timeout functions in the same manner described in the MMIO Response Timeout section.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#15-ofs-design-hierarchy","title":"15 OFS Design Hierarchy","text":"
                                            Files for design, build and unit test simulation are found at https://github.com/OFS/ofs-agx7-pcie-attach, release branch ${{ N6001_Release }}.
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#151-design-guidance","title":"15.1 Design Guidance","text":"
                                            The OFS FIM is designed with configurability and scalability in mind.  At a high level, these are the necessary steps for a user to customize the design.  Please refer to the F[Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs] for detaaled design guidance. 
                                            "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                            Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                             
                                            OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122.  \u00a0
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/","title":"Platform Evaluation Script: Open FPGA Stack for Intel Agilex FPGA","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel Max10 or Intel Cyclone10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implemented on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Management Component Transport Protocol MCTP A standardized model for communication with management controllers. Defines the transport protocol carrying PLDM messages through the BMC. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE-SDK The OPAE-SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Level Data Model PLDM A specification for reporting telemetry data to the host, such as board temperature, voltage, and current. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#1-overview","title":"1 Overview","text":""},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#11-about-this-document","title":"1.1 About this Document","text":"
                                            This document serves as a set-up and user guide for the checkout and evaluation of an Intel\u00ae FPGA SmartNIC N6001-PL development platform using Open FPGA Stack (OFS). After reviewing the document, you will be able to:
                                             
                                               
                                            • Set-up and modify the script to your environment
                                              •  
                                             
                                               
                                            • Compile and simulate an OFS reference design
                                              •  
                                             
                                               
                                            • Run hardware and software tests to evaluate the complete OFS flow
                                              •  
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#table-1-2-software-version-summary","title":"Table 1-2: Software Version Summary","text":"Component Version Description FPGA Platform Intel\u00ae FPGA SmartNIC N6001-PL Intel platform you can use for your custom board development OFS FIM Source Code Branch: https://github.com/OFS/ofs-agx7-pcie-attach, Tag: ofs-2023.3-2 OFS Shell RTL for Intel Agilex FPGA (targeting Intel\u00ae FPGA SmartNIC N6001-PL) OFS FIM Common Branch: https://github.com/OFS/ofs-fim-common/releases/tag/ofs-2023.3-2, Tag: https://github.com/OFS/ofs-fim-common/releases/tag/ofs-2023.3-2 Common RTL across all OFS-based platforms AFU Examples Branch: examples-afu , Tag:ofs-examples-https://github.com/OFS/examples-afu/releases/tag/ofs-2023.3-2 Tutorials and simple examples for the Accelerator Functional Unit region (workload region) OPAE SDK Branch: 2.10.0-1, Tag: 2.10.0-1 Open Programmable Acceleration Engine Software Development Kit Kernel Drivers Branch: ofs-2023.3-6.1-3, Tag: ofs-2023.3-6.1-3 OFS specific kernel drivers OPAE Simulation Branch: opae-sim, Tag: 2.10.0-1 Accelerator Simulation Environment for hardware/software co-simulation of your AFU (workload) Intel Quartus Prime Pro Edition Design Software 23.3 [Intel\u00ae Quartus\u00ae Prime Pro Edition Linux] Software tool for Intel FPGA Development Operating System RHEL 8.6 Operating system on which this script has been tested 
                                            A download page containing the release and already-compiled FIM binary artifacts that you can use for immediate evaluation on the Intel\u00ae FPGA SmartNIC N6001-PL can be found on the OFS ofs-2023.3-2 official release drop on GitHub.
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#2-introduction-to-ofs-evaluation-script","title":"2 Introduction to OFS Evaluation Script","text":"
                                            By following the setup steps and using the OFS evaluation script you can quickly evaluate many features that the OFS framework provides and also leverage this script for your own development.  
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#21-pre-requisites","title":"2.1 Pre-Requisites","text":"
                                            This script uses the following set of software tools which should be installed using the directory structure below. Tool versions can vary.
                                             
                                               
                                            • Intel Quartus\u00ae Prime Pro Software
                                              •  
                                            • Synopsys\u00ae VCS Simulator
                                              •  
                                            • Siemens\u00ae Questa\u00ae Simulator
                                              •  
                                             
                                            Figure 2-1 Folder Hierarchy for Software Tools
                                             
                                             
                                               
                                            1.  
                                              You must create a directory named \"ofs-X.X.X\" where the X represents the current release number, for example ofs-2023.3-2. 
                                               
                                              2.  
                                            2.  
                                              You must clone the required OFS repositories as per Figure 2-2.  Please refer to the BKC table for locations. When cloning the FIM repository, please follow the instructions in section 1.3.2 of the FPGA Interface Manager Developer Guide for Open FPGA Stack: Intel\u00ae FPGA SmartNIC N6001-PL PCIe Attach.  Additionally, please go to FPGA Developer Journey Guide: Open FPGA Stack for the instructions for the BKC installation.
                                               
                                              3.  
                                            3.  
                                              Once the repositories are cloned, download the evaluation script folder (ofs-agx7-pcie-attach_eval.tar.gz) which is available in the OFS release page and copy the evaluation script ofs-agx7-pcie-attach_eval.sh in the location as shown in the example below:
                                               
                                              4.  
                                             
                                            Figure 2-2 Directory Structure for OFS Project
                                             
                                            ## ofs-2023.3-2\n##  -> examples-afu\n##  -> linux-dfl\n##  -> ofs-n6001\n##  -> oneapi-asp\n##  -> oneAPI-samples\n##  -> opae-sdk\n##  -> opae-sim\n##  -> ofs_n6001_eval.sh\n
                                             
                                               
                                            1. Open the README file named (README_ofs-agx7-pcie-attach_eval.txt) which is in  the evaluation script folder(ofs-agx7-pcie-attach_eval.tar.gz) which informs the user which sections to modify in the script prior to building the FIM and running hardware, software and simulation tests.
                                              2.  
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#22-n6001-evaluation-script-modification","title":"2.2 n6001 Evaluation Script modification","text":"
                                            To adapt this script to the user environment please follow the instructions below which explains which line numbers to change in the ofs-agx7-pcie-attach_eval.sh script 
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#user-directory-creation","title":"User Directory Creation","text":"
                                            The user must create the top-level source directory and then clone the OFS repositories
                                             
                                            mkdir ofs-2023.3-2\n
                                             
                                            In the example above we have used ofs-2023.3-2 as the directory name
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#set-up-proxy-server-lines-67-69","title":"Set-Up Proxy Server (lines 67-69)","text":"
                                            Please enter the location of your proxy server to allow access to external internet to build software packages.
                                             
                                            Note: Failing to add proxy server will prevent cloning of repositories and the user will be unable to build the OFS framework.
                                             
                                            export http_proxy=<user_proxy>\nexport https_proxy=<user_proxy>\nexport no_proxy=<user_proxy>\n
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#license-files-lines-72-74","title":"License Files (lines 72-74)","text":"
                                            Please enter the the license file locations for the following tool variables
                                             
                                            export LM_LICENSE_FILE=<user_license>\nexport DW_LICENSE_FILE=<user_license>\nexport SNPSLMD_LICENSE_FILE=<user_license>\n
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#tools-location-line-87-88-89-90","title":"Tools Location (line 87, 88, 89, 90)","text":"
                                            Set Location of Quartus, Synopsys, Questasim and oneAPI Tools
                                             
                                            export QUARTUS_TOOLS_LOCATION=/home\nexport SYNOPSYS_TOOLS_LOCATION=/home\nexport QUESTASIM_TOOLS_LOCATION=/home\nexport ONEAPI_TOOLS_LOCATION=/opt\n
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#quartus-tools-version-line-95","title":"Quartus Tools Version (line 95)","text":"
                                            Set version of Quartus
                                             
                                            export QUARTUS_VERSION=23.3\n
                                             
                                            In the example above \"23.3\" is used as the Quartus tools version
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#opae-tools-line-108","title":"OPAE Tools (line 108)","text":"
                                            change OPAE SDK VERSION
                                             
                                            export OPAE_SDK_VERSION=2.10.0-1\n
                                             
                                            In the example above \"2.10.0-1\" is used as the OPAE SDK tools version
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#ofs-platform-line-173","title":"OFS Platform (line 173)","text":"
                                            Select the oFS platform the evaluation script needs to be run on. It could be one of these : n6000, n6001, fseries-dk, iseries-dk, custom_board.
                                             
                                            export OFS_PLATFORM=n6001\n
                                             
                                            In the example above, n6001 is the platform selected to be tested.
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#pcie-bus-number","title":"PCIe (Bus Number)","text":"
                                            The Bus number must be entered by the user after installing the hardware in the chosen server, in the example below \"b1\" is the Bus Number for a single card as defined in the evaluation script.
                                             
                                            export OFS_CARD0_BUS_NUMBER=b1\n
                                             
                                            The evaluation script uses the bus number as an identifier to interrogate the card. The command below will identify the accelerater card plugged into a server. 
                                             
                                            lspci | grep acc\n\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)<br>\nb1:00.1 Processing accelerators: Intel Corporation Device bcce<br>\nb1:00.2 Processing accelerators: Intel Corporation Device bcce<br>\nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device<br>\nb1:00.4 Processing accelerators: Intel Corporation Device bcce<br>\n
                                             
                                            The result identifies the card as being assigned \"b1\" as the bus number so the entry in the script changes to
                                             
                                            export OFS_CARD0_BUS_NUMBER=b1\n
                                             
                                            The user can also run the following command on the ofs-agx7-pcie-attach_eval.sh script to automatically change the bus number to b1 in the ofs-agx7-pcie-attach_eval.sh script.
                                             
                                            grep -rli 'b1' * | xargs -i@ sed -i 'b1' @
                                             
                                            if the bus number is 85 for example 
                                             
                                            85:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)<br>\n85:00.1 Processing accelerators: Intel Corporation Device bcce<br>\n85:00.2 Processing accelerators: Intel Corporation Device bcce<br>\n85:00.3 Processing accelerators: Red Hat, Inc. Virtio network device<br>\n85:00.4 Processing accelerators: Intel Corporation Device bcce<br>\n
                                             
                                            the command to change to 85 in the evaluation script would be
                                             
                                            grep -rli 'b1' * | xargs -i@ sed -i '85' @
                                             
                                            The ofs-agx7-pcie-attach_eval.sh script has now been modified to the server set-up and the user can proceed to build, compile and simulate the OFS stack
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#3-n6001-evaluation-script","title":"3 n6001 Evaluation Script","text":""},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#31-overview","title":"3.1 Overview","text":"
                                            The figure below shows a snapshot of the full evaluation script menu showing all 62 options and each one one of 11 sub-menus which focus on different areas of evaluation. Each of these menu options are described in the next section.
                                             
                                            Figure 3-1 ofs-agx7-pcie-attach_eval.sh Evaluation Menu
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#311-adp-tools-menu","title":"3.1.1 ADP TOOLS MENU","text":"
                                            By selecting \"List of Documentation for ADP n6001 Project,\" a list of links to the latest OFS documentation appears. Note that these links will take you to documentation for the most recent release which may not correspond to the release version you are evaluating. To find the documentation specific to your release, ensure you clone the intel-ofs-docs tag that corresponds to your OFS version.
                                             
                                            By selecting \"Check Versions of Operating System and Quartus Premier Design Suite\", the tool verifies correct Operating System, Quartus version, kernel parameters, license files and paths to installed software tools.
                                             
                                             Menu Option Example Output 1 - List of Documentation for ADP n6001 Project  Open FPGA Stack Overview                  Guides you through the setup and build steps to evaluate the OFS solution                 https://ofs.github.io 2 - Check versions of Operating System and Quartus Premier Design Suite (QPDS) Checking Linux release                 Linux version 5.15.52-dfl (guest@hw-rae-svr4-l) (gcc (GCC) 8.5.0 20210514 (Red Hat 8.5.0-4), GNU ld version 2.30-79.el8) #1 SMP Fri Sep 23 17:19:37 BST 2022                  Checking RedHat release                 CentOS Linux release 8.3.2011                  Checking Ubuntu release                 cat: /etc/lsb-release: No such file or directory                  Checking Kernel parameters                 BOOT_IMAGE=(hd0,gpt2)/vmlinuz-5.15.52-dfl root=/dev/mapper/cl-root ro crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200                  Checking Licenses                 LM_LICENSE_FILE is set to port@socket number:port@socket number                 DW_LICENSE_FILE is set to port@socket number:port@socket number                 SNPSLMD_LICENSE_FILE is set to port@socket number:port@socket number                  Checking Tool versions                 QUARTUS_HOME is set to /home/intelFPGA_pro/23.3/quartus                 QUARTUS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus                 IMPORT_IP_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../ip                 QSYS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../qsys/bin                  Checking QPDS Patches                 Quartus Prime Shell                 Version 23.3 Build XXX XX/XX/XXXX Patches X.XX SC Pro Edition                 Copyright (C) XXXX  Intel Corporation. All rights reserved."},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#312-adp-hardware-menu","title":"3.1.2 ADP HARDWARE MENU","text":"
                                            Identifies card by PCIe number, checks power, temperature and current firmware configuration. 
                                             
                                             Menu Option Example Output 3 - Identify Acceleration Development Platform (ADP) n6001 Hardware via PCIe PCIe card detected as                 b1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)                 b1:00.1 Processing accelerators: Intel Corporation Device bcce                 b1:00.2 Processing accelerators: Intel Corporation Device bcce                 b1:00.4 Processing accelerators: Intel Corporation Device bcce                 Host Server is connected to SINGLE card configuration 4 - Identify the Board Management Controller (BMC) Version and check BMC sensors Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.15.0                 Board Management Controller Build version: 3.15.0                 //****** BMC SENSORS ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2 5 - Identify the FPGA Management Engine (FME) Version Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.15.0                 Management Controller Build version: 3.15.0                 //****** FME ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                 Boot Page                        : user1                 Factory Image Info               : a7c6879683182ce61084c420e51f50b6                 User1 Image Info                 : 8a7440ddff52e0e27dbb989d5eb954f4                 User2 Image Info                 : a7c6879683182ce61084c420e51f50b6 6 - Check Board Power and Temperature Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.15.0                 Board Management Controller Build version: 3.15.0                 //****** POWER ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                 ( 1) VCCRT_GXER_0V9 Voltage      : 0.91 Volts                 etc ......................                  Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.15.0                 Board Management Controller Build version: 3.15.0                 //****** TEMP ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                 ( 1) FPGA E-Tile Temperature [Remote] : 33.50 Celsius                 etc ...................... 7 - Check Accelerator Port status //****** PORT ******//                 Object Id                        : 0xED00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00 8 - Check MAC and PHY status Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.15.0                 Board Management Controller Build version: 3.15.0                 //****** MAC ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                 Number of MACs                   : 255                 mac info is not supported                  Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.2.0                 Board Management Controller Build version: 3.2.0                 //****** PHY ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                  //****** HSSI information ******//                 HSSI version                     : 1.0                 Number of ports                  : 8                 Port0                            :25GbE        DOWN                 Port1                            :25GbE        DOWN                 Port2                            :25GbE        DOWN                 Port3                            :25GbE        DOWN                 Port4                            :25GbE        DOWN                 Port5                            :25GbE        DOWN                 Port6                            :25GbE        DOWN                 Port7                            :25GbE        DOWN"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#313-adp-pfvf-mux-menu","title":"3.1.3 ADP PF/VF MUX MENU","text":"
                                            This menu reports the number of PF/VF functions in the reference example and also allows you to reduce the number to 1PF and 1VF to reduce resource utilisation and create a larger area for your workload development. This selection is optional and if the user wants to implement the default number of PF's and VF's then option 9, 10 and 11 should not be used.  Additionally the code used to make the PF/VF modification can be leveraged to increase or modify the number of PF/VFs in the existing design within the limits that the PCIe Subsystem supports (8PF/2K VFs)
                                             
                                             Menu Option Description 9 - Check PF/VF Mux Configuration  This menu selection displays the current configuration of the pcie_host.ofss file which is located at the following directory $OFS_ROOTDIR/tools/pfvf_config_tool                  [ProjectSettings]                 platform = n6001                 family = Agilex                 fim = base_x16                 Part = AGFB014R24A2E2V                 IpDeployFile = pcie_ss.sh                 IpFile = pcie_ss.ip                 OutputName = pcie_ss                 ComponentName = pcie_ss                 is_host = True                  [pf0]                 num_vfs = 3                 pg_enable = True                  [pf1]                  [pf2]                  [pf3]                  [pf4] 10 - Modify PF/VF Mux Configuration As an example this menu selection modifies the pcie_host.ofss file to 1 PF located in the following directory $OFS_ROOTDIR/tools/pfvf_config_tool             This option also displays the the modified pcie_host.ofss file 11 - Build PF/VF Mux Configuration If option 10 is not used then then the default number of PF's and VF's is used to build the FIM, if option 10 is selected then only 1 VF is built to reduce logic utilisation"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#314-adp-fimpr-build-menu","title":"3.1.4 ADP FIM/PR BUILD MENU","text":"
                                            Builds FIM, Partial Reconfiguration Region and Remote Signal Tap
                                             
                                             Menu Option Description 12 - Check ADP software versions for ADP n6001 Project OFS_ROOTDIR is set to /home/user_area/ofs-X.X.X/ofs-n6001                 OPAE_SDK_REPO_BRANCH is set to release/X.X.X                 OPAE_SDK_ROOT is set to /home/user_area/ofs-X.X.X/ofs-n6001/../opae-sdk                 LD_LIBRARY_PATH is set to /home/user_area/ofs-X.X.X/ofs-n6001/../opae-sdk/lib64: 13 - Build FIM for n6001 Hardware This option builds the FIM based on the setting for the $ADP_PLATFORM, $FIM_SHELL environment variable. Check this variable in the following file ofs_n6001_eval.sh 14 - Check FIM Identification of FIM for n6001 Hardware The FIM is identified by the following file fme-ifc-id.txt located at $OFS_ROOTDIR/$FIM_WORKDIR/syn/syn_top/ 15 - Build Partial Reconfiguration Tree for n6001 Hardware This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development and also for the oneAPI build flow  16 - Build Base FIM Identification(ID) into PR Build Tree template This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU and oneAPI workloads 17 - Build Partial Reconfiguration Tree for n6001 Hardware with Remote Signal Tap This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development and also for the oneAPI build flow for the Remote Signal Tap flow 18 - Build Base FIM Identification(ID) into PR Build Tree template with Remote Signal Tap This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree for Remote Signal Tap to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU and oneAPI workloads"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#315-adp-hardware-programmingdiagnostic-menu","title":"3.1.5 ADP HARDWARE PROGRAMMING/DIAGNOSTIC MENU","text":"
                                            The following submenu allows you to: * Program and check flash  * Perform a remote system update (RSU) of the FPGA image into the FPGA * Bind virtual functions to VFIO PCIe driver  * Run host exerciser (HE) commands such as loopback to test interfaces VFIO PCI driver binding * Read the control and status registers (CSRs) for bound modules that are part of the OFS reference design.
                                             
                                             Menu Option Description 19 - Program BMC Image into n6001 Hardware The user must place a new BMC flash file in the following directory $OFS_ROOTDIR/bmc_flash_files. Once the user executes this option a new BMC image will be programmed. A remote system upgrade command is initiated to store the new BMC image 20 - Check Boot Area Flash Image from n6001 Hardware This option checks which location area in FLASH the image will boot from, the default is user1              Boot Page : user1 21 - Program FIM Image into user1 area for n6001 Hardware This option programs the FIM image \"ofs_top_page1_unsigned_user1.bin\" into user1 area in flash 22 - Initiate Remote System Upgrade (RSU) from user1 Flash Image into n6001 Hardware This option initiates a Remote System Upgrade and soft reboots the server and re-scans the PCIe bus for the new image to be loaded              2022-11-10 11:26:24,307 - [[pci_address(0000:b1:00.0), pci_id(0x8086, 0xbcce)]] performing RSU operation             2022-11-10 11:26:24,310 - [[pci_address(0000:b0:02.0), pci_id(0x8086, 0x347a)]] removing device from PCIe bus             2022-11-10 11:26:24,357 - waiting 10 seconds for boot             2022-11-10 11:26:34,368 - rescanning PCIe bus: /sys/devices/pci0000:b0/pci_bus/0000:b0             2022-11-10 11:26:35,965 - RSU operation complete 23 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 24 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 25 - Create Virtual Functions (VF) and bind driver to vfio-pci n6001 Hardware This option creates vfio-pci driver binding for the PF's and VF's             Once the VF's have been bound to the driver the user can select menu option 23 to check that the new drivers are bound 26 - Verify FME Interrupts with hello_events The hello_events utility is used to verify FME interrupts. This tool injects FME errors and waits for error interrupts, then clears the errors 27 - Run HE-LB Test This option runs 5 tests              1) checks and generates traffic with the intention of exercising the path from the AFU to the Host at full bandwidth             2) run a loopback throughput test using one cacheline per request             3) run a loopback read test using four cachelines per request             4) run a loopback write test using four cachelines per request             5) run a loopback throughput test using four cachelines per request 28 - Run HE-MEM Test This option runs 2 tests              1) Checking and generating traffic with the intention of exercising the path from FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host             2) run a loopback throughput test using one cacheline per request 29 - Run HE-HSSI Test This option runs 1 test              HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs, and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic              1) Send traffic through the 10G AFU              30 - Run Traffic Generator AFU Test This option runs 3 tests              TG AFU has an OPAE application to access & exercise traffic, targeting a specific bank              1) Run the preconfigured write/read traffic test on channel 0             2) Target channel 1 with a 1MB single-word write only test for 1000 iterations             3) Target channel 2 with 4MB write/read test of max burst length for 10 iterations 31 - Read from CSR (Command and Status Registers) for n6001 Hardware This option reads from the following CSR's             HE-LB Command and Status Register Default Definitions             HE-MEM Command and Status Register Default Definitions             HE-HSSI Command and Status Register Default Definitions"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#316-adp-hardware-afu-testing-menu","title":"3.1.6  ADP HARDWARE AFU TESTING MENU","text":"
                                            This submenu tests partial reconfiguration by building and loading an memory-mapped I/O example AFU/workload, executes software from host, and tests remote signal tap.
                                             
                                             Menu Option Description 32 - Build and Compile host_chan_mmio example This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS (Green Bit Stream) ready for hardware programming 33 - Execute host_chan_mmio example This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test 34 - Modify host_chan_mmio example to insert Remote Signal Tap This option inserts a pre-defined host_chan_mmio.stp Signal Tap file into the OFS code to allow a user to debug the host_chan_mmio AFU example 35 - Build and Compile host_chan_mmio example with Remote Signal Tap This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS(Green Bit Stream) ready for hardware programming with Remote Signal tap enabled 36 - Execute host_chan_mmio example with Remote Signal Tap This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test. The user must open the Signal Tap window when running the host code to see the transactions in the Signal Tap window"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#317-adp-hardware-afu-bbb-testing-menu","title":"3.1.7 ADP HARDWARE AFU BBB TESTING MENU","text":"
                                            This submenu tests partial reconfiguration using a hello_world example AFU/workload, executes sw from host
                                             
                                             Menu Option Description 37 - Build and Compile hello_world example This option builds the hello_ world example from the following repo $FPGA_BBB_CCI_SRC/samples/tutorial/afu_types/01_pim_ifc/$AFU_BBB_TEST_NAME, where AFU_BBB_NAME=hello_world. This produces a GBS(Green Bit Stream) ready for hardware programming 38 - Execute hello_world example This option builds the host code for hello_world example and programs the GBS file and then executes the test"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#318-adp-oneapi-project-menu","title":"3.1.8 ADP ONEAPI PROJECT MENU","text":"
                                            Builds oneAPI kernel, executes sw from host and runs diagnostic tests
                                             
                                             Menu Option Result 39 - Check oneAPI software versions for n6001 Project This option checks the setup of the oneAPI software and adds the relevant oneAPI environment variables to the terminal. This option also informs the user to match the oneAPI software version to the oneAPI-samples version 40 - Build and clone shim libraries required by oneAPI host This option builds the oneAPI directory structure 41 - Install oneAPI Host Driver This option Installs the oneAPI Host driver at the following location /opt/Intel/OpenCLFPGA/oneAPI/Boards/, and requires sudo permission 42 - Uninstall oneAPI Host Driver This option Uninstall's the oneAPI Host driver, and requires sudo permissions 43 - Diagnose oneAPI Hardware This option Checks ICD (Intel Client Driver) and FCD (FPGA Client Driver), oneAPI library locations and detects whether oneAPI BSP is loaded into the FPGA 44 - Build oneAPI BSP ofs_n6001 Default Kernel (hello_world) This option Builds the oneAPI BSP using hello_world kernel 45 - Build oneAPI MakeFile Environment This option Builds the oneAPI environment using a Makefile for kernel insertion  46 - Compile oneAPI Sample Application (board_test) for Emulation This option compiles the board_test kernel for Emulation 47 - Run oneAPI Sample Application (board_test) for Emulation This option executes the board_test kernel for Emulation 48 - Generate oneAPI Optimization report for (board_test) This option generates an optimization report for the board_test kernel 49 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 50 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 51 - Create Virtual Function (VF) and bind driver to vfio-pci n6001 Hardware This option creates vfio-pci driver binding for the PF's and VF's             Once the VF's have been bound to the driver the user can select menu option 45 to check that the new drivers are bound 52 - Program OpenCL BSP ofs_n6001 Default Kernel (hello_world) This option programs the FPGA with a aocf file based on the hello_world kernel 53 - Compile oneAPI Sample Application (board_test) for Hardware This option compiles the board_test kernel for Hardware 54 - Run oneAPI Sample Application (board_test) for Hardware This option builds the host code for board_test kernel and executes the program running through kernel and host bandwidth tests"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#319-adp-unit-test-project-menu","title":"3.1.9 ADP UNIT TEST PROJECT MENU","text":"
                                            Builds, compiles and runs standalone simulation block tests.  More unit test examples are found at the following location ofs_n6001/sim/unit_test 
                                             
                                             Menu Option Result 55 - Generate Simulation files for Unit Test This option builds the simulation file set for running a unit test simulation 56 - Simulate Unit Test dfh_walker and log waveform This option runs the dfh_walker based on the environment variable \"UNIT_TEST_NAME=dfh_walker\" in the evaluation script. A user can change the test being run by modifying this variable"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#3110-adp-uvm-project-menu","title":"3.1.10 ADP UVM PROJECT MENU","text":"
                                            Builds, compiles and runs full chip simulation tests.  The user should execute the options sequentially ie 68,69, 70 and 71
                                             
                                             Menu Option Description 57 - Check UVM software versions for n6001 Project DESIGNWARE_HOME is set to /home/synopsys/vip_common/vip_Q-2020.03A                 UVM_HOME  is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm                 VCS_HOME is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel                 VERDIR is set to /home/user_area/ofs-X.X.X/ofs-n6001/verification                 VIPDIR is set to /home/user_area/ofs-X.X.X/ofs-n6001/verification 58 - Compile UVM IP This option cmpiles the UVM IP  59 - Compile UVM RTL and Testbench This option compiles the UVM RTL and Testbench 60 - Simulate UVM dfh_walking_test and log waveform This option runs the dfh_walking test based on the environment variable \"UVM_TEST_NAME=dfh_walking_test\" in the evaluation script. A user can change the test being run by modifying this variable 61 - Simulate all UVM test cases (Regression Mode) This option runs the n6001 regression mode, cycling through all UVM tests defined in verification/tests/test_pkg.svh file"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#3111-adp-build-all-project-menu","title":"3.1.11 ADP BUILD ALL PROJECT MENU","text":"
                                            Builds the complete OFS flow, good for regression testing and overnight builds
                                             
                                            For this menu a user can run a sequence of tests (compilation, build and simulation) and executes them sequentially. After the script is successfully executed, a set of binary files is produced which a you can use to evaluate your hardware. Log files are also produced which checks whether the tests passed.
                                             
                                            A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 62 from the main menu the script will execute 24 tests ie (main menu options 2, 9, 12, 13, 14, 15, 16, 17, 18, 32, 34, 35, 37, 39, 40, 44, 45, 53, 55, 56, 57, 58, 59 and 60. These 24 menu options are chosen to build the complete OFS flow covering build, compile and simulation.
                                             
                                             Menu Option Result 62 - Build and Simulate Complete n6001 Project Generating Log File with date and timestamp                 Log file written to /home/guest/ofs-2.3.1/log_files/n6001_log_2022_11_10-093649/ofs_n6001_eval.log"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#definition-of-multi-test-set-up","title":"Definition of Multi-Test Set-up","text":"
                                            Menu Option 62 above in the evaluation script can be refined to tailor it to the users need and is principally defined by the variable below
                                             
                                            MULTI_TEST[A,B]=C
                                             
                                            where
                                             
                                            A= Total Number of menu options in script B= Can be changed to a number to select the test order C= Menu Option in Script
                                             
                                            Example 1 MULTI_TEST[62,0]=2
                                             
                                            A= 62 is the total number of options in the script B= 0 indicates that this is the first test to be run in the script C= Menu option in Script ie 2- List of Documentation for ADP n6001 Project
                                             
                                            Example 2 MULTI_TEST[62,0]=2 MULTI_TEST[62,1]=9
                                             
                                            In the example above two tests are run in order ie 0, and 1 and the following menu options are executed ie 2- List of Documentation for ADP n6001 Project and 9 - Check ADP software versions for ADP n6001 Project
                                             
                                            The user can also modify the build time by de-selecting options they do not wish to use, see below for a couple of use-case scenarios.
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#default-user-case","title":"Default User Case","text":"
                                            A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 62 from the main menu the script will execute 24 tests ie (main menu options 2, 9, 12, 13, 14, 15, 16, 17, 18, 32, 34, 35, 37, 39, 40, 44, 45, 53, 55, 56, 57, 58, 59 and 60. All other tests with an \"X\" indicates do not run that test.
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#user-case-for-adp-fimpr-build-menu","title":"User Case for ADP FIM/PR BUILD MENU","text":"
                                            In the example below when the user selects option 62 from the main menu the script will only run options from the ADP FIM/PR BUILD MENU (7 options, main menu options 12, 13, 14, 15, 16, 17 and 18). All other tests with an \"X\" indicates do not run that test.
                                             
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#4-n6001-common-test-scenarios","title":"4 n6001 Common Test Scenarios","text":"
                                            This section will describe the most common compile build scenarios if a user wanted to evaluate an acceleration card on their server. The Pre-requisite column indcates the menu comamnds that must be run befere executing the test eg To run Test 5 then a user needs to have run option 13, 15 and 16 before running options 23, 24, 25, 32 and 33.
                                             Test Test Scenario Pre-Requisite Menu Option Menu Option Test 1 FIM Build - 13 Test 2 Partial Reconfiguration Build 13 15, 16 Test 3 Program FIM and perform Remote System Upgrade 13 21, 22 Test 4 Bind PF and VF to vfio-pci drivers - 23, 24, 25 Test 5 Build, compile and test AFU on hardware 13, 15, 16 23, 24, 25, 32, 33 Test 6 Build, compile and test AFU Basic Building Blocks on hardware 13, 15, 16 23, 24, 25, 37, 38 Test 7 Build, compile and test oneAPI on hardware 13, 15, 16 39, 40, 41, 44, 45, 49, 50, 51, 52, 53, 54 Test 8 Build and Simulate Unit Tests - 55, 56 Test 9 Build and Simulate UVM Tests - 57, 58, 59, 60"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                            Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                             
                                            OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/","title":"Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex FPGAs Targeting the Intel\u00ae FPGA SmartNIC N6001-PL","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#10-introduction","title":"1.0 Introduction","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#11-about-this-document","title":"1.1 About This Document","text":"
                                            The purpose of this document is to help users get started in evaluating the 2023.3-2  version of the Open FPGA Stack (OFS) for the Intel Agilex FPGA targeting the Intel N6001-PL FPGA SmartNIC Platform. After reviewing the document a user shall be able to:
                                             
                                               
                                            • Set up their server environment according to the Best Known Configuration (BKC)
                                              •  
                                            • Build and install the OFS Linux Kernel drivers
                                              •  
                                            • Build and install the Open Programmable Acceleration Engine Software Development Kit (OPAE SDK) software on top of the OFS Linux kernel drivers
                                              •  
                                            • Load and Verify the Firmware and FIM versions loaded on their boards
                                              •  
                                            • Verify the full stack functionality offered by the OFS solution
                                              •  
                                            • Know where to find additional information on other OFS ingredients
                                              •  
                                             
                                            The following flow charts show a high level overview of the initial bring up process, split into three sequential diagrams. Detailed instructions for each installation process are shown in their respective sections.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#diagram-1-installing-the-opae-sdk","title":"Diagram 1: Installing the OPAE SDK","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#diagram-2-installing-the-linux-dfl-drivers","title":"Diagram 2: Installing the Linux DFL Drivers","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#diagram-3-bringing-up-the-intel-fpga-smartnic-n6001-pl","title":"Diagram 3: Bringing up the Intel\u00ae FPGA SmartNIC N6001-PL","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#12-terminology","title":"1.2 Terminology","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-1-glossary","title":"Table 1: Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel Max10 or Intel Cyclone10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implemented on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Management Component Transport Protocol MCTP A standardized model for communication with management controllers. Defines the transport protocol carrying PLDM messages through the BMC. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE-SDK The OPAE-SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Level Data Model PLDM A specification for reporting telemetry data to the host, such as board temperature, voltage, and current. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#13-introduction-to-ofs","title":"1.3 Introduction to OFS","text":"
                                            Each OFS FIM targets a specific platform, but the modular hardware, software, simulation and test infrastructure allow users to modify each part of the design and test environment for their own custom acceleration platform card. This OFS release targets the Intel\u00ae FPGA SmartNIC N6001-PL.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#14-intended-audience","title":"1.4 Intended Audience","text":"
                                            The information in this document is intended for customers evaluating the Intel\u00ae FPGA SmartNIC N6001-PL. The card is an acceleration development platform (ADP) intended to be used as a starting point for evaluation and development. This document will cover key topics related to initial bring up and development, with links for deeper dives on the topics discussed therein.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#15-reference-documents","title":"1.5 Reference Documents","text":"
                                            Documentation for N6001 is collected on GitHub. The document list is as follows:
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-2-reference-documents","title":"Table 2: Reference Documents","text":"Document Purpose Getting Started Guide: Intel Open FPGA Stack for Intel Agilex FPGA Guides you through the setup and build steps to evaluate the OFS solution targeting an Intel N6001-PL FPGA SmartNIC Platform FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA Describes the OFS FIM architecture and features. Software Reference Manual: Intel Open FPGA Stack Describes the Open Programmable Acceleration Engine (OPAE) Software Development Kit, the OPAE C++ and Python API and management interfaces.  This document also covers building the OPAE SDK, how to add a new PCIe device, and debugging the software stack. FPGA Interface Manager Developer Guide: Intel Open Stack for Intel Agilex FPGA Provides guidance on developing an FPGA Interface Manager (FIM) for a custom FPGA acceleration board. Hard Processor System Software Developer Guide: Intel OFS for Intel Agilex FPGAs Describes the HPS configuration and software package for the Intel OFS release targeting Intel Agilex OFS and guides you through the steps to build your own Yocto application image for HPS. Accelerator Functional Unit Developer Guide: Intel Open FPGA Stack Provides guidance on how to build and test an AFU when designing to an OFS-based FPGA Interface Manager Simulation User Guide: Intel Open FPGA Stack for Intel Agilex FPGA Provides steps for setting up the UVM verification tool suite and running UVM unit tests Security User Guide: Intel Open FPGA Stack (request access by emailing ofs.github@intel.com) Describes how to create keys and sign bitstreams for your custom design; includes new VAB feature oneAPI Accelerator Support Package (ASP): Getting Started User Guide Describes how to get started using OneAPI with the OFS FIM BSP. oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack Describes how to use the provided shim for OFS for creating your custom OneAPI board support package. Platform Evaluation Script: Open FPGA Stack for Intel Agilex FPGA Serves as a set-up and user guide for the checkout and evaluation of an Intel FPGA SmartNIC N6001 Platform using Open FPGA Stack (OFS)."},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#16-component-version-summary","title":"1.6 Component Version Summary","text":"
                                            The OFS 2023.3-2 Release targeting the Intel\u00ae FPGA SmartNIC N6001-PL is built upon tightly coupled software and firmware versions. Use this section as a general reference for the versions which compose this release.
                                             
                                            The following table highlights the hardware which makes up the Best Known Configuration (BKC) for the OFS 2023.3-2 release.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-3-hardware-bkc","title":"Table 3: Hardware BKC","text":"Component Version 1 x Intel\u00ae FPGA SmartNIC N6001-PL, SKU2 1 x Supermicro Server SYS-220HE 1 x Intel FPGA Download Cable II (Only Required for manual flashing) 1 x 2x5 Extension header - Samtech Part No: ESQ-105-13-L-D (Only Required for manual flashing) 
                                            The following table highlights the versions of the software which compose the OFS stack. The installation of the OPAE SDK on top of the Linux DFL drivers will be discussed in their relevant sections in this document.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-4-software-version-summary","title":"Table 4: Software Version Summary","text":"Component Version FPGA Platform Intel\u00ae FPGA SmartNIC N6001-PL, release notes: https://github.com/OFS/ofs-n6001/releases/tag/ofs-2023.3-2 under \"Known Issues\" OPAE SDK 2.10.0-1 Kernel Drivers ofs-2023.3-6.1-3 OneAPI-ASP ofs-2023.3-2 OFS FIM Source Code for N6001 ofs-2023.3-2 OFS FIM Common Resources Tag: ofs-fim-common-1.1.0-rc2 OFS Platform AFU BBB ofs-2023.3-2 Intel Quartus Prime Pro Edition Design Software* Quartus Prime Pro Version 23.3 for Linux Operating System RedHatEnterprise Linux\u00ae (RHEL) 8.6 
                                            The following table highlights the differences between N6000/1 PL FPGA SmartNIC platforms (SKU1/SKU2). Use this table to identify which version of the N6000/1-PL FPGA SmartNIC platform you have. The board identification printed by the fpgainfo fme commands depends on both the OPAE SDK and Linux DFL drivers from sections 3.0 OFS DFL Kernel Drivers through 4.0 OPAE Software Development Kit to be installed before it can be run.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-5-intel-n60001-pl-fpga-smartnic-platform-sku-mapping","title":"Table 5: Intel N6000/1-PL FPGA SmartNIC Platform SKU Mapping","text":"SKU Mapping SKU Value Primary Difference fpgainfo Identification N6000 Q1613314XXXXX PCIe Gen 4 1x16 mechanical bifurcated 2x8 logical to host, with one PCIe Gen 4x8 endpoint reserved for Intel E810-C-CAM2 NIC, the other reserved for FIM \"Intel Acceleration Development Platform N6000\" N6001 Q0216514XXXXX PCIe Gen 4 1x16 mechanical and logical connection between host and FIM \"Intel Acceleration Development Platform N6001\" 
                                            The following table highlights the programmable firmware versions that are supported on the Intel N6001-PL FPGA SmartNIC Platform in the OFS 2023.3-2 release. Programming and verifying these components is discussed in their respective sections.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-6-intel-fpga-smartnic-n6001-pl-programmable-component-version-summary","title":"Table 6: Intel\u00ae FPGA SmartNIC N6001-PL Programmable Component Version Summary","text":"Component Version PR Interface ID 1d6beb4e-86d7-5442-a763-043701fb75b7 Bitstream ID 00x50102023508A422 BMC RTL 3.15.0 BMC NIOS FW 3.15.0"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#17-initial-server-setup","title":"1.7 Initial Server Setup","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#171-server-bmc-and-bios-updates","title":"1.7.1 Server BMC and BIOS Updates","text":"
                                            Both the server BIOS and BMC need to match the versions listed below in Table 7: Supermicro Server BMC BKC. These updates only apply for this specific Best Known Configuration (BKC) - other server manufacturers may require different BIOS updates. Please consult your server's user guide and release notes for update information.
                                             
                                            Information about the server\u2019s currently loaded firmware can be found on the BMC web portal dashboard. Accessing this page requires an Ethernet cable to be attached to an open port on the server labelled \u201cIPMI\u201d. During boot the BMC\u2019s login IP will be presented on the screen.
                                             
                                            Open this IP address in a browser and enter your login credentials. The default username is ADMIN, and the default password has been printed on the service tag that pulls out from the front of the server case. It is recommended the user change their BMC\u2019s default username as soon as they are able
                                             
                                            After logging in you should be able to review information about the BMC and BIOS by referring to the System box, visible upon initial loading of the page. Double check that the values match those in Table 7 Supermicro Server BMC BKC. If they do not, you may download the appropriate versions from the Supermicro product page by selecting the BIOS option and downloading the most recent \u201cBundled Software File Name\u201d. Follow the BMC and BIOS update instructions included in the Supermicro manuals page in the document X12/H12 BMC Manual in Appendix A.2 Updating Firmware Using BMC Web GUI.
                                             
                                            If using a different server model, refer to that server\u2019s user guide for instructions on remote system management.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-7-supermicro-server-bmc-bkc","title":"Table 7: Supermicro Server BMC BKC","text":"Component Version BIOS Version American Megatrends International, LLC(1.4)"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#172-server-fan-speed","title":"1.7.2 Server Fan Speed","text":"
                                            The recommended fan speed setting is to use the 100% preset. If using a different server model, refer to that server\u2019s user guide for instructions on changing fan speed. The following steps will help users on the Supermicro platform.
                                             
                                               
                                            1. Log in to the Supermicro server BMC. (This requires an Ethernet cable to be attached to an open port on the server labelled \u201cIPMI\u201d.)
                                              2.  
                                            2. During boot the BMC\u2019s login IP will be presented on the screen. Open this IP address in a browser and enter your login credentials. The default username is ADMIN, and the default password has been printed on the service tag that pulls out from the front of the server case.
                                              3.  
                                            3. On the left menu select System -> Component Info, select the Fan tab, under Advanced Settings click the circle next to Full Speed.
                                              4.  
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#173-server-bios-configuration-changes","title":"1.7.3 Server BIOS Configuration Changes","text":"
                                            To enter the Supermicro server\u2019s BIOS setup page, reboot, and press \\<Delete> when prompted. You can browse the tabs / options with a combination of arrow keys along with \\<Escape> and \\<Enter>.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#1731-enabling-intel-vt-d-technology","title":"1.7.3.1 Enabling Intel VT-d Technology","text":"
                                            Navigate right to the Advanced tab, then select the following menu options: Chipset Configuration -> North Bridge -> IIO Configuration -> Intel VT for Directed I/O (VT-d). If not already, enable the option Intel VT for Directed I/O (VT-d).
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#1732-pcie-slot-mapping","title":"1.7.3.2 PCie Slot Mapping","text":"
                                            The Intel N6001-PL FPGA SmartNIC Platform is officially verified in the upper middle PCIe x16 slot (Slot 3). If using a different slot, refer to the information in Table 8 PCIe Slot Mapping for which port to manually change in the server BIOS.
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-8-pcie-slot-mapping","title":"Table 8: PCIe Slot Mapping","text":"CPU Number Port Number (in BIOS) PCIe Slot CPU1 Port 2 5 and 6 CPU1 Port 4 7 and 8 CPU2 Port 2 1 and 2 CPU2 Port 4 3 and 4"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#174-intel-fpga-smartnic-n6001-pl-installation-procedure","title":"1.7.4 Intel\u00ae FPGA SmartNIC N6001-PL Installation Procedure","text":"
                                            The following instructions will help to ensure safe installation of the Intel\u00ae FPGA SmartNIC N6001-PL into a supported server. Refer to Table 8 PCIe Slot Mapping for a breakdown of the available PCIe slots on a SuperMicro SYS-220HE.
                                             
                                               
                                            1. Position the board over the selected connector on the motherboard.
                                              2.  
                                            2. Press down gently and firmly to seat the card in the PCIe slot, and then secure the bracket to the system chassis with the retention screw.
                                              3.  
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-9-intel-fpga-smartnic-n6001-pl-installation-procedure","title":"Table 9: Intel\u00ae FPGA SmartNIC N6001-PL Installation Procedure","text":"Callout Description 1 Retention screw 2 Press down here gently 3 Press down here gently 4 Motherboard 
                                            Do not bend the card while inserting into a slot. Do not apply much pressure in regions 2 or 3 while inserting.
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#175-intel-fpga-smartnic-n6001-pl-removal-procedure","title":"1.7.5 Intel\u00ae FPGA SmartNIC N6001-PL Removal Procedure","text":"
                                            The following instructions will help to ensure safe removal of the platform from a supported server.
                                             
                                               
                                            1. Disconnect all power cords from the server power supply(s).
                                              2.  
                                            2. Remove the retention bracket screw.
                                              3.  
                                            3. Carefully lift the card out of the PCIe slot.
                                              4.  
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-10-intel-fpga-smartnic-n6001-pl-removal-procedure","title":"Table 10: Intel\u00ae FPGA SmartNIC N6001-PL Removal Procedure","text":"Callout Description 1 Retention screw 2 Pull up here gently 3 Motherboard 
                                            Do not bend the card while removing it from the slot.
                                             
                                             
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#20-ofs-stack-architecture-overview-for-reference-platform","title":"2.0 OFS Stack Architecture Overview for Reference Platform","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#21-hardware-components","title":"2.1 Hardware Components","text":"
                                            The OFS hardware architecture decomposes all designs into a standard set of modules, interfaces, and capabilities. Although the OFS infrastructure provides a standard set of functionality and capability, the user is responsible for making the customizations to their specific design in compliance with the specifications outlined in the FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA.
                                             
                                            OFS is a hardware and software infrastructure that provides an efficient approach to developing a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#211-fpga-interface-manager","title":"2.1.1 FPGA Interface Manager","text":"
                                            The FPGA Interface Manager (FIM), or shell of the FPGA provides platform management functionality, clocks, resets, and interface access to the host and peripheral features on the acceleration platform. The OFS architecture for Intel Agilex FPGA provides modularity, configurability and scalability.  The primary components of the FPGA Interface Manager or shell of the reference design are:
                                             
                                               
                                            • PCIe Subsystem - a hierarchical design that targets the P-tile PCIe hard IP and is configured to support Gen4 speeds and Arm AXI4-Stream Data Mover functional mode.
                                              •  
                                            • Ethernet Subsystem - provides portability to different Ethernet configurations across platforms and generations and reusability of the hardware framework and software stack.
                                              •  
                                            • Memory Subsystem - composed of 5 DDR4 channels; one HPS DDR4 bank, x40 (x32 Data and x8 ECC), 1200 MHz, 1GB each, and four Fabric DDR4 banks, x32 (no ECC), 1200 MHz, 4GB
                                              •  
                                            • Hard Processor System - 64-bit quad core ARM\u00ae Cortex*-A53 MPCore with integrated peripherals.
                                              •  
                                            • Reset Controller
                                              •  
                                            • FPGA Management Engine - Provides a way to manage the platform and enable acceleration functions on the platform.
                                              •  
                                            • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                                              •  
                                            • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                                              •  
                                            • Platform Management Controller Interface (PMCI) to the board management controller
                                              •  
                                             
                                            The FPGA Management Engine (FME) provides management features for the platform and the loading/unloading of accelerators through partial reconfiguration. Each feature of the FME exposes itself to the kernel-level OFS drivers on the host through a Device Feature Header (DFH) register that is placed at the beginning of Control Status Register (CSR) space. Only one PCIe link can access the FME register space in a multi-host channel design architecture at a time.
                                             
                                            Note: For more information on the FIM and its external connections, refer to the FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#212-afu","title":"2.1.2 AFU","text":"
                                            An AFU is an acceleration workload that interfaces to the FIM. The AFU boundary in this reference design comprises both static and partial reconfiguration (PR) regions.  You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region. A port gasket within the design provides all the PR specific modules and logic required to support partial reconfiguration. Only one partial reconfiguration region is supported in this design.
                                             
                                            Similar to the FME, the port gasket exposes its capability to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space.  
                                             
                                            You can compile your design in one of the following ways:
                                             
                                               
                                            • Your entire AFU resides in a partial reconfiguration region of the FPGA.
                                              •  
                                            • The AFU is part of the static region and is compiled as a flat design.
                                              •  
                                            • Your AFU contains both static and PR regions.
                                              •  
                                             
                                            In this design, the AFU region is comprised of:
                                             
                                               
                                            • AFU Interface handler to verify transactions coming from AFU region.
                                              •  
                                            • PF/VF Mux to route transactions to and from corresponding AFU components: ST2MM module, Virtio LB stub, PCIe loopback host exerciser (HE-LB), HSSI host exerciser (HE-HSSI), Memory Host Exerciser (HE-MEM), Traffic Generator to memory (HE-MEM-TG), Port Gasket (PRG) and HPS Copy Engine.
                                              •  
                                            • AXI4 Streaming to Memory Map (ST2MM) Module that routes MMIO CSR accesses to FME and board peripherals.
                                              •  
                                            • Host exercisers to test PCIe, memory and HSSI interfaces (these can be removed from the AFU region after your FIM design is complete to provide more resource area for workloads)
                                              •  
                                            • Basic HPS Copy Engine to copy second-stage bootloader and Linux OS image from Host DDR to HPS DDR.  
                                              •  
                                            • Port gasket and partial reconfiguration support.
                                              •  
                                            • Component for handling PLDM over MCTP over PCIe Vendor Defined Messages (VDM)
                                              •  
                                             
                                            The AFU has the option to consume native packets from the host or interface channels or to instantiate a shim provided by the Platform Interface Manager (PIM) to translate between protocols.
                                             
                                            Note: For more information on the Platform Interface Manager and AFU development and testing, refer to the AFU Development Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#22-ofs-software-overview","title":"2.2 OFS Software Overview","text":"
                                            The responsibility of the OFS kernel drivers is to act as the lowest software layer in the FPGA software stack, providing a minimalist driver implementation between the host software and functionality that has been implemented on the development platform. This leaves the implementation of IP-specific software in user-land, not the kernel. The OFS software stack also provides a mechanism for interface and feature discovery of FPGA platforms.
                                             
                                            The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space, and can be found on the OPAE SDK Github.
                                             
                                            The OFS drivers decompose implemented functionality, including external FIM features such as HSSI, EMIF and SPI, into sets of individual Device Features. Each Device Feature has its associated Device Feature Header (DFH), which enables a uniform discovery mechanism by software. A set of Device Features are exposed through the host interface in a Device Feature List (DFL). The OFS drivers discover and \"walk\" the Device Features in a Device Feature List and associate each Device Feature with its matching kernel driver.
                                             
                                            In this way the OFS software provides a clean and extensible framework for the creation and integration of additional functionalities and their features.
                                             
                                            *Note: A deeper dive on available SW APIs and programming model is available in the Software Reference Manual: Intel\u00ae Open FPGA Stack and on kernel.org.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#30-ofs-dfl-kernel-drivers","title":"3.0 OFS DFL Kernel Drivers","text":"
                                            OFS DFL driver software provides the bottom-most API to FPGA platforms. Libraries such as OPAE and frameworks like DPDK are consumers of the APIs provided by OFS. Applications may be built on top of these frameworks and libraries. The OFS software does not cover any out-of-band management interfaces. OFS driver software is designed to be extendable, flexible, and provide for bare-metal and virtualized functionality. An in depth look at the various aspects of the driver architecture such as the API, an explanation of the DFL framework, and instructions on how to port DFL driver patches to other kernel distributions can be found on https://github.com/OPAE/linux-dfl/wiki.
                                             
                                            An in-depth review of the Linux device driver architecture can be found on opae.github.io.
                                             
                                            The DFL driver suite can be automatically installed using a supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3-2 Release Page.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#31-ofs-dfl-kernel-driver-environment-setup","title":"3.1 OFS DFL Kernel Driver Environment Setup","text":"
                                            All OFS DFL kernel driver code resides in the Linux DFL GitHub repository. This repository is open source and should not require any permissions to access. It includes a snapshot of the Linux kernel with the OFS driver included in /drivers/fpga/*. Downloading, configuration, and compilation will be discussed in this section. The only operating systems supported out of the box by the Linux DFL kernel drivers is RHEL 8.6.
                                             
                                            This installation process assumes the user has access to an internet connection in order to clone specific GitHub repositories, and to satisfy package dependencies.
                                             
                                            1. Make the following changes on your installation machine to satisfy all dependencies:
                                             
                                            $ subscription-manager release --set=8.6\n$ sudo dnf update\n
                                             
                                            If you wish to install the pre-built linux-dfl package available on the OFS 2023.3-2 Release Page skip to section 3.3 Installing the OFS DFL Kernel Drivers from Pre-Built Packages.
                                             
                                            2. You must satisfy the following package dependencies if building and installing the drivers from source. Double check that all packages have been found and installed. The following section assumes you require use of a proxy to download from remote repositories. If you do not, then you may safely ignore all references to proxies in the following code block.
                                             
                                            # If you require the use of a proxy, add it to DNF using by editing the following file\n$ sudo nano /etc/dnf/dnf.conf\n# Include your proxy by adding the following line, replacing the URL with your proxy's URL\n# proxy=http://proxy.server.com:port``\n$ sudo dnf update\n$ subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms\n$ sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n\n$ sudo dnf install -y python3 python3-pip python3-devel \\\ngdb vim git gcc gcc-c++ make cmake libuuid-devel rpm-build systemd-devel nmap \\\npython3-jsonschema json-c-devel tbb-devel rpmdevtools libcap-devel \\\npython3-pyyaml hwloc-devel libedit-devel git kernel-headers kernel-devel elfutils-libelf-devel ncurses-devel openssl-devel bison flex cli11-devel spdlog-devel numactl-devel\n\n$ python3 -m pip install --user jsonschema virtualenv pudb pyyaml\n\n$ sudo pip3 uninstall setuptools\n\n$ sudo pip3 install Pybind11==2.10.0 --proxy http://yourproxy:xxx\n\n$ sudo pip3 install setuptools==59.6.0 --prefix=/usr --proxy http://yourproxy:xxx\n\n$ wget http://ftp.pbone.net/mirror/archive.fedoraproject.org/epel/8.4/Everything/x86_64/Packages/p/pybind11-devel-2.4.3-2.el8.x86_64.rpm\n\n$ wget http://ftp.pbone.net/mirror/archive.fedoraproject.org/epel/8.4/Everything/x86_64/Packages/p/python3-pybind11-2.4.3-2.el8.x86_64.rpm\n\n$ sudo dnf localinstall ./python3-pybind11-2.4.3-2.el8.x86_64.rpm ./pybind11-devel-2.4.3-2.el8.x86_64.rpm -y\n
                                             
                                            It is recommended you create an empty top level directory for your OFS related repositories to keep the working environment clean. All steps in this installation will use a generic top-level directory at /home/OFS/. If you have created a different top-level directory, replace this path with your custom path.
                                             
                                            3. Initialize an empty git repository and clone the LTS tagged DFL driver source code:
                                             
                                            $ mkdir /home/OFS/\n$ cd /home/OFS/\n$ git init\n$ git clone https://github.com/OFS/linux-dfl\n$ cd /home/OFS/linux-dfl\n$ git checkout tags/ofs-2023.3-6.1-3\n
                                             
                                            4. Verify that the correct tag/branch have been checkout out.
                                             
                                            $ git describe --tags\nofs-2023.3-6.1-3\n
                                             
                                            Note: If two different tagged releases are tied to the same commit, running git describe tags may report the other release's tag. This is why the match is made explicit.
                                             
                                            Note: The linux-dfl repository is roughly 5 GB in size.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#32-building-and-installing-the-ofs-dfl-kernel-drivers-from-source","title":"3.2 Building and Installing the OFS DFL Kernel Drivers from Source","text":"
                                            1. The following set of instructions walk you through copying an existing kernel configuration file on your machine and changing the minimal required configuration settings.:
                                             
                                            $ cd /home/OFS/linux-dfl\n$ cp /boot/config-`uname -r` .config\n$ cat configs/dfl-config >> .config\n$ echo 'CONFIG_LOCALVERSION=\"-dfl\"' >> .config\n$ echo 'CONFIG_LOCALVERSION_AUTO=y' >> .config\n$ sed -i -r 's/CONFIG_SYSTEM_TRUSTED_KEYS=.*/CONFIG_SYSTEM_TRUSTED_KEYS=\"\"/' .config\n$ sed -i '/^CONFIG_DEBUG_INFO_BTF/ s/./#&/' .config\n$ echo 'CONFIG_DEBUG_ATOMIC_SLEEP=y' >> .config\n$ export LOCALVERSION=\n$ make olddefconfig\n
                                             
                                            1.1. The above command may report errors resembling symbol value 'm' invalid for CHELSIO_IPSEC_INLINE. These errors indicate that the nature of the config has changed between the currently executing kernel and the kernel being built. The option \"m\" for a particular kernel module is no longer a valid option, and the default behavior is to simply turn the option off. However the option can likely be turned back on by setting it to 'y'. If the user wants to turn the option back on, change it to 'y' and re-run \"make olddefconfig\":
                                             
                                            $ cd /home/OFS/linux-dfl\n$ echo 'CONFIG_CHELSIO_IPSEC_INLINE=y' >> .config\n$ make olddefconfig\n
                                             
                                            (Optional) To use the built-in GUI menu for editing kernel configuration parameters, you can opt to run make menuconfig.
                                             
                                            2. Linux kernel builds take advantage of multiple processors to parallelize the build process. Display how many processors are available with the nproc command, and then specify how many make threads to utilize with the -j option. Note that number of threads can exceed the number of processors. In this case, the number of threads are set to the number of processors in the system.
                                             
                                            $ cd /home/OFS/linux-dfl\n$ make -j $(nproc)\n
                                             
                                            3. The following options are available to build a set of packages. The package options for this flow as as follows:
                                             
                                               
                                            • rpm-pkg: Build both source and binary RPM kernel packages
                                              •  
                                            • binrpm-pkg: Build only the binary kernel RPM package
                                              •  
                                            • deb-pkg: Build both source and binary deb kernel packages
                                              •  
                                            • bindeb-pkg: Build only the binary kernel deb package
                                              •  
                                             
                                            If you are concerned about the size of the resulting package and binaries, they can significantly reduce the size of the package and object files by using the make variable INSTALL_MOD_STRIP. If this is not a concern, feel free to skip this step. The below instructions will build a set of binary RPM packages:
                                             
                                            cd /home/OFS/linux-dfl\nmake INSTALL_MOD_STRIP=1 binrpm-pkg\n
                                             
                                            4. By default a directory is created in your home directory called rpmbuild. This directory will house all of the kernel packages which have been built. You need to navigate to the newly built kernel packages and install them. The following files were generated using the build command executed in the previous step:
                                             
                                            $ cd ~/rpmbuild/RPMS/x86_64\n$ ls\nkernel-6.1.41_dfl.x86_64.rpm  kernel-headers-6.1.41_dfl.x86_64.rpm\n$ sudo dnf localinstall kernel*.rpm\n
                                             
                                            5. The system will need to be rebooted in order for changes to take affect. After a reboot, select the newly built kernel as the boot target. This can be done pre-boot using the command grub2-reboot, which removes the requirement for user intervention. After boot, verify that the currently running kernel matches expectation.
                                             
                                            $ uname -r\n6.1.41-dfl\n
                                             
                                            6. Verify the DFL drivers have been successfully installed by reading version information directly from /lib/modules. Recall that the name of the kernel built as apart of this section is 6.1.41-dfl. If the user set a different name for their kernel, change this path as needed:
                                             
                                            $ cd /usr/lib/modules/6.1.41-dfl/kernel/drivers/fpga\n$ ls\ndfl-afu.ko     dfl-fme.ko      dfl-fme-region.ko  dfl.ko             dfl-pci.ko      fpga-mgr.ko     intel-m10-bmc-sec-update.ko\ndfl-fme-br.ko  dfl-fme-mgr.ko  dfl-hssi.ko        dfl-n3000-nios.ko  fpga-bridge.ko  fpga-region.ko\n
                                             
                                            If an N6000/1-PL FPGA SmartNIC Platform is present on the server, you can also double check driver versions using the lsmod command:
                                             
                                            $ lsmod | grep dfl\nuio_dfl                20480  0\ndfl_emif               16384  0\nuio                    20480  1 uio_dfl\nptp_dfl_tod            16384  0\ndfl_intel_s10_iopll    20480  0\n8250_dfl               20480  0\ndfl_fme_region         20480  0\ndfl_fme_br             16384  0\ndfl_fme_mgr            20480  2\ndfl_fme                49152  0\ndfl_afu                36864  0\ndfl_pci                20480  0\ndfl                    40960  11 dfl_pci,uio_dfl,dfl_fme,intel_m10_bmc_pmci,dfl_fme_br,8250_dfl,qsfp_mem,ptp_dfl_tod,dfl_afu,dfl_intel_s10_iopll,dfl_emif\nfpga_region            20480  3 dfl_fme_region,dfl_fme,dfl\nfpga_bridge            20480  4 dfl_fme_region,fpga_region,dfl_fme,dfl_fme_br\nfpga_mgr               20480  4 dfl_fme_region,fpga_region,dfl_fme_mgr,dfl_fme\n
                                             
                                            Refer to the following table for a brief description of each driver:
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-11-dfl-driver-modules","title":"Table 11: DFL Driver Modules","text":"Module Name Short Description uio_dfl Generic DFL driver for Userspace I/O devices dfl_intel_s10_iopll DFL Intel S10 IOPLL driver uio UIO Driver dfl_fme_region FPGA Region for DFL FPGA Management Engine ptp_dfl_tod DFL ToD driver dfl_emif DFL EMIF driver dfl_fme_br FPGA Bridge for DFL FPGA Management Engine 8250_dfl DFL Intel UART driver dfl_fme_mgr FPGA Manager for DFL FPGA Management Engine dfl_fme FPGA Management Engine driver dfl_afu FPGA Accelerated Function Unit driver dfl_pci FPGA DFL PCIe Device Driver dfl FPGA Device Feature List (DFL) Support fpga_region FPGA Region Driver fpga_bridge FPGA Bridge Driver fpga_mgr FPGA manager framework 
                                            7. Two kernel parameters must be added to the boot commandline for the newly installed kernel. First, open the file grub:
                                             
                                            $ sudo vim /etc/default/grub\n
                                             
                                            8. In the variable GRUB_CMDLINE_LINUX add the following parameters in bold: GRUB_CMDLINE_LINUX=\"crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\"
                                             
                                            Note: If you wish to instead set hugepages on a per session bassis, you can perform the following steps. These settings will be lost on reboot.
                                             
                                            $ mkdir -p /mnt/huge \n$ mount -t hugetlbfs nodev /mnt/huge \n$ echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages \n$ echo 2048 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages \n
                                             
                                            9. Save your edits, then apply them to the GRUB2 configuration file.
                                             
                                            $ sudo grub2-mkconfig\n
                                             
                                            10. Warm reboot. Your kernel parameter changes should have taken affect.
                                             
                                            $ cat /proc/cmdline\nBOOT_IMAGE=(hd1,gpt2)/vmlinuz-6.1.41-dfl root=/dev/mapper/cl-root ro crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200 rhgb quiet\n
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#33-installing-the-ofs-dfl-kernel-drivers-from-pre-built-packages","title":"3.3 Installing the OFS DFL Kernel Drivers from Pre-Built Packages","text":"
                                            1. Make the following changes on your installation machine to satisfy all dependencies:
                                             
                                            $ subscription-manager release --set=8.6\n$ sudo dnf update\n
                                             
                                            2. To use the pre-built Linux DFL packages, the user will need to download the files from the OFS 2023.3-2 Release Page. You can choose to either install using the SRC RPMs, or to use the pre-built RPM packages targeting the official supported release platform.
                                             
                                            $ tar xf kernel-6.1.41_dfl-1.x86_64-<<version>>.tar.gz\n$ sudo dnf localinstall kernel-6.1.41_dfl_<<version>>.x86_64.rpm \\\nkernel-devel-6.1.41_dfl_<<version>>.x86_64.rpm \\\nkernel-headers-6.1.41_dfl_<<version>>.x86_64.rpm\n### OR\n$ sudo dnf localinstall kernel-6.1.41_dfl_<<version>>.src.rpm\n
                                             
                                            3. After installation has completed you should continue with steps 4-9 in previous section 3.2 Building and Installing the OFS DFL Kernel Drivers from Source.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#40-opae-software-development-kit","title":"4.0 OPAE Software Development Kit","text":"
                                            The OPAE SDK software stack sits in user space on top of the OFS kernel drivers. It is a common software infrastructure layer that simplifies and streamlines integration of programmable accelerators such as FPGAs into software applications and environments. OPAE consists of a set of drivers, user-space libraries, and tools to discover, enumerate, share, query, access, manipulate, and reconfigure programmable accelerators. OPAE is designed to support a layered, common programming model across different platforms and devices. To learn more about OPAE, its documentation, code samples, an explanation of the available tools, and an overview of the software architecture, visit the opae reference page.
                                             
                                            The OPAE SDK source code is contained within a single GitHub repository hosted at the OPAE Github. This repository is open source and does not require any permissions to access. You have two options to install OPAE as discussed below - using pre-built packages offered by Intel, or building the source code locally.
                                             
                                            The OPAE SDK can be automatically installed using a supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3-2 Release Page.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#41-opae-sdk-build-environment-setup","title":"4.1 OPAE SDK Build Environment Setup","text":"
                                            This installation process assumes the you have access to an internet connection in order to pull specific GitHub repositories, and to satisfy package dependencies.
                                             
                                            1. Before OPAE SDK installation you must remove any prior OPAE frameworks.  To remove these packages:
                                             
                                            sudo dnf remove opae*\n
                                             
                                            2. The following repository changes must be enabled in order to install all dependencies:
                                             
                                            $ subscription-manager release --set=8.6\n$ sudo dnf update\n$ subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms\n$ sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n
                                             
                                            It is recommended you create an empty top level directory for their OFS related repositories to keep the working environment clean. All steps in this installation will use a generic top-level directory at /home/OFS/. If the you have created a different top-level directory, replace this path with your custom path.
                                             
                                            3. Initialize an empty git repository and clone the tagged OPAE SDK source code:
                                             
                                            $ cd /home/OFS/\n$ git init\n$ git clone https://github.com/OFS/opae-sdk opae-sdk\n$ cd /home/OFS/opae-sdk\n$ git checkout tags/2.10.0-1\n
                                             
                                            5. Verify that the correct tag/branch have been checkout out.
                                             
                                            $ git describe --tags\n2.10.0-1\n
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#42-installing-the-opae-sdk-with-pre-built-packages","title":"4.2 Installing the OPAE SDK with Pre-Built Packages","text":"
                                            You can skip the entire build process and use a set of pre-built binaries supplied by Intel. Visit the OFS ofs-2023.3-2 Release Page and navigate to the bottom of the page, under the Assets tab you will see a file named opae-2.10.0-1.x86_64-<>_<>.tar.gz. Download this package and extract its contents: 
                                            $ dnf install --enablerepo=codeready-builder-for-rhel-8-x86_64-rpms -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel\n\n$ pip3 install --upgrade --prefix=/usr pip setuptools pybind11\n\n$ tar xf opae-2.10.0-1.x86_64-<<date>>_<<build>>.tar.gz\n
                                             
                                            For a fast installation you can delete the source RPM as it isn't necessary, and install all remaining OPAE RPMs:
                                             
                                            $ rm opae-*.src.rpm\n$ sudo dnf localinstall opae*.rpm\n
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#43-building-and-installing-the-opae-sdk-by-script","title":"4.3 Building and Installing the OPAE SDK by Script","text":"
                                            1. Build the OPAE SDK source code, and pack it into several local RPM packages. Building the code into packages allows for easier installation and removal. This build script can take advantage of multiple processors to parallelize the build process. Display how many processors are available with the $(nproc) command, and then specify how many make threads to utilize with the -j option. Note that number of threads can exceed the number of processors. In this case, the number of threads are set to the number of processors in the system.
                                             
                                            cd /home/OFS/\n\n$ podman pull registry.access.redhat.com/ubi8:8.6\n$ podman run -ti -v \"$PWD\":/src:Z -w /src registry.access.redhat.com/ubi8:8.6\n\n# Everything after runs within container:\n\n# Enable EPEL\n$ dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n\n$ dnf install --enablerepo=codeready-builder-for-rhel-8-x86_64-rpms -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel numactl-devel\n\n$ pip3 install --upgrade --prefix=/usr pip setuptools pybind11\n\n$ ./opae-sdk/packaging/opae/rpm/create unrestricted\n\n$ exit\n
                                             
                                            2. After a successful compile there should be packages generated:
                                             
                                            The below table lists a short description for each package:
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-12-opae-package-description","title":"Table 12: OPAE Package Description","text":"Package Name Description opae OPAE SDK is a collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. It provides a library implementing the OPAE C API for presenting a streamlined and easy-to-use interface for software applications to discover, access, and manage FPGA devices and accelerators using the OPAE software stack. opae-debuginfo This package provides debug information for package opae. Debug information is useful when developing applications that use this package or when debugging this package. opae-debugsource This package provides debug sources for package opae. Debug sources are useful when developing applications that use this package or when debugging this package. opae-devel OPAE headers, tools, sample source, and documentation opae-devel-debuginfo This package provides debug information for package opae-devel. Debug information is useful when developing applications that use this package or when debugging this package. opae-tools This package contains OPAE base tools binaries opae-extra-tools Additional OPAE tools opae-extra-tools-debuginfo This package provides debug information for package opae-extra-tools. Debug information is useful when developing applications that use this package or when debugging this package. 
                                            3. Install the OPAE SDK packages:
                                             
                                            $ cd /home/OFS/opae-sdk/packaging/opae/rpm\n$ rm -rf opae-2.10.0-1.el8.src.rpm \n$ sudo dnf localinstall -y opae*.rpm\n
                                             
                                            4. Check that all packages have been installed:
                                             
                                            $ rpm -qa | grep opae\nopae-packager-2.10.0-1.x86_64\nopae-devel-2.10.0-1.x86_64\nopae-PACSign-2.10.0-1.x86_64\nopae-tools-extra-2.10.0-1.x86_64\nopae-2.10.0-1.x86_64\nopae-tools-2.10.0-1.x86_64\nopae-libs-2.10.0-1.x86_64\nopae-opae.admin-2.10.0-1.x86_64\nopae-tests-2.10.0-1.x86_64\n
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#44-fpga-device-access-permissions","title":"4.4 FPGA Device Access Permissions","text":"
                                            Access to FPGA accelerators and devices is controlled using file access permissions on the Intel\u00ae FPGA device files, /dev/dfl-fme.* and /dev/dfl-port.*, as well as to the files reachable through /sys/class/fpga_region/.
                                             
                                            In order to allow regular (non-root) users to access accelerators, you need to grant them read and write permissions on /dev/dfl-port.* (with * denoting the respective socket, i.e. 0 or 1). E.g.:
                                             
                                            sudo chmod a+rw /dev/dfl-port.0\n
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#45-memlock-limit","title":"4.5 Memlock limit","text":"
                                            Depending on the requirements of your application, you may also want to increase the maximum amount of memory a user process is allowed to lock. The exact way to do this depends on your Linux distribution.
                                             
                                            You can check the current memlock limit using
                                             
                                            ulimit -l\n
                                             
                                            A way to permanently remove the limit for locked memory for a regular user is to add the following lines to your /etc/security/limits.conf:
                                             
                                            user1    hard   memlock           unlimited\nuser1    soft   memlock           unlimited\n
                                             
                                            This removes the limit on locked memory for user user1. To remove it for all users, you can replace user1 with *:
                                             
                                            *    hard   memlock           unlimited\n*    soft   memlock           unlimited\n
                                             
                                            Note that settings in the /etc/security/limits.conf file don't apply to services. To increase the locked memory limit for a service you need to modify the application's systemd service file and add the line:
                                             
                                            [Service]\nLimitMEMLOCK=infinity\n
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#46-opae-tools-overview","title":"4.6 OPAE Tools Overview","text":"
                                            The following section offers a brief introduction including expected output values for the utilities included with OPAE. A full explanation of each command with a description of its syntax is available in the opae-sdk GitHub repo.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#461-board-management-with-fpgainfo","title":"4.6.1 Board Management with fpgainfo","text":"
                                            The fpgainfo utility displays FPGA information derived from sysfs files.
                                             
                                            Displays FPGA information derived from sysfs files. The command argument is one of the following: errors, power, temp, port, fme, bmc, phy or mac, security. Some commands may also have other arguments or options that control their behavior.
                                             
                                            For systems with multiple FPGA devices, you can specify the BDF to limit the output to the FPGA resource with the corresponding PCIe configuration. If not specified, information displays for all resources for the given command.
                                             
                                            Note: Your BItstream ID and PR Interface Id may not match the below examples.
                                             
                                            The following examples walk through sample outputs generated by fpgainfo.
                                             
                                            $ sudo fpgainfo fme\n\nIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : a2b5fd0e7afca4ee6d7048f926e75ac2\nUser1 Image Info                 : 1d6beb4e-86d7-5442-a763-043701fb75b7\nUser2 Image Info                 : 1d6beb4e-86d7-5442-a763-043701fb75b7\n
                                             
                                            $ sudo fpgainfo bmc\n\nIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\n( 1) VCCRT_GXER_0V9 Voltage                             : 0.91 Volts\n( 2) FPGA VCCIO_1V2 Voltage                             : 1.21 Volts\n( 3) Inlet 12V Aux Rail Current                         : 0.87 Amps\n( 4) FPGA E-Tile Temperature [Remote]                   : 47.00 Celsius\n( 5) AVDD_ETH_0V9_CVL Voltage                           : 1.48 Volts\n( 6) FPGA E-TILE Temperature #3                         : 51.00 Celsius\n...\n(77) FPGA FABRIC Remote Digital Temperature#3           : 47.00 Celsius\n(78) MAX10 & Board CLK PWR 3V3 Inlet Current            : 0.97 Amps\n(79) CVL Non Core Rails Inlet Current                   : 0.01 Amps\n(80) FPGA Core Voltage Phase 0 VR Temperature           : 49.50 Celsius\n
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#462-sensor-monitoring-with-fpgad","title":"4.6.2 Sensor Monitoring with fpgad","text":"
                                            The fpgad is a service that can help you protect the server from crashing when the hardware reaches an upper non-recoverable or lower non-recoverable sensor threshold (also called as fatal threshold). The fpgad is capable of monitoring each of the 80 sensors reported by the Board Management Controller. This service is only available once the installation instructions in sections 3.2 Building and Installing the OFS DFL Kernel Drivers and 4.1 OPAE SDK Build Environment Setup have been completed . Note: Qualified OEM server systems should provide the required cooling for your workloads. Therefore, using fpgad may be optional.
                                             
                                            When the opae-tools-extra-2.10.0-1.x86_64  package is installed, fpgad is placed in the OPAE binaries directory (default: /usr/bin). The configuration file fpgad.cfg is located at /etc/opae. The log file fpgad.log which monitors fpgad actions is located at /var/lib/opae/. The fpgad periodically reads the sensor values and if the values exceed the warning threshold stated in the fpgad.conf or the hardware defined warning threshold, it masks the PCIe Advanced Error Reporting (AER) registers for the Intel N6000/1-PL FPGA SmartNIC Platform to avoid system reset. Use the following command to start the fpgad service:
                                             
                                            Use the following command to start the fpgad service:
                                             
                                            $ sudo systemctl start fpgad\n
                                             The configuration file only includes the threshold setting for critical sensor 12V Aux Rail Voltage (sensor 29). This sensor does not have a hardware defined warning threshold and hence fpgad relies on the configuration file. The fpgad uses information contained within this file to mask the PCIe AER register when the sensor reaches the warning threshold.
                                             
                                            You may create another entry below the 12V Aux Voltage entry for any other sensors on the board. The updated configuration file includes a new entry for (18) Board Front Side Temperature with arbitrary values:
                                             
                                            {\n\"configurations\": {\n    \"fpgad-xfpga\": {\n        \"configuration\": {\n        },\n        \"enabled\": true,\n        \"plugin\": \"libfpgad-xfpga.so\",\n        \"devices\": [\n            [ \"0x8086\", \"0xbcc0\" ],\n            [ \"0x8086\", \"0xbcc1\" ]\n        ]\n    },\n    \"fpgad-vc\": {\n        \"configuration\": {\n            \"cool-down\": 30,\n            \"get-aer\": [\n                \"setpci -s %s ECAP_AER+0x08.L\",\n                \"setpci -s %s ECAP_AER+0x14.L\"\n            ],\n            \"disable-aer\": [\n                \"setpci -s %s ECAP_AER+0x08.L=0xffffffff\",\n                \"setpci -s %s ECAP_AER+0x14.L=0xffffffff\"\n            ],\n            \"set-aer\": [\n                \"setpci -s %s ECAP_AER+0x08.L=0x%08x\",\n                \"setpci -s %s ECAP_AER+0x14.L=0x%08x\"\n            ],\n            \"config-sensors-enabled\": true,\n            \"sensors\": [\n                {\n                    \"name\": \"12V AUX Voltage\",\n                    \"low-warn\": 11.40,\n                    \"low-fatal\": 10.56\n                },\n                {\n                    \u201cname\u201d: \u201c3V3 VR Temperature\u201d,\n                    \u201clow-warn\u201d: 50.00,\n                    \u201clow-fatal\u201d: 100.00\n\n                }\n            ]\n        },\n        \"enabled\": true,\n        \"plugin\": \"libfpgad-vc.so\",\n        \"devices\": [\n            [ \"0x8086\", \"0x0b30\" ],\n            [ \"0x8086\", \"0x0b31\" ],\n            [ \"0x8086\", \"0xaf00\" ],\n            [ \"0x8086\", \"0xbcce\" ]\n            ]\n    }\n},\n\"plugins\": [\n    \"fpgad-xfpga\",\n    \"fpgad-vc\"\n]\n}\n
                                             
                                            You can monitor the log file to see if upper or lower warning threshold levels are hit. For example:
                                             
                                            $ tail -f /var/lib/opae/fpgad.log | grep \u201csensor.*warning\u201d\nfpgad-vc: sensor ' Columbiaville Die Temperature ' warning\n
                                             
                                            You must take appropriate action to recover from this warning before the sensor value reaches upper or lower fatal limits. On reaching the warning threshold limit, the daemon masks the AER registers and the log file will indicate that the sensor is tripped. Sample output: Warning message when the 'CVL Core0 Voltage VR Temperature' exceeds the upper warning threshold limit
                                             
                                            $ tail -f /var/lib/opae/fpgad.log \nfpgad-vc: sensor 'CVL Core Voltage VR Temperature' warning.\nfpgad-vc: saving previous ECAP_AER+0x08 value 0x057ff030 for 0000:b0:02.0\nfpgad-vc: saving previous ECAP_AER+0x14 value 0x0000f1c1 for 0000:b0:02.0\nfpgad-vc: sensor 'CVL Core Voltage VR Temperature' still tripped.\n
                                             
                                            If the upper or lower fatal threshold limit is reached, then a power cycle of server is required to recover the Intel N6001-PL SmartNIC FPGA Platform. AER is unmasked by the fpgad after the sensor values are within the normal range which is above the lower warning or below the upper warning threshold.
                                             
                                            To stop fpgad:
                                             
                                            $ sudo systemctl stop fpgad.service\n
                                             
                                            To check status of fpgad:
                                             
                                            $ sudo systemctl status fpgad.service\n
                                             
                                            Optional: To enable fpgad to re-start on boot, execute
                                             
                                            $ sudo systemctl enable fpgad.service\n
                                             
                                            For a full list of systemctl commands, run the following command:
                                             
                                            $ systemctl -h\n
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#463-updating-with-fpgasupdate","title":"4.6.3 Updating with fpgasupdate","text":"
                                            The fpgasupdate tool updates the Intel Max10 Board Management Controller (BMC) image and firmware (FW), root entry hash, and FPGA Static Region (SR) and user image (PR). The fpgasupdate tool only accepts images that have been formatted using PACsign. If a root entry hash has been programmed onto the board, then you must also sign the image using the correct keys. Refer to the Security User Guide: Intel Open FPGA Stack for information on created signed images and on programming and managing the root entry hash.  
                                             
                                            The Intel\u00ae FPGA SmartNIC N6001-PL ships with a factory, user1, and user2 programmed image for both the FIM and BMC FW and RTL on all cards. The platform ships with a single FIM image that can be programmed into either user1 or user2, depending in the image selected.
                                             
                                            Use the following chart for information on the Bitstream ID and Pr Interface ID, two unique values reported by fpgainfo which can be used to identify the loaded FIM.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-13-fim-version-summary-for-ofs-20233-2-release","title":"Table 13: FIM Version Summary for OFS 2023.3-2 Release","text":"FIM Version Bitstream ID Pr Interface ID File Name Download Location ofs-2023.3-2 00x50102023508A422 1d6beb4e-86d7-5442-a763-043701fb75b7 ofs_top_page[\u00bd]_unsigned_user[\u00bd].bin ofs-2023.3-2 Release Page ofs-n6001-0.9.0-rc2 0x50102025AD3DD11 92ec8960-2f2f-5544-9804-075d2e8a71a1 ofs_top_page[\u00bd]_unsigned_user[\u00bd].bin ofs-2.3.0 Release Page OFS-2.3.0 0x50102022267A9ED f59830f7-e716-5369-a8b0-e7ea897cbf82 ofs_top_page[\u00bd]_unsigned_user[\u00bd].bin ofs-2.3.0 Release Page OFS-2.2.0 0x501020295B081F0 8c157a52-1cf2-5d37-9514-944af0a060da ofs_top_page[\u00bd]_unsigned_user[\u00bd].bin ofs-2.2.0-beta Release Page"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-14-bmc-version-summary-for-ofs-20233-2-release","title":"Table 14: BMC Version Summary for OFS 2023.3-2 Release","text":"BMC FW and RTL Version File Name Download Location 3.15.0 AC_BMC_RSU_user_retail_3.15.0_unsigned.rsu n/a 
                                               
                                            1.  
                                              Example loading a new version of the BMC RTL and FW.
                                               
                                              $ sudo fpgasupdate AC_BMC_RSU_user_retail_3.11.0_unsigned.rsu <PCI ADDRESS>\n[2022-04-14 16:32:47.93] [WARNING ] Update starting. Please do not interrupt.                                           \n[2022-04-14 16:32:47.93] [INFO    ] updating from file /home/user/AC_BMC_RSU_user_retail_3.11.0_unsigned.rsu with size 904064                                   \n[2022-04-14 16:32:47.94] [INFO    ] waiting for idle                                                                    \n[2022-04-14 16:32:47.94] [INFO    ] preparing image file                                                                \n[2022-04-14 16:33:26.98] [INFO    ] writing image file                                                                  \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [904064/904064 bytes][Elapsed Time: 0:00:00.00]                                           \n[2022-04-14 16:33:26.98] [INFO    ] programming image file                                                              \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [Elapsed Time: 0:00:26.02]                                                                 \n[2022-04-14 16:33:53.01] [INFO    ] update of 0000:b1:00.0 complete                                                     \n[2022-04-14 16:33:53.01] [INFO    ] Secure update OK                                                                    \n[2022-04-14 16:33:53.01] [INFO    ] Total time: 0:01:05.07\nsudo rsu bmcimg\n
                                               
                                              2.  
                                            2.  
                                              Example for loading a Static Region (SR) update image. This process will take up to 20 minutes.
                                               
                                              3.  
                                             
                                            $ sudo fpgasupdate ofs_top_page1_unsigned_user1.bin <PCI ADDRESS>\n[2022-04-14 16:42:31.58] [WARNING ] Update starting. Please do not interrupt.                                           \n[2022-04-14 16:42:31.58] [INFO    ] updating from file ofs_top_page1_pacsign_user1.bin with size 19928064               \n[2022-04-14 16:42:31.60] [INFO    ] waiting for idle                                                                    \n[2022-04-14 16:42:31.60] [INFO    ] preparing image file                                                                \n[2022-04-14 16:42:38.61] [INFO    ] writing image file                                                                  \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [19928064/19928064 bytes][Elapsed Time: 0:00:16.01]                                       \n[2022-04-14 16:42:54.63] [INFO    ] programming image file                                                              \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588][Elapsed Time: 0:06:16.40]                                                                 \n[2022-04-14 16:49:11.03] [INFO    ] update of 0000:b1:00.0 complete                                                     \n[2022-04-14 16:49:11.03] [INFO    ] Secure update OK                                                                    \n[2022-04-14 16:49:11.03] [INFO    ] Total time: 0:06:39.45\n
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#464-signing-images-with-pacsign","title":"4.6.4 Signing Images with PACSign","text":"
                                            PACSign is an OPAE utility which allows users to insert authentication markers into bitstreams targeted for the Intel\u00ae FPGA SmartNIC N6001-PL. PACSign also allows users updating their Static Region (SR) to designate which partition of flash (user1, user2, factory) to overwrite given a specific FIM binary image. All binary images must be signed using PACSign before fpgasupdate can use them for an update. Assuming no Root Entry Hash (REH) has been programmed on the device, the following examples demonstrate how to prepend the required secury authentication data, and specifiy which region of flash to update. More information, including charts detailing the different certification types and their required options, are fully described in the PACsign README on GitHub.
                                             
                                            For more information on PACSign and on general security practices surrounding the Intel N6001-PL FPGA SmartNIC device, visit the Security User Guide: Intel Open FPGA Stack.
                                             
                                            PACSign can be run on images that have previously been signed. It will overwrite any existing authentication data.
                                             
                                            The following example creates an unsigned SR image from an existing signed SR binary update image, targeting the user1 partition in flash.
                                             
                                            $ PACSign SR -t UPDATE -s 0 -H openssl_manager -i ofs_top_page1_pacsign_user1.bin -o new_image.bin\nNo root key specified.  Generate unsigned bitstream? Y = yes, N = no: y\nNo CSK specified.  Generate unsigned bitstream? Y = yes, N = no: y\nNo root entry hash bitstream specified.  Verification will not be done.  Continue? Y = yes, N = no: y\n2021-10-18 14:42:54,490 - PACSign.log - WARNING - Bitstream is already signed - removing signature blocks\n
                                             
                                            --->
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#465-loading-images-with-rsu","title":"4.6.5 Loading Images with rsu","text":"
                                            The rsu performs a Remote System Update operation on a N6000/1-PL device, given its PCIe address. An rsu operation sends an instruction to the device to trigger a power cycle of the card and forces reconfiguration from flash for either the BMC or FPGA image.
                                             
                                            The Intel\u00ae FPGA SmartNIC N6001-PL contains two regions of flash you may store FIM images. These locations are referred to as user1 and user2. After an image has been programmed with fpgasupdate in either of these regions you may choose to perform an rsu to switch. This operation indicates to the BMC which region to configure the FPGA device from after power-on.
                                             
                                            If the factory image has been updated, Intel strongly recommends the user to immediately RSU to the factory image to ensure the image is functional.
                                             
                                            The user can determine which region of flash was used to configure their FPGA device using the command fpgainfo fme and referring to the row labelled Boot Page.
                                             
                                            $ sudo fpgainfo fme | grep Boot\nBoot Page                        : user1\n
                                             
                                            Swapping between user1 and user2 skips load times that are created when using fpgasupdate to flash a new FIM image.
                                             
                                            rsu Overview
                                             
                                            Mode 1: RSU
                                             
                                            Perform RSU (remote system update) operation on a development platform given its PCIe address. An RSU operation sends an instruction to the device to trigger a power cycle of the card only. This will force reconfiguration from flash for either BMC, Retimer, SDM, (on devices that support these) or the FPGA.
                                             
                                            Mode 2: Default FPGA Image
                                             
                                            Set the default FPGA boot sequence. The --page option determines the primary FPGA boot image. The --fallback option allows a comma-separated list of values to specify fallback images.
                                             
                                            The following example will load an image stored in user2.
                                             
                                            $ sudo rsu fpga --page=user2 0000:b1:00.0\n2022-04-15 09:25:22,951 - [[pci_address(0000:b1:00.0), pci_id(0x8086, 0xbcce)]] performing RSU operation\n2022-04-15 09:25:22,955 - [[pci_address(0000:b0:02.0), pci_id(0x8086, 0x347a)]] removing device from PCIe bus\n2022-04-15 09:25:22,998 - waiting 10 seconds for boot\n2022-04-15 09:25:33,009 - rescanning PCIe bus: /sys/devices/pci0000:b0/pci_bus/0000:b0\n2022-04-15 09:25:34,630 - RSU operation complete\n
                                             
                                            Note: As a result of using the rsu command, the host rescans the PCI bus and may assign a different Bus/Device/Function (B/D/F) value than the originally assigned value.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#466-verify-fme-interrupts-with-hello_events","title":"4.6.6 Verify FME Interrupts with hello_events","text":"
                                            The hello_events utility is used to verify FME interrupts. This tool injects FME errors and waits for error interrupts, then clears the errors.
                                             
                                            Sample output from sudo hello_events.
                                             
                                            $ sudo hello_events\nWaiting for interrupts now...\ninjecting error\nFME Interrupt occurred\nSuccessfully tested Register/Unregister for FME events!\nclearing error\n
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#467-host-exercisor-modules","title":"4.6.7 Host Exercisor Modules","text":"
                                            The reference FIM and unchanged FIM compilations contain Host Exerciser Modules (HEMs). These are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. There are three HEMs present in the Intel OFS Reference FIM - HE-LPBK, HE-MEM, and HE-HSSI. These exercisers are tied to three different VFs that must be enabled before they can be used. Execution of these exercisers requires you bind specific VF endpoint to vfio-pci. The host-side software looks for these endpoints to grab the correct FPGA resource.
                                             
                                            Refer to the Intel FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA for a full description of these modules.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-15-module-pfvf-mappings","title":"Table 15: Module PF/VF Mappings","text":"Module PF/VF ST2MM PF0 HE-MEM PF0-VF0 HE-HSSI PF0-VF1 HE-MEM_TG PF0-VF2 HE-LB Stub PF1-VF0 HE-LB PF2 VirtIO LB Stub PF3 HPS Copy Engine PF4"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#4671-he-mem-he-lb","title":"4.6.7.1 HE-MEM / HE-LB","text":"
                                            The host exerciser used to exercise and characterize the various host-FPGA interactions eg. MMIO, Data transfer from host to FPGA , PR, host to FPGA memory etc. Host Exerciser Loopback (HE-LBK) AFU can move data between host memory and FPGA.
                                             
                                            HE-LBK supports: - Latency (AFU to Host memory read) - MMIO latency (Write+Read) - MMIO BW (64B MMIO writes) - BW (Read/Write, Read only, Wr only)
                                             
                                            Host Exerciser Loopback Memory (HE-MEM) AFU is used to exercise use of FPGA connected DDR, data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host.
                                             
                                            HE-LB is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth. HE-MEM is used to exercise use of FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host. HE-MEM uses external DDR memory (i.e. EMIF) to store data. It has a customized version of the AVMM interface to communicate with the EMIF memory controller. Both exercisers rely on the user-space tool host_exerciser. When using the Intel N6001-PL FPGA SmartNIC Platform, optimal performance requires the exercisers be run at 400 MHz.
                                             
                                            Execution of these exercisers requires you to bind specific VF endpoint to vfio-pci. The following commands will bind the correct endpoint for a device with B/D/F 0000:b1:00.0 and run through a basic loopback test.
                                             
                                            Note: While running the opae.io init command listed below, if no output is present after completion then the command has failed. Double check that Intel VT-D and IOMMU have been enabled in the kernel as discussed in step 12 in section 3.1 Intel OFS DFL Kernel Driver Environment Setup.
                                             
                                            $ sudo pci_device  0000:b1:00.0 vf 3\n$ sudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci                                                                  \nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci \niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188                                                                  \nAssigning /dev/vfio/188 to DCPsupport                                                                                  \nChanging permissions for /dev/vfio/188 to rw-rw----\nsudo host_exerciser --clock-mhz 400 lpbk\n    starting test run, count of 1\nAPI version: 1\nAFU clock from command line: 400 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Number of clocks: 5224\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1022\n    Bandwidth: 5.018 GB/s\n    Test lpbk(1): PASS\n
                                             
                                            The following example will run a loopback throughput test using one cacheline per request.
                                             
                                            $ sudo pci_device  0000:b1:00.0 vf 3\n$ sudo opae.io init -d 0000:b1:00.2 user:user\n$ sudo host_exerciser --clock-mhz 400 --mode trput --cls cl_1 lpbk\n    starting test run, count of 1\nAPI version: 1\nAFU clock from command line: 400 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 512\n    Host Exerciser numWrites: 513\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Number of clocks: 3517\n    Total number of Reads sent: 512\n    Total number of Writes sent: 512\n    Bandwidth: 7.454 GB/s\n    Test lpbk(1): PASS\n
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#4672-traffic-generator-afu-test-application","title":"4.6.7.2 Traffic Generator AFU Test Application","text":"
                                            Beginning in OPAE version 2.0.11-1+ the TG AFU has an OPAE application to access & exercise traffic, targeting a specific bank. The supported arguments for test configuration are:
                                             
                                               
                                            • Number of test loops: --loops
                                              •  
                                            • Number of read transfers per test loop: -r,--read
                                              •  
                                            • Number of write transfers per test loop: -w,--write
                                              •  
                                            • Burst size of each transfer: -b,--bls
                                              •  
                                            • Address stride between each transfer: --stride
                                              •  
                                            • Target memory TG: -m,--mem-channel
                                              •  
                                             
                                            Below are some example commands for how to execute the test application. To run the preconfigured write/read traffic test on channel 0:
                                             
                                            $ mem_tg tg_test\n
                                             
                                            Target channel 1 with a 1MB single-word write only test for 1000 iterations
                                             
                                            $ mem_tg --loops 1000 -r 0 -w 2000 -m 1 tg_test\n
                                             
                                            Target channel 2 with 4MB write/read test of max burst length for 10 iterations
                                             
                                            $ mem_tg --loops 10 -r 8 -w 8 --bls 255 -m 2 tg_test\n
                                             
                                            $ sudo mem_tg --loops 1000 -r 2000 -w 2000 --stride 2 --bls 2  -m 1 tg_test\n[2022-07-15 00:13:16.349] [tg_test] [info] starting test run, count of 1\nMemory channel clock frequency unknown. Assuming 300 MHz.\nTG PASS\nMem Clock Cycles: 17565035\nWrite BW: 4.37232 GB/s\nRead BW: 4.37232 GB/s\n
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#4673-he-hssi","title":"4.6.7.3 HE-HSSI","text":"
                                            HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs, and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic.
                                             
                                            The hssi application provides a means of interacting with the 10G and with the 100G HSSI AFUs. In both 10G and 100G operating modes, the application initializes the AFU, completes the desired transfer as described by the mode- specific options, and displays the ethernet statistics by invoking ethtool --statistics INTERFACE.
                                             
                                            The following example walks through the process of binding the VF corresponding with the HE-HSSI exerciser to vfio-pci, sending traffic, and verifying that traffic was received.
                                             
                                            1. Create 3 VFs in the PR region.
                                             
                                            $ sudo pci_device b1:00.0 vf 3 \n
                                             
                                            2. Verify all 3 VFs were created.
                                             
                                            $ lspci -s b1:00 \nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01) \nb1:00.1 Processing accelerators: Intel Corporation Device bcce \nb1:00.2 Processing accelerators: Intel Corporation Device bcce \nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device \nb1:00.4 Processing accelerators: Intel Corporation Device bcce \nb1:00.5 Processing accelerators: Intel Corporation Device bccf \nb1:00.6 Processing accelerators: Intel Corporation Device bccf \nb1:00.7 Processing accelerators: Intel Corporation Device bccf \n
                                             
                                            3. Bind all of the PF/VF endpoints to the vfio-pci driver.
                                             
                                            $ sudo opae.io init -d 0000:b1:00.1 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 187\nAssigning /dev/vfio/187 to DCPsupport\nChanging permissions for /dev/vfio/187 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188\nAssigning /dev/vfio/188 to DCPsupport\nChanging permissions for /dev/vfio/188 to rw-rw----\n\n...\n\n$ sudo opae.io init -d 0000:b1:00.7 user:user\nBinding (0x8086,0xbccf) at 0000:b1:00.7 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.7 is 319\nAssigning /dev/vfio/319 to DCPsupport\nChanging permissions for /dev/vfio/319 to rw-rw----\n
                                             
                                            4. Check that the accelerators are present using fpgainfo. Note your port configuration may differ from the below.
                                             
                                            $ sudo fpgainfo port \n//****** PORT ******//\nObject Id                        : 0xEC00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0xE0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.7\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 4dadea34-2c78-48cb-a3dc-5b831f5cecbb\n//****** PORT ******//\nObject Id                        : 0xC0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.6\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 823c334c-98bf-11ea-bb37-0242ac130002\n//****** PORT ******//\nObject Id                        : 0xA0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.5\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n//****** PORT ******//\nObject Id                        : 0x80B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.4\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 44bfc10d-b42a-44e5-bd42-57dc93ea7f91\n//****** PORT ******//\nObject Id                        : 0x40B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n//****** PORT ******//\nObject Id                        : 0x20B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 3e7b60a0-df2d-4850-aa31-f54a3e403501\n
                                             
                                            The following table contains a mapping between each VF, Accelerator GUID, and component.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-16-accelerator-pfvf-and-guid-mappings","title":"Table 16: Accelerator PF/VF and GUID Mappings","text":"Component VF Accelerator GUID Intel N6001-PL FPGA SmartNIC Platform base PF XXXX:XX:XX.0 N/A VirtIO Stub XXXX:XX:XX.1 3e7b60a0-df2d-4850-aa31-f54a3e403501 HE-MEM Stub XXXX:XX:XX.2 56e203e9-864f-49a7-b94b-12284c31e02b Copy Engine XXXX:XX:XX.4 44bfc10d-b42a-44e5-bd42-57dc93ea7f91 HE-MEM XXXX:XX:XX.5 8568ab4e-6ba5-4616-bb65-2a578330a8eb HE-HSSI XXXX:XX:XX.6 823c334c-98bf-11ea-bb37-0242ac130002 MEM-TG XXXX:XX:XX.7 4dadea34-2c78-48cb-a3dc-5b831f5cecbb 
                                            5. Check Ethernet PHY settings with fpgainfo.
                                             
                                            $ sudo fpgainfo phy -B 0xb1 \nIIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\n//****** HSSI information ******//\nHSSI version                     : 1.0\nNumber of ports                  : 8\nPort0                            :25GbE        DOWN\nPort1                            :25GbE        DOWN\nPort2                            :25GbE        DOWN\nPort3                            :25GbE        DOWN\nPort4                            :25GbE        DOWN\nPort5                            :25GbE        DOWN\nPort6                            :25GbE        DOWN\nPort7                            :25GbE        DOWN\n
                                             
                                            6. Set loopback mode.
                                             
                                            $ sudo hssiloopback --loopback enable  --pcie-address 0000:b1:00.0 \nargs Namespace(loopback='enable', pcie_address='0000:b1:00.0', port=0)\nsbdf: 0000:b1:00.0\nFPGA dev: {'segment': 0, 'bus': 177, 'dev': 0, 'func': 0, 'path': '/sys/class/fpga_region/region0', 'pcie_address': '0000:b1:00.0'}\nargs.hssi_grps{0: ['dfl_dev.6', ['/sys/bus/pci/devices/0000:b1:00.0/fpga_region/region0/dfl-fme.0/dfl_dev.6/uio/uio0']]}\nfpga uio dev:dfl_dev.6\n\n--------HSSI INFO START-------\nDFH                     :0x3000000010002015\nHSSI ID                 :0x15\nDFHv                    :0.5\nguidl                   :0x99a078ad18418b9d\nguidh                   :0x4118a7cbd9db4a9b\nHSSI version            :1.0\nFirmware Version        :1\nHSSI num ports          :8\nPort0                   :25GbE\nPort1                   :25GbE\nPort2                   :25GbE\nPort3                   :25GbE\nPort4                   :25GbE\nPort5                   :25GbE\nPort6                   :25GbE\nPort7                   :25GbE\n--------HSSI INFO END-------\n\nhssi loopback enabled to port0\n
                                             
                                            7. Send traffic through the 10G AFU.
                                             
                                            $ sudo hssi --pci-address b1:00.6 hssi_10g --num-packets 100       \n10G loopback test\n  port: 0\n  eth_loopback: on\n  he_loopback: none\n  num_packets: 100\n  packet_length: 64\n  src_address: 11:22:33:44:55:66\n    (bits): 0x665544332211\n  dest_address: 77:88:99:aa:bb:cc\n    (bits): 0xccbbaa998877\n  random_length: fixed\n  random_payload: incremental\n  rnd_seed0: 5eed0000\n  rnd_seed1: 5eed0001\n  rnd_seed2: 25eed\n  eth:\n\nNo eth interface, so not honoring --eth-loopback.\n0x40000           ETH_AFU_DFH: 0x1000010000001000\n0x40008          ETH_AFU_ID_L: 0xbb370242ac130002\n0x40010          ETH_AFU_ID_H: 0x823c334c98bf11ea\n0x40030      TRAFFIC_CTRL_CMD: 0x0000000000000000\n0x40038     TRAFFIC_CTRL_DATA: 0x0000000100000000\n0x40040 TRAFFIC_CTRL_PORT_SEL: 0x0000000000000000\n0x40048        AFU_SCRATCHPAD: 0x0000000045324511\n\n0x3c00         number_packets: 0x00000064\n0x3c01          random_length: 0x00000000\n0x3c02         random_payload: 0x00000000\n0x3c03                  start: 0x00000000\n0x3c04                   stop: 0x00000000\n0x3c05           source_addr0: 0x44332211\n0x3c06           source_addr1: 0x00006655\n0x3c07             dest_addr0: 0xaa998877\n0x3c08             dest_addr1: 0x0000ccbb\n0x3c09        packet_tx_count: 0x00000064\n0x3c0a              rnd_seed0: 0x5eed0000\n0x3c0b              rnd_seed1: 0x5eed0001\n0x3c0c              rnd_seed2: 0x00025eed\n0x3c0d             pkt_length: 0x00000040\n0x3cf4          tx_end_tstamp: 0x000003d2\n0x3d00                num_pkt: 0xffffffff\n0x3d01               pkt_good: 0x00000064\n0x3d02                pkt_bad: 0x00000000\n0x3d07            avst_rx_err: 0x00000000\n0x3d0b          rx_sta_tstamp: 0x00000103\n0x3d0c          rx_end_tstamp: 0x0000053b\n0x3e00               mac_loop: 0x00000000\n\nHSSI performance:\n        Selected clock frequency : 402.832 MHz\n        Latency minimum : 642.948 ns\n        Latency maximum : 896.155 ns\n        Achieved Tx throughput : 18.4528 GB/s\n        Achieved Rx throughput : 16.7101 GB/s\n\nNo eth interface, so not showing stats.\n
                                             
                                            The hssi_loopback utility works in conjunction with a packet generator accelerator function unit (AFU) to test high-speed serial interface (HSSI) cards. The hssi_loopback utility tests both external and internal loopbacks.
                                             
                                            The hssistats tool provides the MAC statistics.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#50-upgrading-the-intel-fpga-smartnic-n6001-pl-with-20233-2-version-of-the-bmc-and-fim","title":"5.0 Upgrading the Intel\u00ae FPGA SmartNIC N6001-PL with 2023.3-2 Version of the BMC and FIM","text":"
                                            If your Intel\u00ae FPGA SmartNIC N6001-PL does not have the 2022.3.1 version of the FIM and BMC, use this section to begin your upgrade process. The upgrade process depends on both the OPAE SDK and kernel drivers, which were installed in sections 3.0 Intel OFS DFL Kernel Drivers and 4.0 OPAE Software Development Kit. Use the output of fpgainfo and compare against the table below to determine if an upgade is necessary.
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-17-fim-version-summary-for-intel-ofs-20233-2-release","title":"Table 17: FIM Version Summary for Intel OFS 2023.3-2 Release","text":"FIM Version Bitstream ID Pr Interface ID File Name Download Location 1 00x50102023508A422 1d6beb4e-86d7-5442-a763-043701fb75b7 ofs_top_page[1 / 2]_unsigned_user[1 / 2].bin ofs-2023.3-2 Release Page"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-18-bmc-version-summary-for-intel-ofs-20233-2-release","title":"Table 18: BMC Version Summary for Intel OFS 2023.3-2 Release","text":"BMC FW and RTL Version File Name Download Location 3.15.0 AC_BMC_RSU_user_retail_3.15.0_unsigned.rsu n/a 
                                            Sample output of fpgainfo with matching values:
                                             
                                            Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : a2b5fd0e7afca4ee6d7048f926e75ac2\nUser1 Image Info                 : 1d6beb4e-86d7-5442-a763-043701fb75b7\nUser2 Image Info                 : 1d6beb4e-86d7-5442-a763-043701fb75b7\n
                                             
                                               
                                            1. If your output does not match the table above, download the appropriate FIM image from the Intel OFS 2023.3-2 (Intel Agilex) release page. Once downloaded transfer the file over to the server and use the fpgasupdate utility to perform an upgrade of the BMC.
                                              2.  
                                             
                                            $ sudo fpgasupdate AC_BMC_RSU_user_retail_3.15.0_unsigned.rsu\n[2022-04-14 16:32:47.93] [WARNING ] Update starting. Please do not interrupt.                                           \n[2022-04-14 16:32:47.93] [INFO    ] updating from file /home/user/AC_BMC_RSU_user_retail_3.15.0_unsigned.rsu with size 904064                                   \n[2022-04-14 16:32:47.94] [INFO    ] waiting for idle                                                                    \n[2022-04-14 16:32:47.94] [INFO    ] preparing image file                                                                \n[2022-04-14 16:33:26.98] [INFO    ] writing image file                                                                  \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [904064/904064 bytes][Elapsed Time: 0:00:00.00]                                           \n[2022-04-14 16:33:26.98] [INFO    ] programming image file                                                              \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [Elapsed Time: 0:00:26.02]                                                                 \n[2022-04-14 16:33:53.01] [INFO    ] update of 0000:b1:00.0 complete                                                     \n[2022-04-14 16:33:53.01] [INFO    ] Secure update OK                                                                    \n[2022-04-14 16:33:53.01] [INFO    ] Total time: 0:01:05.07\nsudo rsu bmcimg\n
                                             2. Load the new FIM image.
                                             
                                            ```bash\n$ sudo fpgasupdate ofs_top_page1_unsigned_user1.bin <PCI ADDRESS>\n[2022-04-14 16:42:31.58] [WARNING ] Update starting. Please do not interrupt.                                           \n[2022-04-14 16:42:31.58] [INFO    ] updating from file ofs_top_page1_pacsign_user1.bin with size 19928064               \n[2022-04-14 16:42:31.60] [INFO    ] waiting for idle                                                                    \n[2022-04-14 16:42:31.60] [INFO    ] preparing image file                                                                \n[2022-04-14 16:42:38.61] [INFO    ] writing image file                                                                  \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [19928064/19928064 bytes][Elapsed Time: 0:00:16.01]                                       \n[2022-04-14 16:42:54.63] [INFO    ] programming image file                                                              \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588][Elapsed Time: 0:06:16.40]                                                                 \n[2022-04-14 16:49:11.03] [INFO    ] update of 0000:b1:00.0 complete                                                     \n[2022-04-14 16:49:11.03] [INFO    ] Secure update OK                                                                    \n[2022-04-14 16:49:11.03] [INFO    ] Total time: 0:06:39.45\nsudo rsu fpga --page=user1 <PCI ADDRESS>\n```\n
                                             
                                               
                                            1. Verify output of fpgainfo matches the table above.
                                              2.  
                                            "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                            Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                             
                                            OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/","title":"Simulation User Guide: Open FPGA Stack for Intel Intel\u00ae Agilex\u00ae 7 FPGA","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel Max10 or Intel Cyclone10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implemented on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Management Component Transport Protocol MCTP A standardized model for communication with management controllers. Defines the transport protocol carrying PLDM messages through the BMC. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE-SDK The OPAE-SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Level Data Model PLDM A specification for reporting telemetry data to the host, such as board temperature, voltage, and current. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#1-overview","title":"1 Overview","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#11-about-this-document","title":"1.1 About this Document","text":"
                                            This document serves as a set-up and user guide for the UVM simulation tool using OFS. After reviewing the document, you will be able to:
                                             
                                               
                                            • Set-up the UVM verification tool suite
                                              •  
                                             
                                               
                                            • Run pre-existing UVM unit tests and also create new UVM tests for your design
                                              •  
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#2-introduction-to-uvm","title":"2 Introduction to UVM","text":"
                                            The Functional Verification Environment for OFS is UVM (Universal Verification Methodology) compliant and provides configurable setup for verifying various FIM features in simulation.
                                             
                                            The purpose of this document is to demonstrate a full chip level and unit level UVM based verification environment for the current base shell FIM architecture as well as providing examples to extend the setup for different OFS variants by reusing the existing architecture and infrastructure for UVM based verification.
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#3-universal-testbench-architecture","title":"3 Universal Testbench Architecture","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#31-overview","title":"3.1 Overview","text":"
                                            The main idea behind UVM is to develop modular, reusable, and a scalable testbench structure by providing an API framework that can be deployed across multiple projects.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#32-core-verification-concepts","title":"3.2 Core Verification Concepts","text":"
                                            The following is the list of verification components that will be used to design a UVM testbench architecture:
                                             
                                            \u2022   Sequencer \u2022   Driver \u2022   Monitor \u2022   Scoreboard \u2022   Virtual Sequencer \u2022   Interface \u2022   Verification Environment \u2022   TOP Testbench
                                             
                                            Figure 1 provides the general UVM Testbench and the verification components involved in the top-level architecture.
                                             
                                             
                                            Figure 1 Typical UVM Testbench
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4-ofs-testbench-architecture","title":"4 OFS Testbench Architecture","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#41-overview","title":"4.1 Overview","text":"
                                            OFS (Open FPGA Stack) provides a UVM (Universal Verification Methodology) environment for the FIM with a modular, reusable, and scalable testbench structure via an API framework.
                                             
                                            The framework consists of a FIM Testbench which is UVM compliant and integrates third party VIPs from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this Testbench. UVM RAL(Register Abstraction Level) is used for CSR (Command and Status Registers) verification. 
                                             
                                            The qualified verification IPs will help to detect incorrect protocol behavior, help to focus on FIM features and accelerate the verification process.
                                             
                                            Verification components include:
                                             
                                            \u2022 FIM monitor to detect correct design behavior\n\u2022 FIM assertions for signal level integrity testing\n\u2022 Arm AMBA Arm\u00ae AMBA\u00ae 4 AXI4 scoreboards to check data integrity\n\u2022 FIM coverage to collect functional data\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#42-base-fim-dut","title":"4.2 Base FIM DUT","text":"
                                            The hardware architecture of an Agilex FIM is based on the OFS hardware architecture. The following is the list of features and subsystems supported in the base shell.
                                             
                                            \u2022   PCIe Subsystem\n\u2022   HSSI Subsystem\n\u2022   Memory Subsystem\n\u2022   HPS Subsystem\n\u2022   FME\n\u2022   AFU with PR support\n\u2022   QSFP Controllers\n\u2022   PMCI Controller, MCTP\n
                                             
                                             
                                            Figure 2 DUT Base Shell Diagram
                                             
                                            Figure 2 shows the high level architecture of an Agilex Base Shell. It has a Gen4x16, 100G Ethernet Datapath in a 2x4x25G configuration. The Agilex Base Shell is a shell that will enable a user to build other shell variants for a custom configuration. For the N6001 board there is one shell variant
                                             
                                            base_x16
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#43-full-chip-level-verification-archiecture-for-fim","title":"4.3 Full Chip Level Verification Archiecture for FIM","text":"
                                            Figure 3 shows a graphical representation a full chip testbench that includes major RTL blocks depicted in a OFS Agilex based UVM environment 
                                             
                                             
                                            Figure 3 OFS FIM Testbench
                                             
                                            The major connection is the interface between the Xeon CPU and the FPGA where the PCIe Verification IP is connected to PCIe Subsystem. Therefore, as a full chip simulation environment, PCIe host VIP is the sole VIP/BFM used. PCIe host VIP connects to PCIe device which resides in FPGA in serial mode.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#431-testbench-components","title":"4.3.1 Testbench components","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4311-tb-top","title":"4.3.1.1 TB TOP","text":"
                                            TOP is the top level testbench and consists of a FIM DUT instance and top-level UVM Verification Environment instance. It binds RTL inputs with the verification environmnet interfaces to drive stimulus. It also has clock generation and reset driving logic.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4312-fim-verification-environment","title":"4.3.1.2 FIM Verification Environment","text":"
                                            This is the top most verification environment class and consists of the protocol specific PCI Express and AXI UVM environment VIP instances, Monitors, Scoreboards, Assertions, Functional coverage Modules and other verification components. It instantiates Virtual sequencers to control stimuli for FIM traffic from different sequencers of the VIPs.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4313-synopsys-vips","title":"4.3.1.3 Synopsys VIPs","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#43131-pci-vip-as-host","title":"4.3.1.3.1 PCI VIP as Host","text":"
                                            This is Synopsys Licensed PCI Express Gen4 VIP and acts as Root Port. The PCI Express link is connected to the DUT using TX-RX lanes. Agent is an active component and includes a sequencer to generate TLPs, Driver to drive it on a PCI Express link and Monitor to check the protocol correctness.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#43132-axi-streaming-vip-monitors","title":"4.3.1.3.2 AXI-Streaming VIP Monitors","text":"
                                            This is Synopsys Licensed AXI streaming interface Verification IP used as a Passive Agent to monitor AXI-ST links at various points. Please refer to Figure 3 to see all the AXI-ST monitors connected to different modules.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#43133-axi4-memory-mapped-vip-monitors","title":"4.3.1.3.3 AXI4-Memory Mapped VIP Monitors","text":"
                                            This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 memory mapped interface Verification IP used in passive mode to observe memory requests. For each master-slave pair, the verification environment has a VIP instantiated in passive mode for monitoring protocol violations and performing data integrity checks.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#43134-axi4-lite-vip-monitors","title":"4.3.1.3.4 AXI4-Lite VIP Monitors","text":"
                                            This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 Lite interface Verification IP used in passive mode to observe MMIO requests. For each master-slave pair, the verification environment has a VIP instantiated in passive mode for monitoring protocol violations and perfoming data integrity checks. Please refer to Figure 3 to see all the Arm\u00ae AMBA\u00ae 4 AXI4-Lite monitors connected to different modules.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#43135-axi4-lite-vip-as-pmci-master","title":"4.3.1.3.5 AXI4-Lite VIP as PMCI Master","text":"
                                            This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 Lite interface Verification IP used to drive and observe MMIO requests as PMCI master. For each master-slave pair, the verification environment has a VIP instantiated in active mode and includes a sequencer to generate MMIO requests, driver to drive these requests on AXI-4 Lite interface to BPF and a monitor for observing protocol violations and data integrity checks. 
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4314-axi4-s-scoreboard","title":"4.3.1.4 AXI4-S Scoreboard","text":"
                                            The Arm\u00ae AMBA\u00ae 4 AXI4-S scoreboard checks data integrity of source and sink components. It has input transactions from VIP monitors and a TLP to AXI converter for PCIe TLP packets. It makes sure the valid TLPs or AXI transactions are not missed while traversing from Host to AFU and reverse. The scoreboard will be disabled for error testing especially for invalid TLP requests and UR responses.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4315-virtual-sequencer","title":"4.3.1.5 Virtual Sequencer","text":"
                                            The virtual sequencer is the top-level sequencer which controls Enumeration, MMIO Requests, downstream and Upstream traffic as well as HSSI and Memory transactions. It makes sure that the transactions are ordered correctly as per the design specification while running a UVM test simulation.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4316-fim-monitor","title":"4.3.1.6 FIM Monitor","text":"
                                            The FIM monitor is used for checking the correctness of a specific FIM feature. As an example, a user can add interrupt related checks, error generation and clearing checks, Protocol checker impacts etc. in this component. 
                                             
                                            This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4317-fim-assertions","title":"4.3.1.7 FIM Assertions","text":"
                                            The assertion component is used for checking the signal level integrity of a specific FIM feature or behavior. As an example, we can add interrupt signal triggering and clear assertions, Error detection to error register bit assertions, protocol checker and clearing logic, FLR behavior assertion etc. in this top-level module. There are alos assertions to make sure the inputs are driven correctly to catch errors in a users simulation. 
                                             
                                            This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4318-fim-functional-coverage","title":"4.3.1.8 FIM Functional Coverage","text":"
                                            The FIM functional coverage component will have the functional coverage of each feature like interrupts, CSR's, MMIO requests, Byte align transactions, error reporting etc. to demonstrate a variety of FIM features. This will help us to find holes in the design as well as wide verification coverage to make sure thorough testing of a design. It will also provide a user what the FIM is capable of and how much code coverage they are testing.
                                             
                                            This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#5-uvm-verification-set-up","title":"5 UVM Verification set-up","text":"
                                            To run the tutorial steps in this guide requires the following development environment:
                                             Item Version Intel Quartus Prime Pro Intel Quartus Prime Pro 23.3 Simulator Synopsys VCS P-2019.06-SP2-5 or newer for UVM simulation of top level FIM Simulator (Questasim) Questasim 2023.2 or newer for UVM simulation of top level FIM"},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#51-uvm-prerequisite","title":"5.1 UVM Prerequisite","text":"
                                            Retrieve OFS repositories.
                                             
                                            The OFS FIM source code is included in the OTCShare GitHub repository. Create a new directory to use as a clean starting point to store the retrieved files.  The following is a short description of each repository, followed by the git commands for cloning.  The instructions section uses the HTTPS git method for cloning repositories.  Cloning the repo using the HTTPS method requires a personal access token.  Please see this blog post for information about obtaining a personal access token Token authentication requirements for Git operations.
                                             
                                            Navigate to location for storage of OFS source, create the top-level source directory and clone OFS repositories.
                                             
                                            $ mkdir ofs-2023.3-2\n$ cd ofs-2023.3-2\n$ export OFS_BUILD_ROOT=$PWD\n$ git clone --branch --recurse-submodules https://github.com/ofs-n6001.git\n\nCloning into 'ofs-n6001'...'\nUsername for 'https://github.com': <<Enter your git hub username>>\nPassword for 'https://<<Your username>>': <<Enter your personal access token>>\nremote: Enumerating objects:  ....\n...\n...\nResolving deltas  ..., done.\n\n$ cd ofs-n6001\n$ git checkout tags/ofs-2023.3-2\n
                                             
                                            Verify that the correct tag/branch have been checked out        
                                             
                                            $ git describe --tags\n\n$ ofs-2023.3-2\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#52-license-requirements","title":"5.2 License Requirements","text":"
                                            The FIM Testbench is UVM compliant and integrates third party VIP's from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this TB. UVM RAL (Register Abstraction Layer) is used for CSR Verification.
                                             
                                            The Qualified Verification IPs will help to detect incorrect protocol behavior easily, help to focus on FIM features and accelerate the verification process.
                                             
                                            \u2022 VCS & DVE\n\u2022 SNPS-Assertions\n\u2022 Verdi\n\u2022 VerdiCoverage\n\u2022 VerdiSimDB\n\u2022 VerdiTransactionDebugUltra\n\u2022 VIP-AMBA-AXI-SVT\n\u2022 VIP-AMBA-STREAM-SVT\n\u2022 VIP-PCIE-SVT\n\u2022 VIP-PCIE-TS-SVT\n\u2022 VIP-PCIE-G3-OPT-SVT\n\u2022 VIP-Ethernet-SVT\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#53-software-tools-requirements","title":"5.3 Software Tools Requirements","text":"
                                            The following tools are required for successful UVM set-up
                                             
                                               
                                            • Python 3.6.8
                                              •  
                                            • Synopsys PCIE and AMBA AXI UVM VIP Q-2020.03A License
                                              •  
                                            • Synopsys Verdi R-2020.12-SP2 License      Note: Makefile can be modified to use DVE instead of Verdi
                                              •  
                                             
                                               
                                            • VCS R-2020.12-SP2 License 
                                              •  
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#54-creating-a-software-tools-script","title":"5.4 Creating a Software Tools Script","text":"
                                            The UVM tool set-up is best done by creating a simple set-up script so all applicable tools are sourced before running the tests.
                                             
                                            The following environment variables can be pasted into a script and used prior to running the UVM verification environment
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#license-files","title":"License Files","text":"
                                            export LM_LICENSE_FILE=\nexport SNPSLMD_LICENSE_FILE=\n
                                             
                                            The license environment variables LM_LICENSE_FILE and SNPSLMD_LICENSE_FILE can point to a server license on your system.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#general-environment-variables","title":"General Environment Variables","text":"
                                            export IOFS_BUILD_ROOT=$PWD\nexport OFS_ROOTDIR=<user_path>/ofs-n6001\nexport WORKDIR=$OFS_ROOTDIR\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#quartus-tools","title":"Quartus Tools","text":"
                                            export QUARTUS_HOME=<user_path>/intelFPGA_pro/23.3/quartus\nexport QUARTUS_ROOTDIR=$QUARTUS_HOME\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$PATH\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-verification-tools","title":"Synopsys Verification Tools","text":"
                                            export DESIGNWARE_HOME=<user_path>/synopsys/vip_common/vip_Q-2020.03A\nexport PATH=$DESIGNWARE_HOME/bin:$PATH\nexport UVM_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport PATH=$VCS_HOME/bin:$PATH\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-verification-tools","title":"QuestaSIM Verification Tools","text":"
                                            export MTI_HOME=<user_path>/mentor/questasim/2023.2/linux64\nexport PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\nexport QUESTA_HOME=$MTI_HOME\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#6-running-a-uvm-simulation-test-and-analysing-results","title":"6 Running a UVM Simulation Test and Analysing Results","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#61-simulation","title":"6.1 Simulation","text":"
                                            The default simulator used in the simulation script is Synopsys VCS-MX. Users can refer to the options and adopt the options for other simulators. The script is a makefile that calls vlogan, vcs and simv for compilation, elaboration and simulation, respectively.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#62-file-structure","title":"6.2 File Structure","text":"
                                            After cloning the repo, the verification and ofs-common directories contain all UVM verification related files. The directory structure is shown in Figure 4 below.
                                             
                                             
                                            Figure 4 UVM Verification Directory File Structure
                                             
                                            ofs-n6001/verification/testbench has a testbench, uvm env, virtual sequencer, RAL etc. 
                                             
                                            ofs-n6001/tests contains all uvm tests and sequences. 
                                             
                                            Users can run the simulation under \"ofs-n6001/verification/scripts\" directory and the simulation result is outputted to a \"sim\" directory for Synopsys VCS or sim_msim for Questasim.
                                             
                                            The simulation result folder is named after the test name with increasing suffix number. If user runs the same test multiple times, the suffix is incremented by 1 each time.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#63-uvm-test-suite","title":"6.3 UVM Test Suite","text":"
                                            The UVM environment contains a variety of tests that have been developed to test out the FIM portion of OFS.
                                             
                                            The table below has four columns which describe the \"Test Name\", \"DUT Scope\", \"Test Scenario\" and the \"Checking Criteria\".
                                             
                                            Tests are located at ofs-n6001/ofs-common/verification/fpga_family/agilex/tests
                                             Test Name DUT Scope Test Scenario Checking Criteria afu_mmio_flr_pf0_test PF0 FLR Reset Apply FLR Reset for PF0 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF0 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf0_vf0_test PF0_VF0_FLR Reset Apply FLR Reset for PF0_VF0 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR       access are working fine afu_mmio_flr_pf0_vf1_test PF0_VF1_FLR Reset Apply FLR Reset for PF0_VF1 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf0_vf2_test PF0_VF2_FLR Reset Apply FLR Reset for PF0_VF2 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf2_test PF2 FLR Reset Apply FLR Reset for PF2 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF2 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf3_test PF3 FLR Reset Apply FLR Reset for PF3 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF3 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf4_test PF4 FLR Reset Apply FLR Reset for PF4 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF4 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_stress_5bit_tag_test AFU-Stress To check the AFU Stress by sending traffic from all PF/VF. i.e Send traffic on HE-LPK/HE-MEM and initiating MMIO access to other PF/VFs Data checking afu_stress_8bit_tag_test AFU-Stress To check the AFU Stress by sending traffic from all PF/VF. i.e Send traffic on HE-LPK/HE-MEM and initiating MMIO access to other PF/VFs Data checking afu_stress_test AFU-Stress 1. Initiate transactions to all the supported PF/VF from PCIE VIP and ensure that traffic is sent to all blocks of the AFU.                 2. Ensure that CE/HE-LB/HE-MEM/HSSI/BPF/FME are seeing traffic.                 3. Ensure that HE-LB/HE-MEM/CE sends DMWR/DMRD requests to PCIE VIP.                 4. Ensure the Mux/DeMux blocks is able to handle the traffic based on the PF's/VF's and proper muxing/demuxing happens. Data checking bar_32b_test PCIe MMIO Path Set the BAR values to 32bit and test mmio access BAR address, Register Base Offset bar_64b_test PCIe MMIO Path Set the BAR values to 64bit and test mmio access BAR address, Register Base Offset dfh_walking_test DFH DFH walking offset checking, eol checking -> tb emif_csr_test EMIF CSR access data checking fme_csr_test FME CSR CSR accesses data checking fme_err_intr_test Interrupt FME Interrupt request using FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read fme_intr_test Interrupt FME interrupt request using RAS ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read fme_multi_err_intr_test Interrupt FME interrupt request using RAS ERROR and FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests generated from FME and RAS ERROR bits he_hssi_csr_test HE-HSSI CSR accesses data checking he_hssi_rx_lpbk_25G_10G_test HE-HSSI Sending back to back ethernet data traffic with 25G speed on HSSI RX Port0-7 lanes using Ethernet VIPs Enable the loopback mode in HE-HSSI and compare the pkts recived on HSSI TX Port(DATA CHECKING) he_hssi_tx_lpbk_P0_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port0  lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P1_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port1 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P2_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port2 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P3_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port3 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P4_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port4 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P5_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port5 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P6_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port6 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P7_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port7 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_lpbk_cont_test HE-LPBK Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking he_lpbk_csr_test HE-LPBK CSR accesses data checking he_lpbk_flr_rst_test HE-LPBK Set HE_LPK in  all the modes randomly and iterate the transactions in loop. At the end of every loop assert the Soft reset in the middle of the transactions data checking, counter checking he_lpbk_long_rst_test HE-LPBK Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_lpbk_long_test HE-LPBK Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_lpbk_multi_user_intr_test HE-LPBK generate user HE_LB interrupt interrupt checking he_lpbk_rd_cont_test HE-LPBK Read only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking he_lpbk_rd_test HE-LPBK Read only mode. Randomize num_lines, addresses, req_len counter checking he_lpbk_reqlen16_test HE-LPBK To check the behavior of HE_LPK block when req_length 16 and num_lines set to 1024 Cache lines in Loopback mode data checking, counter checking he_lpbk_reqlen1_test HE-LPBK Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_lpbk_reqlen2_test HE-LPBK Loopback mode. 128 CLs, req_len = 2CL, random addresses data checking, counter checking he_lpbk_reqlen4_test HE-LPBK Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_lpbk_reqlen8_test HE-LPBK Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_lpbk_rst_in_middle_test PCIe MMIO Path Set HE_LPK in  all the modes randomly and iterate the transactions in loop. At the end of every loop assert the Soft reset in the middle of the transactions Register Base Offset he_lpbk_test HE-LPBK Loopback mode.  Randomize num_lines, addresses, req_len data checking, counter checking he_lpbk_thruput_contmode_test HE-LPBK Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking he_lpbk_thruput_test HE-LPBK generate user HE_LB interrupt counter checking he_lpbk_user_intr_test HE-LPBK Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_lpbk_wr_cont_test HE-LPBK Write only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking he_lpbk_wr_test HE-LPBK Write only mode. Randomize num_lines, addresses, req_len counter checking he_mem_cont_test HE-MEM Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking, counter checking he_mem_csr_test HE-MEM CSR accesses data checking he_mem_lpbk_long_rst_test HE-MEM Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_mem_lpbk_long_test HE-MEM Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_mem_lpbk_reqlen16_test HE-MEM To check the behavior of HE_MEM block when req_length 16 and num_lines set to 1024 Cache lines in Loopback mode data checking, counter checking he_mem_lpbk_reqlen1_test HE-MEM Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_mem_lpbk_reqlen2_test HE-MEM Loopback mode. 128 CLs, req_len = 2CL, random addresses data checking, counter checking he_mem_lpbk_reqlen4_test HE-MEM Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_mem_lpbk_reqlen8_test HE-MEM Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_mem_lpbk_test HE-MEM Loopback mode.  Randomize num_lines, addresses, req_len data checking, counter checking he_mem_multi_user_intr_test Interrupt Back to back multiple User interrupt request from HE MEM Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read for multiple back to back request he_mem_rd_cont_test HE-MEM Read only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_rd_test HE-MEM Read only mode. Randomize num_lines, addresses, req_len counter checking he_mem_thruput_contmode_directed_test HE-MEM Set HE_LPK in thruput mode and send traffic with req len 1 and num_lines set to 40 data checking, counter checking he_mem_thruput_contmode_test HE-MEM Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_thruput_test HE-MEM Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_mem_user_intr_test Interrupt FME interrupt request using RAS ERROR and FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests generated from FME and RAS ERROR bits he_mem_wr_cont_test HE-MEM Write only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_wr_test HE-MEM Write only mode. Randomize num_lines, addresses, req_len counter checkingt he_random_test All HEs Enable all HEs and randomize modes data checking if in lpbk mode, counter checking helb_rd_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Read only mode Measure the performance helb_rd_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Read only mode Measure the performance helb_rd_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Read only mode Measure the performance helb_thruput_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Thruput mode Measure the performance helb_thruput_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Thruput mode Measure the performance helb_thruput_4cl_5bit_tag_test Performance Set HE_LPK in thruput mode and send traffic with req len 4 and num_lines set to 1024. Measure the Read/Write performance Measure the performance helb_thruput_4cl_8bit_tag_test Performance Set HE_LPK in thruput mode and send traffic with req len 4 and num_lines set to 1024. Measure the Read/Write performance Measure the performance helb_thruput_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Thruput mode Measure the performance helb_wr_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Write only mode Measure the performance helb_wr_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Write only mode Measure the performance helb_wr_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Write only mode Measure the performance hssi_ss_test HSSI CSR accesses data checking malformedtlp_pcie_rst_test Protocol Checker generate malformed TLP protocol error and initiate pcie reset to clear the error Check for error malformedtlp_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MaxTagError_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transactions are completing. mem_tg_csr_test MEM-TG CSR access data checking mem_tg_traffic_gen_test MEM-TG This test checks the MEM_TG traffic generator flow for 1 bank data checking mini_smoke_test All HEs shorter simpler version of random test for turn-in sanity check data checking if in lpbk mode, counter checking mix_intr_test Interrupt Mix interrupt testcase to send multiple FME and User interrupt request simultaneously Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests through different sources - FME and HE-MEM modules mmio_pcie_mrrs_128B_mps_128B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_128B_mps_256B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_256B_mps_128B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_256B_mps_256B_test PCIe - Max Payload/Max Read Req Size Random length mmio Read Checking valid possible combination of MPS & MRRS mmio_stress_nonblocking_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux with non-blocking MMIO reads data checking mmio_stress_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux data checking mmio_test PCIe MMIO Path MMIO targeting PF0(ST2MM, FME, PMCI, QSFP, HSSI SS), PF1, PF2,PF3, PF4, PF1.VF1, PF1.VF2 data checking mmio_unimp_test PCIe MMIO Path MMIO acccess to unimplemented addresses data checking MMIODataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MMIOInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MMIOTimedout_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. pcie_csr_test PCIESS Earlier msix registers were in fme block but now it has moved from fme to pciess. Hence coded a seperate test for msix data checking pcie_pmci_mctp_multi_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM multiple packets received from PMCI controller over APF and BPF to PCIe HIA subsystem pcie_pmci_mctp_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM (Vendor Defined Message) single packet received from PCIe HIA subsystem to PMCI controller over APF and BPF via AXI4-lite memory write request pmci_csr_test PMCI CSR CSR accesses data checking pmci_fme_csr_test PMCI FME CSR CSR accesses data checking pmci_pcie_mctp_multi_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM multiple packets received from PMCI controller over APF and BPF to PCIe HIA subsystem pmci_pcie_mctp_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM single packet received from PMCI controller over APF and BPF to PCIe HIA subsystem pmci_pciess_csr_test PMCI PCIESS CSR CSR accesses data checking pmci_qsfp_csr_test PMCI QSFP CSR CSR accesses data checking port_gasket_csr_test PORT GASKET CSR accesses data checking qsfp_csr_test QSFP CSR accesses data checking TxMWrDataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. TxMWrInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. uart_intr_test UART Checking Generates UART interrupt Check interrupt UnexpMMIORspErr_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. vdm_err_vid_test Vendor ID check generate a packet with undefined Vendor-ID from Host to PMCI_SS ID check 
                                             
                                            The next section describes how to compile and build the UVM environment prior to running each UVM test and analyinsg the results in the log files 
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#64-ip-compile","title":"6.4 IP Compile","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-vcs","title":"Synopsys VCS","text":"
                                            To compile all IPs for the Synopsys VCS simulater:
                                             
                                                cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk cmplib_adp\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-tbd","title":"Questasim (TBD)","text":"
                                            To compile all IPs for the Questasim simulater:
                                             
                                                cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk cmplib_adp\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#65-rtl-test-bench-compile","title":"6.5 RTL & Test Bench Compile","text":"
                                            The RTL file list for compilation is located here: verification/scripts/rtl_comb.f
                                             
                                            The TB file list for compilation is located here: verification/scripts/ver_list.f
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-vcs_1","title":"Synopsys VCS","text":"
                                            To compile RTL and Testbench for the Synopsys VCS simulater
                                             
                                                cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_adp DUMP=1\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-tbd_1","title":"Questasim (TBD)","text":"
                                            To compile RTL and Testbench for the Questasim simulater
                                             
                                                cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_adp DUMP=1\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#66-ip-and-rtl-test-bench-compile","title":"6.6 IP and RTL & Test Bench Compile","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-vcs_2","title":"Synopsys VCS","text":"
                                            If the user wants to compile all IPs and RTL Testbench in one command for Synopsys VCS then follow the procedure below
                                             
                                                cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_all DUMP=1\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-tbd_2","title":"Questasim (TBD)","text":"
                                            If the user wants to compile all IPs and RTL Testbench in one command for Questasim then follow the procedure below
                                             
                                                cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_all DUMP=1\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-vcs_3","title":"Synopsys VCS","text":"
                                            To run a simulation for Synopsys VCS:
                                             
                                                cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk run TESTNAME=ofs_mmio_test DUMP=1\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-tbd_3","title":"Questasim (TBD)","text":"
                                            To run a simulation for Questasim:
                                             
                                                cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=ofs_mmio_test DUMP=1\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-vcs_4","title":"Synopsys VCS","text":"
                                            To dump the waveform, \"DUMP=1\" must be added into the command line for Synopsys VCS build and simulation. 
                                             
                                                gmake -f Makefile_VCS.mk build_adp DUMP=1\n\n    gmake -f Makefile_VCS.mk run TESTNAME=<test_case_name> DUMP=1\n
                                             
                                            Or
                                             
                                                gmake -f Makefile_VCS.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-tbd_4","title":"Questasim (TBD)","text":"
                                            To dump the waveform, \"DUMP=1\" must be added into the command line for Questasim build and simulation. 
                                             
                                                gmake -f Makefile_MSIM.mk build_adp DUMP=1\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=<test_case_name> DUMP=1\n
                                             
                                            Or
                                             
                                                gmake -f Makefile_MSIM.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                                             
                                            There are some optimizations in the Table below for convenience if you want to bypass some commands for both Synopsys VCS and Questasim:
                                             Command (Synopsys VCS) Command (Questasim) Details gmake -f Makefile_VCS.mk build_all DUMP=1 gmake -f Makefile_MSIM.mk build_all DUMP=1 compile IP + compile RTL gmake -f Makefile_VCS.mk build_run TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk build_run TESTNAME= DUMP=1 compile RTL + run test gmake -f Makefile_VCS.mk do_it_all TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk do_it_all TESTNAME= DUMP=1 compile IP, RTL and run test gmake -f Makefile_VCS.mk rundb TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk rundb TESTNAME= DUMP=1 run test in sim dir + over-writes content"},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#67-uvm-regression-test","title":"6.7 UVM Regression Test","text":"
                                            cd $VERDIR/scripts\n\npython uvm_regress.py -l -n 8 -p adp -k top_pkg -s vcs -c none\n\nFor Regression in VCS with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -p adp -k top_pkg -s vcs -c none\n\nResults are created in a sim directory ($VERDIR/sim) with individual testcase log dir\n\nFor Regression in MSIM with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -p adp -k top_pkg -s msim -c none\n
                                             
                                            Results are created in a sim directory ($VERDIR/sim_msim) with individual testcase log dir
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#68-uvm-waveform-and-transcript-analysis","title":"6.8 UVM Waveform and Transcript Analysis","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-vcs_5","title":"Synopsys VCS","text":"
                                            Running Synopsys VCS UVM tests will generate a ofs-n6001/verification/sim directory
                                             
                                            \u2022 All build time logs are located at ofs-n6001/verification/sim\n\n\u2022 Each testcase will have separate directory inside sim ofs-n6001/verification/sim/<test_case_name>\n
                                             
                                            There are two tracker or log files that are available: runsim.log and trans.log. 
                                             
                                            runsim.log is the simulation log file generated from Synopsys VCS. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 5
                                             
                                             
                                            Figure 5 runsim.log
                                             
                                            trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 6
                                             
                                             
                                            Figure 6 trans.log
                                             
                                            The waveform generated is named as \"inter.vpd\". To open the waveform, go to simulation result directory and run 
                                             
                                                dve -full64 -vpd inter.vpd &\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-tbd_5","title":"Questasim (TBD)","text":"
                                            Running Questasim UVM tests will generate a ofs-n6001/verification/sim_msim directory
                                             
                                            \u2022 All build time logs are at ofs-n6001/verification/sim_msim\n\n\u2022 Each testcase will have separate directory inside sim ofs-n6001/verification/sim_msim/<test_case_name>\n
                                             
                                            There are two tracker or log files that are available: runsim.log and trans.log. 
                                             
                                            runsim.log is the simulation log file generated from Questasim. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 7
                                             
                                             
                                            Figure 7 runsim.log
                                             
                                            trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 8
                                             
                                             
                                            Figure 8 trans.log
                                             
                                            The waveform generated is named as \"vsim.wlf\". To open the waveform, go to simulation result directory and run 
                                             
                                                vsim -view vsim.wlf &\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#69-uvm-coverage-analysis","title":"6.9 UVM Coverage Analysis","text":"
                                            The following command allows to run a single testcase with coverage enabled
                                             
                                                gmake -f Makefile_VCS.mk cmplib_adp && gmake -f Makefile_VCS.mk build_adp DUMP=1 DEBUG=1 COV_FUNCTIONAL=1&& gmake -f Makefile_VCS.mk run TESTNAME=<TESTCASE-NAME> DUMP=1 DEBUG=1 COV_FUNCTIONAL=1 &\n
                                             
                                            The following command shows how to merge and generate the coverage report
                                             
                                                urg -dir <$VERDIR/sim/simv.vdb> <$VERDIR/sim/regression.vdb> -format both -dbname <regression_database_name>\n
                                             
                                            This will generate both urgreport directory and .vdb file Multiple regression.vdb from different regressions can be merged with the same command. 
                                                e.g \"urg -dir <path1_till_simv.vdb> <path1_till_regression.vdb> <path2_till_regression.vdb> -report <dir> -format both -dbname <dirname>\"\n
                                             
                                            The following commands shows how to launch DVE and check the coverage reports
                                             
                                            To open DVE of a single regression or testcase, execute:\n\n    dve -full64 -cov -covdir simv.vdb regression.vdb &\n\nTo open DVE of a merged regression test, execute:\n\n    dve -full64 -cov -covdir <dirname.vdb> &\n
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#7-csr-verification-using-uvm-ral","title":"7 CSR Verification using UVM RAL","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#71-overview","title":"7.1 Overview","text":"
                                            The UVM Register Layer provides a standard base class library that enable users to implement the object-oriented model to access the DUT registers and memories. The UVM Register Layer is also referred to as UVM Register Abstraction Layer (UVM RAL). Design registers can be accessed independently of the physical bus interface. i.e. by calling read/write methods. This is shown in Figure 9 below.
                                             
                                             
                                            Figure 9 RAL UVM Testbench
                                             
                                            The RAL register models for different CSR's mimics the design registers. All RAL files are located here.
                                             
                                                ofs-n6001/verification/testbench/ral\n
                                             
                                            The RAL model is generated through the Synopsys RALGEN tool and is used for CSR verification.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#72-ral-integration","title":"7.2 RAL Integration","text":"
                                            For UVM RAL model integration to the environment, adapters for each CSR is implemented and integrated into the Testbench Environment. It is used to convert the PCIe bus sequence items into uvm_reg_bus_op and vice versa. The CSR test cases pick up all the registers from the respective CSR blocks and perform a default value, wr/rd check on them. 
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#73-ral-model-generation","title":"7.3 RAL Model Generation","text":"
                                            Steps for RAL model generation
                                             
                                            Excel(xls) file containing the registers is required. Make sure there are separate xls files for each CSR and the worksheet name must contain the name \"reg_fields\".  
                                             
                                            Excel sheet snapshot example below for EMIF_CSR.xls located at /ipss/mem/rtl/adp
                                             
                                             
                                            \u2022 Navigate to ofs-n6001/ofs-common/verification/common/scripts/ral\n\u2022 Copy the excel file (xls) to the above area\n\u2022 In the bash terminal run mk_ral.sh <Excel sheet name without extension > <output *.sv file name without ral_  prepended >\n\u2022 The above steps generate two ral *.sv files. File with _cov suffix is a coverage enabled ral model. \n\u2022 Copy *.sv files to ofs-n6001/verification/testbench/ral\n
                                             
                                             
                                            \u2022 As an example to generate ral_ac_ce.sv from AC_CE_CSR.xls file the command is\n\n    mk_ral.sh AC_CE_CSR ac_ce\n
                                             
                                            This generates two ral models (ral_ac_ce.sv and ral_ac_ce_cov.sv)
                                             
                                            To add new registers
                                             
                                            \u2022 To create new registers, copy existing ones and modify the cells in the xls. Make sure the last line is also a copied blank line\n\u2022 Follow all the steps of RAL model generation\n
                                             
                                            To Generate a RAL model when a new xls sheet is created for a new component
                                             
                                            \u2022 Copy the relevant xls sheet to ofs-n6001/ofs-common/verification/common/scripts/ral\n\u2022 Follow all the steps of RAL model generation\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#74-top-level-verification-architecture-for-csr-testing","title":"7.4 Top Level Verification Architecture for CSR testing","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#741-testbench-components","title":"7.4.1 Testbench components","text":"
                                            The testbench components for RAL are defined below
                                             
                                            \u2022 ral_reg_iofs_* (uvm_reg) generated by the steps as mentioned in section 5.3\n
                                             
                                            The uvm register class is written by extending the uvm_reg. A register represents a set of fields that are accessible as a single entity Each register contains any number of fields, which mirror the values of the corresponding elements in hardware
                                             
                                            \u2022 ral_block_iofs_* (uvm_block) generated in the same register file\n
                                             
                                            A register model is an instance of a register block, which may contain any number of registers, register files, memories, and other blocks
                                             
                                            \u2022 ral_block_ofs (uvm_block) \u2013 Contains all the CSR block instantiations\n\u2022 Reg2vip_*_adapter (uvm_reg_adapter) \u2013 This class defines an interface for converting between uvm_reg_bus_op and a specific bus transaction. For each CSR a respective adapter is present\n
                                             
                                            All the components are defined in ofs-n6001/ofs-common/verification/testbench
                                             
                                            Integration of components in testbench
                                             
                                            \u2022 The RAL blocks and adapters for each CSR is instantiated in tb_env\n\u2022 The bar range for each CSR is also set in the tb_env\n
                                             
                                            Sample Environment Integration snippets 
                                             
                                             
                                             
                                             
                                             
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#8-modifying-uvm-testbench","title":"8 Modifying UVM Testbench","text":"
                                            The next sections describe what needs to be considered when modifying the UVM, adding a new interface to the testbench and creating a new UVM test for a customised OFS Accelerator platform.
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#81-modifying-uvm-environment-for-new-shell-variant","title":"8.1 Modifying UVM environment for new Shell Variant","text":"
                                            OFS n6001 comprises a shell based on PCIe Gen4x16 and is named base_x16
                                             
                                            This base_x16 shell is described by an RTL file list, IP File lists and setup scripts to complete the build flow
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#82-modifying-uvm-environment-and-setting-up-compile-and-run-flow","title":"8.2 Modifying UVM environment and setting up Compile and Run flow","text":"
                                            All the variants can mostly reuse the existing UVM infrastructure to setup the build and run flow
                                             
                                            \u2022 Create directory under $OFS_BUILD_ROOT new variant e.g ofs-n9000\n\u2022 Change directory to $OFS_BUILD_ROOT/ofs-n9000/verification/scripts\n\u2022 modify Makefile it to point to the new RTL, IP and script files.\n
                                             
                                            Following these three steps above will enable the build and sim flow to run the existing UVM TB and tests with new IOFS n9000 variant.
                                             
                                            Adding a new interface requires signal connections in the testbench. An additional BFM or verification IP is needed to drive the new interface. The main testbench file tb_top.sv is found at the following location
                                             
                                                ofs-n6001/verification/testbench\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#83-adding-a-new-ral-directory","title":"8.3 Adding a new RAL directory","text":"
                                            In case the design has many register changes and the user decides to generate all the new RAL models instead of reusing from existing base RAL models, the following steps will help to create and integrate a new RALDIR in the UVM environment.
                                             
                                            \u2022 Generate the new RAL files in desired directory. Preferably under the \"ofs-n6001/verification/common/testbench\" \n\u2022 By default, the Makefile points to base FIM RAL so set the RALDIR path in the Makefile to the new generated RAL file directory\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#84-modifying-tbtest-files-for-new-variant","title":"8.4 Modifying TB/Test files for new Variant","text":"
                                            Create a define for the new variant. e.g 'define FIM_NEW. If you are modifying common files then add new logic under this define so other projects will not get affected with variant specific change.
                                             
                                            If there are more changes, please create separate \"testbench\" and \"test\" folders under this new variant. 
                                             
                                            \u2022 Extend variant specific env file from base env file to add variant specific changes.\n\u2022 Create new test/seq lib files in \"tests\" folder.\n\u2022 Create new variant package, add new TB/Tests/Sequence lib files and also import the base package files.\n
                                             
                                            If you are adding new files then make sure it's included in Makefile for the build+run flow. 
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#85-uvm-pcie-drivers","title":"8.5 UVM PCIe Drivers","text":"
                                            The \"svt_pcie_driver_app_transaction_base_sequence\" is part of Synopsys PCIe VIP library. You can find the sequence definition in the following two directories
                                             
                                            \u2022 Navigate to \"$DESIGNWARE_HOME/vip/svt/pcie_svt/Q-2020.03/sverilog/src/vcs/svt_pcie_driver_app_transaction_sequence_collection.svp\" file. All the base and PCIe sequences are defined in this file.\n\n\u2022 When the OFS UVM build command is executed, it creates \"vip\" directory under \"$OFS_BUILD_ROOT/ofs-n6001/verification\". You can also find the same sequence file at \"$IOFS_BUILD_ROOT/ofs-n6001/verification/vip/pcie_vip/src/sverilog/vcs/svt_pcie_driver_app_transaction_sequence_collection.sv\"\n
                                            "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                            Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                             
                                            OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                            "},{"location":"sw/build_chain/fpga_api/api_build/","title":"Building OPAE SDK Artifacts","text":""},{"location":"sw/build_chain/fpga_api/api_build/#steps","title":"Steps","text":"
                                               
                                            1. Fetch the OPAE SDK source tree
                                              2.  
                                            2. Configure the OPAE SDK CMake project
                                              3.  
                                            3. Build OPAE SDK targets
                                              4.  
                                             
                                            The example below lists commands that can be used to fetch and build OPAE SDK.
                                             
                                            # fetch the source\ngit clone https://github.com/OPAE/opae-sdk.git\ncd opae-sdk\n# configure CMake\ncmake ..\n# build\nmake\n
                                             
                                            For a list of targets that can be built, type make help from the build directory.
                                             
                                            CMake options that may be set during the configuration include the following:
                                             
                                            |----------------------------|-----------------------|-------------------------------------|---------------------------------------|----------------|\n| cmake flag                 | Optional or Mandatory | Purpose                             | Valid values                          | Default value  |\n|----------------------------|-----------------------|-------------------------------------|---------------------------------------|----------------|\n| -DCMAKE_BUILD_TYPE         | Optional              | Set compiler flags                  | Debug/Release/Coverage/RelWithDebInfo | RelWithDebInfo |\n| -DOPAE_BUILD_LEGACY        | Optional              | Enable/disable opae-legacy.git      | ON/OFF                                | OFF            |\n| -DOPAE_BUILD_SPHINX_DOC    | Optional              | Enable/disable documentation build  | ON/OFF                                | OFF            |\n| -DOPAE_BUILD_TESTS         | Optional              | Enable/disable building unit tests  | ON/OFF                                | OFF            |\n| -DOPAE_INSTALL_RPATH       | Optional              | Enable/disable rpath for install    | ON/OFF                                | OFF            |\n| -DOPAE_BUILD_LIBOPAE_CXX   | Optional              | Enable/disable OPAE C++ bindings    | ON/OFF                                | ON             | \n| -DOPAE_WITH_PYBIND11       | Optional              | Enable/disable pybind11 binaries    | ON/OFF                                | ON             |\n| -DOPAE_BUILD_PYTHON_DIST   | Optional              | Enable/disable Python Distribution  | ON/OFF                                | OFF            |\n| -DOPAE_ENABLE_MOCK         | Optional              | Enable/disable mocks for unit tests | ON/OFF                                | OFF            |\n| -DOPAE_BUILD_SIM           | Optional              | Enable/disable opae-sim.git         | ON/OFF                                | OFF            |\n
                                            "},{"location":"sw/build_chain/fpga_driver/driver_build/","title":"Building the OPAE Intel FPGA driver (out-of-tree)","text":"
                                            The Intel FPGA driver included with OPAE SDK releases is packaged as an RPM or DEB package as well as a source tarball. Starting with OPAE SDK release of 1.4, the driver can be built from source out-of-tree but requires the following packages:
                                             
                                            For RPM package managers (Red Hat, CentOS, Fedora, etc.) * kernel-headers * kernel-devel * gcc * make
                                             
                                            For DEB package managers (Debian, Ubuntu, etc.) * kernel-headers-generic * gcc * make
                                             
                                            After installation of necessary distribution packages, follow the steps in the example below to build the Intel Kernel driver. NOTE The example below references Intel FPGA Kernel driver version 2.0.2. but can be applied to later versions.
                                             
                                            tar zxf opae-intel-fpga-driver-2.0.2-1.tar.gz\ncd opae-intel-fpga-driver-2.0.2\nmake\n
                                            "},{"location":"sw/drv_arch/drv_arch/","title":"Open Programmable Accelerator Engine (OPAE) Linux Device Driver Architecture","text":"
                                            The OPAE FPGA Linux Device Driver provides interfaces for user-space applications to configure, enumerate, open, and access FPGA accelerators on platforms equipped with Intel FPGA solutions. The OPAE FPGA driver also enables system-level management functions such as FPGA reconfiguration and virtualization.
                                            "},{"location":"sw/drv_arch/drv_arch/#hardware-architecture","title":"Hardware Architecture","text":"
                                            The Linux Operating System treats the FPGA hardware as a PCIe* device. A predefined data structure, Device Feature List (DFL), allows for dynamic feature discovery in an Intel FPGA solution.
                                             
                                             
                                            The Linux Device Driver implements PCIe Single Root I/O Virtualization (SR-IOV) for the creation of Virtual Functions (VFs). The device driver can release individual accelerators for assignment to virtual machines (VMs).
                                             
                                            "},{"location":"sw/drv_arch/drv_arch/#fpga-management-engine-fme","title":"FPGA Management Engine (FME)","text":"
                                            The FPGA Management Engine provides error reporting, reconfiguration, performance reporting, and other infrastructure functions. Each FPGA has one FME which is always accessed through the Physical Function (PF). The Intel Xeon\u00ae Processor with Integrated FPGA also performs power and thermal management. These functions are not available on the Intel Programmable Acceleration Card (PAC).
                                             
                                            User-space applications can acquire exclusive access to the FME using open(), and release it using close(). Device access may be managed by standard Linux interfaces and tools.
                                             
                                            If an application terminates without freeing the FME or Port resources, Linux closes all file descriptors owned by the terminating process, freeing those resources.
                                            "},{"location":"sw/drv_arch/drv_arch/#port","title":"Port","text":"
                                            A Port represents the interface between two components: * The FPGA Interface Manager (FIM) which is part of the static FPGA fabric * The Accelerator Function Unit (AFU) which is the partially reconfigurable region
                                             
                                            The Port controls the communication from software to the AFU and makes features such as reset and debug available.
                                            "},{"location":"sw/drv_arch/drv_arch/#accelerator-function-unit-afu","title":"Accelerator Function Unit (AFU)","text":"
                                            An AFU attaches to a Port. The AFU provides a 256 KB memory mapped I/O (MMIO) region for accelerator-specific control registers.
                                             
                                               
                                            • Use open() on the Port device to acquire access to an AFU associated with the Port device.
                                              •  
                                            • Use close()on the Port device to release the AFU associated with the Port device.
                                              •  
                                            • Use mmap() on the Port device to map accelerator MMIO regions.
                                              •  
                                            "},{"location":"sw/drv_arch/drv_arch/#partial-reconfiguration-pr","title":"Partial Reconfiguration (PR)","text":"
                                            Use PR to reconfigure an AFU from a bitstream file. Successful reconfiguration has the following requirement:
                                             
                                               
                                            • You must generate the reconfiguration AFU for the exact FIM. The AFU and FIM are compatible if their interface IDs match. You can verify this match by comparing the interface ID in the bitstream header against the interface ID that is exported by the driver in sysfs.
                                              •  
                                             
                                            In all other cases PR fails and may cause system instability.
                                             
                                            Platforms that support 512-bit Partial Reconfiguration require binutils >= version 2.25.
                                             
                                            Close any software programs accessing the FPGA, including those running in a virtualized host before initiating PR. For virtualized environments, the recommended sequence is as follows:
                                             
                                               
                                            1. Unload the driver from the guest
                                              2.  
                                            2. Release the VF from the guest
                                              3.  
                                             
                                            Releasing the VF from the guest while an application on the guest is still accessing its resources may lead to VM instabilities. We recommend closing all applications accessing the VF in the guest before releasing the VF.
                                             
                                               
                                            1. Disable SR-IOV
                                              2.  
                                            2. Perform PR
                                              3.  
                                            3. Enable SR-IOV
                                              4.  
                                            4. Assign the VF to the guest
                                              5.  
                                            5. Load the driver in the guest
                                              6.  
                                            "},{"location":"sw/drv_arch/drv_arch/#fpga-virtualization","title":"FPGA Virtualization","text":"
                                            To enable accelerator access from applications running on a VM, create a VF for the port using the following process:
                                             
                                               
                                            1.  
                                              Release the Port from the PF using the associated ioctl on the FME device.
                                               
                                              2.  
                                            2.  
                                              Use the following command to enable SR-IOV and VFs. Each VF can own a single Port with an AFU. In the following command, N is the number of Port released from the PF.
                                               
                                              3.  
                                             
                                                echo N > $PCI_DEVICE_PATH/sriov_numvfs\n
                                             
                                            The number, 'N', cannot be greater than the number of supported VFs. This can be read from $PCI_DEVICE_PATH/sriov_totalvfs.
                                             
                                               
                                            1.  
                                              Pass the VFs through to VMs using hypervisor interfaces.
                                               
                                              2.  
                                            2.  
                                              Access the AFU on a VF from applications running on the VM using the same driver inside the VM.
                                               
                                              3.  
                                             
                                            Creating VFs is only supported for port devices. Consequently, PR and other management functions are only available through the PF.    
                                             
                                             
                                            If assigning multiple devices to the same VM on a guest IOMMU, you may need to increase the hard_limit option in order to avoid hitting a limit of pinned memory. The hard limit should be more than (VM memory size x Number of PCIe devices).
                                            "},{"location":"sw/drv_arch/drv_arch/#driver-organization","title":"Driver Organization","text":""},{"location":"sw/drv_arch/drv_arch/#pcie-module-device-driver","title":"PCIe Module Device Driver","text":"
                                            FPGA devices appear as a PCIe devices. Once enumeration detects a PCIe PF or VF, the Linux OS loads the FPGA PCIe device driver. The device driver performs the following functions:
                                             
                                               
                                            1. Walks through the Device Feature List in PCIe device base address register (BAR) memory to discover features and their sub-features and creates necessary platform devices.
                                              2.  
                                            2. Enables SR-IOV.
                                              3.  
                                            3. Introduces the feature device infrastructure, which abstracts operations for sub-features and provides common functions to feature device drivers.
                                              4.  
                                            "},{"location":"sw/drv_arch/drv_arch/#pcie-module-device-driver-functions","title":"PCIe Module Device Driver Functions","text":"
                                            The PCIe Module Device Driver performs the following functions:
                                             
                                               
                                            1. PCIe discovery, device enumeration, and feature discovery.
                                              2.  
                                            2. Creates sysfs directories for the device, FME, and Port.
                                              3.  
                                            3. Creates the platform driver instances, causing the Linux kernel to load their respective drivers.
                                              4.  
                                            "},{"location":"sw/drv_arch/drv_arch/#fme-platform-module-device-driver","title":"FME Platform Module Device Driver","text":"
                                            The FME Platform Module Device Driver loads automatically after the PCIe driver creates the FME Platform Module. It provides the following features for FPGA management:
                                             
                                               
                                            1.  
                                              Power and thermal management, error reporting, performance reporting, and other infrastructure functions. You can access these functions via sysfs interfaces the FME driver provides.
                                               
                                              2.  
                                            2.  
                                              Partial Reconfiguration. During PR sub-feature initialization, the FME driver registers the FPGA Manager framework to support PR. When the FME receives the relevant ioctl request from user-space, it invokes the common interface function from the FPGA Manager to reconfigure the AFU using PR.
                                               
                                              3.  
                                            3.  
                                              Port management for virtualization (releasing/assigning port device).
                                               
                                              4.  
                                             
                                            After a port device is released, you can use the PCIe driver SR-IOV interfaces to create/destroy VFs.
                                             
                                            For more information, refer to \"FPGA Virtualization\".
                                            "},{"location":"sw/drv_arch/drv_arch/#fme-platform-module-device-driver-functions","title":"FME Platform Module Device Driver Functions","text":"
                                            The FME Platform Module Device Driver performs the the following functions:
                                             
                                               
                                            • Creates the FME character device node.
                                              •  
                                            • Creates the FME sysfs files and implements the FME sysfs file accessors.
                                              •  
                                            • Implements the FME private feature sub-drivers.
                                              •  
                                            • FME private feature sub-drivers:     * FME Header     * Partial Reconfiguration     * Global Error     * Global Performance
                                              •  
                                            "},{"location":"sw/drv_arch/drv_arch/#port-platform-module-device-driver","title":"Port Platform Module Device Driver","text":"
                                            After the PCIe Module Device Driver creates the Port Platform Module device, the FPGA Port and AFU driver are loaded.  This module provides an interface for user-space applications to access the individual accelerators, including basic reset control on the Port, AFU MMIO region export, DMA buffer mapping service, and remote debug functions.
                                            "},{"location":"sw/drv_arch/drv_arch/#port-platform-module-device-driver-functions","title":"Port Platform Module Device Driver Functions","text":"
                                            The Port Platform Module Device Driver performs the the following functions:
                                             
                                               
                                            • Creates the Port character device node.
                                              •  
                                            • Creates the Port sysfs files and implements the Port sysfs file accessors.
                                              •  
                                            • Implements the following Port private feature sub-drivers.     * Port Header     * AFU     * Port Error     * Signal Tap
                                              •  
                                            "},{"location":"sw/drv_arch/drv_arch/#opae-fpga-driver-interface","title":"OPAE FPGA Driver Interface","text":"
                                            The user-space interface consists of a sysfs hierarchy and ioctl requests. Most kernel attributes can be accessed/modified via sysfs nodes in this hierarchy. More complex I/O operations are controlled via ioctl requests. The OPAE API implementation, libopae-c, has been designed to use this interface to interact with the OPAE FPGA kernel drivers.
                                            "},{"location":"sw/fpga_api/fpga_api/","title":"OPAE C API Reference","text":"
                                            The reference documentation for the OPAE C API is grouped into the following sections:
                                             
                                               
                                            • Types
                                                 
                                              • types.h
                                                •  
                                              • types_enum.h
                                                •  
                                               
                                              •  
                                            • Enumeration API
                                                 
                                              • enum.h
                                                •  
                                              • properties.h
                                                •  
                                               
                                              •  
                                            • Access API
                                                 
                                              • access.h
                                                •  
                                               
                                              •  
                                            • Event API
                                                 
                                              • event.h
                                                •  
                                               
                                              •  
                                            • MMIO and Shared Memory APIs
                                                 
                                              • mmio.h
                                                •  
                                              • buffer.h
                                                •  
                                              • umsg.h
                                                •  
                                               
                                              •  
                                            • Management API
                                                 
                                              • manage.h
                                                •  
                                               
                                              •  
                                            • Metrics API
                                                 
                                              • metrics.h
                                                •  
                                               
                                              •  
                                            • SysObject
                                                 
                                              • sysobject.h
                                                •  
                                               
                                              •  
                                            • Utilities
                                                 
                                              • utils.h
                                                •  
                                               
                                              •  
                                            • Samples
                                                 
                                              • hello_fpga.c
                                                •  
                                              • hello_events.c
                                                •  
                                               
                                              •  
                                            "},{"location":"sw/fpga_api/fpga_api/#types","title":"Types","text":"
                                            The OPAE C API defines a number of types; most prominent are the types fpga_token, fpga_handle, and fpga_properties. All regular types are defined in types.h, while the values of enumeration types are defined in types_enum.h.
                                            "},{"location":"sw/fpga_api/fpga_api/#typesh","title":"types.h","text":"
                                            types.h
                                            "},{"location":"sw/fpga_api/fpga_api/#types_enumh","title":"types_enum.h","text":"
                                            types_enum.h
                                            "},{"location":"sw/fpga_api/fpga_api/#enumeration-api","title":"Enumeration API","text":"
                                            The OPAE enumeration API allows selective discovery of FPGA resources. When enumerating resources, a list of filter criteria can be passed to the respective function to select a subset of all resources in the system. The fpgaEnumerate() function itself then returns a list of fpga_tokens denoting resources, which can be used in subsequent API calls.
                                             
                                            Filter criteria are specified using one or more fpga_properties object. These objects need to be created using fpgaGetProperties() (defined in ) before being passed to fpgaEnumerate(). Individual attributes of an fpga_properties object are set using specific accessors, which are also defined in ."},{"location":"sw/fpga_api/fpga_api/#enumh","title":"enum.h","text":"
                                            enum.h
                                            "},{"location":"sw/fpga_api/fpga_api/#propertiesh","title":"properties.h","text":"
                                            properties.h
                                            "},{"location":"sw/fpga_api/fpga_api/#access-api","title":"Access API","text":"
                                            The access API provides functions for opening and closing FPGA resources. Opening a resource yields an fpga_handle, which denotes ownership and can be used in subsequent API calls to interact with a specific resource. Ownership can be exclusive or shared.
                                            "},{"location":"sw/fpga_api/fpga_api/#accessh","title":"access.h","text":"
                                            access.h
                                            "},{"location":"sw/fpga_api/fpga_api/#event-api","title":"Event API","text":"
                                            The event API provides functions and types for handling asynchronous events such as errors or accelerator interrupts.
                                             
                                            To natively support asynchronous event, the driver for the FPGA platform needs to support events natively (in which case the OPAE C library will register the event directly with the driver). For some platforms that do not support interrupt-driven event delivery, you need to run the FPGA Daemon (fpgad) to enable asynchronous OPAE events. fpgad will act as a proxy for the application and deliver asynchronous notifications for registered events.
                                            "},{"location":"sw/fpga_api/fpga_api/#eventh","title":"event.h","text":"
                                            event.h
                                            "},{"location":"sw/fpga_api/fpga_api/#mmio-and-shared-memory-apis","title":"MMIO and Shared Memory APIs","text":"
                                            These APIs feature functions for mapping and accessing control registers through memory-mapped IO (mmio.h), allocating and sharing system memory buffers with an accelerator (buffer.h), and using low-latency notifications (umsg.h).
                                            "},{"location":"sw/fpga_api/fpga_api/#mmioh","title":"mmio.h","text":"
                                            mmio.h
                                            "},{"location":"sw/fpga_api/fpga_api/#bufferh","title":"buffer.h","text":"
                                            buffer.h
                                            "},{"location":"sw/fpga_api/fpga_api/#umsgh","title":"umsg.h","text":"
                                            umsg.h
                                            "},{"location":"sw/fpga_api/fpga_api/#management-api","title":"Management API","text":"
                                            The management APIs define functions for reconfiguring an FPGA (writing new partial bitstreams) as well as assigning accelerators to host interfaces.
                                            "},{"location":"sw/fpga_api/fpga_api/#manageh","title":"manage.h","text":"
                                            manage.h
                                            "},{"location":"sw/fpga_api/fpga_api/#metrics-api","title":"Metrics API","text":"
                                            The metrics APIs define functions for discovery/enumeration of metrics information and reading metrics values.
                                            "},{"location":"sw/fpga_api/fpga_api/#metricsh","title":"metrics.h","text":"
                                            metrics.h
                                            "},{"location":"sw/fpga_api/fpga_api/#sysobject","title":"SysObject","text":"
                                            The SysObject API can be used to get system objects by name. Names used with the SysObject API are driver-specific and may not be compatible across plugins and/or drivers. For example, SysObject names used with the xfpga plugin will apply to the OPAE Linux Kernel driver and refer to sysfs nodes under the sysfs tree for the resource used with the SysObject API.
                                            "},{"location":"sw/fpga_api/fpga_api/#sysobjecth","title":"sysobject.h","text":"
                                            sysobject.h
                                            "},{"location":"sw/fpga_api/fpga_api/#utilities","title":"Utilities","text":"
                                            Functions for mapping fpga_result values to meaningful error strings are provided by the utilities API.
                                            "},{"location":"sw/fpga_api/fpga_api/#utilsh","title":"utils.h","text":"
                                            utils.h
                                            "},{"location":"sw/fpga_api/fpga_api/#samples","title":"Samples","text":"
                                            Code samples demonstrate how to use OPAE C API.
                                            "},{"location":"sw/fpga_api/fpga_api/#hello_fpgac","title":"hello_fpga.c","text":"
                                            hello_fpga.c
                                            "},{"location":"sw/fpga_api/fpga_api/#hello_eventsc","title":"hello_events.c","text":"
                                            hello_events.c
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/","title":"OPAE C++ Core API Reference","text":"
                                            The reference documentation for the OPAE C++ Core API is grouped into the following sections:
                                             
                                               
                                            • Overview
                                              •  
                                            • Goals
                                                 
                                              • Simplicity
                                                •  
                                              • Extensibility and Interoperability
                                                •  
                                              • Modern C++ Coding Practices
                                                •  
                                              • Error Handling
                                                •  
                                              • Coding Style
                                                •  
                                               
                                              •  
                                            • Fundamental Types
                                                 
                                              • Properties
                                                •  
                                              • pvalue.h
                                                •  
                                              • properties.h
                                                •  
                                              • Resource Classes
                                                •  
                                              • token.h
                                                •  
                                              • handle.h
                                                •  
                                              • shared_buffer.h
                                                •  
                                              • errors.h
                                                •  
                                              • events.h
                                                •  
                                              • sysobject.h
                                                •  
                                              • Exceptions
                                                •  
                                              • except.h
                                                •  
                                              • Misc
                                                •  
                                              • version.h
                                                •  
                                               
                                              •  
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#overview","title":"Overview","text":"
                                            The OPAE C++ API enables C++ developers with the means to use FPGA resources by integrating the OPAE software stack into C++ applications.
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#goals","title":"Goals","text":""},{"location":"sw/fpga_api/fpga_cxx_api/#simplicity","title":"Simplicity","text":"
                                            Keep the API as small and lightweight as possible. Although features such as system validation and orchestration are beyond the scope of this API, using this API for their development should be relatively easy.
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#extensibility-and-interoperability","title":"Extensibility and Interoperability","text":"
                                            While keeping to the goal of simplicity, the OPAE C++ API is designed to allow for better reuse by either extending the API or by integrating with other languages.
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#modern-c-coding-practices","title":"Modern C++ Coding Practices","text":"
                                            The OPAE C++ API uses the C++ 11 standard library and makes use of its features whenever practical. The OPAE C++ API is also designed to require the minimum number of third-party libraries/dependencies.
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#error-handling","title":"Error Handling","text":"
                                            The OPAE C++ API is designed to throw exceptions when appropriate. The structure of OPAE C++ exceptions is similar to the error codes in the OPAE C API. This gives users of the API more freedom on error handling while providing better debug information in cases of failure.
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#coding-style","title":"Coding Style","text":"
                                            For formatting of the OPAE C++ API complies with most of the recommendations of the Google C++ style. For example, the OPAE C++ API uses:
                                             
                                               
                                            • opening braces on the same line as their scope definition
                                              •  
                                            • spaces instead of tabs for indentation
                                              •  
                                            • indentation of two spaces
                                              •  
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#fundamental-types","title":"Fundamental Types","text":"
                                            Basic types for the OPAE C++ API are found in the opae::fpga::types namespace. They serve as an adapter layer between the OPAE C API and the OPAE C++ layer. Aside from providing a C++ binding to the C fundamental types, these types also:
                                             
                                               
                                            • manage the lifetime and scope of the corresponding C struct.
                                              •  
                                             
                                               
                                            • For example a C++ destructor will take care of calling the appropriate C function to release the data structure being wrapped.
                                              •  
                                             
                                               
                                            • provide a friendly syntax for using the OPAE C type.
                                              •  
                                             
                                            Most classes in this namespace have a c_type() method that returns the C data structure being wrapped, making it easy to use the OPAE C++ type with the OPAE C API. Alternatively, most classes in this namespace have implicit conversion operators that enable interoperability with the OPAE C API.
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#properties","title":"Properties","text":"
                                            C++ class properties wraps fpga_properties and uses pvalue and guid_t to get and set properties stored in an instance of an fpga_properties. pvalue and guid_t are designed to call an accessor method in the OPAE C API to either read property values or write them. Most accessor methods in the OPAE C API share a similar signature, so pvalue generalizes them into common operations that translate into calling the corresponding C API function. guid_t follows similar patterns when reading or assigning values.
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#pvalueh","title":"pvalue.h","text":"
                                            pvalue.h
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#propertiesh","title":"properties.h","text":"
                                            properties.h
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#resource-classes","title":"Resource Classes","text":"
                                            The token, handle, and shared_buffer classes are used to enumerate and access FPGA resources. properties are used to narrow the search space for token's. Before enumerating the accelerator resources in the system, applications can produce one or more properties objects whose values are set to the desired characteristics for the resource. For example, an application may search for an accelerator resource based on its guid.
                                             
                                            Once one or more token's have been enumerated, the application must choose which token's to request. The token is then converted to a handle by requesting that a handle object be allocated and opened for it.
                                             
                                            Once a handle has been successfully opened, the application can read and write the associated configuration and status space. Additionally, the application may use the handle to allocate shared_buffer's or to register event's. The shared_buffer and event objects retain a reference to their owning handle so that the handle does not lose scope before freeing the shared_buffer and event objects.
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#tokenh","title":"token.h","text":"
                                            token.h
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#handleh","title":"handle.h","text":"
                                            handle.h
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#shared_bufferh","title":"shared_buffer.h","text":"
                                            shared_buffer.h
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#errorsh","title":"errors.h","text":"
                                            errors.h
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#eventsh","title":"events.h","text":"
                                            events.h
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#sysobjecth","title":"sysobject.h","text":"
                                            sysobject.h
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#exceptions","title":"Exceptions","text":"
                                            When the OPAE C++ API encounters an error from the OPAE C API, it captures the current source code location and the error code into an object of type except, then throws the except. Applications should implement the appropriate catch blocks required to respond to runtime exceptions.
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#excepth","title":"except.h","text":"
                                            except.h
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#misc","title":"Misc","text":"
                                            The version class wraps the OPAE C version API.
                                            "},{"location":"sw/fpga_api/fpga_cxx_api/#versionh","title":"version.h","text":"
                                            version.h
                                            "},{"location":"sw/fpga_api/fpga_python_api/","title":"OPAE Python API Reference","text":"
                                            The reference documentation for the OPAE Python API and is grouped into the following sections:
                                             
                                               
                                            • Module Types, Methods, and Constants
                                              •  
                                            • Fundamental Types
                                                 
                                              • Properties
                                                •  
                                              • Token
                                                •  
                                              • Handle
                                                •  
                                              • Event
                                                •  
                                              • Shared Buffer
                                                •  
                                              • Error
                                                •  
                                              • SysObject
                                                •  
                                               
                                              •  
                                            "},{"location":"sw/fpga_api/fpga_python_api/#module-types-methods-and-constants","title":"Module Types, Methods, and Constants","text":""},{"location":"sw/fpga_api/fpga_python_api/#fundamental-types","title":"Fundamental Types","text":""},{"location":"sw/fpga_api/fpga_python_api/#properties","title":"Properties","text":""},{"location":"sw/fpga_api/fpga_python_api/#token","title":"Token","text":""},{"location":"sw/fpga_api/fpga_python_api/#handle","title":"Handle","text":""},{"location":"sw/fpga_api/fpga_python_api/#event","title":"Event","text":""},{"location":"sw/fpga_api/fpga_python_api/#shared-buffer","title":"Shared Buffer","text":""},{"location":"sw/fpga_api/fpga_python_api/#error","title":"Error","text":""},{"location":"sw/fpga_api/fpga_python_api/#sysobject","title":"SysObject","text":""},{"location":"sw/fpga_api/plug_guide/readme/","title":"Plugin Developer's Guide","text":""},{"location":"sw/fpga_api/plug_guide/readme/#overview","title":"Overview","text":"
                                            Beginning with OPAE C library version 1.2.0, OPAE implements a plugin-centric model. This guide serves as a reference to define the makeup of an OPAE C API plugin and to describe a sequence of steps that one may follow when constructing an OPAE C API plugin.
                                            "},{"location":"sw/fpga_api/plug_guide/readme/#plugin-required-functions","title":"Plugin Required Functions","text":"
                                            An OPAE C API plugin is a runtime-loadable shared object library, also known as a module. On Linux systems, the dl family of APIs from libdl are used to interact with shared objects. Refer to \"man dlopen\" and \"man dlsym\" for examples of using the libdl API.
                                             
                                            An OPAE C API plugin implements one required function. This function is required to have C linkage, so that its name is not mangled.
                                             
                                                int opae_plugin_configure(opae_api_adapter_table *table, const char *config);\n
                                             
                                            During initialization, the OPAE plugin manager component loads each plugin, searching for its opae_plugin_configure function. If none is found, then the plugin manager rejects that plugin. When it is found, opae_plugin_configure is called passing a pointer to a freshly-created opae_api_adapter_table and a buffer consisting of configuration data for the plugin.
                                             
                                            The job of the opae_plugin_configure function is to populate the given adapter table with each of the plugin's API entry points and to consume and comprehend the given configuration data in preparation for initialization.
                                            "},{"location":"sw/fpga_api/plug_guide/readme/#opae-api-adapter-table","title":"OPAE API Adapter Table","text":"
                                            The adapter table is a data structure that contains function pointer entry points for each of the OPAE APIs implemented by a plugin. In this way, it adapts the plugin-specific behavior to the more general case of a flat C API. Note that OPAE applications are only required to link with opae-c. In other words, the name of the plugin library should not appear on the linker command line. In this way, plugins are truly decoupled from the OPAE C API, and they are required to adapt to the strict API specification by populating the adapter table only. No other linkage is required nor recommended.
                                             
                                            adapter.h contains the definition of the opae_api_adapter_table. An abbreviated version is depicted below, along with supporting type opae_plugin:
                                             
                                                typedef struct _opae_plugin {\n        char *path;\n        void *dl_handle;\n    } opae_plugin;\n\n    typedef struct _opae_api_adapter_table {\n\n        struct _opae_api_adapater_table *next;\n        opae_plugin plugin;\n\n        fpga_result (*fpgaOpen)(fpga_token token, fpga_handle *handle,\n                                int flags);\n\n        fpga_result (*fpgaClose)(fpga_handle handle);\n\n        ...\n\n        fpga_result (*fpgaEnumerate)(const fpga_properties *filters,\n                                     uint32_t num_filters, fpga_token *tokens,\n                                     uint32_t max_tokens,\n                                     uint32_t *num_matches);\n\n        ...\n\n        // configuration functions\n        int (*initialize)(void);\n        int (*finalize)(void);\n\n        // first-level query\n        bool (*supports_device)(const char *device_type);\n        bool (*supports_host)(const char *hostname);\n\n    } opae_api_adapter_table;\n
                                             
                                            Some points worth noting are that the adapter tables are organized in memory by adding them to a linked list data structure. This is the use of the next structure member. (The list management is handled by the plugin manager.) The plugin structure member contains the handle to the shared object instance, as created by dlopen. This handle is used in the plugin's opae_plugin_configure to load plugin entry points. A plugin need only implement the portion of the OPAE C API that a target application needs. Any API entry points that are not supported should be left as NULL pointers (the default) in the adapter table. When an OPAE API that has no associated entry point in the adapter table is called, the result for objects associated with that plugin will be FPGA_NOT_SUPPORTED.
                                             
                                            The following code illustrates a portion of the opae_plugin_configure for a theoretical OPAE C API plugin libfoo.so:
                                             
                                                /* foo_plugin.c */\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        adapter->fpgaOpen = dlsym(adapter->plugin.dl_handle, \"foo_fpgaOpen\");\n        adapter->fpgaClose =\n                dlsym(adapter->plugin.dl_handle, \"foo_fpgaClose\");\n\n        ...\n\n        adapter->fpgaEnumerate =\n                dlsym(adapter->plugin.dl_handle, \"foo_fpgaEnumerate\");\n\n        ...\n\n        return 0;\n    }\n
                                             
                                            Notice that the implementations of the API entry points for plugin libfoo.so are prefixed with foo_. This is the recommended practice to avoid name collisions and to enhance the debugability of the application. Upon successful configuration, opae_plugin_configure returns 0 to indicate success. A non-zero return value indicates failure and causes the plugin manager to reject the plugin from futher consideration.
                                            "},{"location":"sw/fpga_api/plug_guide/readme/#plugin-optional-functions","title":"Plugin Optional Functions","text":"
                                            Once the plugin manager loads and configures each plugin, it uses the adapter table to call back into the plugin so that it can be made ready for runtime. This is the job of the opae_plugin_initialize entry point, whose signature is defined as:
                                             
                                                int opae_plugin_initialize(void);\n
                                             
                                            The function takes no parameters, as the configuration data was already given to the plugin by opae_plugin_configure. opae_plugin_initialize returns 0 if no errors were encountered during initialization. A non-zero return code indicates that plugin initialization failed. A plugin makes its opae_plugin_initialize available to the plugin manager by populating the adapter table's initialize entry point as shown:
                                             
                                                /* foo_plugin.c */\n\n    int foo_plugin_initialize(void)\n    {\n        ...\n\n        return 0; /* success */\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->initialize =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_initialize\");\n\n        ...\n\n        return 0;\n    }\n
                                             
                                            If a plugin does not implement an opae_plugin_initialize entry point, then the initialize member of the adapter table should be left uninitialized. During plugin initialization, if a plugin has no opae_plugin_initialize entry in its adapter table, the plugin initialization step will be skipped, and the plugin will be considered to have initialized successfully.
                                             
                                            Once plugin initialization is complete for all loaded plugins, the system is considered to be running and fully functional.
                                             
                                            During teardown, the plugin manager uses the adapter table to call into each plugin's opae_plugin_finalize entry point, whose signature is defined as:
                                             
                                                int opae_plugin_finalize(void);\n
                                             
                                            opae_plugin_finalize returns 0 if no errors were encountered during teardown. A non-zero return code indicates that plugin teardown failed. A plugin makes its opae_plugin_finalize available to the plugin manager by populating the adapter table's finalize entry point as shown:
                                             
                                                /* foo_plugin.c */\n\n    int foo_plugin_finalize(void)\n    {\n        ...\n\n        return 0; /* success */\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->finalize =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_finalize\");\n\n        ...\n\n        return 0;\n    }\n
                                             
                                            If a plugin does not implement an opae_plugin_finalize entry point, then the finalize member of the adapter table should be left uninitialized. During plugin cleanup, if a plugin has no opae_plugin_finalize entry point in its adapter table, the plugin finalize step will be skipped, and the plugin will be considered to have finalized successfully.
                                             
                                            In addition to initialize and finalize, an OPAE C API plugin has two further optional entry points that relate to device enumeration. During enumeration, when a plugin is being considered for a type of device, the plugin may provide input on that decision by exporting an opae_plugin_supports_device entry point in the adapter table:
                                             
                                                bool opae_plugin_supports_device(const char *device_type);\n
                                             
                                            opae_plugin_supports_device returns true if the given device type is supported and false if it is not. A false return value from opae_plugin_supports_device causes device enumeration to skip the plugin.
                                             
                                            Populating the opae_plugin_supports_device is done as:
                                             
                                                /* foo_plugin.c */\n\n    bool foo_plugin_supports_device(const char *device_type)\n    {\n        if (/* device_type is supported */)\n            return true;\n\n        ...\n\n        return false;\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->supports_device =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_supports_device\");\n\n        ...\n\n        return 0;\n    }\n
                                             
                                            .. note::\n    The `opae_plugin_supports_device` mechanism serves as a placeholder only.\n    It is not implemented in the current version of the OPAE C API.\n
                                             
                                            Similarly to determining whether a plugin supports a type of device, a plugin may also answer questions about network host support by populating an opae_plugin_supports_host entry point in the adapter table:
                                             
                                                bool opae_plugin_supports_host(const char *hostname);\n
                                             
                                            opae_plugin_supports_host returns true if the given hostname is supported and false if it is not. A false return value from opae_plugin_supports_host causes device enumeration to skip the plugin.
                                             
                                            Populating the opae_plugin_supports_host is done as:
                                             
                                                /* foo_plugin.c */\n\n    bool foo_plugin_supports_host(const char *hostname)\n    {\n        if (/* hostname is supported */)\n            return true;\n\n        ...\n\n        return false;\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->supports_host =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_supports_host\");\n\n        ...\n\n        return 0;\n    }\n
                                             
                                            .. note::\n    The `opae_plugin_supports_host` mechanism serves as a placeholder only.\n    It is not implemented in the current version of the OPAE C API.\n
                                            "},{"location":"sw/fpga_api/plug_guide/readme/#plugin-construction","title":"Plugin Construction","text":"
                                            The steps required to implement an OPAE C API plugin, libfoo.so, are:
                                             
                                               
                                            • Create foo_plugin.c: implements opae_plugin_configure, opae_plugin_initialize, opae_plugin_finalize, opae_plugin_supports_device, and opae_plugin_supports_host as described in the previous sections.
                                              •  
                                            • Create foo_plugin.h: implements function prototypes for each of the plugin-specific OPAE C APIs.
                                              •  
                                             
                                                /* foo_plugin.h */\n\n    fpga_result foo_fpgaOpen(fpga_token token, fpga_handle *handle,\n                             int flags);\n\n    fpga_result foo_fpgaClose(fpga_handle handle);\n\n    ...\n\n    fpga_result foo_fpgaEnumerate(const fpga_properties *filters,\n                                  uint32_t num_filters, fpga_token *tokens,\n                                  uint32_t max_tokens,\n                                  uint32_t *num_matches);\n    ...\n
                                             
                                               
                                            • Create foo_types.h: implements plugin-specific types for opaque data structures.
                                              •  
                                             
                                                /* foo_types.h */\n\n    struct _foo_token {\n        ...\n    };\n\n    struct _foo_handle {\n        ...\n    };\n\n    struct _foo_event_handle {\n        ...\n    };\n\n    struct _foo_object {\n        ...\n    };\n
                                             
                                               
                                            • Create foo_enum.c: implements foo_fpgaEnumerate, foo_fpgaCloneToken, and foo_fpgaDestroyToken.
                                              •  
                                            • Create foo_open.c: implements foo_fpgaOpen.
                                              •  
                                            • Create foo_close.c: implements foo_fpgaClose.
                                              •  
                                            • Create foo_props.c: implements foo_fpgaGetProperties, foo_fpgaGetPropertiesFromHandle, foo_fpgaUpdateProperties
                                              •  
                                            • Create foo_mmio.c: implements foo_fpgaMapMMIO, foo_fpgaUnmapMMIO foo_fpgaWriteMMIO64, foo_fpgaReadMMIO64, foo_fpgaWriteMMIO32, foo_fpgaReadMMIO32.
                                              •  
                                            • Create foo_buff.c: implements foo_fpgaPrepareBuffer, foo_fpgaReleaseBuffer, foo_fpgaGetIOAddress.
                                              •  
                                            • Create foo_error.c: implements foo_fpgaReadError, foo_fpgaClearError, foo_fpgaClearAllErrors, foo_fpgaGetErrorInfo.
                                              •  
                                            • Create foo_event.c: implements foo_fpgaCreateEventHandle, foo_fpgaDestroyEventHandle, foo_fpgaGetOSObjectFromEventHandle, foo_fpgaRegisterEvent, foo_fpgaUnregisterEvent.
                                              •  
                                            • Create foo_reconf.c: implements foo_fpgaReconfigureSlot.
                                              •  
                                            • Create foo_obj.c: implements foo_fpgaTokenGetObject, foo_fpgaHandleGetObject, foo_fpgaObjectGetObject, foo_fpgaDestroyObject, foo_fpgaObjectGetSize, foo_fpgaObjectRead, foo_fpgaObjectRead64, foo_fpgaObjectWrite64.
                                              •  
                                            • Create foo_clk.c: implements foo_fpgaSetUserClock, foo_fpgaGetUserClock.
                                              •  
                                            "},{"location":"sw/fpga_api/prog_guide/readme/","title":"OPAE C API Programming Guide","text":""},{"location":"sw/fpga_api/prog_guide/readme/#overview","title":"Overview","text":"
                                            The OPAE C library (libopae-c) is a lightweight user-space library that provides abstractions for FPGA resources in a compute environment. The OPAE C library  builds on the driver stack that supports the FPGA device, abstracting  hardware- and OS-specific details. It provides access to the underlying FPGA resources as a set of features available to software programs running on the host. These features include the acceleration logic preconfigured on the FPGA and functions to manage and reconfigure the FPGA. The library enables your applications to transparently and seamlessly benefit from FPGA-based acceleration.
                                             
                                             
                                            By providing a unified C API, the library supports different FPGA integration and deployment models, ranging from single-node systems with one or a few FPGA devices to large-scale FPGA deployments in a data center. At one end of the spectrum, the API supports a simple application using a PCIe link to reconfigure  the FPGA with different accelerator functions. At the other end of the spectrum, resource management and orchestration services in a data center can use this API to discover and select FPGA resources and then allocate them for use by acceleration workloads.
                                            "},{"location":"sw/fpga_api/prog_guide/readme/#opae-role","title":"OPAE Role","text":"
                                            The OPAE provides a common base layer for a wide range of applications without sacrificing performance or efficiency. The abstraction layer limits  the details of the FPGA hardware that software applications must handle. 
                                             
                                            The OPAE provides consistent interfaces to crucial components of the platform. The  OPAE does not constrain frameworks and applications by making optimizations with limited applicability. When the OPAE does provide convenience functions or optimizations, they are optional.
                                             
                                            For example, the OPAE provides an interface to allocate physically contiguous buffers in system memory that user-space software and an accelerator can share.  This interface enables the most basic feature set of allocating and sharing a large page of memory in one API call. However, it  does not provide a malloc()-like interface backed by a memory pool or slab allocator. Higher layers of the software stack can make such domain-specific optimizations.
                                            "},{"location":"sw/fpga_api/prog_guide/readme/#intel-accelerator-stack-hardware-terminology","title":"Intel Accelerator Stack Hardware Terminology","text":"
                                            The following terms define the hardware and hardware processes involved in creating an accelerator function. 
                                             
                                               
                                            • FPGA: Field Programmable Gate Array  is a discrete or integrated device connecting to a host CPU via PCIe or other type of interconnects.
                                              •  
                                            • Accelerator Function Unit (AFU): The AFU is the supplied implementation of an accelerator, typically in HDL. AFUs implement a function such as compression, encryption, or mathematical operations. The Quartus Prime Pro software synthesizes the RTL logic into a bitstream. 
                                              •  
                                            • Accelerator Function (AF): The AF is the compiled binary for an AFU. An AF is a raw binary file (.rbf) bitstream. A tool (fpgaconf) reconfigures the FPGA using an AF bitstream.
                                              •  
                                            • Reconfiguration: The process of reprogramming the FPGA with a different AF.
                                              •  
                                            "},{"location":"sw/fpga_api/prog_guide/readme/#opae-software-concepts-reflected-in-the-c-api","title":"OPAE Software Concepts Reflected in the C API","text":"
                                            The following OPAE data structures and functions integrate AFUs into the OPAE environment.  The OPAE C API models these data structures and functions. For more information on the object  models refer to the Object model section.
                                             
                                               
                                            • Accelerator: An accelerator is an allocable accelerator function implemented in an FPGA.  An accelerator tracks the  ownership of an AFU (or part of it) for a process that uses it. Multiple processes can share an accelerator.
                                              •  
                                            • Device: The OPAE enumerates and models two device types: the FPGA and the AFU.
                                              •  
                                            • Events: Events are asynchronous notifications. The FPGA driver triggers particular events to indicate error conditions. Accelerator logic can also define its own events. User applications can choose to be notified when particular events occur and respond appropriately.
                                              •  
                                            • Shared memory buffers: Software allocates shared memory buffers in user process memory on the host. Shared memory buffers facilitate data transfers between the user process and the  accelerator that it owns.
                                              •  
                                            "},{"location":"sw/fpga_api/prog_guide/readme/#opae-library","title":"OPAE Library","text":"
                                            Linking with this library is straightforward. Code using the  OPAE library should include the header file fpga.h. Taking the GCC compiler on Linux as an example, here is the simplest compile and link command:
                                             
                                            gcc myprog.c -I</path/to/fpga.h> -L</path/to/libopae-c.so> -lopae-c -luuid -ljson-c -lpthread
                                             
                                            .. note::
                                             
                                            The OPAE library uses the third-party `libuuid` and `libjson-c` libraries that are not distributed with \nthe OPAE library. Make sure to install these libraries.\n
                                            "},{"location":"sw/fpga_api/prog_guide/readme/#sample-code","title":"Sample Code","text":"
                                            The library source includes two code samples. Use these samples to learn how to call functions in the library. Build and run these samples to determine if your installation and environment are set up properly. 
                                             
                                            Refer to the Running the Hello FPGA Example chapter in the Intel\u00ae Acceleration Stack Quick Start Guide for for Intel Programmable Acceleration Card with Intel Arria\u00ae 10 GX FPGA for more information about using the sample code.  
                                            "},{"location":"sw/fpga_api/prog_guide/readme/#high-level-directory-structure","title":"High-Level Directory Structure","text":"
                                            Building and installing the OPAE library results in the following directory structure on the Linux OS. Windows and MacOS have similar directories and files.
                                             Directory & Files Contents include/opae Directory containing all header files include/opae/fpga.h Top-level header for user code to include include/opae/access.h Header file for accelerator acquire/release, MMIO, memory management, event handling, and so on include/opae/bitstream.h Header file for bitstream manipulation functions include/opae/common.h Header file for error reporting functions include/opae/enum.h Header file for AFU enumeration functions include/opae/manage.h Header file for FPGA management functions include/opae/types.h Various type definitions lib Directory containing shared library files lib/libopae-c.so The shared dynamic library for linking with the user application doc Directory containing API documentation doc/html Directory for documentation of HTML format doc/latex Directory for documentation of LaTex format doc/man Directory for documentation of Unix man page format"},{"location":"sw/fpga_api/prog_guide/readme/#basic-application-flow","title":"Basic Application Flow","text":"
                                            The figure below shows the basic application flow from the viewpoint of a user-process. 
                                             
                                            "},{"location":"sw/fpga_api/prog_guide/readme/#api-components","title":"API Components","text":"
                                            The API object model abstracts the physical FPGA device and available functions. It is a generalized model and extends to  describe any FPGA type. 
                                            "},{"location":"sw/fpga_api/prog_guide/readme/#object-models","title":"Object Models","text":"
                                               
                                            • fpga_objtype: An enum type that represents the type of an FPGA resource, either FPGA_DEVICE or FPGA_ACCELERATOR.  An FPGA_DEVICE object corresponds to a physical FPGA device. Only FPGA_DEVICE objects can invoke management functions. The FPGA_ACCELERATOR represents an instance of an AFU. 
                                              •  
                                            • fpga_token: An opaque type that represents a resource known to, but not necessarily owned by, the calling process. The calling process must own a resource before it can invoke functions of the resource.
                                              •  
                                            • fpga_handle: An opaque type that represents a resource owned by the calling process. The API functions fpgaOpen() and fpgaClose() acquire and release ownership of a resource that an fpga_handle represents. (Refer to the Functions section for more information.)
                                              •  
                                            • fpga_properties: An opaque type for a properties object. Your applications use these properties to query and search for appropriate resources. The  FPGA Resource Properties section documents properties visible to your applications.
                                              •  
                                            • fpga_event_handle: An opaque handle the FPGA driver uses to notify your application about an event. 
                                              •  
                                            • fpga_event_type: An enum type that represents the types of events. The following are valid values:  FPGA_EVENT_INTERRUPT, FPGA_EVENT_ERROR, and FPGA_EVENT_POWER_THERMAL. (The Intel Programmable Acceleration Card (PAC) with Intel Arria 10 GX FPGA does not handle thermal and power events.)
                                              •  
                                            • fpga_result: An enum type to represent the result of an API function. If the function returns successfully the result is FPGA_OK. Otherwise, the result is the appropriate error codes. Function fpgaErrStr() translates an error code into human-readable strings.
                                              •  
                                            "},{"location":"sw/fpga_api/prog_guide/readme/#functions","title":"Functions","text":"
                                            The table below groups important API calls by their functionality. For more information about each of the functions, refer to the  OPAE C API reference manual.
                                             Functionality API Call FPGA Accelerator Description Enumeration fpgaEnumerate() Yes Yes Query FPGA resources that match certain properties Enumeration: Properties fpga[Get, Update, Clear, Clone, Destroy Properties]() Yes Yes Manage fpga_properties life cycle fpgaPropertiesGet[Prop]() Yes Yes Get the specified property Prop, from the FPGA Resource Properties table fpgaPropertiesSet[Prop]() Yes Yes Set the specified property Prop, from the FPGA Resource Properties table Access: Ownership fpga[Open, Close]() Yes Yes Acquire/release ownership Access: Reset fpgaReset() Yes Yes Reset an accelerator Access: Event handling fpga[Register, Unregister]Event() Yes Yes Register/unregister an event to be notified about fpga[Create, Destroy]EventHandle() Yes Yes Manage fpga_event_handle life cycle Access: MMIO fpgaMapMMIO(), fpgaUnMapMMIO() Yes Yes Map/unmap MMIO space fpgaGetMMIOInfo() Yes Yes Get information about the specified MMIO space fpgaReadMMIO[32, 64]() Yes Yes Read a 32-bit or 64-bit value from MMIO space fpgaWriteMMIO[32, 64]() Yes Yes Write a 32-bit or 64-bit value to MMIO space Memory management: Shared memory fpga[Prepare, Release]Buffer() Yes Yes Manage memory buffer shared between the calling process and an accelerator fpgaGetIOAddress() Yes Yes Return the device I/O address of a shared memory buffer fpgaBindSVA() Yes Yes Bind IOMMU shared virtual addressing Management: Reconfiguration fpgaReconfigureSlot() Yes No Replace an existing AFU with a new one Error report fpgaErrStr() Yes Yes Map an error code to a human readable string 
                                            .. note::
                                             
                                            The UMsg APIs are not supported for the Intel(R) PAC cards. They will be deprecated in a future release.\n
                                            "},{"location":"sw/fpga_api/prog_guide/readme/#fpga-resource-properties","title":"FPGA Resource Properties","text":"
                                            Applications query resource properties by specifying the property name for Prop in the  fpgaPropertiesGet[Prop]() and fpgaPropertiesSet[Prop]() functions. The FPGA and Accelerator columns state whether or not the Property is available for the FPGA or Accelerator objects.
                                             Property FPGA Accelerator Description Parent No Yes fpga_token of the parent object ObjectType Yes Yes The type of the resource: either FPGA_DEVICE or FPGA_ACCELERATOR Bus Yes Yes The bus number Device Yes Yes The PCI device number Function Yes Yes The PCI function number SocketId Yes Yes The socket ID DeviceId Yes Yes The device ID NumSlots Yes No Number of AFU slots available on an FPGA_DEVICE resource BBSID Yes No The FPGA Interface Manager (FIM) ID of an FPGA_DEVICE resource BBSVersion Yes No The FIM version of an FPGA_DEVICE resource VendorId Yes No The vendor ID of an FPGA_DEVICE resource GUID Yes Yes The GUID of an FPGA_DEVICE or FPGA_ACCELERATOR resource NumMMIO No Yes The number of MMIO space of an FPGA_ACCELERATOR resource NumInterrupts No Yes The number of interrupts of an FPGA_ACCELERATOR resource AcceleratorState No Yes The state of an FPGA_ACCELERATOR resource: either FPGA_ACCELERATOR_ASSIGNED or FPGA_ACCELERATOR_UNASSIGNED"},{"location":"sw/fpga_api/prog_guide/readme/#opae-c-api-return-codes","title":"OPAE C API Return Codes","text":"
                                            The OPAE C library returns a code for every exported public API function.  FPGA_OK indicates successful completion of the requested operation. Any return code other than FPGA_OK indicates an error or unexpected behavior. When using the OPAE C API, always check the API return codes. 
                                             Error Code Description FPGA_OK Operation completed successfully FPGA_INVALID_PARAM Invalid parameter supplied FPGA_BUSY Resource is busy FPGA_EXCEPTION An exception occurred FPGA_NOT_FOUND A required resource was not found FPGA_NO_MEMORY Not enough memory to complete operation FPGA_NOT_SUPPORTED Requested operation is not supported FPGA_NO_DRIVER Driver is not loaded FPGA_NO_DAEMON FPGA Daemon (fpgad) is not running FPGA_NO_ACCESS Insufficient privileges or permissions FPGA_RECONF_ERROR Error while reconfiguring FPGA"},{"location":"sw/fpga_api/prog_guide/readme/#usage-models","title":"Usage Models","text":""},{"location":"sw/fpga_api/prog_guide/readme/#query-and-search-for-a-resource","title":"Query and Search for a Resource","text":"
                                            The user-code first populates an fpga_properties object with the required properties. Then, fpgaEnumerate() searches for matching resources. fpgaEnumerate() may return more than one matching resource.
                                             
                                            #include \"fpga/fpga.h\"\n\nfpga_guid               guid;\nfpga_properties         filter = NULL;\nfpga_result             res;\nfpga_token              tokens[MAX_NUM_TOKENS];\nuint32_t                num_matches = 0;\n\n/* Start with an empty properties object */\nres = fpgaGetProperties(NULL, &filter);\n\n/* Populate the properties object with required values.\n   In this case, search for accelerators that matches \n   the specified GUID.\n*/\nuuid_parse(GUID, guid);\nres = fpgaPropertiesSetObjectType(filter, FPGA_ACCELERATOR);\nres = fpgaPropertiesSetGuid(filter, guid);\n\n/* Query the number of matching resources */\nres = fpgaEnumerate(&filter, 1, NULL, 1, &num_matches);\n\n/* Return tokens for all matching resources */\nres = fpgaEnumerate(&filter, 1, tokens, num_matches, &num_matches);\n\n/* Destroy the properties object */\nres = fpgaDestroyProperties(&filter);\n\n/* More code */\n......\n\n/* Destroy tokens */\nfor (uint32_t i = 0; i < num_matches; ++i) {\n    res = fpgaDestroyToken(tokens[i]);\n}\n
                                             
                                            The fpgaEnumerate() function can take multiple fpga_propertiesobjects in an array. In such cases, the function performs a logical OR of  the properties object and returns resources that match any of  the multiple properties. The  fpga_token objects that fpgaEnumerate() returns, do not signify ownership. To acquire ownership of a resource represented by a token, pass the token to fpgaOpen().
                                            "},{"location":"sw/fpga_api/prog_guide/readme/#acquire-and-release-a-resource","title":"Acquire and Release a Resource","text":"
                                            Use fpgaOpen() and fpgaClose() to acquire and release ownership of a resource.  The calling process must own the resource before it can initiate MMIO, access share memory buffers,  and use functions offered by the resource.
                                             
                                                #include \"fpga/fpga.h\"\n\n    fpga_handle             handle;\n    fpga_result             res;\n\n    /* Acquire ownership of a resource that \n    `fpgaEnumerate()` previously returned as a token */\n\n    res = fpgaOpen(token, &handle);\n\n    /* More code */\n    ......\n\n    /* Release the ownership */\n    res = fpgaClose(handle);\n
                                            "},{"location":"sw/fpga_api/prog_guide/readme/#shared-memory-buffer","title":"Shared Memory Buffer","text":"
                                            This code snippet shows how to prepare a memory buffer to be shared between the calling process and an accelerator.
                                             
                                                #include \"fpga/fpga.h\"\n\n    fpga_handle             handle;\n    fpga_result             res;\n\n    /* Hint for the virtual address of the buffer */\n    volatile uint64_t       *addr_hint;\n    /* An ID we can use to reference the buffer later */\n    uint32_t                bufid;\n    /* Flag to indicate whether or not the buffer is preallocated */\n    int                     flag = 0;\n\n    /* Allocate (if necessary), pin, and map a buffer to be accessible\n       by an accelerator\n    */\n    res = fpgaPrepareBuffer(handle, BUF_SIZE, (void **) &addr_hint,\n                            &bufid, flag);\n\n    /* The actual address mapped to the buffer */\n    uint64_t                iova;\n    /* Get the IO virtual address for the buffer */\n    res = fpgaGetIOAddress(handle, bufid, &iova);\n\n    /* Inform the accelerator about the virtual address by writing to its mapped\n       register file\n    */\n    ......\n\n    /* More code */\n    ......\n\n    /* Release the shared buffer */\n    res = fpgaReleaseBuffer(handle, bufid);\n
                                             
                                            .. note::
                                             
                                            The `flag` variable can take a constant `FPGA_BUF_PREALLOCATED` to\nindicate that the calling process has already allocated the address space\nthat `addr_hint` points to.\n
                                            "},{"location":"sw/fpga_api/prog_guide/readme/#mmio","title":"MMIO","text":"
                                            This code snippet shows how to map and unmap the register file of an accelerator into the calling process's virtual memory space.
                                             
                                                #include \"fpga/fpga.h\"\n\n    fpga_handle             handle;\n    fpga_result             res;\n\n    /* Index of the MMIO space. There might be multiple spaces on an accelerator */\n    uint32_t                mmio_num = 0;\n    /* Mapped address */\n    uint64_t                mmio_addr;\n\n    /* Map MMIO */\n    res = fpgaMapMMIO(handle, mmio_num, &mmio_addr);\n\n    /* Write to a 32-bit value to the mapped register file at a certain byte\n       offset.\n\n       CSR_CTL is the offset in the mapped space to where the value will be\n       written. It's defined elsewhere.\n    */\n    res = fpgaWriteMMIO32(handle, mmio_num, CSR_CTL, value);\n\n    /* More code */\n    ......\n\n    /* Unmap MMIO */\n    res = fpgaUnmapMMIO(handle, mmio_num);\n
                                             
                                            .. Note::
                                             
                                            Every AFU has its own register adress space and its own protocol to control operation through \nthe registers. \n
                                            "},{"location":"sw/fpga_api/quick_start/readme/","title":"Quick Start Guide","text":""},{"location":"sw/fpga_api/quick_start/readme/#overview","title":"Overview","text":"
                                            The OPAE C library is a lightweight user-space library that provides an abstraction for FPGA resources in a compute environment. Built on top of the OPAE Intel\u00ae FPGA driver stack that supports Intel\u00ae FPGA platforms, the library abstracts away hardware-specific and OS-specific details and exposes the underlying FPGA resources as a set of features accessible from within software programs running on the host.
                                             
                                            These features include the acceleration logic preconfigured on the device, as well as functions to manage and reconfigure the device. Hence, the library can enable user applications to transparently and seamlessly leverage FPGA-based acceleration.
                                             
                                            In this document, we will explore the initial steps on how to set up the required libraries and utilities to use the FPGA devices.
                                             
                                            If you do not have access to an Intel\u00ae Xeon\u00ae processor with integrated FPGA, or a programmable FPGA acceleration card for Intel\u00ae Xeon\u00ae processors, you will not be able to run the examples below. However, you can still make use of the AFU simulation environment (ASE) to develop and test accelerator RTL with OPAE applications.
                                             
                                            The source for the OPAE SDK Linux device drivers is available at the OPAE Linux DFL drivers repository.
                                            "},{"location":"sw/fpga_api/quick_start/readme/#build-the-opae-linux-device-drivers-from-the-source","title":"Build the OPAE Linux device drivers from the source","text":"
                                            For building the OPAE kernel and kernel driver, the kernel development environment is required. So before you build the kernel, you must install the required packages. Run the following commands (we are using Fedora 32 as an example):
                                             
                                            sudo yum install gcc gcc-c++ make kernel-headers kernel-devel elfutils-libelf-devel ncurses-devel openssl-devel bison flex\n
                                             
                                            Download the OPAE upstream kernel tree from GitHub:
                                             
                                            git clone https://github.com/OPAE/linux-dfl.git -b fpga-upstream-dev-5.8.0\n
                                             
                                            Configure the kernel:
                                             
                                            cd linux-dfl\ncp /boot/config-`uname -r` .config\ncat configs/n3000_d5005_defconfig >> .config \necho 'CONFIG_LOCALVERSION=\"-dfl\"' >> .config\necho 'CONFIG_LOCALVERSION_AUTO=y' >> .config\nmake olddefconfig\n
                                             
                                            Compile and install the new kernel:
                                             
                                            make -j\nsudo make modules_install\nsudo make install\n
                                             
                                            After the installation finishes, reboot your system. Log back into the system, and confirm the correct version for the kernel:
                                             
                                            $ uname -a\nLinux localhost.localdomain 5.8.0-rc1-dfl-g73e16386cda0 #6 SMP Wed Aug 19 08:38:32 EDT 2020 x86_64 x86_64 x86_64 GNU/Linux\n
                                            "},{"location":"sw/fpga_api/quick_start/readme/#building-and-installing-the-opae-sdk-from-the-source","title":"Building and installing the OPAE SDK from the source","text":"
                                            Download the OPAE SDK source package:
                                             
                                               
                                            1. Go to the section corresponding to the desired release on GitHub:
                                              2.  
                                            2. Click the Source code (tar.gz) link under the section's Assets.
                                              3.  
                                            3.  
                                              On the command line, go through the following steps to install it:
                                               
                                              # Unpack\ntar xfvz opae-sdk-<release>.tar.gz\n# Configure\ncd opae-sdk-<release>\nmkdir build\ncd build\n# The default installation prefix is `/usr/local`;\n# You have the option to configure for a different location\ncmake [-DCMAKE_INSTALL_PREFIX=<prefix>] ..\n# Compile\nmake\n# Install: you need system administration privileges (`sudo`)\n# if you have elected to install in the default location\n[sudo] make install\n
                                               
                                              4.  
                                             
                                            The remainder of this guide assumes you installed into /usr/local.
                                            "},{"location":"sw/fpga_api/quick_start/readme/#configuring-the-fpga-loading-an-fpga-afu","title":"Configuring the FPGA (loading an FPGA AFU)","text":"
                                            The fpgaconf tool exercises the AFU reconfiguration functionality. It shows how to read a bitstream from a disk file, check its validity and compatibility, and then injects it into FPGA to be configured as a new AFU, which can then be discovered and used by user applications.
                                             
                                            For this step, you require a valid green bitstream (GBS) file. To reconfigure the FPGA slot, you can issue the following command as system administrator:
                                             
                                            sudo fpgaconf -b 0x5e <filename>.gbs\n
                                             
                                            The -b option to fpgaconf indicates the target bus number of the FPGA slot to be reconfigured. Alternatively, you can also specify the target socket number of the FPGA using the -s option.
                                             
                                            $ sudo fpgaconf --help\nUsage:\n        fpgaconf [-hvn] [-b <bus>] [-d <device>] [-f <function>] [-s <socket>] <gbs>\n\n                -h,--help           Print this help\n                -v,--verbose        Increase verbosity\n                -n,--dry-run        Don't actually perform actions\n                -b,--bus            Set target bus number\n                -d,--device         Set target device number\n                -f,--function       Set target function number\n                -s,--socket         Set target socket number\n
                                             
                                            The sample application on the Building a Sample Application\nsection requires loading of an AFU called \"Native Loopback\nAdapter\" (NLB) on the FPGA. Please refer to the NLB documentation\nfor the location of the NLB's green bitstream. You also can verify\nif the NLB green bitstream has already been loaded into the FPGA\nslot by typing the following command and checking the output\nmatches the following:\n\n```bash\n$ cat /sys/class/fpga_region/region0/dfl-port.0/afu_id\nd8424dc4a4a3c413f89e433683f9040b\n```\n
                                             
                                            The fpgaconf tool is not available for the Intel PAC N3000. The NLB is\nalready included in the AFU.\n
                                            "},{"location":"sw/fpga_api/quick_start/readme/#building-a-sample-application","title":"Building a sample application","text":"
                                            The library source includes code samples. Use these samples to learn how to call functions in the library. Build and run these samples as quick sanity checks to determine if your installation and environment are set up properly.
                                             
                                            In this guide, we will build hello_fpga.c. This is the \"Hello World!\" example of using the library.  This code searches for a predefined and known AFU called \"Native Loopback Adapter\" on the FPGA. If found, it acquires ownership and then interacts with the AFU by sending it a 2MB message and waiting for the message to be echoed back. This code exercises all major components of the API except for AFU reconfiguration: AFU search, enumeration, access, MMIO, and memory management.
                                             
                                            You can also find the source for hello_fpga in the samples directory of the OPAE SDK repository on GitHub.
                                             
                                                int main(int argc, char *argv[])\n    {\n        fpga_properties    filter = NULL;\n        fpga_token         afu_token;\n        fpga_handle        afu_handle;\n        fpga_guid          guid;\n        uint32_t           num_matches;\n\n        volatile uint64_t *dsm_ptr    = NULL;\n        volatile uint64_t *status_ptr = NULL;\n        volatile uint64_t *input_ptr  = NULL;\n        volatile uint64_t *output_ptr = NULL;\n\n        uint64_t        dsm_wsid;\n        uint64_t        input_wsid;\n        uint64_t        output_wsid;\n        fpga_result     res = FPGA_OK;\n\n        if (uuid_parse(NLB0_AFUID, guid) < 0) {\n            fprintf(stderr, \"Error parsing guid '%s'\\n\", NLB0_AFUID);\n            goto out_exit;\n        }\n\n        /* Look for accelerator by its \"afu_id\" */\n        res = fpgaGetProperties(NULL, &filter);\n        ON_ERR_GOTO(res, out_exit, \"creating properties object\");\n\n        res = fpgaPropertiesSetObjectType(filter, FPGA_ACCELERATOR);\n        ON_ERR_GOTO(res, out_destroy_prop, \"setting object type\");\n\n        res = fpgaPropertiesSetGuid(filter, guid);\n        ON_ERR_GOTO(res, out_destroy_prop, \"setting GUID\");\n\n        /* TODO: Add selection via BDF / device ID */\n\n        res = fpgaEnumerate(&filter, 1, &afu_token, 1, &num_matches);\n        ON_ERR_GOTO(res, out_destroy_prop, \"enumerating accelerators\");\n\n        if (num_matches < 1) {\n            fprintf(stderr, \"accelerator not found.\\n\");\n            res = fpgaDestroyProperties(&filter);\n            return FPGA_INVALID_PARAM;\n        }\n\n        /* Open accelerator and map MMIO */\n        res = fpgaOpen(afu_token, &afu_handle, 0);\n        ON_ERR_GOTO(res, out_destroy_tok, \"opening accelerator\");\n\n        res = fpgaMapMMIO(afu_handle, 0, NULL);\n        ON_ERR_GOTO(res, out_close, \"mapping MMIO space\");\n\n        /* Allocate buffers */\n        res = fpgaPrepareBuffer(afu_handle, LPBK1_DSM_SIZE,\n                    (void **)&dsm_ptr, &dsm_wsid, 0);\n        ON_ERR_GOTO(res, out_close, \"allocating DSM buffer\");\n\n        res = fpgaPrepareBuffer(afu_handle, LPBK1_BUFFER_ALLOCATION_SIZE,\n                   (void **)&input_ptr, &input_wsid, 0);\n        ON_ERR_GOTO(res, out_free_dsm, \"allocating input buffer\");\n\n        res = fpgaPrepareBuffer(afu_handle, LPBK1_BUFFER_ALLOCATION_SIZE,\n                   (void **)&output_ptr, &output_wsid, 0);\n        ON_ERR_GOTO(res, out_free_input, \"allocating output buffer\");\n\n        printf(\"Running Test\\n\");\n\n        /* Initialize buffers */\n        memset((void *)dsm_ptr,    0,    LPBK1_DSM_SIZE);\n        memset((void *)input_ptr,  0xAF, LPBK1_BUFFER_SIZE);\n        memset((void *)output_ptr, 0xBE, LPBK1_BUFFER_SIZE);\n\n        cache_line *cl_ptr = (cache_line *)input_ptr;\n        for (uint32_t i = 0; i < LPBK1_BUFFER_SIZE / CL(1); ++i) {\n            cl_ptr[i].uint[15] = i+1; /* set the last uint in every cacheline */\n        }\n\n        /* Reset accelerator */\n        res = fpgaReset(afu_handle);\n        ON_ERR_GOTO(res, out_free_output, \"resetting accelerator\");\n\n        /* Program DMA addresses */\n        uint64_t iova;\n        res = fpgaGetIOAddress(afu_handle, dsm_wsid, &iova);\n        ON_ERR_GOTO(res, out_free_output, \"getting DSM IOVA\");\n\n        res = fpgaWriteMMIO64(afu_handle, 0, CSR_AFU_DSM_BASEL, iova);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_AFU_DSM_BASEL\");\n\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 0);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 1);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        res = fpgaGetIOAddress(afu_handle, input_wsid, &iova);\n        ON_ERR_GOTO(res, out_free_output, \"getting input IOVA\");\n        res = fpgaWriteMMIO64(afu_handle, 0, CSR_SRC_ADDR, CACHELINE_ALIGNED_ADDR(iova));\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_SRC_ADDR\");\n\n        res = fpgaGetIOAddress(afu_handle, output_wsid, &iova);\n        ON_ERR_GOTO(res, out_free_output, \"getting output IOVA\");\n        res = fpgaWriteMMIO64(afu_handle, 0, CSR_DST_ADDR, CACHELINE_ALIGNED_ADDR(iova));\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_DST_ADDR\");\n\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_NUM_LINES, LPBK1_BUFFER_SIZE / CL(1));\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_NUM_LINES\");\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CFG, 0x42000);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        status_ptr = dsm_ptr + DSM_STATUS_TEST_COMPLETE/8;\n\n        /* Start the test */\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 3);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        /* Wait for test completion */\n        while (0 == ((*status_ptr) & 0x1)) {\n            usleep(100);\n        }\n\n        /* Stop the device */\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 7);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        /* Check output buffer contents */\n        for (uint32_t i = 0; i < LPBK1_BUFFER_SIZE; i++) {\n            if (((uint8_t*)output_ptr)[i] != ((uint8_t*)input_ptr)[i]) {\n                fprintf(stderr, \"Output does NOT match input \"\n                    \"at offset %i!\\n\", i);\n                break;\n            }\n        }\n\n        printf(\"Done Running Test\\n\");\n\n        /* Release buffers */\n    out_free_output:\n        res = fpgaReleaseBuffer(afu_handle, output_wsid);\n        ON_ERR_GOTO(res, out_free_input, \"releasing output buffer\");\n    out_free_input:\n        res = fpgaReleaseBuffer(afu_handle, input_wsid);\n        ON_ERR_GOTO(res, out_free_dsm, \"releasing input buffer\");\n    out_free_dsm:\n        res = fpgaReleaseBuffer(afu_handle, dsm_wsid);\n        ON_ERR_GOTO(res, out_unmap, \"releasing DSM buffer\");\n\n        /* Unmap MMIO space */\n    out_unmap:\n        res = fpgaUnmapMMIO(afu_handle, 0);\n        ON_ERR_GOTO(res, out_close, \"unmapping MMIO space\");\n\n        /* Release accelerator */\n    out_close:\n        res = fpgaClose(afu_handle);\n        ON_ERR_GOTO(res, out_destroy_tok, \"closing accelerator\");\n\n        /* Destroy token */\n    out_destroy_tok:\n        res = fpgaDestroyToken(&afu_token);\n        ON_ERR_GOTO(res, out_destroy_prop, \"destroying token\");\n\n        /* Destroy properties object */\n    out_destroy_prop:\n        res = fpgaDestroyProperties(&filter);\n        ON_ERR_GOTO(res, out_exit, \"destroying properties object\");\n\n    out_exit:\n        return res;\n\n    }\n
                                             
                                            Linking with the OPAE library is straightforward.  Code using this library should include the header file fpga.h. Taking the GCC compiler on Linux as an example, the minimalist compile and link line should look like:
                                             
                                            gcc -std=c99 hello_fpga.c -I/usr/local/include -L/usr/local/lib -lopae-c -luuid -ljson-c -lpthread -o hello_fpga\n
                                             
                                            The API uses some features from the C99 language standard. The\n`-std=c99` switch is required if the compiler does not support C99 by\ndefault.\n
                                             
                                            Third-party library dependency: The library internally uses\n`libuuid` and `libjson-c`. But they are not distributed as part of the\nlibrary. Make sure you have these libraries properly installed.\n
                                             
                                            The layout of AFU is different between the N3000 card and Rush Creek/Darby Creek.\nIn the N3000 card, the NLB and DMA are contained in the AFU, so we need to do\nenumeration again in AFU to discover the NLB.\nTo run the hello_fpga application on the N3000 card, it should use the `-c`\noption to support the N3000 card:\n\n```bash\n$ sudo ./hello_fpga -c\nRunning Test\nRunning on bus 0x08.\nAFU NLB0 found @ 28000\nDone Running Test\n```\n
                                             
                                            To run the hello_fpga application; just issue:
                                             
                                            $ sudo ./hello_fpga\nRunning Test\nDone\n
                                            "},{"location":"sw/fpga_api/quick_start/readme/#setup-iofs-release1-bitstream-on-fpga-pcie-card","title":"Setup IOFS Release1 Bitstream on FPGA PCIe card","text":"
                                            Program IOFS Release1 bitstream on the FPGA D5005 or N6000 cards and reboot the system.
                                             
                                            Run this command:
                                             
                                            $ lspci | grep acc\n3b:00.0 Processing accelerators: Intel Corporation Device af00 (rev 01)\n
                                             
                                            Number of virtual functions supported by bitstream:
                                             
                                            $ cat /sys/bus/pci/devices/0000:3b:00.0/sriov_totalvfs \noutput: 3\n
                                             
                                            Enable FPGA virtual functions:
                                             
                                            sudo sh -c \"echo 3 > /sys/bus/pci/devices/0000:3b:00.0/sriov_numvfs\"\n
                                             
                                            List of FPGA PF and VF's:
                                             
                                            Physical Functions (PFs):\n  3b:00.0 Processing accelerators: Intel Corporation Device af00 (rev 01)\n\nVirtual Functions (VFs).\n  3b:00.1 Processing accelerators: Intel Corporation Device af01 (rev 01)\n  3b:00.2 Processing accelerators: Intel Corporation Device af01 (rev 01)\n  3b:00.3 Processing accelerators: Intel Corporation Device af01 (rev 01)\n
                                             
                                            Bind vfio-pcie driver to FPGA virtual functions:
                                             
                                            sudo opaevfio  -i 0000:3b:00.1 -u userid -g groupid\nsudo opaevfio  -i 0000:3b:00.2 -u userid -g groupid\nsudo opaevfio  -i 0000:3b:00.3 -u userid -g groupid\n
                                             
                                            List of fpga accelerators:
                                             
                                            $ fpgainfo port\n\n  //****** PORT ******//\n  Object Id                        : 0x600D000000000000\n  PCIe s:b:d.f                     : 0000:3b:00.3\n  Device Id                        : 0xAF00\n  Socket Id                        : 0xFF\n  Accelerator Id                   : 43425ee6-92b2-4742-b03a-bd8d4a533812\n  Accelerator GUID                 : 43425ee6-92b2-4742-b03a-bd8d4a533812\n  //****** PORT ******//\n  Object Id                        : 0x400D000000000000\n  PCIe s:b:d.f                     : 0000:3b:00.2\n  Device Id                        : 0xAF00\n  Socket Id                        : 0xFF\n  Accelerator Id                   : 8568AB4E-6bA5-4616-BB65-2A578330A8EB\n  Accelerator GUID                 : 8568AB4E-6bA5-4616-BB65-2A578330A8EB\n  //****** PORT ******//\n  Object Id                        : 0x200D000000000000\n  PCIe s:b:d.f                     : 0000:3b:00.1\n  Device Id                        : 0xAF00\n  Socket Id                        : 0xFF\n  Accelerator Id                   : 56e203e9-864f-49a7-b94b-12284c31e02b\n  Accelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n\nFPGA VF1/3b:00.1/Host Exerciser Loopback Accelerator GUID: 56E203E9-864F-49A7-B94B-12284C31E02B\nFPGA VF2/3b:00.2/Host Exerciser Memory Accelerator GUID: 8568AB4E-6bA5-4616-BB65-2A578330A8EB\nFPGA VF3/3b:00.3/Host Exerciser HSSI Accelerator GUID: 43425ee6-92b2-4742-b03a-bd8d4a533812\n
                                             
                                            Unbind pcie-vfio dirver to FPGA virtual functions:
                                             
                                            sudo opaevfio  -r 0000:3b:00.1\n
                                             
                                            Host Exerciser Loopback (HE-LBK) AFU can move data between host memory and FPGA:
                                             
                                            $ host_exerciser lpbk\n\n  [lpbk] [info] starting test run, count of 1\n  Input Config:0\n  Allocate SRC Buffer\n  Allocate DST Buffer\n  Allocate DSM Buffer\n  Start Test\n  Test Completed\n  Host Exerciser swtest msg:0\n  Host Exerciser numReads:32\n  Host Exerciser numWrites:32\n  Host Exerciser numPendReads:0\n  Host Exerciser numPendWrites:0\n  [lpbk] [info] Test lpbk(1): PASS\n
                                             
                                              In order to successfully run hello\\_fpga, the user needs to configure\n  system hugepage to reserve 2M-hugepages.\n  For example, the command below reserves 20 2M-hugepages:\n\n  ```bash\n  echo 20 | sudo tee /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages\n  ```\n\n  For x86_64 architecture CPU, user can use the following command to find out available huge page sizes:\n\n  ```bash\n  $ grep pse /proc/cpuinfo | uniq\n  flags : ... pse ...\n  ```\n\n  If this command returns a non-empty string, 2MB pages are supported:\n\n  ```bash\n  $ grep pse /proc/cpuinfo | uniq\n  flags : ... pdpe1gb ...\n  ```\n\n  If this commands returns a non-empty string, 1GB pages are supported.\n  ````\n\n````{note}\nThe default configuration for many Linux distributions currently sets a\nrelatively low limit for pinned memory allocations per process \n(RLIMIT_MEMLOCK, often set to a default of 64kiB).\n\nTo run an OPAE application that attempts to share more memory than specified\nby this limit between software and an accelerator, you can either:\n\n* Run the application as root, or\n* Increase the limit for locked memory via `ulimit`:\n\n```bash\nulimit -l unlimited\n```\n\nSee the Installation Guide for how to permanently adjust the memlock limit.\n
                                            "},{"location":"sw/fpga_dfl_drv/fpga_dfl_drv/","title":"Enable OPAE on FPGA PCIe drivers","text":"
                                            .. toctree::\n.. highlight:: c\n.. highlight:: console\n
                                             
                                            FPGA PCIe driver for PCIe-based Field-Programmable Gate Array (FPGA) solutions which implement the Device Feature List (DFL). This driver provides interfaces for user space applications to configure, enumerate, open and access FPGA accelerators on the FPGA DFL devices. additionally, it also enables system level management functions such as FPGA partial reconfiguration, power management, virtualization with DFL framework and DFL feature device drivers.
                                             
                                            OPAE 1.4.0 release supports both FPGA Intel Linux driver as well as Linux FPGA DFL driver patch set2. Linux PCIe FPGA DFL driver supports Intel FPGA devices.
                                             
                                            FPGA DFL Linux driver source code patchset2 available  https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers?h=linux-5.4.y
                                             
                                            FPGA DFL Linux driver source code patchset1 available  https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/fpga?h=v4.19.14
                                             
                                            "},{"location":"sw/fpga_tools/readme/","title":"fpga_tools","text":""},{"location":"sw/fpga_tools/readme/#fpgainfo","title":"fpgainfo","text":""},{"location":"sw/fpga_tools/readme/#name","title":"NAME","text":"
                                            fpgainfo - FPGA information tool
                                            "},{"location":"sw/fpga_tools/readme/#synopsis","title":"SYNOPSIS","text":"
                                            fpgainfo [-h | --help] [-s | --socket-id] <command> [<args>]\n
                                            "},{"location":"sw/fpga_tools/readme/#description","title":"DESCRIPTION","text":"
                                            fpgainfo is a tool to show FPGA information derived from sysfs files. The command argument is one of the following: errors, power, temp and is used to specify what type of information to report. Some commands may also have other arguments/options that can be used to control the behavior of that command.
                                            "},{"location":"sw/fpga_tools/readme/#common-options","title":"COMMON OPTIONS","text":"
                                            --help, -h
                                             
                                            Print help information and exit.\n
                                             
                                            --socket-id, -s
                                             
                                            Socket ID encoded in BBS. Default=0\n
                                            "},{"location":"sw/fpga_tools/readme/#fpgainfo-commands","title":"FPGAINFO COMMANDS","text":"
                                            errors
                                             
                                            Show/clear errors of an FPGA resource as specified by the first argument.\nError information is parsed to display in human readable form.\n
                                             
                                            power
                                             
                                            Show total power consumed by the FPGA hardware in watts\n
                                             
                                            temp
                                             
                                            Show FPGA temperature values in degrees Farenheit\n
                                            "},{"location":"sw/fpga_tools/readme/#errors-options","title":"ERRORS OPTIONS","text":"
                                            --clear, -c
                                             
                                            Clear errors for the given FPGA resource\n
                                            "},{"location":"sw/fpga_tools/readme/#errors-arguments","title":"ERRORS ARGUMENTS","text":"
                                            The first argument to the errors command is used to specify what kind of resource to act on. It must be one of the following: fme,port,first_error,pcie0,pcie1,bbs,gbs,all More details on the errors reported for the resource can be found below:
                                            "},{"location":"sw/fpga_tools/readme/#errors-resources","title":"ERRORS RESOURCES","text":"
                                            fme
                                             
                                            Show/clear errors pertaining to the FME\n
                                             
                                            port
                                             
                                            Show/clear errors pertaining to the PORT\n
                                             
                                            first_error
                                             
                                            Show/clear first errors encountered by the FPGA\n
                                             
                                            pcie0
                                             
                                            Show/clear errors pertaining to the PCIE0 lane\n
                                             
                                            pcie1
                                             
                                            Show/clear errors pertaining to the PCIE1 lane\n
                                             
                                            bbs
                                             
                                            Show/clear errors pertaining to the BBS (blue bitstream)\n
                                             
                                            gbs
                                             
                                            Show/clear errors pertaining to the GBS (green bitstream)\n
                                             
                                            all
                                             
                                            Show/clear errors for all resources\n
                                            "},{"location":"sw/fpga_tools/readme/#examples","title":"EXAMPLES","text":"
                                            This command shows the current power consumtion: 
                                            ./fpgainfo power\n
                                             
                                            This command shows the current temperature reading: 
                                            ./fpgainfo temp\n
                                             
                                            This command shows the errors for the FME resource: 
                                            ./fpgainfo errors fme\n
                                             This command clears all the errors on all resources: 
                                            ./fpgainfo errors all -c\n
                                            "},{"location":"sw/fpga_tools/readme/#fpgaconf","title":"fpgaconf","text":""},{"location":"sw/fpga_tools/readme/#name_1","title":"NAME","text":"
                                            fpgadiag - Configure a green bitstream to an FPGA
                                            "},{"location":"sw/fpga_tools/readme/#synopsis_1","title":"SYNOPSIS","text":"
                                            fpgaconf [-hvn] [-b <bus>] [-d <device>] [-f <function>] [-s <socket>] <gbs>
                                            "},{"location":"sw/fpga_tools/readme/#description_1","title":"DESCRIPTION","text":"
                                            fpgaconf writes accelerator configuration bitstreams (also referred to as \"green bitstreams\" to an FPGA device recognized by OPAE. In the process, it also checks the green bitstream file for compatibility with the targeted FPGA and its current infrastructure bitstream (the \"blue bistream\"). fpgaconf takes the following arguments:
                                             
                                            -h, --help
                                             
                                            Print usage information\n
                                             
                                            -v, --verbose
                                             
                                            Print more verbose messages while enumerating and configuring. Can be\ngiven more than once\n
                                             
                                            -n, --dry-run
                                             
                                            Perform enumeration, but skip any operations with side-effects (like the\nactual configuration of the bitstream\n
                                             
                                            -b, --bus
                                             
                                            PCI bus number of the FPGA to target\n
                                             
                                            -d, --device
                                             
                                            PCI device number of the FPGA to target\n
                                             
                                            -f, --function
                                             
                                            PCI function number of the FPGA to target\n
                                             
                                            -s, --socket
                                             
                                            Socket number of the FPGA to target\n
                                             
                                            fpgaconf will enumerate available FPGA devices in the system and select compatible FPGAs for configuration. If there are more than one candidate FPGAs that are compatible with the given green bitstream, fpgaconf will exit and ask you to be more specific in selecting the target FPGAs (e.g. by specifying a socket number, or a PCIe bus/device/function).
                                            "},{"location":"sw/fpga_tools/readme/#examples_1","title":"EXAMPLES","text":"
                                            fpgaconf my_green_bitstream.gbs
                                             
                                            Program \"my_green_bitstream.gbs\" to a compatible FPGA\n
                                             
                                            fpgaconf -v -s 0 my_green_bitstream.gbs
                                             
                                            Program \"my_green_bitstream.gbs\" to the FPGA in socket 0, if compatible,\nwhile printing out slightly more verbose information\n
                                            "},{"location":"sw/fpga_tools/readme/#fpgad","title":"fpgad","text":""},{"location":"sw/fpga_tools/readme/#name_2","title":"NAME","text":"
                                            fpgad - log errors and generate events
                                            "},{"location":"sw/fpga_tools/readme/#synopsis_2","title":"SYNOPSIS","text":"
                                            fpgad --daemon [--directory=<dir>] [--logfile=<file>] [--pidfile=<file>] [--umask=<mode>] [--socket=<sock>] [--null-bitstream=<file>] fpgad [--socket=<sock>] [--null-bitstream=<file>]
                                            "},{"location":"sw/fpga_tools/readme/#description_2","title":"DESCRIPTION","text":"
                                            Periodically monitors/reports the error status reflected in the device driver's error status sysfs files. Establishes the channel by which events are communicated to the OPAE application. Programs a NULL bitstream in response to AP6 event.
                                             
                                            fpgad is required to be running before API calls fpgaRegisterEvent and fpgaUnregisterEvent will succeed.
                                             
                                            Use SIGINT to stop fpgad.
                                             
                                            -d, --daemon
                                             
                                            When given, fpgad executes as a system demon process.\n
                                             
                                            -D, --directory <dir>
                                             
                                            When running in daemon mode, execute from the given directory.\nIf omitted when daemonizing, /tmp is used.\n
                                             
                                            -l, --logfile <file>
                                             
                                            When running in daemon mode, send output to file. When not in daemon mode, the output is sent to stdout.\nIf omitted when daemonizaing, /tmp/fpgad.log is used.\n
                                             
                                            -p, --pidfile <file>
                                             
                                            When running in daemon mode, write the daemon's process id to file.\nIf omitted when daemonizing, /tmp/fpgad.pid is used.\n
                                             
                                            -m, --umask <mode>
                                             
                                            When running in daemon mode, use the mode value as the file mode creation mask passed to umask.\nIf omitted when daemonizing, 0 is used.\n
                                             
                                            -s, --socket <sock>
                                             
                                            Listen for event API registration requests on sock. The default socket value used by the API is\n/tmp/fpga_event_socket.\n
                                             
                                            -n, --null-bitstream <file>
                                             
                                            Specify the NULL bitstream to program when an AP6 event occurs. This option may be given multiple\ntimes. The bitstream, if any, that matches the FPGA's PR interface id will be programmed when AP6\nis detected.\n
                                            "},{"location":"sw/fpga_tools/readme/#troubleshooting","title":"TROUBLESHOOTING","text":"
                                            If any issues are encountered, try the following for additional debug information:
                                             
                                               
                                            1. Examine the log file when in daemon mode.
                                              2.  
                                            2. Run in non-daemon mode and view stdout.
                                              3.  
                                            "},{"location":"sw/fpga_tools/readme/#examples_2","title":"EXAMPLES","text":"
                                            fpgad --daemon --null-bitstream=my_null_bits.gbs
                                            "},{"location":"sw/fpga_tools/readme/#see-also","title":"SEE ALSO","text":"
                                            umask
                                            "},{"location":"sw/fpga_tools/readme/#fpgadiag","title":"fpgadiag","text":""},{"location":"sw/fpga_tools/readme/#name_3","title":"NAME","text":"
                                            fpgadiag - FPGA diagnosis and testing tool.
                                            "},{"location":"sw/fpga_tools/readme/#synopsis_3","title":"SYNOPSIS","text":"
                                            fpgadiag [-m | --mode=] <mode> [-t | --target=] <target> [options]\n
                                            "},{"location":"sw/fpga_tools/readme/#description_3","title":"DESCRIPTION","text":"
                                            fpgadiag includes several tests to diagnose, test and report on the FPGA hardware.
                                             
                                            <mode> chooses which test to run. <target> specifies on what platform to run the test. <target> can be either fpga or ase, where ase stands for \"AFU Simulation Environment\".
                                             
                                            The tests that can be selected by <mode> include:
                                             
                                            lpbk1
                                             
                                            The test performs loopback test on the number of cachelines specified with \nthe `BEGIN` option. _fpgadiag_ sets up source and  destination buffers in \nmain memory. The FPGA then performs a memcpy from a source buffer to the \ndestination buffer, one cacheline at a time.\n\nA cacheline is 64 bytes. When `BEGIN = END`, you perform one iteration. When \n`BEGIN = END + x`, you perform `x` iterations. The first iteration consists \nof copying `BEGIN` cachelines; the second iteration consists of copying \n`BEGIN+1` cache lines; the third iteration consists of copying `BEGIN+3` \ncache lines, etc.\n\nThe latency is shown as the number of clock ticks.\n\nWhen you specify `MULTI-CL`, you copy `MULTI-CL` cache lines at a time.\nThere is always a WrFence. `WR-FENCE` chooses what virtual channel the \nWrFence occurs on.\n\nIf you specify continuous mode with `--cont`, the program runs an iteration\nuntil the timeout specified in `TIMEOUT` completes.\n
                                             
                                            read
                                             
                                            This test performs only a read, not a memcpy. It is used to measure read \nbandwidth.\n
                                             
                                            write 
                                             
                                            This test is used to measure write bandwidth.\n
                                             
                                            trput
                                             
                                            This test measures both read and write bandwidth by performing 50% read and \n50% write tests.\n
                                             
                                            sw
                                             
                                            This is a send-and-respond (ping-pong) test where one side sends data and \nwaits for answer.\n
                                             
                                            Each test requires presence of one of these bitstreams, as documented below. Before running a test, make sure its required bitstream is properly configured on the platform.
                                             
                                               
                                            • nlb mode 0 for the lpbk1 test.
                                              •  
                                            • nlb mode 3 for the trput, read, and write tests.
                                              •  
                                            • nlb mode 7 for the sw test.
                                              •  
                                            "},{"location":"sw/fpga_tools/readme/#options","title":"OPTIONS","text":""},{"location":"sw/fpga_tools/readme/#common-options_1","title":"Common options","text":"
                                            --help, -h
                                             
                                            Print help information and exit.\n
                                             
                                            --target=, -t
                                             
                                            Values accepted for this switch are fpga or ase. Default=fpga\n
                                             
                                            --mode=, -m
                                             
                                            The test to run. Values accepted for this switch are `lpbk1`, `read`,\n`write`, `trput`, `sw`\n
                                             
                                            --config=, -c
                                             
                                            A configuration file in the JSON format that specifies options for a test.\nIf an option is specified both in the configuration file and on the command \nline, the value in the configuration file prevails\n
                                             
                                            --socket-id=, -s
                                             
                                            Socket ID encoded in BBS. Default=0\n
                                             
                                            --bus-number=, -B
                                             
                                            Bus number of the PCIe device. Default=0\n
                                             
                                            --device=, -D
                                             
                                            Device number of the PCIe device. Default=0\n
                                             
                                            --function=, -F
                                             
                                            Function number of the PCIe device. Default=0\n
                                             
                                            --freq=, -T
                                             
                                            Clock frequency in Hz. Default=400 MHz\n
                                             
                                            --suppress-hdr, -S
                                             
                                            Suppress column headers for text output. Default=off\n
                                             
                                            --csv, -V
                                             
                                            Comma separated value format. Default=off\n
                                            "},{"location":"sw/fpga_tools/readme/#lpbk1-test-options","title":"lpbk1 test options","text":"
                                            --guid=, -g
                                             
                                            Accelerator ID to enumerate. Default=D8424DC4-A4A3-C413-F89E-433683F9040B\n
                                             
                                            --begin=B, -b
                                             
                                            1 <= B <= 65535. Default=1, B = number of cache lines\n
                                             
                                            --end=E, -e
                                             
                                            1 <= E <= 65535. Default=B, B and E designate number of cache lines\n
                                             
                                            --multi-cl=M, -U
                                             
                                            M can equal 1, 2, or 4. Default=1\n
                                             
                                            --cont, -L
                                             
                                            Continuous mode. Default=off\n
                                             
                                            --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                             
                                            timeout for --cont mode (microseconds portion default=0; milliseconds \nportion default=0; seconds portion default=1; minutes portion default=0;\nhours portion default=0)\n
                                             
                                            --cache-policy=, -p
                                             
                                            Can be wrline-I, wrline-M, or wrpush-I Default=wrline-M\n
                                             
                                            --cache-hint=, -i
                                             
                                            Can be rdline-I or rdline-S. Default=rdline-I\n
                                             
                                            --read-vc=, -r
                                             
                                            Can be auto, vl0, vh0, vh1, random. Default=auto\n
                                             
                                            --write-vc=, -w
                                             
                                            Can be auto, vl0, vh0, vh1, random. Default=auto\n
                                             
                                            --wrfence-vc=, -f
                                             
                                            Can be auto, vl0, vh0, vh1. Default=auto\n
                                            "},{"location":"sw/fpga_tools/readme/#read-test-options","title":"read test options","text":"
                                            --guid=, -g
                                             
                                            Accelerator ID to enumerate. Default=F7DF405C-BD7A-CF72-22F1-44B0B93ACD18\n
                                             
                                            --begin=B, -b
                                             
                                            1 <= B <= 65535. Default=1, B = number of cache lines\n
                                             
                                            --end=E, -e
                                             
                                            1 <= E <= 65535. Default=B, B and E designate number of cache lines\n
                                             
                                            --multi-cl=M, -u
                                             
                                            M can equal 1, 2, or 4. Default=1\n
                                             
                                            --strided-access=S, -a
                                             
                                            1<= S <= 64. Default=1\n
                                             
                                            --cont, -L
                                             
                                            Continuous mode. Default=off\n
                                             
                                            --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                             
                                            timeout for --cont mode (microseconds portion default=0; milliseconds \nportion default=0; seconds portion default=1; minutes portion default=0;\nhours portion default=0)\n
                                             
                                            --cache-hint=, -i
                                             
                                            Can be rdline-I or rdline-S. Default=rdline-I\n
                                             
                                            --warm-fpga-cache -H; --cool-fpga-cache -M
                                             
                                            Attempt to prime the cache with hits. Default=off, Attempt to prime the \ncache with misses. Default=off\n
                                             
                                            --cool-cpu-cache, -C
                                             
                                            Attempt to prime the cpu cache with misses. Default=off\n
                                             
                                            --read-vc=, -r
                                             
                                            Can be auto, vl0, vh0, vh1, random. Default=auto\n
                                            "},{"location":"sw/fpga_tools/readme/#write-test-options","title":"write test options","text":"
                                            --guid=, -g
                                             
                                            Accelerator ID to enumerate. Default=F7DF405C-BD7A-CF72-22F1-44B0B93ACD18\n
                                             
                                            --begin=B, -b
                                             
                                            1 <= E <= 65535. Default=B, B and E designate number of cache lines\n
                                             
                                            --multi-cl=M, -u
                                             
                                            M can equal 1, 2, or 4. Default=1\n
                                             
                                            --strided-access=S, -a
                                             
                                            1<= S <= 64. Default=1\n
                                             
                                            --cont, -L
                                             
                                            Continuous mode. Default=off\n
                                             
                                            --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                             
                                            timeout for --cont mode (microseconds portion default=0; milliseconds \nportion default=0; seconds portion default=1; minutes portion default=0;\nhours portion default=0)\n
                                             
                                            --cache-policy=, -p
                                             
                                            Can be wrline-I, wrline-M, or wrpush-I Default=wrline-M\n
                                             
                                            --warm-fpga-cache -H; --cool-fpga-cache -M
                                             
                                            Attempt to prime the cache with hits. Default=off, Attempt to prime the \ncache with misses. Default=off\n
                                             
                                            --cool-cpu-cache, -C
                                             
                                            Attempt to prime the cpu cache with misses. Default=off\n
                                             
                                            --write-vc=, -w
                                             
                                            Can be auto, vl0, vh0, vh1, random. Default=auto\n
                                             
                                            --wrfence-vc=, -f
                                             
                                            Can be auto, vl0, vh0, vh1, random. Default=`WRITE-VC`\n
                                             
                                            --alt-wr-pattern, -l
                                             
                                            Alternate Write Pattern. Default=off\n
                                            "},{"location":"sw/fpga_tools/readme/#trput-test-options","title":"trput test options","text":"
                                            --guid=, -g
                                             
                                            Accelerator ID to enumerate. Default=F7DF405C-BD7A-CF72-22F1-44B0B93ACD18\n
                                             
                                            --begin=B, -b
                                             
                                            1 <= B <= 65535. Default=1, B = number of cache lines\n
                                             
                                            --end=E, -e
                                             
                                            1 <= E <= 65535. Default=B, B and E designate number of cache lines\n
                                             
                                            --multi-cl=M, -u
                                             
                                            M can equal 1, 2, or 4. Default=1\n
                                             
                                            --strided-access=S, -a
                                             
                                            1<= S <= 64. Default=1\n
                                             
                                            --cont, -L
                                             
                                            Continuous mode. Default=off\n
                                             
                                            --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                             
                                            timeout for --cont mode (microseconds portion default=0; milliseconds \nportion default=0; seconds portion default=1; minutes portion default=0;\nhours portion default=0)\n
                                             
                                            --cache-policy=, -p
                                             
                                            Can be wrline-I, wrline-M, or wrpush-I Default=wrline-M\n
                                             
                                            --cache-hint=, -i
                                             
                                            Can be rdline-I or rdline-S. Default=rdline-I\n
                                             
                                            --read-vc=, -r
                                             
                                            Can be auto, vl0, vh0, vh1, random. Default=auto\n
                                             
                                            --write-vc=, -w
                                             
                                            Can be auto, vl0, vh0, vh1, random. Default=auto\n
                                             
                                            --wrfence-vc=, -f
                                             
                                            Can be  auto, vl0, vh0, vh1. Default=`WRITE-VC`\n
                                            "},{"location":"sw/fpga_tools/readme/#sw-test-options","title":"sw test options","text":"
                                            --guid=, -g
                                             
                                            Accelerator ID to enumerate. Default=7BAF4DEA-A57C-E91E-168A-455D9BDA88A3\n
                                             
                                            --begin=B, -b
                                             
                                            1 <= B <= 65535. Default=1, B = number of cache lines\n
                                             
                                            --end=E, -e
                                             
                                            1 <= E <= 65535. Default=B, B and E designate number of cache lines\n
                                             
                                            --multi-cl=M, -u
                                             
                                            M can equal 1, 2, or 4. Default=1\n
                                             
                                            --strided-access=S, -a
                                             
                                            1<= S <= 64. Default=1\n
                                             
                                            --cache-policy=, -p
                                             
                                            Can be wrline-I, wrline-M, or wrpush-I. Default=wrline-M\n
                                             
                                            --cache-hint= -i
                                             
                                            Can be rdline-I or rdline-S. Default=rdline-I\n
                                             
                                            --read-vc=, -r
                                             
                                            Can be auto, vl0, vh0, vh1, random Default=auto\n
                                             
                                            --write-vc=, -w
                                             
                                            Can be auto, vl0, vh0, vh1, random Default=auto\n
                                             
                                            --wrfence-vc=, -f
                                             
                                            Can be auto, vl0, vh0, vh1. Default=`WRITE-VC`\n
                                             
                                            --notice=, -N
                                             
                                            Can be poll, csr-write, umsg-data, or umsg-hint. Default=poll\n
                                            "},{"location":"sw/fpga_tools/readme/#examples_3","title":"EXAMPLES","text":"
                                            This command starts an lpbk1 test on the FPGA on bus 0x5e. The test  copies 57535, 57536, 57537, ..., up to 65535 cache lines, one line at a time. The test output is printed in the CSV format with header suppressed. 
                                            ./fpgadiag --mode=lpbk1 --target=fpga -SV --bus-number=0x5e --begin=57535\n--end=65535 --cache-hint=rdline-I --cache-policy=wrpush-I --multi-cl=1\n--write-vc=vl0 --read-vc=vh1 --wrfence-vc=auto\n
                                             
                                            This command starts a read test on the FPGA located on bus 0xbe. The test reads 2045 cache lines in the continuous mode with a 15-second timeout period.  Data is accessed with a strided pattern with a 10-byte stride length. 
                                            ./fpgadiag --mode=read --target=fpga -SV --bus-number=0xbe --begin=2045 --cont\n--timeout-sec=15   --cache-hint=rdline-I --multi-cl=1 -a=10 --write-vc=vh1\n--read-vc=auto --wrfence-vc=auto\n
                                             
                                            This command starts an sw test on the FPGA located on bus 0xbe. The test notifies completion using a CSR write. 
                                            ./fpgadiag --mode=sw --target=fpga -SV --bus-number=0xbe --begin=4 --end=8192\n--cache-hint=rdline-I --cache-policy=wrline-I --notice=csr-write --write-vc=vl0\n--wrfence-vc=auto --read-vc=random \n
                                            "},{"location":"sw/fpga_tools/readme/#troubleshooting_1","title":"TROUBLESHOOTING","text":"
                                            When a test fails to run or gives errors, check the following:
                                             
                                               
                                            • Is Intel FPGA driver properly installed?  See Installation Guide  for driver installation instructions.
                                              •  
                                            • Are FPGA port permissions set properly? Check the permission bits of the port, for example, /dev/intel-fpga-port-0. Users need READ and WRITE permissions to run fpgadiag tests.
                                              •  
                                            • Is hugepage properly configured on the system?  See Installation Guide for hugepage configuration steps.
                                              •  
                                            • Is the required bitstream loaded? See DESCRIPTION for information about what bitstream is required by which test.
                                              •  
                                            • Are --begin and --end values set properly? --end must be no smaller than the --begin. Also, --begin must be a multiple of the --multi-cl value.
                                              •  
                                            • The --warm-fpga-cache and --cool-fpga-cache options in the read and write tests are mutually exclusive.
                                              •  
                                            • The timeout options are only meaningful for the continuous mode  (with the --cont option).
                                              •  
                                            "},{"location":"sw/fpga_tools/readme/#mmlink","title":"mmlink","text":""},{"location":"sw/fpga_tools/readme/#name_4","title":"NAME","text":"
                                            MMLink - Debugging RTL.
                                            "},{"location":"sw/fpga_tools/readme/#synopsis_4","title":"SYNOPSIS","text":"
                                            mmlink [-B <bus>] [-D <device>] [-F <function>] [-S <socket>] [-P <TCP port>] [-I <IP Address>]
                                            "},{"location":"sw/fpga_tools/readme/#description_4","title":"DESCRIPTION","text":"
                                            Remote signaltap is software  tool used for debug RTL (AFU), effectively a signal trace capability that Quartus places into a green bitstream. Remote Signal Tap provides  access the RST part of the Port MMIO space, and then runs the remote protocol on top.
                                            "},{"location":"sw/fpga_tools/readme/#examples_4","title":"EXAMPLES","text":"
                                            ./mmlink  -B 0x5e -P 3333
                                             
                                            MMLink app starts and listens for connection.
                                            "},{"location":"sw/fpga_tools/readme/#options_1","title":"OPTIONS","text":"
                                            -B,--bus FPGA Bus number.
                                             
                                            -D,--device FPGA Device number.
                                             
                                            -F,--functio FPGA function number.
                                             
                                            -S,--socket FPGA socket number.
                                             
                                            -P,--port TCP port number.
                                             
                                            -I,--ip IP address of FPGA system. 
                                            "},{"location":"sw/fpga_tools/readme/#notes","title":"NOTES","text":"
                                            Driver privilege:
                                             
                                            Change AFU driver privilege to user .
                                             
                                            command: chmod 777 /dev/intel-fpga-port.0
                                             
                                            set memlock:    
                                             
                                            command: ulimit -l 10000
                                            "},{"location":"sw/fpga_tools/readme/#coreidle","title":"coreidle","text":""},{"location":"sw/fpga_tools/readme/#name_5","title":"NAME","text":"
                                            coreidle - idles cores for shared TDP sockets to run online cores at maximum capacity.
                                            "},{"location":"sw/fpga_tools/readme/#synopsis_5","title":"SYNOPSIS","text":"
                                            coreidle [-B <bus>] [-D <device>] [-F <function>] [-S <socket>] [-G <GBS path>]
                                            "},{"location":"sw/fpga_tools/readme/#description_5","title":"DESCRIPTION","text":"
                                            This tools parses input GBS, extracts power from metadata ,calculates fpga power, number of online and idle cores. It moves threads from idle cores to online cores.
                                            "},{"location":"sw/fpga_tools/readme/#examples_5","title":"EXAMPLES","text":"
                                            ./coreidle  -B 0x5e -G /home/lab/gbs/mode0.gbs
                                             
                                            Idle cores to run online cores at maximum capacity.
                                            "},{"location":"sw/fpga_tools/readme/#options_2","title":"OPTIONS","text":"
                                            -B,--bus FPGA Bus number.
                                             
                                            -D,--device FPGA Device number.
                                             
                                            -F,--functio FPGA function number.
                                             
                                            -S,--socket FPGA socket number.
                                             
                                            -G,--gbs Green bitstream file path.
                                            "},{"location":"sw/fpga_tools/readme/#fpgamux","title":"fpgamux","text":""},{"location":"sw/fpga_tools/readme/#name_6","title":"NAME","text":"
                                            fpgamux - Software MUX for running multiple AFU (accelerator functional unit) tests in one GBS (green bitsream)\n
                                            "},{"location":"sw/fpga_tools/readme/#synopsis_6","title":"SYNOPSIS","text":"
                                            fpgamux [-h] [-S|--socket-id SOCKET_ID] [-B|--bus-number BUS] [-D|--device DEVICE] [-F|--function FUNCTION]\n          [-G|--guid GUID] -m|--muxfile MUXFILE.json\n
                                            "},{"location":"sw/fpga_tools/readme/#description_6","title":"DESCRIPTION","text":"
                                            fpgamux is a testing tool to interact with multiple AFUs that have been synthesized into one GBS along with the CCIP-MUX BBB (basic building block). The CCIP-MUX uses upper bits in the MMIO addresses to route MMIO reads/writes to the AFU running on the corresponding CCIP-MUX port. fpgamux uses a configuration file that lists the software components and configuration to use.
                                             
                                            .. note::
                                             
                                              Only one (the first) AFU is discoverable by the OPAE driver. Enumerating acceleration on an FPGA will find\n  the accelerator associated with the first AFU only. The first software component in the configuration will\n  be used to determine the GUID to use for enumeration. This can be overridden with the -G|--guid option.\n
                                            "},{"location":"sw/fpga_tools/readme/#options_3","title":"OPTIONS","text":"
                                            -S SOCKET_ID, --socket-id SOCKET_ID\n   socket id of FPGA resource\n\n-B BUS, --bus BUS\n   bus id of FPGA resource\n\n-D DEVICE, --device DEVICE\n   device id of FPGA resource\n\n\n-F FUNCTION, --function FUNCTION\n   function id of FPGA resource\n\n-G, --guid\n   specify what guid to use for the accelerator enumeration\n
                                            "},{"location":"sw/fpga_tools/readme/#configuration","title":"CONFIGURATION","text":"
                                            fpgamux uses a configuration file (in JSON format) to determine what software components to instantiate and how to configure them for interacting with the AFUs in the GBS. This schema for this is listed below:
                                             
                                            [\n    {\n        \"app\" : \"fpga_app\",\n        \"name\" : \"String\",\n        \"config\" : \"Object\"\n    }\n]\n
                                            "},{"location":"sw/fpga_tools/readme/#examples_6","title":"EXAMPLES","text":"
                                            An example configuration with two components is listed below:
                                             
                                            [\n    {\n        \"app\" : \"nlb0\",\n        \"name\" : \"nlb0\",\n        \"config\" :\n        {\n            \"begin\" : 1,\n            \"end\" : 1,\n            \"multi-cl\" : 1,\n            \"cont\" : false,\n            \"cache-policy\" : \"wrline-M\",\n            \"cache-hint\" : \"rdline-I\",\n            \"read-vc\" : \"vh0\",\n            \"write-vc\" : \"vh1\",\n            \"wrfence-vc\" : \"write-vc\",\n            \"timeout-usec\" : 0,\n            \"timeout-msec\" : 0,\n            \"timeout-sec\" : 1,\n            \"timeout-min\" : 0,\n            \"timeout-hour\" : 0,\n            \"freq\" : 400000000\n        }\n    },\n    {\n        \"app\" : \"nlb3\",\n        \"name\" : \"nlb3\",\n        \"config\" :\n        {\n            \"mode\" : \"read\",\n            \"begin\" : 1,\n            \"end\" : 1,\n            \"multi-cl\" : 1,\n            \"strided-access\" : 1,\n            \"cont\" : false,\n            \"warm-fpga-cache\" : false,\n            \"cool-fpga-cache\" : false,\n            \"cool-cpu-cache\" : false,\n            \"cache-policy\" : \"wrline-M\",\n            \"cache-hint\" : \"rdline-I\",\n            \"read-vc\" : \"vh0\",\n            \"write-vc\" : \"vh1\",\n            \"wrfence-vc\" : \"write-vc\",\n            \"alt-wr-pattern\" : false,\n            \"timeout-usec\" : 0,\n            \"timeout-msec\" : 0,\n            \"timeout-sec\" : 1,\n            \"timeout-min\" : 0,\n            \"timeout-hour\" : 0,\n            \"freq\" : 400000000\n        }\n    }\n]\n
                                            "},{"location":"sw/fpga_tools/readme/#userclk","title":"userclk","text":""},{"location":"sw/fpga_tools/readme/#name_7","title":"NAME","text":"
                                            userclk - to set afu high and low clock frequency.
                                            "},{"location":"sw/fpga_tools/readme/#synopsis_7","title":"SYNOPSIS","text":"
                                            userclk [-B <bus>] [-D <device>] [-F <function>] [-S <socket>] [-P <Port id>] [-H <User clock high frequency>] -L <User clock low frequency>]
                                            "},{"location":"sw/fpga_tools/readme/#description_7","title":"DESCRIPTION","text":"
                                            userclk tool used to set high and low clock frequency to acceleration function unit.
                                            "},{"location":"sw/fpga_tools/readme/#examples_7","title":"EXAMPLES","text":"
                                            ./userclk  -B 0x5e -H 400 -L 200
                                             
                                            Sets AFU frequency.
                                            "},{"location":"sw/fpga_tools/readme/#options_4","title":"OPTIONS","text":"
                                            -B,--bus FPGA Bus number.
                                             
                                            -D,--device FPGA Device number.
                                             
                                            -F,--functio FPGA function number.
                                             
                                            -S,--socket FPGA socket number.
                                             
                                            -P,--port Port id.
                                             
                                            -H,--freq-high User clock high frequency. 
                                             
                                            -L,--freq-low User clock low frequency. 
                                            "},{"location":"sw/fpga_tools/coreidle/coreidle/","title":"coreidle","text":""},{"location":"sw/fpga_tools/coreidle/coreidle/#synopsis","title":"SYNOPSIS","text":"
                                            coreidle [-v] [-B <bus>] [-D <device>] [-F <function>] [-S <socket>] [-G <GBS path>]
                                            "},{"location":"sw/fpga_tools/coreidle/coreidle/#description","title":"DESCRIPTION","text":"
                                            coreidle parses the Accelerator Function Unit (AFU) metadata and extracts power information. coreidle calculates the FPGA power and calculates the number of online and idle cores. It moves threads from idle cores to online cores.  coreidle is only available the Integrated FPGA Platform. You cannot run coreidle on the PCIe Accelerator Card (PAC).
                                            "},{"location":"sw/fpga_tools/coreidle/coreidle/#examples","title":"EXAMPLES","text":"
                                            ./coreidle  -B 0x5e -G /home/lab/gbs/mode0.gbs
                                             
                                            Idle cores to run online cores at maximum capacity.
                                            "},{"location":"sw/fpga_tools/coreidle/coreidle/#options","title":"OPTIONS","text":"
                                            -v,--version Prints version information and exit.
                                             
                                            -B,--bus FPGA bus number.
                                             
                                            -D,--device FPGA device number.
                                             
                                            -F,--functio FPGA function number.
                                             
                                            -S,--socket FPGA socket number.
                                             
                                            -G,--gbs Green bitstream file path.
                                            "},{"location":"sw/fpga_tools/fecmode/fecmode/","title":"fecmode (N3000 specific tool)","text":""},{"location":"sw/fpga_tools/fecmode/fecmode/#synopsis","title":"SYNOPSIS","text":"
                                            fecmode [<mode>][<args>]\n
                                            "},{"location":"sw/fpga_tools/fecmode/fecmode/#description","title":"DESCRIPTION","text":"
                                            Fecmode changes FEC mode of external ethernet PHY, this tool only support on N3000 Card.
                                            "},{"location":"sw/fpga_tools/fecmode/fecmode/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                            --segment, -S
                                             
                                            segment number of the PCIe device.
                                             
                                            --bus, -B 
                                             
                                            bus number of the PCIe device. 
                                             
                                            --device, -D 
                                             
                                            device number of the PCIe device.
                                             
                                            --function, -F function number of the PCIe device
                                             
                                            --rsu, -r reboot card only if mode is not configured
                                             
                                            --debug, -d output debug information 
                                            "},{"location":"sw/fpga_tools/fecmode/fecmode/#fec-mode","title":"FEC Mode","text":"
                                            no no FEC.
                                             
                                            kr BaseR FEC (Fire-Code) correction \u2013 4 orders
                                             
                                            rs Reed-Solomon FEC correction \u2013 7 orders
                                            "},{"location":"sw/fpga_tools/fecmode/fecmode/#example","title":"EXAMPLE","text":"
                                            This command change FEC mode to \u201ckr\u201d: 
                                            # fecmode -B 0x25 kr\n
                                             
                                            This command reboot card (no need to specify bus number if there is only one card): 
                                            # fecmode -r\n
                                             
                                            This command display the current FEC mode: 
                                            # fecmode\n
                                            "},{"location":"sw/fpga_tools/fpgabist/fpgabist/","title":"fpgabist","text":""},{"location":"sw/fpga_tools/fpgabist/fpgabist/#synopsis","title":"SYNOPSIS","text":"
                                            fpgabist [-h] [-i device_id] [-b bus] [-d device] [-f function] [path_to_gbs1 path_to_gbs2 ...]\n
                                            "},{"location":"sw/fpga_tools/fpgabist/fpgabist/#description","title":"DESCRIPTION","text":"
                                            The fpgabist tool performs self-diagnostic tests on supported FPGA platforms.
                                             
                                            The tool accepts one or more Accelerator Function (AF) binaries from a predetermined set of AFs. Depending on the available binaries,  the tool runs appropriate tests and reports hardware issues.
                                             
                                            fpgabist always uses fpgainfo to report system information before running any hardware tests.
                                             
                                            Currently, fpgabist accepts the following AFs:    1. nlb_mode_3: The native loopback (NLB) test implements a loopback from TX to RX. Use it to verify basic functionality    and to measure bandwidth.    2. dma_afu: The direct memory access (DMA) AFU test transfers data from host memory to FPGA-attached local memory. 
                                             
                                            The installation includes the AF files, but you can also compile the AFs from the source. 
                                             
                                            If there are multiple PCIe\u00ae devices, use -b, -d, -f to specify the BDF for the specific PCIe\u00ae device.
                                            "},{"location":"sw/fpga_tools/fpgabist/fpgabist/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                            [path_to_gbs1 path_to_gbs2 ...]
                                             
                                            Paths to Accelerator Function (AF) files.
                                            "},{"location":"sw/fpga_tools/fpgabist/fpgabist/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                            You can use the single letter or the full parameter name for the command line arguments.
                                             
                                            -h, --help
                                             
                                            Prints usage information
                                             
                                            -i device_id, --device-id device_id
                                             
                                            Device ID for Intel FPGA. Default is: 0x09c4
                                             
                                            -B bus, --bus bus
                                             
                                            Bus number for specific FPGA
                                             
                                            -D device, --device device
                                             
                                            Device number for specific FPGA
                                             
                                            -F function, --function function
                                             
                                            Function number for specific FPGA
                                            "},{"location":"sw/fpga_tools/fpgabist/fpgabist/#examples","title":"EXAMPLES","text":"
                                            fpgabist <path_to_gbs_files>/dma_afu.gbs <path_to_gbs_files>/nlb_3.gbs
                                             
                                            Runs fpgabist on any platform in the system that matches the default device ID. This command runs both the DMA and   NLB_MODE_3 tests.
                                             
                                            fpgabist -i 09c4 -b 5 <path to gbs>/dma_afu.gbs
                                             
                                            Runs fpgabist the DMA test on the PCIe\u00ae  Endpoint with device_id 09c4 on bus 5. 
                                            "},{"location":"sw/fpga_tools/fpgaconf/fpgaconf/","title":"fpgaconf","text":""},{"location":"sw/fpga_tools/fpgaconf/fpgaconf/#synopsis","title":"SYNOPSIS","text":"
                                            fpgaconf [-hvVn] [-S <segment>] [-B <bus>] [-D <device>] [-F <function>] [PCI_ADDR] <gbs>
                                            "},{"location":"sw/fpga_tools/fpgaconf/fpgaconf/#description","title":"DESCRIPTION","text":"
                                            fpgaconf configures the FPGA with the accelerator function (AF). It also checks the AF for compatibility with  the targeted FPGA and the FPGA Interface Manager (FIM). fpgaconf takes the following arguments: 
                                             
                                            -h, --help
                                             
                                            Prints usage information.\n
                                             
                                            -v, --version
                                             
                                            Prints version information and exits.\n
                                             
                                            -V, --verbose
                                             
                                            Prints more verbose messages while enumerating and configuring. Can be\nrequested more than once.\n
                                             
                                            -n, --dry-run
                                             
                                            Performs enumeration. Skips any operations with side-effects such as the\nactual AF configuration.\n
                                             
                                            -S, --segment
                                             
                                            PCIe segment number of the target FPGA.\n
                                             
                                            -B, --bus
                                             
                                            PCIe bus number of the target FPGA.\n
                                             
                                            -D, --device
                                             
                                            PCIe device number of the target FPGA.\n
                                             
                                            -F, --function
                                             
                                            PCIe function number of the target FPGA.\n
                                             
                                            --force
                                             
                                            Reconfigure the AFU even if it is in use.\n
                                             
                                            fpgaconf enumerates available FPGA devices in the system and selects compatible FPGAs for configuration. If more than one FPGA is compatible with the AF, fpgaconf exits and asks you to be more specific in selecting the target FPGAs by specifying a a PCIe BDF.
                                            "},{"location":"sw/fpga_tools/fpgaconf/fpgaconf/#examples","title":"EXAMPLES","text":"
                                            fpgaconf my_af.gbs
                                             
                                            Program \"my_af.gbs\" to a compatible FPGA.\n
                                             
                                            fpgaconf -V -B 0x3b my_af.gbs
                                             
                                            Program \"my_af.gbs\" to the FPGA in bus 0x3b, if compatible,\nwhile printing out slightly more verbose information.\n
                                             
                                            fpgaconf 0000:3b:00.0 my_af.gbs
                                             
                                            Program \"my_af.gbs\" to the FPGA at address 0000:3b:00.0.\n
                                            "},{"location":"sw/fpga_tools/fpgad/fpgad/","title":"fpgad","text":""},{"location":"sw/fpga_tools/fpgad/fpgad/#synopsis","title":"SYNOPSIS","text":"
                                            fpgad --daemon [--version] [--directory=<dir>] [--logfile=<file>] [--pidfile=<file>] [--umask=<mode>] [--socket=<sock>] [--null-bitstream=<file>] fpgad [--socket=<sock>] [--null-bitstream=<file>]
                                            "},{"location":"sw/fpga_tools/fpgad/fpgad/#description","title":"DESCRIPTION","text":"
                                            fpgad monitors the device sensors, checking for sensor values that are out of the prescribed range. 
                                             
                                            When any of the sensors is detected to be out of bounds, fpgad will focus on keeping the server from rebooting by masking PCIE AER, and send a message to system administrator. System administrator can take further actions like stop the application and stop the FPGA, but fpgad just focus on monitor the sensors and will not take any cooling actions. 
                                             
                                            Note: fpgad must be running (as root) and actively monitoring devices when a sensor anomaly occurs in order to initiate Graceful Shutdown.  If fpgad is not loaded during such a sensor anomaly, the out-of-bounds scenario will not be detected, and the resulting effect on the hardware is undefined.
                                            "},{"location":"sw/fpga_tools/fpgad/fpgad/#arguments","title":"ARGUMENTS","text":"
                                            -v, --version
                                             
                                            Prints version information and exits.\n
                                             
                                            -d, --daemon
                                             
                                            When specified, fpgad executes as a system daemon process.\n
                                             
                                            -D, --directory <dir>
                                             
                                            When running in daemon mode, run from the specified directory.\nIf omitted when daemonizing, `fpgad` uses /tmp.\n
                                             
                                            -l, --logfile <file>
                                             
                                            When running in daemon mode, send output to file. When not in daemon mode, the output goes to stdout.\nIf omitted when daemonizaing, fpgad uses /tmp/fpgad.log.\n
                                             
                                            -p, --pidfile <file>
                                             
                                            When running in daemon mode, write the daemon's process id to a file.\nIf omitted when daemonizing, fpgad uses /tmp/fpgad.pid.\n
                                             
                                            -m, --umask <mode>
                                             
                                            When running in daemon mode, use the mode value as the file mode creation mask passed to umask.\nIf omitted when daemonizing, fpgad uses 0.\n
                                             
                                            -s, --socket <sock>
                                             
                                            Listen for event API registration requests on the UNIX domain socket on the specified path. \nThe default=/tmp/fpga_event_socket.\n
                                             
                                            -n, --null-bitstream <file>
                                             
                                            Specify the NULL bitstream to program when an AP6 event occurs. This option may be specified multiple\ntimes. The AF, if any, that matches the FPGA's PR interface ID is programmed when an AP6\nevent occurs.\n
                                            "},{"location":"sw/fpga_tools/fpgad/fpgad/#troubleshooting","title":"TROUBLESHOOTING","text":"
                                            If you encounter any issues, you can get debug information in two ways:
                                             
                                               
                                            1. By examining the log file when in daemon mode.
                                              2.  
                                            2. By running in non-daemon mode and viewing stdout.
                                              3.  
                                            "},{"location":"sw/fpga_tools/fpgad/fpgad/#examples","title":"EXAMPLES","text":"
                                            fpgad --daemon --null-bitstream=my_null_bits.gbs
                                             
                                            This command starts fpgad as a system daemon process: 
                                            sudo systemctl start fpgad\n
                                            "},{"location":"sw/fpga_tools/fpgadiag/","title":"fpgadiag","text":""},{"location":"sw/fpga_tools/fpgadiag/#synopsis","title":"SYNOPSIS","text":"
                                            fpgadiag [-m | --mode=] <mode> [-t | --target=] <target> [options]\n
                                            "},{"location":"sw/fpga_tools/fpgadiag/#description","title":"DESCRIPTION","text":"
                                            Includes several tests to diagnose, test, and report on the FPGA hardware.
                                             
                                            <mode> chooses which test to run.  <target> specifies the platform that runs the test. <target> can be either fpga or ase where ase.  <ase> is the abbreviation for Accelerator Simulation Environment.
                                             
                                            The <mode> selects from the  following tests:
                                             
                                            lpbk1
                                             
                                            This test runs a loopback test on the number of cachelines specified with  the BEGIN option. fpgadiag sets up source and  destination buffers in  main memory. The FPGA then performs a memcpy from a source buffer to the  destination buffer, one cacheline at a time. 
                                             
                                            A cacheline is 64 bytes. When BEGIN = END, the test performs one iteration. When  BEGIN = END + x, the test performs x iterations. The first iteration consists  of copying BEGIN cachelines; the second iteration consists of copying  BEGIN+1 cache lines. The third iteration consists of copying BEGIN+2  cache lines, and so on. 
                                             
                                            The latency is shown as the number of clock cycles. 
                                             
                                            When you specify MULTI-CL, you copy MULTI-CL cache lines at a time. The WR-FENCE chooses on which virtual channel the WrFence occurs.
                                             
                                            If you specify continuous mode with --cont, the program iterates until the timeout specified in TIMEOUT completes.
                                             
                                            read
                                             
                                            This test performs reads. Use this test to measure read bandwidth. 
                                             
                                            write 
                                             
                                            This test performs writes. Use it to measure write bandwidth. 
                                             
                                            trput
                                             
                                            This test measures both read and write bandwidth by performing 50% read and  50% write tests.
                                             
                                            sw
                                             
                                            This is a send-and-respond (ping-pong) test. One side sends data and  waits for response.
                                             
                                            Each test requires a particular AF. Before running a test, make sure the required AF is properly configured on the platform. 
                                             
                                               
                                            • The lpbk1 test requires the nlb mode 0 AF.
                                              •  
                                            • The trput test requires the nlb mode 3 AF. 
                                              •  
                                            • The sw test requires the nlb mode 7 AF. This AF is only available for the integrated FPGA platform.      You cannot run it on the PCIe accelerator card (PAC).
                                              •  
                                             
                                            fpgalpbk
                                             
                                            This enable/disable FPGA loopback.
                                             
                                            fpgastats
                                             
                                            This get fpga mac statistics.
                                             
                                            mactest
                                             
                                            This compare mac addresses that read from MAC ROM with mac addresses read from Host side.
                                            "},{"location":"sw/fpga_tools/fpgadiag/#options","title":"OPTIONS","text":""},{"location":"sw/fpga_tools/fpgadiag/#common-options","title":"Common options","text":"
                                            --help, -h
                                             
                                            Print help information and exit.\n
                                             
                                            --target=, -t
                                             
                                            This switch specifies fpga (hardware) or ase (simulation). The default=fpga.\n
                                             
                                            --mode=, -m
                                             
                                            The test to run. The valid values are `lpbk1`, `read`,\n`write`, `trput`, and `sw`.\n
                                             
                                            --config=, -c
                                             
                                            A configuration file in the JSON format that specifies options for a test.\nIf an option is specified both in the configuration file and on the command \nline, the value in the configuration file takes precedence.\n
                                             
                                            --dsm-timeout-usec
                                             
                                            Timeout in microseconds for test completion. The test fails if not completed by \nspecified timeout. The default=1000000.\n
                                             
                                            --socket-id=, -s
                                             
                                            Socket ID encoded in FPGA Interface Manager (FIM). The default=0.\n
                                             
                                            --bus=, -B
                                             
                                            Bus number of the PCIe device. The default=0.\n
                                             
                                            --device=, -D
                                             
                                            Device number of the PCIe device. The default=0.\n
                                             
                                            --function=, -F
                                             
                                            Function number of the PCIe device. The default=0.\n
                                             
                                            --freq=, -T
                                             
                                            Clock frequency (in Hz) used for bandwidth calculation. The default=400000000 Hz (400 MHz).\n
                                             
                                            eval_rst .. note::     This frequency is used only when the software cannot infer the frequency from the accelerator.
                                             
                                            --suppress-hdr, -S
                                             
                                            Suppress column headers for text output. The default=off.\n
                                             
                                            --csv, -V
                                             
                                            Comma separated value format. The default=off.\n
                                             
                                            --suppress-stats
                                             
                                            Suppress statistics output at the end of test. The default=off.\n
                                            "},{"location":"sw/fpga_tools/fpgadiag/#lpbk1-test-options","title":"lpbk1 test options","text":"
                                            --guid=, -g
                                             
                                            AFU ID to enumerate. The default=D8424DC4-A4A3-C413-F89E-433683F9040B.\n
                                             
                                            --begin=B, -b
                                             
                                            1 <= B <= 65535. The default=1, B = number of cache lines.\n
                                             
                                            --end=E, -e
                                             
                                            1 <= E <= 65535. The default=B, B and E designate number of cache lines.\n
                                             
                                            --multi-cl=M, -u
                                             
                                            M can equal 1, 2, or 4. The default=1.\n
                                             
                                            --cont, -L
                                             
                                            Continuous mode. The default=off.\n
                                             
                                            --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                             
                                            timeout for --cont mode. The default for all options is 0.\n
                                             
                                            --cache-policy=, -p
                                             
                                            Can be wrline-I, wrline-M, or wrpush-I The default=wrline-M.\n
                                             
                                            --cache-hint=, -i
                                             
                                            Can be rdline-I or rdline-S. The default=rdline-I.\n
                                             
                                            --read-vc=, -r
                                             
                                            Can be auto, vl0, vh0, vh1, random. The default=auto.\n
                                             
                                            --write-vc=, -w
                                             
                                            Can be auto, vl0, vh0, vh1, random. The default=auto.\n
                                             
                                            --wrfence-vc=, -f
                                             
                                            Can be auto, vl0, vh0, vh1. The default=auto.\n
                                            "},{"location":"sw/fpga_tools/fpgadiag/#read-test-options","title":"read test options","text":"
                                            --guid=, -g
                                             
                                            AFU ID to enumerate. The default=F7DF405C-BD7A-CF72-22F1-44B0B93ACD18.\n
                                             
                                            --begin=B, -b
                                             
                                            1 <= B <= 65535. The default=1, B = number of cache lines.\n
                                             
                                            --end=E, -e
                                             
                                            1 <= E <= 65535. The default=B, B and E designate number of cache lines.\n
                                             
                                            --multi-cl=M, -u
                                             
                                            M can equal 1, 2, or 4. The default=1.\n
                                             
                                            --strided-access=S, -a
                                             
                                            1<= S <= 64. The default=1.\n
                                             
                                            --cont, -L
                                             
                                            Continuous mode. The default=off.\n
                                             
                                            --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                             
                                            timeout for --cont mode. The default for all options is 0.\n
                                             
                                            --cache-hint=, -i
                                             
                                            Can be rdline-I or rdline-S. The default=rdline-I.\n
                                             
                                            --warm-fpga-cache -H; --cool-fpga-cache -M
                                             
                                            Try to prime the cache with hits. The default=off. Try to prime the \ncache with misses. The default=off.\n
                                             
                                            --cool-cpu-cache, -C
                                             
                                            Try to prime the cpu cache with misses. The default=off.\n
                                             
                                            --read-vc=, -r
                                             
                                            Can be auto, vl0, vh0, vh1, random. The default=auto\n
                                            "},{"location":"sw/fpga_tools/fpgadiag/#write-test-options","title":"write test options","text":"
                                            --guid=, -g
                                             
                                            AFU ID to enumerate. The default=F7DF405C-BD7A-CF72-22F1-44B0B93ACD18\n
                                             
                                            --begin=B, -b
                                             
                                            1 <= E <= 65535. The default=B, B and E designate number of cache lines.\n
                                             
                                            --multi-cl=M, -u
                                             
                                            M can equal 1, 2, or 4. The default=1.\n
                                             
                                            --strided-access=S, -a
                                             
                                            1<= S <= 64. The default=1.\n
                                             
                                            --cont, -L
                                             
                                            Continuous mode. The default=off.\n
                                             
                                            --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                             
                                            timeout for --cont mode. The default for all options is 0.\n
                                             
                                            --cache-policy=, -p
                                             
                                            Can be wrline-I, wrline-M, or wrpush-I The default=wrline-M\n
                                             
                                            --warm-fpga-cache -H; --cool-fpga-cache -M
                                             
                                            Try to prime the cache with hits. The default=off. Try to prime the \ncache with misses. The default=off.\n
                                             
                                            --cool-cpu-cache, -C
                                             
                                            Try to prime the cpu cache with misses. The default=off.\n
                                             
                                            --write-vc=, -w
                                             
                                            Can be auto, vl0, vh0, vh1, random. The default=auto.\n
                                             
                                            --wrfence-vc=, -f
                                             
                                            Can be auto, vl0, vh0, vh1, random. The default=`WRITE-VC`.\n
                                             
                                            --alt-wr-pattern, -l
                                             
                                            Alternate Write Pattern. The default=off.\n
                                            "},{"location":"sw/fpga_tools/fpgadiag/#trput-test-options","title":"trput test options","text":"
                                            --guid=, -g
                                             
                                            AFU ID to enumerate. The default=F7DF405C-BD7A-CF72-22F1-44B0B93ACD18.\n
                                             
                                            --begin=B, -b
                                             
                                            1 <= B <= 65535. The default=1, B = number of cache lines.\n
                                             
                                            --end=E, -e
                                             
                                            1 <= E <= 65535. The default=B, B and E designate number of cache lines.\n
                                             
                                            --multi-cl=M, -u
                                             
                                            M can equal 1, 2, or 4. The default=1.\n
                                             
                                            --strided-access=S, -a
                                             
                                            1<= S <= 64. The default=1\n
                                             
                                            --cont, -L
                                             
                                            Continuous mode. The default=off.\n
                                             
                                            --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                             
                                            timeout for --cont mode. The default for all options is 0.\n
                                             
                                            --cache-policy=, -p
                                             
                                            Can be wrline-I, wrline-M, or wrpush-I The default=wrline-M.\n
                                             
                                            --cache-hint=, -i
                                             
                                            Can be rdline-I or rdline-S. The default=rdline-I.\n
                                             
                                            --read-vc=, -r
                                             
                                            Can be auto, vl0, vh0, vh1, random. The default=auto.\n
                                             
                                            --write-vc=, -w
                                             
                                            Can be auto, vl0, vh0, vh1, random. The default=auto.\n
                                             
                                            --wrfence-vc=, -f
                                             
                                            Can be  auto, vl0, vh0, vh1. The default=`WRITE-VC`.\n
                                            "},{"location":"sw/fpga_tools/fpgadiag/#sw-test-options","title":"sw test options","text":"
                                            --guid=, -g
                                             
                                            AFU ID to enumerate. The default=7BAF4DEA-A57C-E91E-168A-455D9BDA88A3.\n
                                             
                                            --begin=B, -b
                                             
                                            1 <= B <= 65535. The default=1, B = number of cache lines.\n
                                             
                                            --end=E, -e
                                             
                                            1 <= E <= 65535. The default=B, B and E designate number of cache lines.\n
                                             
                                            --cache-policy=, -p
                                             
                                            Can be wrline-I, wrline-M, or wrpush-I. The default=wrline-M.\n
                                             
                                            --cache-hint= -i
                                             
                                            Can be rdline-I or rdline-S. The default=rdline-I.\n
                                             
                                            --read-vc=, -r
                                             
                                            Can be auto, vl0, vh0, vh1, random The default=auto.\n
                                             
                                            --write-vc=, -w
                                             
                                            Can be auto, vl0, vh0, vh1, random The default=auto.\n
                                             
                                            --wrfence-vc=, -f
                                             
                                            Can be auto, vl0, vh0, vh1. The default=`WRITE-VC`.\n
                                             
                                            --notice=, -N
                                             
                                            Can be poll or csr-write. The default=poll.\n
                                            "},{"location":"sw/fpga_tools/fpgadiag/#enable-fpga-n3000-ethernet-group-vfio-mdev","title":"Enable FPGA N3000 Ethernet group VFIO mdev","text":"
                                            FPGA DFL driver does not support any ioctls to read/write ethernet group info and registers. Users can read/write eth group registers by enabling VFIO mdev. Unbind the dfl_eth_group driver and bind vfio-mdev-dfl driver for ethernet group dfl-device; then userspace can take full control of ethernet group feature id 10.
                                             
                                            Ethernet group must be enabled before running fpgalpbk, mactest tools.
                                            "},{"location":"sw/fpga_tools/fpgadiag/#steps-to-enablecreate-vfio-mdev","title":"Steps to enable/create vfio mdev","text":"
                                            unbind eth group feature id 10:\n    echo dfl-fme.0.8 > /sys/bus/dfl/drivers/dfl-eth-group/unbind\n    echo dfl-fme.0.7 > /sys/bus/dfl/drivers/dfl-eth-group/unbind\nbind to vfio-mdev-dfl:\n    echo vfio-mdev-dfl > /sys/bus/dfl/devices/dfl-fme.0.7/driver_override\n    echo vfio-mdev-dfl > /sys/bus/dfl/devices/dfl-fme.0.8/driver_override\nload vfio driver:\n    modprobe vfio_pci\n    modprobe vfio_iommu_type1\n    modprobe vfio_mdev\n    modprobe vfio_mdev_dfl\ntrigger mdev:\n    echo dfl-fme.0.7 >/sys/bus/dfl/drivers_probe\n    echo dfl-fme.0.8 >/sys/bus/dfl/drivers_probe\n    echo 83b8f4f2-509f-382f-3c1e-e6bfe0fa1001 > /sys/bus/dfl/devices/dfl-fme.0.7/mdev_supported_types/vfio-mdev-dfl-1/create\n    echo 83b8f4f2-509f-382f-3c1e-e6bfe0fa1002 > /sys/bus/dfl/devices/dfl-fme.0.8/mdev_supported_types/vfio-mdev-dfl-1/create\n\nlinux kerenl msg after enabling mdev:\n    i40e 0000:b3:00.0 eth1: NIC Link is Down\n    i40e 0000:b1:00.1 eth0: NIC Link is Down\n    vfio-mdev-dfl dfl-fme.2.7: MDEV: Registered\n    vfio-mdev-dfl dfl-fme.2.8: MDEV: Registered\n    vfio_mdev 83b8f4f2-509f-382f-3c1e-e6bfe0fa1005: Adding to iommu group 140\n    vfio_mdev 83b8f4f2-509f-382f-3c1e-e6bfe0fa1005: MDEV: group_id = 140\n    vfio_mdev 83b8f4f2-509f-382f-3c1e-e6bfe0fa1006: Adding to iommu group 141\n    vfio_mdev 83b8f4f2-509f-382f-3c1e-e6bfe0fa1006: MDEV: group_id = 141\n
                                            "},{"location":"sw/fpga_tools/fpgadiag/#remove-vfio-mdev","title":"Remove vfio mdev","text":"
                                                echo 1 | sudo tee /sys/bus/mdev/devices/83b8f4f2-509f-382f-3c1e-e6bfe0fa1002/remove\n    echo 1 | sudo tee /sys/bus/mdev/devices/83b8f4f2-509f-382f-3c1e-e6bfe0fa1001/remove\n\n    rmmod vfio_mdev_dfl\n    modprobe dfl_eth_group\n\n    echo dfl-fme.0.7 >/sys/bus/dfl/drivers_probe\n    echo dfl-fme.0.8 >/sys/bus/dfl/drivers_probe\n\n    echo dfl-eth-group > /sys/bus/dfl/devices/dfl-fme.0.7/driver_override\n    echo dfl-eth-group > /sys/bus/dfl/devices/dfl-fme.0.8/driver_override\n
                                            "},{"location":"sw/fpga_tools/fpgadiag/#fpgalpbk-test-options","title":"fpgalpbk test options","text":"
                                            --enable
                                             
                                            Enable fpga phy loopback.\n
                                             
                                            --disable
                                             
                                            Disable fpga phy loopback.\n
                                             
                                            --direction
                                             
                                            Can be local, remote.\n
                                             
                                            --type
                                             
                                            Can be serial, precdr, postcdr.\n
                                             
                                            --side
                                             
                                            Can be line, host.\n
                                             
                                            --port
                                             
                                            0 <= port <= 7, the default is all.\n
                                            "},{"location":"sw/fpga_tools/fpgadiag/#mactest-test-options","title":"mactest test options","text":"
                                            --offset
                                             
                                            Read mac addresses from an offset, The default=0.\n
                                            "},{"location":"sw/fpga_tools/fpgadiag/#examples","title":"EXAMPLES","text":"
                                            This command starts a lpbk1 test for the FPGA on bus 0x5e. The test  copies 57535, 57536, 57537 ... up to 65535 cache lines, one line at a time. The test prints output in the comma separated values (CSV) format with the header suppressed. 
                                            ./fpgadiag --mode=lpbk1 --target=fpga -V --bus=0x5e --begin=57535\n--end=65535 --cache-hint=rdline-I --cache-policy=wrpush-I --multi-cl=1\n--write-vc=vl0 --read-vc=vh1 --wrfence-vc=auto\n
                                             
                                            This command starts a read test on the FPGA located on bus 0xbe. The test reads 2045 cache lines in the continuous mode with a 15-second timeout period.  The reads use a strided pattern with a 10-byte stride length. 
                                            ./fpgadiag --mode=read --target=fpga -V --bus=0xbe --begin=2045 --cont\n--timeout-sec=15 --cache-hint=rdline-I --multi-cl=1 -a=10 \n--read-vc=auto --wrfence-vc=auto\n
                                             
                                            This command starts a sw test on the FPGA located on bus 0xbe. The test signals completion using a CSR write. 
                                            ./fpgadiag --mode=sw --target=fpga -V --bus=0xbe --begin=4 --end=8192\n--cache-hint=rdline-I --cache-policy=wrline-I --notice=csr-write --write-vc=vl0\n--wrfence-vc=auto --read-vc=random \n
                                             
                                            This command enable a fpgalpbk on the FPGA located on bus 0xbe. 
                                            ./fpgadiag -m fpgalpbk --bus 0xbe --enable --direction local --type postcdr\n--side host\n
                                             
                                            This command show fpgastats on the FPGA located on bus 0xbe. 
                                            ./fpgadiag -m fpgastats --bus 0xbe\n
                                            "},{"location":"sw/fpga_tools/fpgadiag/#troubleshooting","title":"TROUBLESHOOTING","text":"
                                            When a test fails to run or gives errors, check the following:
                                             
                                               
                                            • Is the Intel FPGA driver properly installed?  See Installation Guide  for driver installation instructions.
                                              •  
                                            • Are FPGA port permissions set properly? Check the permission bits of the port, for example, /dev/intel-fpga-port-0. You need READ and WRITE permissions to run fpgadiag tests.
                                              •  
                                            • Is hugepage properly configured on the system?  See Installation Guide for hugepage configuration steps. In particular, fpgadiag requires a few 1 GB pages. 
                                              •  
                                            • Is the required AFU loaded? See DESCRIPTION for information about what AFU the test requires.
                                              •  
                                            • Are --begin and --end values set properly? --end must be larger than the --begin. Also, --begin must be a multiple of the --multi-cl value.
                                              •  
                                            • The --warm-fpga-cache and --cool-fpga-cache options in the read and write tests are mutually exclusive.
                                              •  
                                            • The timeout options are only meaningful for the continuous mode  (with the --cont option).
                                              •  
                                            "},{"location":"sw/fpga_tools/fpgaflash/fpgaflash/","title":"fpgaflash","text":""},{"location":"sw/fpga_tools/fpgaflash/fpgaflash/#synopsis","title":"SYNOPSIS","text":"
                                            fpgaflash [-h] {user,factory} file [bdf]\n
                                            "},{"location":"sw/fpga_tools/fpgaflash/fpgaflash/#description","title":"DESCRIPTION","text":"
                                            fpgaflash updates the static FIM image loaded from flash at power-on.
                                             
                                            If there are multiple devices in the system, fpgaflash must specify a BDF to select the correct device. If no BDF is specified, fpgaflash prints out the BDFs of any compatible devices.
                                            "},{"location":"sw/fpga_tools/fpgaflash/fpgaflash/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                            {user, factory}
                                             
                                            Specifies the type of flash programming.
                                             
                                            user 
                                             
                                            Only reprograms the user image in flash.  
                                             
                                            factory 
                                             
                                            Reprograms the entire flash.  A catastrophic failure during a factory update such as a power outage  requires a USB cable and quartus_pgm to recover.
                                             
                                            file
                                             
                                            Specifies the Raw Programming Data File (rpd) to program into flash.
                                             
                                            bdf
                                             
                                            Specifies the bus, device and function (BDF) of device to program such as 04:00.0 or 0000:04:00.0. This flag is optional when there is a single device in the system.
                                            "},{"location":"sw/fpga_tools/fpgaflash/fpgaflash/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                            -h, --help
                                             
                                            Print usage information.
                                            "},{"location":"sw/fpga_tools/fpgaflash/fpgaflash/#example","title":"EXAMPLE","text":"
                                            fpgaflash user new_image.rpd 0000:04:00.0
                                             
                                            Programs new_image.rpd to flash of device with BDF 0000:04:00.0.
                                            "},{"location":"sw/fpga_tools/fpgaflash/superrsu/","title":"super-rsu","text":""},{"location":"sw/fpga_tools/fpgaflash/superrsu/#synopsis","title":"SYNOPSIS","text":"
                                            super-rsu [-h] [-n] [--verify] | [ [--log-level {trace,debug,error,warn,info,notset}]\n          [--log-file <filename>] [--rsu-only] [--with-rsu] [--force-flash] ]\n      rsu_config\n
                                            "},{"location":"sw/fpga_tools/fpgaflash/superrsu/#description","title":"DESCRIPTION","text":"
                                            super-rsu is a tool that can be used for flashing image files and commanding an Intel PAC device to perform RSU (remote system update - or a board reboot). Performing an RSU on an Intel PAC device will cause it to reload any firmware or programmable logic and restart its execution, a requirement for any updated firmware or programmable logic to take effect.
                                             
                                            At the core of super-rsu is its configuration file (referred to in this document as 'rsu_config') which is essentially a manifest file for identifying both the target device and the binary images (and their versions) to be flashed.
                                             
                                            At a high level, the flow of super-rsu should be: 1. Read and parse rsu_config file 2. Use product identifiers (like vendor, device and any additional vendor, device    pairs that may be present in the PCIe bus) to locate all compatible    devices on the PCIe bus. 3. For every device found on the system, update the device using the flash    images defined in the \"flash\" section in the rsu_config data (or nvmupdate    section).    Each item in the \"flash\" section is a \"flash spec\" that contains:    * The flash type (\"user\", \"bmc_fw\", \"bmc_img\", ...)    * The filename of the image to flash. super-rsu will look for this file      first in the same directory of the rsu_config file, and then look in the      current working directory.    * The version of the image.    * An optional \"force\" indicator    * An optional \"requires\" indicator    The \"nvmupdate\" section is used to describe an Ethernet firmware file and    its version. 4. Using the data in the \"nvmupdate\" and \"flash\" sections, the update routine    involves:   * If an \"nvmupdate\" section is present:     1. Locate the file on the file system to use to flash the Ethernet device.     2. Call nvmupdate to get an \"inventory\" of devices matching the vendor and        device id in this section.     3. Use this data to dynamically generate an nvmupdate compatible        configuration file.     4. Call nvmupdate with the generated configuration file to flash the        Ethernet interfaces in the Vista Creek card (if version reported by        system does not match the version in this section).   * For each spec in the \"flash\" section:     1. Locate the file on the file system to use to flash.     2. Compare the version listed in the \"flash spec\" to version reported by        the target component.     3. Create a task to call fpgaflash if either of the following conditions is        met (and the revision specified is compatible):        * The \"force\" indicator is present and set to true.        * The version in the spec does not match the version reported by the          system OR the flash type is factory type.   * For each task created from the \"flash\" section:     1. Call fpgaflash with the command line arguments that correspond to the        flash type and the file name in the spec used to create the task.        This opens and controls the execution of fpgaflash in another process.
                                             
                                            NOTE: If the system reports a revision for one of the components being flashed, this revision must be in the set of revisions listed in the manifest. Example: if the system reports 'a' for bmc_img and the manifest includes 'ab', then the image will be flashed.
                                             
                                            NOTE: Each update routine is run in a thread specific to a device located on the PCIe bus. Every task in an update routine involves opening a new process that is controlled and managed by its update routine thread. If a task includes a timeout and the timeout is reached, a termination request will be sent to its process and it will be counted as a failure. If a global timeout is reached in the main thread, a termination request will be sent to each thread performing the update. Consequently, the update routine will give the current task extra time before terminating the process. The RSU operation will only be performed if requested with either --with-rsu command line argument or with the --rsu-only command line argument. The former will perform the RSU command upon successful completion of flash operations. The latter will skip the process of version matching and flashing images and will only perform the RSU command. It is recommended that super-rsu be executed again if any flash operation is interrupted.
                                            "},{"location":"sw/fpga_tools/fpgaflash/superrsu/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                            rsu config
                                             
                                            Specifies the name of the file containing the RSU configuration (in JSON format)
                                            "},{"location":"sw/fpga_tools/fpgaflash/superrsu/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                            -h, --help
                                             
                                            Print usage information.
                                             
                                            --verify
                                             
                                            Compare versions of flashable components on the system against the manifest.   Return non-zero exit if compatible components are not up to date.
                                             
                                            -n, --dry-run
                                             
                                            Don't perform any updates, just a dry run.   This will print out commands that can be executed   in a Linux shell.
                                             
                                            --log-level {trace,debug,error,warn,info,notset}
                                             
                                            Log level to use. Default is 'info'.
                                             
                                            `--log-file  (default: /tmp/super-rsu.log) 
                                            Emit log messages (with DEBUG level) to filename   NOTE: The default log file (/tmp/super-rsu.log) is set to rollover every   time super-rsu is executed. This will create numbered backups before   truncating the log file. The maximum number of backups is 50.
                                             
                                            --rsu-only
                                             
                                            Only perform the RSU command.
                                             
                                            --with-rsu
                                             
                                            Perform RSU after updating flash components(experimental)
                                             
                                            --force-flash
                                             
                                            Flash all images regardless of versions matching or not.
                                            "},{"location":"sw/fpga_tools/fpgaflash/superrsu/#configuration","title":"CONFIGURATION","text":"
                                            The following is the JSON schema expected by super-rsu. Any deviance from this schema may result in errors executing super-rsu.
                                             
                                            {\n  \"definitions\": {},\n  \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n  \"$id\": \"http://example.com/root.json\",\n  \"type\": \"object\",\n  \"title\": \"The Root Schema\",\n  \"required\": [\n    \"product\",\n    \"vendor\",\n    \"device\",\n    \"flash\"\n  ],\n  \"optional\": [\n    \"nvmupdate\",\n  ],\n  \"properties\": {\n    \"product\": {\n      \"$id\": \"#/properties/product\",\n      \"type\": \"string\",\n      \"title\": \"The Product Schema\",\n      \"default\": \"\",\n      \"examples\": [\n        \"n3000\"\n      ],\n      \"pattern\": \"^(.*)$\"\n    },\n    \"vendor\": {\n      \"$id\": \"#/properties/vendor\",\n      \"type\": \"string\",\n      \"title\": \"The Vendor Schema\",\n      \"default\": \"\",\n      \"examples\": [\n        \"0x8086\"\n      ],\n      \"pattern\": \"^((0x)?[A-Fa-f0-9]{4})$\"\n    },\n    \"device\": {\n      \"$id\": \"#/properties/device\",\n      \"type\": \"string\",\n      \"title\": \"The Device Schema\",\n      \"default\": \"\",\n      \"examples\": [\n        \"0x0b30\"\n      ],\n      \"pattern\": \"^((0x)?[A-Fa-f0-9]{4})$\"\n    },\n    \"nvmupdate\": {\n      \"$id\": \"#/properties/nvmupdate\",\n      \"type\": \"object\",\n      \"title\": \"The nvmupdate Schema\",\n      \"required\": [\n        \"vendor\",\n        \"device\",\n        \"filename\",\n        \"version\"\n      ],\n      \"optional\": [\n        \"interfaces\"\n      ],\n      \"properties\": {\n        \"vendor\": {\n          \"$id\": \"#/properties/nvmupdate/vendor\",\n          \"type\": \"string\",\n          \"title\": \"The nvmupdate Vendor Schema\",\n          \"default\": \"\",\n          \"examples\": [\n             \"0x8086\"\n          ],\n          \"pattern\": \"^((0x)?[A-Fa-f0-9]{4})$\"\n        },\n        \"device\": {\n          \"$id\": \"#/properties/nvmupdate/device\",\n          \"type\": \"string\",\n          \"title\": \"The nvmupdate Device Schema\",\n          \"default\": \"\",\n          \"examples\": [\n            \"0x0d58\"\n          ],\n          \"pattern\": \"^((0x)?[A-Fa-f0-9]{4})$\"\n        },\n        \"interfaces\": {\n          \"$id\": \"#/properties/nvmupdate/interfaces\",\n          \"type\": \"number\",\n          \"title\": \"The nvmupdate Interfaces Schema\",\n          \"default\": \"1\",\n          \"examples\": [\n            2, 4\n          ]\n        },\n        \"filename\": {\n          \"$id\": \"#/properties/nvmupdate/filename\",\n          \"type\": \"string\",\n          \"title\": \"The nvmupdate Filename Schema\",\n          \"default\": \"\",\n          \"examples\": [\n            \"PSG_XL710_6p80_XLAUI_NCSI_CFGID2p61_Dual_DID_0D58_800049C6.bin\"\n          ],\n          \"pattern\": \"^(.*)$\"\n        },\n        \"version\": {\n          \"$id\": \"#/properties/nvmupdate/version\",\n          \"type\": \"string\",\n          \"title\": \"The nvmupdate Version Schema\",\n          \"default\": \"\",\n          \"examples\": [\n            \"800049C6\"\n          ],\n          \"pattern\": \"^((0x)?[A-Fa-f0-9]{8})$\"\n        },\n        \"timeout\": {\n          \"$id\": \"#/properties/nvmupdate/timeout\",\n          \"type\": \"string\",\n          \"title\": \"The Timeout Schema\",\n          \"default\": \"\",\n          \"examples\": [\n            \"10m\"\n          ],\n          \"pattern\": \"^([0-9]+(\\\\.[0-9]+)?([dhms]))+$\"\n        }\n      }\n    },\n    \"flash\": {\n      \"$id\": \"#/properties/flash\",\n      \"type\": \"array\",\n      \"title\": \"The Flash Schema\",\n      \"items\": {\n        \"$id\": \"#/properties/flash/items\",\n        \"type\": \"object\",\n        \"title\": \"The Items Schema\",\n        \"required\": [\n          \"filename\",\n          \"type\",\n          \"version\",\n          \"revision\"\n        ],\n    \"optional\": [\n          \"enabled\",\n          \"force\",\n      \"timeout\",\n      \"requires\"\n    ],\n        \"properties\": {\n      \"enabled\": {\n            \"$id\": \"#/properties/flash/items/properties/enabled\",\n        \"type\": \"boolean\",\n        \"title\": \"The Enabled Schema\",\n        \"default\": \"true\"\n      },\n          \"filename\": {\n            \"$id\": \"#/properties/flash/items/properties/filename\",\n            \"type\": \"string\",\n            \"title\": \"The Filename Schema\",\n            \"default\": \"\",\n            \"examples\": [\n              \"vista_creek_qspi_xip_v1.0.6.ihex\"\n            ],\n            \"pattern\": \"^(.*)$\"\n          },\n          \"type\": {\n            \"$id\": \"#/properties/flash/items/properties/type\",\n            \"type\": \"string\",\n            \"title\": \"The Type Schema\",\n            \"default\": \"\",\n            \"examples\": [\n              \"bmc_fw\"\n            ],\n            \"enum\": [\"user\", \"bmc_fw\", \"bmc_img\", \"dtb\", \"factory_only\",\n        \"phy_eeprom\"]\n          },\n          \"version\": {\n            \"$id\": \"#/properties/flash/items/properties/version\",\n            \"type\": \"string\",\n            \"title\": \"The Version Schema\",\n            \"default\": \"\",\n            \"examples\": [\n              \"1.0.6\"\n            ],\n            \"pattern\": \"^\\\\d+\\\\.\\\\d+\\\\.\\\\d+$\"\n          },\n          \"force\": {\n            \"$id\": \"#/properties/flash/items/properties/force\",\n            \"type\": \"boolean\",\n            \"title\": \"The Force Schema\",\n            \"default\": false,\n            \"examples\": [\n              true\n            ]\n          },\n          \"revision\": {\n            \"$id\": \"#/properties/flash/items/properties/revision\",\n            \"type\": \"string\",\n            \"title\": \"The Revision Schema\",\n            \"default\": \"\",\n            \"examples\": [\n              \"C\"\n            ],\n            \"pattern\": \"^([A-Za-z])$\"\n          },\n          \"timeout\": {\n            \"$id\": \"#/properties/nvmupdate/timeout\",\n            \"type\": \"string\",\n            \"title\": \"The Timeout Schema\",\n            \"default\": \"\",\n            \"examples\": [\n              \"10m\"\n            ],\n            \"pattern\": \"^([0-9]+(\\.[0-9]+)?([dhms]))+$\"\n          },\n          \"requires\": {\n            \"$id\": \"#/properties/flash/items/properties/requires\",\n            \"type\": \"string\",\n            \"title\": \"The Requires Schema\",\n            \"default\": \"\",\n            \"examples\": [\n              \"bmc_img >= 1.0.12\"\n            ],\n            \"pattern\": \"^(([a-z_]+) ((<>!=)?=) ([0-9a-z\\\\.]+)$\"\n          }\n        }\n      }\n    }\n  }\n}\n
                                            "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/","title":"fpgainfo","text":""},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#synopsis","title":"SYNOPSIS","text":"
                                               fpgainfo [-h] [-S <segment>] [-B <bus>] [-D <device>] [-F <function>] [PCI_ADDR]\n            {errors,power,temp,fme,port,bmc,mac,phy,security}\n
                                            "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#description","title":"DESCRIPTION","text":"
                                            fpgainfo displays FPGA information derived from sysfs files. The command argument is one of the following: errors, power, temp, port, fme, bmc, phy or mac,security,events. Some commands may also have other arguments or options that control their behavior.
                                             
                                            For systems with multiple FPGA devices, you can specify the BDF to limit the output to the FPGA resource with the corresponding PCIe configuration. If not specified, information displays for all resources for the given command.
                                            "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#fpgainfo-commands","title":"FPGAINFO COMMANDS","text":"
                                            errors
                                             
                                            Show/clear errors of an FPGA resource that the first argument specifies. fpgainfo displays information in human readable form.
                                             Error Description Catfatal Errors Bit 8 indicates an Injected Fatal error, bit 11 indicates an Injected Catastrophic Error. Inject Errors [2:0] are mainly writeable bits. Can read back values. (FME) Next Error [59:0] 60 LSBs are taken from the given error register that was triggered second, [60:61] 0 = FME0 Error, 1 = PCIe0 Error. (FME) First Error [59:0] 60 LSBs are taken from the given error register that was triggered first, [60:61] 0 = FME0 Error, 1 = PCIe0 Error. FME Errors Error from Partial Reconfiguration Block reporting a FIFO Parity Error has occurred. Non-fatal Errors Bit 6 is used to advertise an Injected Warning Error. 
                                            power
                                             
                                            Show total the power in watts that the FPGA hardware consumes.
                                             
                                            temp
                                             
                                            Show FPGA temperature values in degrees Celcius.
                                             
                                            port
                                             
                                            Show information about the port such as the AFU ID of currently loaded AFU.
                                             
                                            fme
                                             
                                            Show information about the FPGA platform including the partial reconfiguration (PR) Interface ID, the OPAE version, and the FPGA Interface Manager (FIM) ID.
                                             
                                            bmc
                                             
                                            Show all Board Management Controller sensor values for the FPGA resource, if available.
                                             
                                            phy
                                             
                                            Show information about the PHY integrated in the FPGA, if available.
                                             
                                            mac
                                             
                                            Show information about the MAC address in ROM attached to the FPGA, if available.
                                             
                                            security
                                             
                                            Show information about the security keys, hashs and flash count, if available.
                                             
                                            events
                                             
                                            Show information about events and sensors, if available.
                                            "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                            --help, -h
                                             
                                            Prints help information and exit.
                                             
                                            --version, -v
                                             
                                            Prints version information and exit.
                                            "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#common-arguments","title":"COMMON ARGUMENTS","text":"
                                            The following arguments are common to all commands and are optional.
                                             
                                            -S, --segment
                                             
                                            PCIe segment number of resource.
                                             
                                            -B, --bus
                                             
                                            PCIe bus number of resource.
                                             
                                            -D, --device
                                             
                                            PCIe device number of resource.
                                             
                                            -F, --function
                                             
                                            PCIe function number of resource.
                                            "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#errors-arguments","title":"ERRORS ARGUMENTS","text":"
                                            The first argument to the errors command specifies the resource type. It must be one of the following:    fme,port,all
                                             
                                            fme
                                             
                                            Show/clear FME errors. 
                                             
                                            port
                                             
                                            Show/clear PORT errors.
                                             
                                            all
                                             
                                            Show/clear errors for all resources.
                                             
                                            The optional <command-args> arguments are:
                                             
                                            --clear, -c
                                             
                                            Clear errors for the given FPGA resource.
                                            "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#phy-arguments","title":"PHY ARGUMENTS","text":"
                                            The optional <command-args> argument is:
                                             
                                            --group, -G
                                             
                                            Select which PHY group(s) information to show.
                                            "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#events-arguments","title":"EVENTS ARGUMENTS","text":"
                                            The optional <command-args> argument is:
                                             
                                            --list,-l
                                             
                                            List boots (implies --all).
                                             
                                            --boot,-b
                                             
                                            Boot index to use, i.e: \u00a0\u00a0\u00a0\u00a00 for current boot (default). \u00a0\u00a0\u00a0\u00a01 for previous boot, etc.
                                             
                                            --count,-c
                                             
                                            Number of events to print.
                                             
                                            --all,-a
                                             
                                            Print all events.
                                             
                                            --sensors,-s
                                             
                                            Print sensor data too.
                                             
                                            --bits,-i
                                             
                                            Print bit values too.
                                             
                                            --help,-h
                                             
                                            Print this help.
                                            "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#examples","title":"EXAMPLES","text":"
                                            This command shows the current power telemetry: 
                                            ./fpgainfo power\n
                                             
                                            This command shows the current temperature readings: 
                                            ./fpgainfo temp\n
                                             
                                            This command shows FME resource errors: 
                                            ./fpgainfo errors fme\n
                                             This command clears all errors on all resources: 
                                            ./fpgainfo errors all -c\n
                                             This command shows information of the FME on bus 0x5e 
                                            ./fpgainfo fme -B 0x5e\n
                                             This command shows information of the FPGA security on bus 0x5e 
                                            ./fpgainfo security -B 0x5e\n
                                             This command shows all events and sensors information including sensor bits: 
                                            ./fpgainfo events -asi\n
                                            "},{"location":"sw/fpga_tools/fpgamux/fpgamux/","title":"fpgamux","text":""},{"location":"sw/fpga_tools/fpgamux/fpgamux/#synopsis","title":"SYNOPSIS","text":"
                                            fpgamux [-h] [-S|--socket-id SOCKET_ID] [-B|--bus-number BUS] [-D|--device DEVICE] [-F|--function FUNCTION]\n        [-G|--guid GUID] -m|--muxfile <filepath.json>\n
                                            "},{"location":"sw/fpga_tools/fpgamux/fpgamux/#description","title":"DESCRIPTION","text":"
                                            fpgamux tests multiple AFUs that are synthesized into a single AFU along with the CCIP-MUX basic building block (BBB). The CCIP-MUX uses the upper bits in the MMIO addresses to route MMIO reads and writes to the AFU running on the corresponding CCIP-MUX port. fpgamux uses a configuration file that lists the software components and correct configuration. fpgamux only runs on the Integrated FPGA Platform.  You cannot run it on the PCIe accelerator card (PAC).
                                             
                                            .. note::
                                             
                                              The OPAE driver discovers only the first AFU. The first software component in the configuration \n  determines the GUID to use for enumeration. Use the -G|--guid option to override the GUID\n  for the first software component.\n
                                            "},{"location":"sw/fpga_tools/fpgamux/fpgamux/#options","title":"OPTIONS","text":"
                                            -S SOCKET_ID, --socket-id SOCKET_ID
                                             
                                            socket id of FPGA resource.
                                             
                                            -B BUS, --bus BUS
                                             
                                            bus id of FPGA resource.
                                             
                                            -D DEVICE, --device DEVICE
                                             
                                            The device id of FPGA resource.
                                             
                                            -F FUNCTION, --function FUNCTION
                                             
                                            The function id of FPGA resource.
                                             
                                            -G, --guid
                                             
                                            Specifies the GUID to use for the resource enumeration.
                                             
                                            -m, --muxfile <filepath.json>
                                             
                                            The path to the fpgamux configuration file. This file must be in JSON format following the schema described below.
                                            "},{"location":"sw/fpga_tools/fpgamux/fpgamux/#configuration","title":"CONFIGURATION","text":"
                                            fpgamux uses a configuration file (in JSON format) to determine what software components to instantiate and how to configure them to work with the AFUs. The schema includes the following elements:
                                             
                                                [\n        {\n            \"app\" : \"fpga_app\",\n            \"name\" : \"String\",\n            \"config\" : \"Object\"\n        }\n    ]\n
                                            "},{"location":"sw/fpga_tools/fpgamux/fpgamux/#examples","title":"EXAMPLES","text":"
                                            The following example shows a configuration with two components: 
                                                [\n        {\n            \"app\" : \"nlb0\",\n            \"name\" : \"nlb0\",\n            \"config\" :\n            {\n                \"begin\" : 1,\n                \"end\" : 1,\n                \"multi-cl\" : 1,\n                \"cont\" : false,\n                \"cache-policy\" : \"wrline-M\",\n                \"cache-hint\" : \"rdline-I\",\n                \"read-vc\" : \"vh0\",\n                \"write-vc\" : \"vh1\",\n                \"wrfence-vc\" : \"write-vc\",\n                \"timeout-usec\" : 0,\n                \"timeout-msec\" : 0,\n                \"timeout-sec\" : 1,\n                \"timeout-min\" : 0,\n                \"timeout-hour\" : 0,\n                \"freq\" : 400000000\n            }\n        },\n        {\n            \"app\" : \"nlb3\",\n            \"name\" : \"nlb3\",\n            \"config\" :\n            {\n                \"mode\" : \"read\",\n                \"begin\" : 1,\n                \"end\" : 1,\n                \"multi-cl\" : 1,\n                \"strided-access\" : 1,\n                \"cont\" : false,\n                \"warm-fpga-cache\" : false,\n                \"cool-fpga-cache\" : false,\n                \"cool-cpu-cache\" : false,\n                \"cache-policy\" : \"wrline-M\",\n                \"cache-hint\" : \"rdline-I\",\n                \"read-vc\" : \"vh0\",\n                \"write-vc\" : \"vh1\",\n                \"wrfence-vc\" : \"write-vc\",\n                \"alt-wr-pattern\" : false,\n                \"timeout-usec\" : 0,\n                \"timeout-msec\" : 0,\n                \"timeout-sec\" : 1,\n                \"timeout-min\" : 0,\n                \"timeout-hour\" : 0,\n                \"freq\" : 400000000\n            }\n        }\n    ]\n
                                            "},{"location":"sw/fpga_tools/fpgaport/fpgaport/","title":"fpgaport","text":""},{"location":"sw/fpga_tools/fpgaport/fpgaport/#synopsis","title":"SYNOPSIS","text":"
                                            fpgaport [-h] [-N NUMVFS] [-X] [--debug] {assign,release} device [port]\n
                                            "},{"location":"sw/fpga_tools/fpgaport/fpgaport/#description","title":"DESCRIPTION","text":"
                                            The fpgaport enables and disables virtualization. It assigns and releases control of the port to the virtual function (VF). By default, the driver assigns the port to the physical function (PF) in the non-virtualization use case.
                                            "},{"location":"sw/fpga_tools/fpgaport/fpgaport/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                            {assign, release}
                                             
                                            Action to perform.\n
                                             
                                            device
                                             
                                            The FPGA device being targeted with this action.\n
                                             
                                            port
                                             
                                            The number of the port.\n
                                            "},{"location":"sw/fpga_tools/fpgaport/fpgaport/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                            -N NUMVFS, --numvfs NUMVFS
                                             
                                            Create NUMVFS virtual functions. The typical value is 1.\n
                                             
                                            -X, --destroy-vfs
                                             
                                            Destroy all virtual functions prior to assigning.\n
                                             
                                            --debug
                                             
                                            Display additional log information.\n
                                             
                                            -h, --help
                                             
                                            Print usage information.\n
                                            "},{"location":"sw/fpga_tools/fpgaport/fpgaport/#example","title":"EXAMPLE","text":"
                                            fpgaport release /dev/dfl-fme.0 0
                                             
                                            Release port 0 from physical function control.\n
                                             
                                            fpgaport assign /dev/dfl-fme.0 0
                                             
                                            Assign port 0 to physical function control.\n
                                            "},{"location":"sw/fpga_tools/fpgasupdate/fpgasupdate/","title":"fpgasupdate","text":""},{"location":"sw/fpga_tools/fpgasupdate/fpgasupdate/#synopsis","title":"SYNOPSIS","text":"
                                            fpgasupdate [--log-level=<level>] file [bdf]
                                            "},{"location":"sw/fpga_tools/fpgasupdate/fpgasupdate/#description","title":"DESCRIPTION","text":"
                                            The fpgasupdate command implements a secure firmware update for the following programmable accelerator cards (PACs): * Intel\u00ae PAC with Intel Arria\u00ae 10 GX FPGA * Intel\u00ae FPGA PAC D5005 * Intel\u00ae PAC N3000 * Intel\u00ae FPGA SmartNIC N6001-PL with Intel&reg Agilex&reg FPGA * Intel\u00ae FPGA IPU F2000X-PL
                                             
                                            --log-level <level>
                                             
                                            Specifies the `log-level` which is the level of information output to your command tool.\nThe following seven levels  are available: `state`, `ioctl`, `debug`, `info`, `warning`,\n`error`, `critical`. Setting `--log-level=state` provides the most verbose output.\nSetting `--log-level=ioctl` provides the second most information, and so on. The default\nlevel is `info`.\n
                                             
                                            file
                                             
                                            Specifies the secure update firmware file to be programmed. This file may be to program a\nstatic region (SR), programmable region (PR), root entry hash, key cancellation, or other\ndevice-specific firmware.\n
                                             
                                            bdf
                                             
                                            The PCIe&reg; address of the PAC to program. `bdf` is of the form `[ssss:]bb:dd:f`,\ncorresponding to PCIe segment, bus, device, function. The segment is optional. If\nyou do not specify a segment, the segment defaults to `0000`. If the system has only\none PAC you can omit the `bdf` and let `fpgasupdate`  determine the address\nautomatically.\n
                                            "},{"location":"sw/fpga_tools/fpgasupdate/fpgasupdate/#troubleshooting","title":"TROUBLESHOOTING","text":"
                                            To gather more debug output, decrease the --log-level parameter. 
                                            "},{"location":"sw/fpga_tools/fpgasupdate/fpgasupdate/#examples","title":"EXAMPLES","text":"
                                            fpgasupdate firmware.bin fpgasupdate firmware.bin 05:00.0 fpgasupdate firmware.bin 0001:04:02.0 --log-level=ioctl
                                            "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/","title":"host_exerciser","text":""},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#synopsis","title":"SYNOPSIS","text":"
                                            Usage: host_exerciser [OPTIONS] SUBCOMMAND\n\n\nOptions:\n  -h,--help                   Print this help message and exit\n  -p,--pci-address TEXT       [<domain>:]<bus>:<device>.<function>\n  -l,--log-level TEXT:{trace,debug,info,warning,error,critical,off}=warning\n                              stdout logging level\n  -s,--shared                 open in shared mode, default is off\n  -t,--timeout UINT=60000     test timeout (msec)\n  -m,--mode UINT:value in {lpbk->0,read->1,trput->3,write->2} OR {0,1,3,2}=lpbk\n                              host exerciser mode {lpbk,read, write, trput}\n  --cls UINT:value in {cl_1->0,cl_2->1,cl_4->2,cl_8->3} OR {0,1,2,3}=cl_1\n                              number of CLs per request{cl_1, cl_2, cl_4, cl_8}\n  --continuousmode BOOLEAN=false\n                              test rollover or test termination\n  --atomic UINT:value in {cas_4->9,cas_8->11,fadd_4->1,fadd_8->3,off->0,swap_4->5,swap_8->7} OR {9,11,1,3,0,5,7}=off\n                              atomic requests (only permitted in combination with lpbk/cl_1)\n  --encoding UINT:value in {default->0,dm->1,pu->2,random->3} OR {0,1,2,3}=default\n                              data mover or power user encoding -- random interleaves both in the same stream\n  -d,--delay BOOLEAN=false    Enables random delay insertion between requests\n  --interleave UINT=0         Interleave requests pattern to use in throughput mode {0, 1, 2}\n                              indicating one of the following series of read/write requests:\n                              0: rd-wr-rd-wr\n                              1: rd-rd-wr-wr\n                              2: rd-rd-rd-rd-wr-wr-wr-wr\n  --interrupt UINT:INT in [0 - 3]\n                              The Interrupt Vector Number for the device\n  --contmodetime UINT=1       Continuous mode time in seconds\n  --testall BOOLEAN=false     Run all tests\n  --clock-mhz UINT=0          Clock frequency (MHz) -- when zero, read the frequency from the AFU\n\nSubcommands:\n  lpbk                        run simple loopback test\n  mem                         run simple mem test\n
                                            "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#description","title":"DESCRIPTION","text":"
                                            The host exerciser used to exercise and characterize the various host-FPGA interactions eg. MMIO, Data transfer from host to FPGA , PR, host to FPGA memory etc.
                                            "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#host-exerciser-loopback-he-lbk","title":"Host Exerciser Loopback (HE-LBK)","text":"
                                            HE-LB is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth.  Host Exerciser Loopback (HE-LBK) AFU can move data between host memory and FPGA.
                                             
                                            HE-LBK supports: 1. Latency (AFU to Host memory read) 2. MMIO latency (Write+Read) 3. MMIO BW (64B MMIO writes) 4. BW (Read/Write, Read only, Wr only)
                                            "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#host-exerciser-memory-he-mem","title":"Host Exerciser Memory (HE-MEM)","text":"
                                            HE-MEM is used to exercise use of FPGA connected DDR; data read from the host is  written to DDR, and the same data is read from DDR before sending it back to the  host. HE-MEM uses external DDR memory (i.e. EMIF) to store data. It has a customized version of the AVMM interface to communicate with the EMIF memory controller.
                                             
                                            Execution of these exercisors requires the user to bind specific VF endpoint to vfio-pci Bind the correct endpoint for a device with B/D/F 0000:b1:00.0
                                             
                                            [user@localhost]: sudo opae.io init -d 0000:b1:00.2 user:user
                                            "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#host-exerciser-sub-commands","title":"HOST EXERCISER SUB COMMANDS","text":"
                                            lpbk
                                             
                                            run host exerciser loopback test
                                             
                                            mem
                                             
                                            run host exerciser memory test
                                            "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                            --help, -h
                                             
                                            Prints help information and exit.
                                            "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#common-arguments-options","title":"COMMON ARGUMENTS / OPTIONS","text":"
                                            The following arguments are common to all commands and are optional.
                                             
                                            -p,--pci-address
                                             
                                            PCIe domain, bus, device, function number of fpga resource.
                                             
                                            -l,--log-level
                                             
                                            set host exerciser tool log level, trace, debug, info, warning, error, critical, off
                                             
                                            -s,--shared
                                             
                                            open FPGA PCIe resource in shared mode
                                             
                                            -t,--timeout
                                             
                                            host exerciser tool time out, by default time out 60000
                                             
                                            -m,--mode
                                             
                                            host exerciser test modes are lpbk, read, write, trput
                                             
                                            --cls
                                             
                                            Number of cachelines per request 1, 2, 3, 4.
                                             
                                            --continuousmode
                                             
                                            Configures test rollover or test termination mode.
                                             
                                            --atomic
                                             
                                            atomic requests.
                                             
                                            --encoding
                                             
                                            select data mover mode or power user mode or random.
                                             
                                            -d,--delay
                                             
                                            Enables random delay insertion between requests.
                                             
                                            --interleave
                                             
                                            Enables interleave requests in throughput mode. Value:3'b000-Rd,Wr,Rd,Wr Value:3'b001-Rd,Rd,Wr,Wr Value:3'b010-Rd,Rd,Rd,Rd,Wr,Wr,Wr,Wr Value:3'b011-Not supported
                                             
                                            --interrupt
                                             
                                            Accelerator interrupt vector Number.
                                             
                                            --contmodetime
                                             
                                            Continuous mode time in seconds.
                                             
                                            --testall
                                             
                                            Run all host exerciser tests.
                                             
                                            --clock-mhz
                                             
                                            pcie clock frequency, default value 350Mhz. When specified by the user, will not check value against AFU clock before calculating performance metrics.
                                            "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#examples","title":"EXAMPLES","text":"
                                            This command exerciser Loopback afu: 
                                            host_exerciser lpbk\n
                                             
                                            This command exerciser memory afu: 
                                            host_exerciser mem\n
                                             
                                            This command exerciser Loopback afu on pcie 000:3b:00.0: 
                                            host_exerciser --pci-address 000:3b:00.0    lpbk\n
                                             
                                            This command exerciser Loopback afu on pcie 000:3b:00.0 and run in write mode: 
                                            host_exerciser --pci-address 000:3b:00.0   --mode write lpbl\n
                                             
                                            This command exerciser Loopback afu on pcie 000:3b:00.0 and run 2 cache lines per request: 
                                            host_exerciser --pci-address 000:3b:00.0   --cls cl_2  lpbk\n
                                             
                                            This command exerciser Loopback afu on pcie 000:3b:00.0 and run continuous mode for 10 seconds: 
                                            host_exerciser --pci-address 000:3b:00.0   -cls cl_1   -m 0 --continuousmode true --contmodetime 10 lpbk\n
                                            "},{"location":"sw/fpga_tools/hssi/hssi/","title":"hssi","text":""},{"location":"sw/fpga_tools/hssi/hssi/#synopsis","title":"SYNOPSIS","text":"
                                            hssi COMMON_OPTIONS MODE MODE_OPTIONS
                                            "},{"location":"sw/fpga_tools/hssi/hssi/#description","title":"DESCRIPTION","text":"
                                            The hssi application provides a means of interacting with the 10G, 100G, and 200G/400F HE-HSSI AFUs. In all operating modes, the application initializes the AFU and completes the desired transfer as described by the mode- specific options.
                                             
                                            COMMON_OPTIONS - application options common to the 10G, 100g, and 200G/400G modes.
                                             
                                            -h, --help
                                             
                                            Display common command-line help and exit.\n
                                             
                                            -p, --pci-address ADDR
                                             
                                            The PCIe address of the desired accelerator in ssss:bb:dd.f format.\n
                                             
                                            -s, --shared on|off
                                             
                                            Whether to open the accelerator in shared mode. The default is off.\n
                                             
                                            -t, --timeout VALUE
                                             
                                            The application timeout value in milliseconds. The default timeout is 60000 msec.\n
                                             
                                            MODE - select AFU. Valid values are hssi_10g, hssi_100g, hssi_200g_400g.
                                             
                                            MODE_OPTIONS [hssi_10g] - application options specific to the 10G AFU.
                                             
                                            -h, --help
                                             
                                            Display 10G AFU specific command-line help and exit.\n
                                             
                                            --port PORT
                                             
                                            Select the QSFP port in the range 0-7. The default is port 0.\n
                                             
                                            --num-packets PACKETS
                                             
                                            The number of packets to transfer. The default is 1 packet.\n
                                             
                                            --random-length fixed|random
                                             
                                            Specify packet length randomization. Valid values are fixed and\nrandom. The default is fixed (no randomization).\n
                                             
                                            --random-payload incremental|random
                                             
                                            Specify payload randomization. Valid values are incremental and\nrandom. The default is incremental.\n
                                             
                                            --packet-length LENGTH
                                             
                                            Specify packet length. The default is 64 bytes.\n
                                             
                                            --src-addr ADDR
                                             
                                            Specify the source MAC address. The default value is 11:22:33:44:55:66.\n
                                             
                                            --dest-addr ADDR
                                             
                                            Specify the destination MAC address. The default value is 77:88:99:aa:bb:cc.\n
                                             
                                            --rnd-seed0 SEED0
                                             
                                            Specify the prbs generator bits [31:0]. The default is 1592590336.\n
                                             
                                            --rnd-seed1 SEED1
                                             
                                            Specify the prbs generator bits [47:32]. The default is 1592590337.\n
                                             
                                            --rnd-seed2 SEED2
                                             
                                            Specify the prbs generator bits [91:64]. The default is 155373.\n
                                             
                                            MODE_OPTIONS [hssi_100g] - application options specific to the 100G AFU.
                                             
                                            --port PORT
                                             
                                            Select the QSFP port in the range 0-7. The default is port 0.\n
                                             
                                            --eth-loopback on|off
                                             
                                            Whether to enable loopback on the ethernet interface. Valid values are\non and off. The default is on.\n
                                             
                                            --num-packets PACKETS
                                             
                                            The number of packets to transfer. The default is 1 packet.\n
                                             
                                            --gap random|none
                                             
                                            Inter-packet gap. Valid values are random and none. The default is none.\n
                                             
                                            --pattern random|fixed|increment
                                             
                                            Pattern mode. Valid values are random, fixed, or increment. The default\nis random.\n
                                             
                                            --src-addr ADDR
                                             
                                            Specify the source MAC address. The default value is 11:22:33:44:55:66.\n
                                             
                                            --dest-addr ADDR
                                             
                                            Specify the destination MAC address. The default value is 77:88:99:aa:bb:cc.\n
                                             
                                            --start-size SIZE
                                             
                                            Specify the packet size in bytes, or the first packet size for --pattern increment.\n
                                             
                                            --end-size SIZE
                                             
                                            Specify the end packet size in bytes.\n
                                             
                                            --end-select pkt_num|gen_idle
                                             
                                            Specify packet generation end mode.\n
                                             
                                            MODE_OPTIONS [pkt_filt_10g] - application options specific to the Packet Filter 10G AFU.
                                             
                                            --dfl-dev DFL_DEV
                                             
                                            Packet Filter DFL device, eg --dfl-dev dfl_dev.0\n
                                             
                                            MODE_OPTIONS [pkt_filt_100g] - application options specific to the Packet Filter 100G AFU.
                                             
                                            --dfl-dev DFL_DEV
                                             
                                            Packet Filter DFL device, eg --dfl-dev dfl_dev.1\n
                                             
                                            MODE_OPTIONS [hssi_200g_400g] - application options specific to the 200G/400G AFU.
                                             
                                            --num-packets PACKETS
                                             
                                            The number of packets to transfer. Must be a multiple of 32. Default value is 32. Increasing the timeout (--timeout) may be necessary if specifying a large number of packets.\n
                                            "},{"location":"sw/fpga_tools/hssi/hssi/#examples","title":"EXAMPLES","text":"
                                            hssi -h hssi hssi_10g -h sudo hssi --pci-address=0000:3b:00.0 hssi_10g --eth-loopback=on --num-packets=500 sudo hssi --pci-address=0000:3b:00.0 hssi_100g --pattern=increment sudo hssi --pci-address=0000:0d:00.6 hssi_200g_400g --num-packets=640000
                                            "},{"location":"sw/fpga_tools/hssi_config/readme/","title":"hssi_config","text":""},{"location":"sw/fpga_tools/hssi_config/readme/#synopsis","title":"Synopsis","text":"
                                            hssi_config reads or writes HSSI registers on either on an Intel\u00ae FPGA using the  FPGA Interface Manager (FIM) or on an HSSI retimer card attached to the board. hssi_config is only available for the Integrated FPGA Platform. You cannot run it  on the PCIe accelerator card (PAC).
                                            "},{"location":"sw/fpga_tools/hssi_config/readme/#usage","title":"Usage","text":"
                                            hssi_config [--resource|-r <sysfs resource>] [--socket-id|s 0|1] command [command options]
                                             
                                            Where command is one of the following:
                                             
                                                dump [outfile.csv] [--input-file inputfile.csv]\n    iread instance (0,1) device-addr byte-address byte-count\n    iwrite instance (0,1) device-addr byte-address byte1 [byte2 [byte3...]]\n    load [inputfile.csv] [--c-header]\n    read lane(0-15) reg-address\n    rread device(0x30, 0x32, 0x34, 0x36) channel(0-3) address\n    rwrite device(0x30, 0x32, 0x34, 0x36) channel(0-3) address value\n    test (rd|rw) inputfile.csv [--acktimes] [--repeat N]\n    write lane(0-15) reg-address value\n
                                             
                                            The first argument is the command and any additional arguments are command arguments. The following options and commands are available:
                                            "},{"location":"sw/fpga_tools/hssi_config/readme/#options","title":"Options","text":"
                                            [--resource|-r <sysfs resource path>
                                             
                                            The resource path in the sysfs pseudo-filesystem. Example:     /sys/devices/pci0000\\:5e/0000\\:5e\\:00.0/resource0
                                             
                                            [--socket-id 0|1]
                                             
                                            The socket id of the target FPGA. Required on two-socket systems to differentiate between the two possible target FPGAs.
                                            "},{"location":"sw/fpga_tools/hssi_config/readme/#commands","title":"Commands","text":"
                                            dump [outfile.csv] [--input-file inputfile.csv]
                                             
                                            Dump registers to stdout or to a file, if provided. hssi_config has a built-in set of registers to dump. The first argument is the path to a file to write. The command dumps to stdout if you do not  specify a file name. Use the --input-file option to specify a different set of registers. 
                                             
                                            load [inputfile.csv] [--c-header]
                                             
                                            Load a set of register values from either stdin or an input file, if provided. The first argument  is the path to a file containing the registers to load. Loads from stdin if omitted.  
                                             
                                            Use --c-header to generate a C header file with an array of 64-bit numbers to write to the  HSSI_CTRL register. This header file can substitute for the input file.  NOTE: You must perform the acknowledge routine after each write.
                                             
                                            read lane(0-15) reg-address
                                             
                                            Read from a single XCVR (transceiver) register. The first command argument is the XCVR lane. Use -1 to specify a read from all lanes. The second argument is the XCVR address (offset).
                                             
                                            write lane(0-15) reg-address value
                                             
                                            Write to a single XCVR register. The first argument is the XCVR lane. The second argument is the XCVR address(offset). The third argument is the value to write to the register.
                                             
                                            rread device(0x30, 0x32, 0x34, 0x36) channel(0-3) address
                                             
                                            Read from a single retimer register. The first argument is the I2C device address. The second argument is the channel. The third argument is the register address (or I2C byte address).
                                             
                                            rwrite device(0x30, 0x32, 0x34, 0x36) channel(0-3) address value
                                             
                                            Write to a single retimer register. The first argument is the I2C device address. The second argument is the channel. The third argument is the register address (or I2C byte address). The fourth argument is the value to write.
                                             
                                            iread instance (0,1) device-addr byte-address byte-count
                                             
                                            Read from a device on the I2C bus. The first argument is the I2C controller instance (0 or 1). The second argument is the device address to read from. The third argument is the byte address of the register to read from the device. The fourth argument is the number of bytes to read.
                                             
                                            iwrite instance (0,1) device-addr byte-address byte1 [byte2 [byte3...]]
                                             
                                            Write to a device on the I2C bus. The first argument is the I2C controller instance (0 or 1). The second argument is the device address to read from. The third argument is the byte address of the register to read from the device. All subsequent arguments are the bytes to write to the device.
                                             
                                            test (rd|rw) inputfile.csv [--acktimes]
                                             
                                            Perform built-in test for reading or writing XCVR registers. The first argument is the path to a file containing the registers to test.
                                            "},{"location":"sw/fpga_tools/hssi_config/readme/#overview","title":"Overview","text":"
                                            The hssi_config utility reads or writes hssi equalization parameters stored in either the transceiver (XCVR) registers or the registers of the retimer on the I2C bus. To access registers,  the hssi controller writes to the HSSI_CTRL register and reads from the HSSI_STAT register in the FPGA Management Engine (FME). These two registers implement the HSSI AUX bus mailbox protocol to access devices on other buses. Because hssi_config maps the FME MMIO space directly, the FPGA driver is not required.
                                            "},{"location":"sw/fpga_tools/hssi_config/readme/#locating-the-fme-device","title":"Locating the FME Device","text":"
                                            The FME reads and writes the HSSI_CTRL and HSSI_STAT registers on the NIOS device. The FME maps the MMIO address space of the FME identified by its resource in the sysfs psuedo-filesystem in Linux operating systems. To identify resource paths, use the lspci utility to query for Intel devices with device id of bcc0.
                                             
                                            Example:
                                             
                                            lspci -d 8086:bcc0
                                             
                                            This command should print out at least one line like the following example:
                                             
                                            5e:00.0 Processing accelerators: Intel Corporation Device bcc0
                                             
                                            Use the first three numbers (bus:device.function) to locate the device resource in the sysfs filesystem:
                                             
                                            /sys/devices/pci0000\\:<bus>/0000\\:<bus>\\:<device>.<function>/resource0
                                             
                                            For example, the example above with bus of 5e, device of 00 and function of 0 would use a resource path as follows:
                                             
                                            /sys/devices/pci0000\\:5e/0000\\:5e\\:00.0/resource0
                                            "},{"location":"sw/fpga_tools/hssi_config/readme/#csv-file-format","title":"CSV File Format","text":"
                                            Any CSV file parsed by hssi_config must meet have at least four columns. The following table provides the column specifications:
                                             Column Name Description 1 Register type. Can be either FPGA_RX, FPGA_TX, RTMR_RX, RTMR_TX (or their corresponding numeric values, 1-4). 2 Lane or Channel 0-15 for XCVR lanes on FPGA, 0-3 for retimer channels. -1 to designate all lanes or channels. 3 Device address (on I2C bus) Only applies to retimer registers. 0 - 3, -1 to designate all devices. 4 Register address (or offset) Examples: 0x213, 0x100. 5 Register value to write Examples: 0x1, 1, 0. Applies only when loading or writing registers."},{"location":"sw/fpga_tools/hssi_config/readme/#examples-of-commands","title":"Examples of Commands","text":""},{"location":"sw/fpga_tools/hssi_config/readme/#dumping-registers","title":"Dumping Registers","text":"
                                            Dump default register set to stdout:
                                             
                                            >hssi_config dump
                                             
                                            Dump default registers to a file, data.csv:
                                             
                                            >hssi_config dump data.csv
                                             
                                            Dump to an output file, data.csv, Registers specified in an input file, regspec.csv:
                                             
                                            >hssi_config dump data.csv --input-file regspec.csv
                                            "},{"location":"sw/fpga_tools/hssi_config/readme/#reading-single-registers","title":"Reading Single Registers","text":"
                                            Read register from XCVR at 0x2e1 on lane 0:
                                             
                                            >hssi_config read 0 0x2e1
                                             
                                            Read register 0x109 from retimer on channel 0, device 0x30, channel 1:
                                             
                                            >hssi_config rread 0 0x30 0x109
                                            "},{"location":"sw/fpga_tools/hssi_config/readme/#loading-registers","title":"Loading Registers","text":"
                                            Load registers specified in an input file called data.csv:
                                             
                                            >hssi_config load data.csv
                                             
                                            Load registers specified from stdin:
                                             
                                            >hssi_config load
                                             
                                            FPGA_RX,1,-1,0x213,0\nFPGA_RX,2,-1,0x213,0\nFPGA_RX,3,-1,0x213,0\n<CTRL-D>\n
                                            "},{"location":"sw/fpga_tools/hssi_config/readme/#writing-single-registers","title":"Writing Single Registers","text":"
                                            Write 1 to XCVR register at 0x2e1 on lane 0:
                                             
                                            >hssi_config write 0 0x2e1 1
                                             
                                            Read register 0x109 from retimer on channel 0, device 0x30, channel 1:
                                             
                                            >hssi_config rread 0 0x30 0x109
                                            "},{"location":"sw/fpga_tools/hssi_config/readme/#testing-hssi-read-and-write","title":"Testing HSSI Read and Write","text":"
                                            >  test (rd|rw) register-file.csv [--acktimes]
                                             
                                            rd|wr
                                             
                                            Specifies either a rd or wr of transceiver registers. For writes, every register in the file is read from and written to in the following sequence:
                                             
                                            1. Read the register, save the value   2. Write 1 to the register   3. Read the register, verify that the register value is 1   4. Write 0 to the register   5. Read the register, verify that the register value is 0   6. Write the original value to the register   7. Read the register, assert it is the original value
                                             
                                            register=file.csv
                                             
                                            Specifies the path to a file containing the set of registers to test. 
                                             
                                            --acktimes 
                                             
                                            Specifies the time spent in the ack routine. When measured, a summary of ack times prints  to stdout. This argument is optional.
                                            "},{"location":"sw/fpga_tools/hssi_ethernet/hssiloopback/","title":"HSSI ethernet loopback","text":""},{"location":"sw/fpga_tools/hssi_ethernet/hssiloopback/#synopsis","title":"SYNOPSIS","text":"
                                            hssiloopback [-h] [--pcie-address PCIE_ADDRESS, -P PCIE_ADDRESS] --loopback [{enable,disable}]\n
                                            "},{"location":"sw/fpga_tools/hssi_ethernet/hssiloopback/#description","title":"DESCRIPTION","text":"
                                            The hssiloopback  tool enables and disable ethernet loopback.
                                            "},{"location":"sw/fpga_tools/hssi_ethernet/hssiloopback/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                            -h, --help
                                             
                                            Prints usage information
                                             
                                            --pcie-address PCIE_ADDRESS, -P PCIE_ADDRESS
                                             
                                            The PCIe address of the desired fpga  in ssss:bb:dd.f format. sbdf of device to program (e.g. 04:00.0 or 0000:04:00.0). Optional when one device in system.
                                             
                                            --loopback [{enable,disable}]
                                             
                                            Ethernet enable or disable loopback.
                                            "},{"location":"sw/fpga_tools/hssi_ethernet/hssiloopback/#examples","title":"EXAMPLES","text":"
                                            hssiloopback --pcie-address  0000:04:00.0 --loopback enable
                                             
                                            Enables ethernet loopback
                                             
                                            hssiloopback --pcie-address  0000:04:00.0 --loopback disable
                                             
                                            Disable ethernet loopback
                                            "},{"location":"sw/fpga_tools/hssi_ethernet/hssimac/","title":"HSSI ethernet mac","text":""},{"location":"sw/fpga_tools/hssi_ethernet/hssimac/#synopsis","title":"SYNOPSIS","text":"
                                            hssimac [-h] --pcie-address PCIE_ADDRESS [--port PORT]\n
                                            "},{"location":"sw/fpga_tools/hssi_ethernet/hssimac/#description","title":"DESCRIPTION","text":"
                                            The hssimac  tool provides Maximum TX and RX frame size.
                                            "},{"location":"sw/fpga_tools/hssi_ethernet/hssimac/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                            -h, --help
                                             
                                            Prints usage information
                                             
                                            --pcie-address PCIE_ADDRESS, -P PCIE_ADDRESS
                                             
                                            The PCIe address of the desired fpga  in ssss:bb:dd.f format. sbdf of device to program (e.g. 04:00.0 or 0000:04:00.0).
                                             
                                            --port PORT
                                             
                                            hssi port number.
                                            "},{"location":"sw/fpga_tools/hssi_ethernet/hssimac/#examples","title":"EXAMPLES","text":"
                                            hssimac --pcie-address  0000:04:00.0 --port 1
                                             
                                            prints Maximum TX and RX frame size for port 1.
                                            "},{"location":"sw/fpga_tools/hssi_ethernet/hssistats/","title":"HSSI ethernet statistics","text":""},{"location":"sw/fpga_tools/hssi_ethernet/hssistats/#synopsis","title":"SYNOPSIS","text":"
                                            hssistats [-h] [--pcie-address PCIE_ADDRESS, -P PCIE_ADDRESS]\n
                                            "},{"location":"sw/fpga_tools/hssi_ethernet/hssistats/#description","title":"DESCRIPTION","text":"
                                            The hssistats  tool provides the MAC statistics.
                                            "},{"location":"sw/fpga_tools/hssi_ethernet/hssistats/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                            -h, --help
                                             
                                            Prints usage information
                                             
                                            --pcie-address PCIE_ADDRESS, -P PCIE_ADDRESS
                                             
                                            The PCIe address of the desired fpga in ssss:bb:dd.f format. sbdf of device to program (e.g. 04:00.0 or 0000:04:00.0). Optional when one device in system.
                                            "},{"location":"sw/fpga_tools/hssi_ethernet/hssistats/#examples","title":"EXAMPLES","text":"
                                            hssistats --pcie-address  0000:04:00.0
                                             
                                            prints the MAC statistics 
                                            "},{"location":"sw/fpga_tools/hssi_loopback/readme/","title":"hssi_loopback","text":""},{"location":"sw/fpga_tools/hssi_loopback/readme/#name","title":"NAME","text":"
                                            hssi_loopback - Software utility to run HSSI loopback tests on FPGA
                                            "},{"location":"sw/fpga_tools/hssi_loopback/readme/#synopsis","title":"SYNOPSIS","text":"
                                            hssi_loopback [[--bus|-b <bus number>] [--device | -d <device number>] [--function | -f <function number>]]|[--socket-id <socket-id>]       [--mode|-m auto|e40|e10]        [send [<source port> [<destination port>] [--packet-count|-c <count>] [--packet-delay|-d <delay>] [--packet-length|-l <length>]] |status [clear] | stop | readmacs
                                            "},{"location":"sw/fpga_tools/hssi_loopback/readme/#description","title":"DESCRIPTION","text":"
                                            The hssi_loopback utility works in conjunction with a packet generator accelerator function unit (AFU) to test high-speed serial interface (HSSI) cards. The hssi_loopback utility tests both external and internal loopbacks. hssi_loopback runs an external loopback test when the command line arguments include both source and destination ports. hssi_loopback runs an internal loopback test when command line arguments include a single port. hssi_loopback only runs on the Intel Xeon with Arria 10 FPGA. You cannot run it on the Intel PAC (programmable accelerator card).
                                             
                                            NOTE: The following limitations apply to the current version of hssi_loopback:
                                             
                                               
                                            • For the external loopback the two port arguments can be the same. For the e10 design, the ports should be the same.
                                              •  
                                            • The hssi_loopback test supports only the e40 and e10 E2E AFUs.  The e10 E2E AFU tests HSSI with a retimer card.
                                              •  
                                            • The hssi_loopback test uses the control and status registers (CSRs) defined in the AFU.
                                              •  
                                            "},{"location":"sw/fpga_tools/hssi_loopback/readme/#options","title":"OPTIONS","text":"
                                            -S SOCKET_ID, --socket-id SOCKET_ID
                                             
                                            Socket ID FPGA resource.
                                             
                                            -B BUS, --bus BUS
                                             
                                            Bus ID of FPGA resource.
                                             
                                            -D DEVICE, --device DEVICE
                                             
                                            Device ID of FPGA resource.
                                             
                                            -F FUNCTION, --function FUNCTION
                                             
                                            Function ID of FPGA resource.
                                             
                                            -G, --guid
                                             
                                            Specifies guid for the resource enumeration.
                                             
                                            -m, --mode
                                             
                                            One of the following: [auto, e40, e10] auto is the default and indicates that the software runs the mode based on the first accelerator functional unit it identifies.
                                             
                                            -t, --timeout
                                             
                                            Timeout (in seconds) before the application terminates in continuous mode. Continuous mode is the default when you do not specify the number of packets.
                                             
                                            -y, --delay
                                             
                                            Delay (in seconds) between printing out a simple status line. Default is 0.100 seconds (100 milliseconds).
                                             
                                            -c, --packet-count
                                             
                                            The number of packets to send.
                                             
                                            -d, --packet-delay
                                             
                                            The delay in between packets. This delay is the number of 100 MHz clock cycles, roughly 10 nanoseconds.
                                             
                                            -s, --packet-size
                                             
                                            The packet size to send. The minimum is 46 bytes and the maximum is 1500 bytes. The default is 46 bytes.
                                            "},{"location":"sw/fpga_tools/hssi_loopback/readme/#commands","title":"COMMANDS","text":"
                                            send <source port> [<destination port>] [--packet-count|-c <count>] [--packet-delay|-d <delay>] [--packet-length|-l <length>]
                                             
                                            Send packets from one port to the other. If the command line does not specify a destination port, the test runs an internal  loopback. Otherwise, the test runs an external loopback from the source port to the destination port.
                                             
                                            status [clear]
                                             
                                            Read and interpret the status registers and print to the screen. clear clears the status registers.
                                             
                                            stop
                                             
                                            Issue a stop command to all Ethernet controllers in the AFU.
                                             
                                            readmacs
                                             
                                            Read and display the port MAC addresses. An EEPROM stores the MAC addresses.
                                            "},{"location":"sw/fpga_tools/hssi_loopback/readme/#exit-codes","title":"EXIT CODES","text":"
                                            0    Success - Number of packets received are equal to the number of packets sent and no errors           are reported.
                                             
                                            -1    Loopback failure - Either number of packets does not match or the test detected errors.
                                             
                                            -2    Errors parsing arguments.
                                            "},{"location":"sw/fpga_tools/hssi_loopback/readme/#examples","title":"EXAMPLES","text":"
                                            Read the MAC addresses of the AFU loaded on bus 0x5e:
                                             
                                            >sudo hssi_loopback readmacs -B 0x5e\n
                                             
                                            Run an external loopback, sending 100 packets from port 0 to port 1. The AFU is on bus 0x5e:
                                             
                                            >sudo hssi_loopback -B 0x5e send 0 1 -c 100\n
                                             
                                            Run an internal loopback until a timeout of 5 seconds is reached. The AFU is on bus 0x5e:
                                             
                                            >sudo hssi_loopback -B 0x5e send 0 -t 5\n
                                            "},{"location":"sw/fpga_tools/mem_tg/mem_tg/","title":"mem_tg","text":""},{"location":"sw/fpga_tools/mem_tg/mem_tg/#synopsis","title":"SYNOPSIS","text":"
                                            Usage: mem_tg [OPTIONS] SUBCOMMAND\n\nOptions:\n  -h,--help                   Print this help message and exit\n  -g,--guid TEXT=4DADEA34-2C78-48CB-A3DC-5B831F5CECBB\n                              GUID\n  -p,--pci-address TEXT       [<domain>:]<bus>:<device>.<function>\n  -l,--log-level TEXT:{trace,debug,info,warning,error,critical,off}=info\n                              stdout logging level\n  -s,--shared                 open in shared mode, default is off\n  -t,--timeout UINT=60000     test timeout (msec)\n  -m,--mem-channel UINT=0     Target memory bank for test to run on (0 indexed)\n  --loops UINT=1              Number of read/write loops to be run\n  -w,--writes UINT=1          Number of unique write transactions per loop\n  -r,--reads UINT=1           Number of unique read transactions per loop\n  -b,--bls UINT=1             Burst length of each request\n  --stride UINT=1             Address stride for each sequential transaction\n  --data UINT:value in {fixed->0,prbs15->2,prbs31->3,prbs7->1,rot1->3} OR {0,2,3,1,3}=fixed\n                              Memory traffic data pattern: fixed, prbs7, prbs15, prbs31, rot1\n  -f,--mem-frequency UINT=0   Memory traffic clock frequency in MHz\n\nSubcommands:\n  tg_test                     configure & run mem traffic generator test\n
                                            "},{"location":"sw/fpga_tools/mem_tg/mem_tg/#description","title":"DESCRIPTION","text":"
                                            The memory traffic generator (TG) used to exercise and test available memory channels with a configurable traffic pattern.
                                             
                                            Execution of this application requires the user to bind the specific VF endpoint containing the mem_tg AFU id to vfio-pci
                                             
                                            In the TG, read responses are checked against a specified pattern. If the application is configured to perform a read only test on a region of memory that has not previously been initialized to contain that pattern it will flag a test failure.
                                            "},{"location":"sw/fpga_tools/mem_tg/mem_tg/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                            --help, -h
                                             
                                            Prints help information and exit.
                                            "},{"location":"sw/fpga_tools/mem_tg/mem_tg/#common-arguments-options","title":"COMMON ARGUMENTS / OPTIONS","text":"
                                            The following arguments are common to all commands and are optional.
                                             
                                            -p,--pci-address
                                             
                                            PCIe domain, bus, device, function number of fpga resource.
                                             
                                            -l,--log-level
                                             
                                            set application log level, trace, debug, info, warning, error, critical, off
                                             
                                            -s,--shared
                                             
                                            open FPGA PCIe resource in shared mode
                                             
                                            -t,--timeout
                                             
                                            mem_tg application time out, by default time out 60000
                                             
                                            -m,--mem-channel
                                             
                                            Target memory bank for test to run on (0 indexed)  default: 0
                                             
                                            --loops
                                             
                                            Number of read/write loops to be run  default: 1
                                             
                                            -w,--writes
                                             
                                            Number of unique write transactions per loop.  default: 1
                                             
                                            -r,--reads
                                             
                                            Number of unique read transactions per loop  default: 1
                                             
                                            -b,--bls
                                             
                                            AXI4 burst length of each request.  Supports 1-256 transfers beginning from 0. default: 0
                                             
                                            --stride
                                             
                                            Address stride for each sequential transaction (>= burst length)  default: 1
                                             
                                            --data
                                             
                                            Memory traffic data pattern. 0 = fixed {0xFF, 0x00} 1 = prbs7 2 = prbs15 3 = prbs31 4 = rot1
                                             
                                            default: fixed
                                             
                                            -f, --mem-frequency
                                             
                                            Memory traffic clock frequency in MHz  default: 300 MHz
                                            "},{"location":"sw/fpga_tools/mem_tg/mem_tg/#examples","title":"EXAMPLES","text":"
                                            This command will run a basic read/write test on the channel 0 traffic generator: 
                                            mem_tg tg_test\n
                                             
                                            This command will run the application for an afu on pcie 000:b1:00.7: 
                                            mem_tg --pci-address 000:b1:00.7 tg_test\n
                                             
                                            This command will test channel 2 write bandwidth: 
                                            mem_tg -loops 1000 -w 1000 -r 0 -m 2 tg_test\n
                                             
                                            This command will perform a read bandwidth test with a burst of 16 on channel 1 and perform a data comparison with the prbs7 pattern: 
                                            mem_tg -loops 1000 -w 0 -r 1000 -b 0xF --data prbs7 -m 1 tg_test\n
                                             
                                            This command will perform a read/write test with 1 MB strided access to channel 0 memory: 
                                            mem_tg -loops 10000 --stride 0x100000 tg_test\n
                                            "},{"location":"sw/fpga_tools/mmlink/mmlink/","title":"mmlink","text":""},{"location":"sw/fpga_tools/mmlink/mmlink/#synopsis","title":"Synopsis","text":"
                                            mmlink [-v] [-B <bus>] [-D <device>] [-F <function>] [-S <socket>] [-P <TCP port>] [-I <IP Address>]
                                            "},{"location":"sw/fpga_tools/mmlink/mmlink/#description","title":"Description","text":"
                                            The Remote Signal Tap logic analyzer provides real-time hardware debugging for the Accelerator Function Unit (AFU).  It provides a signal trace capability that the Quartus Prime software adds to the AFU. The Remote Signal Tap logic analyzer provides access to the Remote Signal Tap part of the Port MMIO space and then runs the remote protocol.
                                            "},{"location":"sw/fpga_tools/mmlink/mmlink/#examples","title":"Examples","text":"
                                            ./mmlink  -B 0x5e -P 3333
                                             
                                            MMLink app starts and listens for connection.
                                            "},{"location":"sw/fpga_tools/mmlink/mmlink/#options","title":"Options","text":"
                                            -v,--version
                                             
                                            Prints version information and exits.
                                             
                                            -B,--bus 
                                             
                                            FPGA Bus number.
                                             
                                            -D,--device 
                                             
                                            FPGA Device number.
                                             
                                            -F,--function 
                                             
                                            FPGA function number.
                                             
                                            -S,--socket 
                                             
                                            FPGA socket number.
                                             
                                            -P,--port 
                                             
                                            TCP port number.
                                             
                                            -I,--ip 
                                             
                                            IP address of FPGA system. 
                                            "},{"location":"sw/fpga_tools/mmlink/mmlink/#notes","title":"Notes","text":"
                                            Driver privilege:
                                             
                                            Change AFU driver privilege to user:
                                             
                                            $ chmod 777 /dev/intel-fpga-port.0\n
                                             
                                            Change locked memory size:
                                             
                                            edit the file /etc/security/limits.conf
                                             
                                            $ sudo vi /etc/security/limits.conf\n\nuser    hard   memlock           10000\n\nuser    soft   memlock           10000\n
                                             
                                            Exit terminal and log into a new terminal.
                                             
                                            Verify that the locked memory is now set:  ``` $ ulimit -l 10000
                                            "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/","title":"ofs.uio","text":""},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#synopsis","title":"SYNOPSIS","text":"
                                            ofs.uio [-h] [--pcie-address PCIE_ADDRESS] [--uio uiox] [--feature-id FEATURE_ID] [--region-index REGION_INDEX]                   [--mailbox-cmdcsr offset] [--bit-size {8,16,32,64}] [--peek offset] [--poke offset value]                   [--mailbox-read offset] [--mailbox-dump address size] [--mailbox-write address value]
                                             
                                            ofs.uio [--uio uiox] [--peek offset] ofs.uio [--uio uiox] [--poke offset value] ofs.uio [--uio uiox] [--mailbox-read address] ofs.uio [--uio uiox] [--mailbox-write address value] ofs.uio [--uio uiox] [--mailbox-dump address size] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--peek offset] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--poke offset value] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--mailbox-read address] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--mailbox-write address value] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--mailbox-dump address size]
                                            "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#description","title":"DESCRIPTION","text":"
                                            ofs.uio is a tool that provides user space access to DFL UIO devices, command line options like peek, poke, mailbox-read, mailbox-write, mailbox-dump to  access Configuration and Status Registers (CSRs).
                                            "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#options","title":"OPTIONS","text":""},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#peek","title":"Peek","text":"
                                            Peek/Read UIO CSR offset ofs.uio [--uio uio] [--peek offset] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--peek offset]
                                            "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#poke","title":"Poke","text":"
                                            Poke/Write value to UIO CSR offset ofs.uio [--uio uio] [--poke offset value] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--poke offset value]
                                            "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#mailbox-read","title":"Mailbox Read","text":"
                                            Read CSR address using mailbox ofs.uio [--uio uio] [--mailbox-read address] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--mailbox-read address]
                                            "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#mailbox-write","title":"Mailbox Write","text":"
                                            Write value to CSR address using mailbox  ofs.uio [--uio uio] [--mailbox-write address value] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--mailbox-write address value]
                                            "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#mailbox-dump","title":"Mailbox Dump","text":"
                                            Reads/Dumps block size of CSR address using mailbox ofs.uio [--uio uio] [--mailbox-dump address size] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--mailbox-dump address size]
                                            "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#bit-size","title":"Bit size","text":"
                                            Read/Write bit-field 8,16,32,64 sizes ofs.uio [--uio uio] --bit-size 8 [--peek offset] ofs.uio [--uio uio] --bit-size 32 [--peek offset]
                                            "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#pcie-address","title":"PCIe Address","text":"
                                            PCIE_ADDR PCIe address of FPGA device ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--peek offset]
                                            "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#uio-region-index","title":"UIO region index","text":"
                                            UIO region index, default region index is 0  ofs.uio [--uio uio] --region-index 0 [--peek offset] ofs.uio [--uio uio] --region-index 1 [--peek offset]
                                            "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#mailbox-command-status-csr-offset","title":"Mailbox command status csr offset","text":"
                                            Mailbox command status csr offset,  default value set to dfl pcie subsystem system feature mailbox command status register offset 0x28  ofs.uio [--uio uio] --mailbox-cmdcsr 0xa8 [--mailbox-read address] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] --mailbox-cmdcsr 0xa8  [--mailbox-read address]
                                            "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#examples","title":"EXAMPLES","text":"
                                            Peek/Read 
                                            ofs.uio --uio uio0 --peek 0x0\npeek(0x0): 0x3000000010002015\n\nofs.uio --uio uio6 --peek 0x0\npeek(0x0): 0x3000000100000020\n\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x15 --peek 0x0\npeek(0x0): 0x3000000010002015\n\nofs.uio --uio uio0 --peek 0x0 --bit-size 32\npeek(0x0): 0x10002015\n
                                             
                                            Poke/Write 
                                            ofs.uio --uio uio6 --peek 0x8\npeek(0x8): 0x0\nofs.uio --uio uio6 --poke  0x8 0xabcdd12345\npoke(0x8):0xabcdd12345\n\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x15 --peek 0x0\npeek(0x8): 0x0\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x15 --poke  0x8 0x1234\npoke(0x8):0x1234\n
                                             
                                            Mailbox Read 
                                            ofs.uio --uio uio6 --mailbox-read 0x0\nMailboxRead(0x0): 0x1000000\nofs.uio --uio uio6 --mailbox-read 0x8\nMailboxRead(0x8): 0x110c000\n\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x20 --mailbox-read 0x0\nMailboxRead(0x0): 0x1000000\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x20 --mailbox-read 0x8 \nMailboxRead(0x8): 0x110c000\n
                                             
                                            Mailbox Write 
                                            ofs.uio --uio uio6 --mailbox-write 0x0 0x1234\nMailboxWrite(0x0):0x1234\nofs.uio --uio uio6 --mailbox-read 0x0\nMailboxRead(0x0):0x1234\n\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x20 --mailbox-write 0x0 0x1234\nMailboxWrite(0x0):0x1234\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x20 --mailbox-read 0x0 \nMailboxRead(0x0):0x1234\n
                                             
                                            Mailbox Dump 
                                            ofs.uio --uio uio6 --mailbox-dump 0x0 0x10\nMailboxDump(0x0): 0x1000000\nMailboxDump(0x4): 0x1000000\nMailboxDump(0x8): 0x110c000\nMailboxDump(0xc): 0x110c000\nMailboxDump(0x10): 0x0\nMailboxDump(0x14): 0x0\nMailboxDump(0x18): 0x0\nMailboxDump(0x1c): 0x0\nMailboxDump(0x20): 0x0\nMailboxDump(0x24): 0x0\nMailboxDump(0x28): 0x0\nMailboxDump(0x2c): 0x0\nMailboxDump(0x30): 0x0\nMailboxDump(0x34): 0x0\nMailboxDump(0x38): 0x0\nMailboxDump(0x3c): 0x0\n\n\n\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x20 --mailbox-dump 0x0 0x10\nMailboxDump(0x0): 0x1000000\nMailboxDump(0x4): 0x1000000\nMailboxDump(0x8): 0x110c000\nMailboxDump(0xc): 0x110c000\nMailboxDump(0x10): 0x0\nMailboxDump(0x14): 0x0\nMailboxDump(0x18): 0x0\nMailboxDump(0x1c): 0x0\nMailboxDump(0x20): 0x0\nMailboxDump(0x24): 0x0\nMailboxDump(0x28): 0x0\nMailboxDump(0x2c): 0x0\nMailboxDump(0x30): 0x0\nMailboxDump(0x34): 0x0\nMailboxDump(0x38): 0x0\nMailboxDump(0x3c): 0x0\n
                                            "},{"location":"sw/fpga_tools/opae.io/opae.io/","title":"opae.io","text":""},{"location":"sw/fpga_tools/opae.io/opae.io/#synopsis","title":"SYNOPSIS","text":"
                                            opae.io ls [-v,--viddid VIDDID] [-s,--sub-viddid SUB_VIDDID] [--all] [--system-class] opae.io init [-d,--device PCI_ADDR] [USER[:GROUP]] opae.io release [-d,--device PCI_ADDR] opae.io [-d,--device PCI_ADDR] [-r,--region REGION] walk [--offset [OFFSET]] [-u,--show-uuid] [-D,--dump] [-c,--count COUNT] [-y,--delay DELAY] [-s,--safe] opae.io [-d,--device PCI_ADDR] [-r,--region REGION] dump [--offset [OFFSET]] [-o,--output OUTPUT] [-f,--format {bin,hex}] [-c,--count COUNT] opae.io [-d,--device PCI_ADDR] [-r,--region REGION] peek OFFSET opae.io [-d,--device PCI_ADDR] [-r,--region REGION] poke OFFSET VALUE opae.io [-d,--device PCI_ADDR] [-r,--region REGION] script SCRIPT ARG1 ARG2 ... ARGN opae.io [-d,--device PCI_ADDR] [-r,--region REGION]
                                            "},{"location":"sw/fpga_tools/opae.io/opae.io/#description","title":"DESCRIPTION","text":"
                                            opae.io is an interactive Python environment packaged on top of libopaevfio.so, which provides user space access to PCIe devices via the vfio-pci driver. The main feature of opae.io is its built-in Python command interpreter, along with some Python bindings that provide a means to access Configuration and Status Registers (CSRs) that reside on the PCIe device.
                                             
                                            opae.io has two operating modes: command line mode and interactive mode.
                                            "},{"location":"sw/fpga_tools/opae.io/opae.io/#command-line-mode","title":"COMMAND LINE MODE","text":"
                                            To view the accelerator devices that are present on the system, opae.io provides the ls command option.
                                             
                                            opae.io ls [-v,--viddid VIDDID] [-s,--sub-viddid SUB_VIDDID] [--all] [--system-class]
                                             
                                            Each accelerator device is listed along with the PCIe address, the PCIe vendor/device ID, a brief description of the device, and the driver to which the device is currently bound.
                                             
                                            Device filtering is available by providing a Vendor ID:Device ID pair, eg -v 8086:bcce. Further filtering can be done by providing a sub- Vendor ID:sub-Device ID pair, eg -s 8086:1771. The --all option provides a list of all of the PCIe devices in the system, which an be quite verbose. The --system-class option prints the PCIe database class of the accelerator device, rather than the product name.
                                             
                                            opae.io provides an option to initialize a PCIe device for use with the vfio-pci driver. In order for the device CSRs to be accessed from user space, the device must first be bound to the vfio-pci driver. This is the job of the init command option.
                                             
                                            opae.io init [-d,--device PCI_ADDR] [USER[:GROUP]]
                                             
                                            The init command unbinds the specified device from its current driver and binds it to vfio-pci. This creates a new vfio group under /dev/vfio. This group path is then used by the libopaevfio.so library to interact with the device.
                                             
                                            To release the PCIe device from vfio-pci and return it to use with its previous driver, the release command option is used.
                                             
                                            opae.io release [-d,--device PCI_ADDR]
                                             
                                            The release command option reverses the actions of the last init command, releasing the device from vfio-pci and binding it to the driver which was bound at the time the init command was issued.
                                             
                                            The walk command option traverses and displays the Device Feature List of the given region.
                                             
                                            opae.io walk [--offset [OFFSET]] [-u,--show-uuid] [-D,--dump] [-c,--count COUNT] [-y,--delay DELAY] [-s,--safe]
                                             
                                            The various fields of each Device Feature Header are displayed. The --show-uuid option additionally displays the GUID for each feature. OFFSET can be used to specify the beginning of the DFL in the MMIO region. --dump displays the raw DFH contents in hex format. COUNT limits the number of DFH entries traversed. DELAY causes a pause between each printout. --safe examines each DFH offset for proper alignment.
                                             
                                            The dump command provides a means to dump the MMIO space in ASCII hex or binary format.
                                             
                                            opae.io dump [--offset [OFFSET]] [-o,--output OUTPUT] [-f,--format {bin,hex}] [-c,--count COUNT]
                                             
                                            OFFSET specifies the starting MMIO offset. OUTPUT gives the name of a file to capture the dump output, where sys.stdout is used by default. --format allows changing the output format. COUNT specifies the number of qwords to dump.
                                             
                                            The peek command option reads and displays a CSR value.
                                             
                                            opae.io peek OFFSET
                                             
                                            The poke command option writes a given value to a CSR.
                                             
                                            opae.io poke OFFSET VALUE
                                             
                                            opae.io can also execute Python scripts from the command line. These Python scripts may contain calls to the device built-in functions that are available during an interactive session. Refer to the description of interactive mode for details.
                                             
                                            opae.io script myscript.py a b c
                                             
                                            In order to enter the interactive mode of opae.io, simply invoke it and optionally pass the desired device address and MMIO region options.
                                             
                                            opae.io [-d,--device PCI_ADDR] [-r,--region REGION]
                                            "},{"location":"sw/fpga_tools/opae.io/opae.io/#interactive-mode","title":"INTERACTIVE MODE","text":"
                                            Upon entering interactive mode, opae.io begins a Python interpreter session and displays the command prompt shown below:
                                             
                                            0000:3f:00.0[0]>>
                                             
                                            The first portion of the prompt shows the address of the active PCIe device, here 0000:3f:00.0. The part in square brackets shows the active MMIO region, here [0].
                                             
                                            The interpreter waits for a valid Python command, then attempts to execute the given command in the usual way. The only differences between the traditional Python command intepreter and opae.io are that opae.io provides 1) the notion of an active PCIe device and MMIO region and 2) several built-in functions and objects that allow manipulating the active device and MMIO region.
                                            "},{"location":"sw/fpga_tools/opae.io/opae.io/#built-in-functions","title":"BUILT-IN FUNCTIONS","text":"
                                            The opae.io built-in functions assume an active device and MMIO region. Attempting to use the built-in functions without first opening a device and region will result in errors.
                                             
                                            peek(OFFSET)
                                             
                                            The peek built-in function reads and displays a CSR value from the active device and region, at the offset supplied by its argument.
                                             
                                            0000:3f:00.0[0]>> peek(0x28) 0xdeadbeef
                                             
                                            poke(OFFSET, VALUE)
                                             
                                            The poke built-in function writes the given VALUE to the current MMIO region at the given OFFSET.
                                             
                                            0000:3f:00.0[0]>> poke(0x28, 0xdeadbeef)
                                             
                                            read_csr(OFFSET)
                                             
                                            The read_csr built-in function returns the value of the CSR at the active MMIO region and the given OFFSET.
                                             
                                            0000:3f:00.0[0]>> print('0x{:0x}'.format(read_csr(0x28))) 0xdeadbeef
                                             
                                            write_csr(OFFSET, VALUE)
                                             
                                            The write_csr built-in function writes the given VALUE to the current MMIO region at the given OFFSET.
                                             
                                            0000:3f:00.0[0]>> write_csr(0x28, 0xdeadbeef)
                                             
                                            device(PCI_ADDR)
                                             
                                            The device built-in function allows changing the active PCIe device.
                                             
                                            0000:3f:00.0[0]>> device('0000:2b:00.0') 0000:2b:00.0>>
                                             
                                            region(REGION)
                                             
                                            The region built-in function allows changing the active MMIO region.
                                             
                                            0000:2b:00.0>> region(0) 0000:2b:00.0[0]>>
                                             
                                            allocate_buffer(SIZE)
                                             
                                            The allocate_buffer built-in function creates and returns a DMA buffer object. The underlying buffer will be SIZE bytes in length.
                                             
                                            0000:2b:00.0[0]>> b1 = allocate_buffer(4096) 0000:2b:00.0[0]>> print(b1.size, '0x{:0x}'.format(b1.address), b1.io_address) 4096 0x7f9361c66000 0
                                             
                                            version()
                                             
                                            The version built-in function returns a tuple containing the four components used to identify the opae.io version:
                                             
                                            0000:2b:00.0[0]>> print(version()) ('opae.io', 0, 2, 0)
                                            "},{"location":"sw/fpga_tools/opae.io/opae.io/#built-in-objects","title":"BUILT-IN OBJECTS","text":"
                                            opae.io interactive mode provides two global objects corresponding to the current device and that device's current MMIO region. These objects are referred to by global variables the_device and the_region, respectively.
                                             
                                            The device class:
                                             
                                            the_device.descriptor() : method that returns the integer file descriptor of the VFIO container.
                                             
                                            0000:2b:00.0[0]>> print(the_device.descriptor()) 5
                                             
                                            the_device.repr() : method that is invoked when a device object is printed.
                                             
                                            0000:2b:00.0[0]>> print(the_device) 0000:2b:00.0
                                             
                                            the_device.allocate(SIZE) : method that allocates and returns a system_buffer object. The buffer will be mapped into the DMA space of the_device.
                                             
                                            0000:2b:00.0[0]>> b1 = the_device.allocate(4096)
                                             
                                            the_device.pci_address() : read-only property that returns the PCIe address of the_device.
                                             
                                            0000:2b:00.0[0]>> print(the_device.pci_address) 0000:2b:00.0
                                             
                                            the_device.num_regions : read-only property that returns the number of MMIO regions in the_device.
                                             
                                            0000:2b:00.0[0]>> print(the_device.num_regions) 2
                                             
                                            the_device.regions : read-only property that returns a list of the active MMIO regions of the_device:
                                             
                                            0000:2b:00.0[0]>> print(the_device.regions) [0, 2]
                                             
                                            The region class:
                                             
                                            the_region.write32(OFFSET, VALUE) : method that writes a 32-bit VALUE to the CSR at OFFSET.
                                             
                                            the_region.read32(OFFSET) : method that returns a 32-bit CSR at the given OFFSET.
                                             
                                            0000:2b:00.0[0]>> the_region.write32(0x28, 0xdeadbeef) 0000:2b:00.0[0]>> print('0x{:0x}'.format(the_region.read32(0x28))) 0xdeadbeef
                                             
                                            the_region.write64(OFFSET, VALUE): method that writes a 64-bit VALUE to the CSR at OFFSET.
                                             
                                            the_region.read64(OFFSET): method that returns a 64-bit CSR at the given OFFSET.
                                             
                                            0000:2b:00.0[0]>> the_region.write64(0x28, 0xbaddecaf) 0000:2b:00.0[0]>> print('0x{:0x}'.format(the_region.read64(0x28))) 0xbaddecaf
                                             
                                            the_region.index(): method that returns the MMIO index of the_region.
                                             
                                            0000:2b:00.0[0]>> print(the_region.index()) 0
                                             
                                            the_region.repr(): method that is invoked when a region object is printed.
                                             
                                            0000:2b:00.0[0]>> print(the_region) 0
                                             
                                            the_region.len(): method that is invoked to determine the MMIO region size.
                                             
                                            0000:2b:00.0[0]>> print(len(the_region)) 524288
                                             
                                            The allocate_buffer() built-in function and the device.allocate() method return objects of type system_buffer.
                                             
                                            The system_buffer class is as follows:
                                             
                                            buf.size: read-only property that gives the buffer size.
                                             
                                            0000:2b:00.0[0]>> print(b1.size) 4096
                                             
                                            buf.address: read-only property that gives the buffer's user mode virtual address.
                                             
                                            0000:2b:00.0[0]>> print('0x{:0x}'.format(b1.address)) 0x7f2c15d8200
                                             
                                            buf.io_address: read-only property that gives the buffer's IO address.
                                             
                                            0000:2b:00.0[0]>> print('0x{:0x}'.format(b1.io_address)) 0x0
                                             
                                            buf.__getitem__ and buf.__setitem__: indexing get/set of 64-bit data item.
                                             
                                            0000:2b:00.0[0]>> b1[0] = 0xdecafbad 0000:2b:00.0[0]>> print('0x{:0x}'.format(b1[0])) 0xdecafbad
                                             
                                            buf.read8(OFFSET) buf.read16(OFFSET) buf.read32(OFFSET) buf.read64(OFFSET) : methods that read the given size data item from the given buffer OFFSET.
                                             
                                            buf.fill8(VALUE) buf.fill16(VALUE) buf.fill32(VALUE) buf.fill64(VALUE) : methods that fill the buffer with the given VALUE, using the given size.
                                             
                                            b1.compare(b2): method that compares buffers. The method returns the index of the first byte that miscompares, or the length of b1.
                                            "},{"location":"sw/fpga_tools/opaeuio/opaeuio/","title":"opaeuio","text":""},{"location":"sw/fpga_tools/opaeuio/opaeuio/#synopsis","title":"SYNOPSIS","text":"
                                            opaeuio [-h] [-i] [-r] [-d DRIVER] [-u USER] [-g GROUP] [-v] [device]
                                            "},{"location":"sw/fpga_tools/opaeuio/opaeuio/#description","title":"DESCRIPTION","text":"
                                            The opaeuio command enables the binding/unbinding of a DFL device to/from the dfl-uio-pdev device driver. See https://kernel.org/doc/html/v4.14/driver-api/uio-howto.html for a description of uio.
                                            "},{"location":"sw/fpga_tools/opaeuio/opaeuio/#options","title":"OPTIONS","text":"
                                            device     The DFL device name, eg dfl_dev.10
                                             
                                            -h, --help
                                             
                                            Display command-line help and exit.\n
                                             
                                            -i, --init
                                             
                                            Specifies binding mode operation - initialize the given device for uio.\nUsed in conjunction with -u, -g, and -d.\n
                                             
                                            -r, --release
                                             
                                            Specifies unbinding mode operation - release the given device from uio.\n
                                             
                                            -d DRIVER, --driver DRIVER
                                             
                                            Specifies the device driver to bind to when binding to uio.\nThe default value is dfl-uio-pdev.\n
                                             
                                            -u USER, --user USER
                                             
                                            The user ID to assign when binding to uio. A new device node is created in\n/dev when the device is bound to uio. Use this option to specify\nthe new device owner.\n
                                             
                                            -g GROUP, --group GROUP
                                             
                                            The group ID to assign when binding to uio. Use this option to specify the\nnew device group for the device created in /dev.\n
                                             
                                            -v, --version
                                             
                                            Display script version information and exit.\n
                                            "},{"location":"sw/fpga_tools/opaeuio/opaeuio/#examples","title":"EXAMPLES","text":"
                                            opaeuio -h opaeuio -v sudo opaeuio -i -u lab -g labusers dfl_dev.10 sudo opaeuio -r dfl_dev.10
                                            "},{"location":"sw/fpga_tools/opaevfio/opaevfio/","title":"opaevfio","text":""},{"location":"sw/fpga_tools/opaevfio/opaevfio/#synopsis","title":"SYNOPSIS","text":"
                                            opaevfio [-h] [-i] [-r] [-d DRIVER] [-u USER] [-g GROUP] [-n] [-v] [addr]
                                            "},{"location":"sw/fpga_tools/opaevfio/opaevfio/#description","title":"DESCRIPTION","text":"
                                            The opaevfio command enables the binding/unbinding of a PCIe device to/from the vfio-pci device driver. See https://kernel.org/doc/Documentation/vfio.txt for a description of vfio-pci.
                                            "},{"location":"sw/fpga_tools/opaevfio/opaevfio/#options","title":"OPTIONS","text":"
                                            addr     The PCIe address of the device in ssss:bb:dd.f format, eg 0000:7f:00.0
                                             
                                            -h, --help
                                             
                                            Display command-line help and exit.\n
                                             
                                            -i, --init
                                             
                                            Specifies binding mode operation - initialize the given addr for vfio.\nUsed in conjunction with -u, -g, and -n.\n
                                             
                                            -r, --release
                                             
                                            Specifies unbinding mode operation - release the given addr from vfio.\nUsed in conjunction with -d.\n
                                             
                                            -d DRIVER, --driver DRIVER
                                             
                                            Specifies the device driver to bind to when releasing from vfio.\nWhen omitted, the device is not rebound to a driver (default).\n
                                             
                                            -u USER, --user USER
                                             
                                            The user ID to assign when binding to vfio. A new device node is created in\n/dev/vfio when the device is bound to vfio-pci. Use this option to specify\nthe new device owner.\n
                                             
                                            -g GROUP, --group GROUP
                                             
                                            The group ID to assign when binding to vfio. Use this option to specify the\nnew device group for the device created in /dev/vfio.\n
                                             
                                            -n, --no-sriov
                                             
                                            Do not enable SR-IOV when binding to vfio. The default value for this option\nis FALSE, ie the script should specify SR-IOV functionality when binding to\nthe vfio-pci driver. When omitted, the modprobe command which loads the vfio-pci\ndriver will contain the `enable_sriov=1` option. When given, it will not.\n
                                             
                                            -v, --version
                                             
                                            Display script version information and exit.\n
                                            "},{"location":"sw/fpga_tools/opaevfio/opaevfio/#examples","title":"EXAMPLES","text":"
                                            opaevfio -h opaevfio -v sudo opaevfio -i -u lab -g labusers 0000:7f:00.0 sudo opaevfio -r 0000:7f:00.0
                                            "},{"location":"sw/fpga_tools/pac_hssi_config/pac_hssi_config/","title":"Pac hssi config","text":"
                                            # pac_hssi_config #\n\n## SYNOPSIS ##\n```console\npac_hssi_config.py [-h] subcommand [subarg] [bdf]\n
                                            "},{"location":"sw/fpga_tools/pac_hssi_config/pac_hssi_config/#description","title":"DESCRIPTION","text":"
                                            The pac_hssi_config.py tool exercises the Ethernet 10 Gbps (10GbE) and 40GbE transceivers for designs using the Intel\u00ae Programmable Acceleration Card (PAC) with Intel Arria\u00ae 10 GX FPGA. This tool does not support the  Intel Xeon\u00ae Processor with Integrated FPGA. 
                                             
                                             
                                            The two required arguments to the pac_hssi_config.py tool specify the subcommand and bus, device, and function (BDF) for the PCIe device under test. You must provide the BDF parameter for systems with more than one PCIe card. 
                                             
                                            .. note::\n    If you do not provide the BDF when required, the command prints a list of valid BDFs for the system. You can also\n    determine the BDF using the ``lspci`` command.\n
                                             
                                            For usage help, type the following at a command prompt:
                                             
                                            pac_hssi_config.py [-h|--help]
                                             
                                            To configure the network ports, send data, and read statistics, use the following form of the pac_hssi_config.py script:
                                             
                                            pac_hssi_config.py subcommand [subarg] [bdf]
                                             
                                            Only a subset of subcommand arguments support subarg. 
                                            "},{"location":"sw/fpga_tools/pac_hssi_config/pac_hssi_config/#table-1-general-subcommands","title":"Table 1. General Subcommands","text":"Subcommand Subarg Description stat N/A Prints high speed serial interface (HSSI) controller statistics. eeprom N/A Reads the 128-bit unique board ID, MAC address, and board-specific IDs from EEPROM."},{"location":"sw/fpga_tools/pac_hssi_config/pac_hssi_config/#table-2-1040-gbe-traffic-generation-subcommands","title":"Table 2. 10/40 GbE Traffic Generation Subcommands","text":"Subcommand Subarg Description e10init and e40init N/A Initializes HSSI PHY to 10GbE or 40GbE mode.  Clears statistics and enable internal HSSI transceiver loopback. e10loop and e40loop On/Off Turns on or off internal HSSI transceiver loopback. e10reset and e40reset On/Off Asserts or deasserts AFU reset.  Clears packet statistics and disables internal HSSI transceiver loopback. e10send and e40send N/A Sends 1,000,000 1500-byte packets. For 10GbE sends packets on all four ports. 40GbE has a single port. e10stat and e40stat N/A Prints packet statistics. e10statclr and e40statclr N/A Clears packet statistics.  Use this command after switching loopback modes to clear any transient statistics accumulated during the mode switch. 
                                            The transceiver equalization eqwrite and eqread subcommands write and read transceiver equalization settings.  These subcommands require you to specify the transceiver channel, the equalization setting, and the value (for writes).  Use the following form for the eqwrite command:
                                             
                                            pac_hssi_config.py eqwrite [transceiver channel number] [equalization setting] [equalization value] [bdf]
                                             
                                            Use the following form for the eqreadcommand:
                                             
                                            pac_hssi_config.py eqread [transceiver channel number] [equalization setting] [bdf] 
                                            "},{"location":"sw/fpga_tools/pac_hssi_config/pac_hssi_config/#table-3-transceiver-equalization-subcommands","title":"Table 3. Transceiver Equalization Subcommands","text":"Subcommand Channel Number Equalization Setting Value eqwrite 0-3 0 = Continuous time-linear equalization (CTLE)  1 = Variable gain amplifier (VGA)  2 = DCGAIN  3 = Pre-emphasis first post-tap  4 = Pre-emphasis second post-tap  5 = Pre-emphasis first pre-tap  6 = Pre-emphasis second pre-tap  7 = Differential output voltage (VOD) Specifies the value for the specified equalization setting. eqread 0-3 0 = Continuous time-linear equalization (CTLE)  1 = Variable gain amplifier (VGA)  2 = DCGAIN  3 = Pre-emphasis first post-tap  4 = Pre-emphasis second post-tap  5 = Pre-emphasis first pre-tap  6 = Pre-emphasis second pre-tap  7 = Differential output voltage (VOD) N/A 
                                            For more information about reconfiguring transceiver analog parameter settings In Arria\u00ae 10 devices, refer to \"Changing PMA Analog Parameters\" in the  Intel\u00ae Arria\u00ae 10 Transceiver PHY User Guide. 
                                            "},{"location":"sw/fpga_tools/packager/packager/","title":"packager","text":""},{"location":"sw/fpga_tools/packager/packager/#synopsis","title":"SYNOPSIS","text":"
                                            packager <cmd> [arguments]
                                            "},{"location":"sw/fpga_tools/packager/packager/#description","title":"Description","text":"
                                            The packager provides tools that Accelerator Functional Unit (AFU) developers use to create Accelerator Function (AF)  files. The AF file is the programming file for an AFU on Intel\u00ae FPGA platforms. The packager tool concatenates the metadata from the JSON file to a raw binary file (.rbf) that the Intel Quartus\u00ae Prime software generates. 
                                             
                                            The packager's only function is to create an AF file. Refer to Packager Command Syntax for more information about invoking the packager. The packager depends on a JSON file to describe AFU metadata. Refer to  Accelerator Description File for more information about the JSON file.
                                             
                                            The packager requires Python 2.7.1 and Python 2.7.3. The tool indicates if it is being called with a compatible  of Python.
                                            "},{"location":"sw/fpga_tools/packager/packager/#packager-command-syntax","title":"Packager Command Syntax","text":"
                                            The packager is a command line tool with the following syntax:
                                             
                                            $ packager <cmd> [arguments]
                                             
                                            The following table describes the <CMD> arguments:
                                             Command Arguments Description create-gbs --rbf=<RBF_PATH> --afu=<AFU_JSON_PATH> --gbs=<GBS_PATH> --set-value=<key>.<value> Creates the AF file. The engineering name for this file is the green bit stream, abbreviated gbs. The --rbf and --afu   arguments are required.  <RBF_PATH> is the path to the RBF file for the AFU. The Quartus\u00ae Prime software generates this RBF by compiling the AFU design. <AFU_JSON_PATH> is the path to the Accelerator Description file. This is a JSON file that describes the metadata that create-gbs appends to the RBF. <GBS_PATH> is the path to the RBF file for the FPGA Interface Manager (FIM) that contains the FPGA interface unit and other interfaces. If you do not specify the --gbs, the command defaults to <rbf_name>.gbs. You can use the optional --set-value=<key>.<value> argument to set values for JSON metadata. To set more than one JSON value, list a series of <key>.<value>  pairs. modify-gbs --gbs=<gbs_PATH> Modifies the AF file. The --input-gbsargument is required. If you do not provide the --output-gbs argument, modify-gbs overwrites the --input-gbs file. Use the --set-value=<key>.<value> argument to set values for JSON metadata. To set more than one JSON value, list a series of <key>.<value>  pairs. gbs-info --input-gbs=<gbs_PATH> Prints information about the AF file. The --input-gbs argument is required. get-rbf --gbs=<GBS_PATH> --rbf=<RBF_PATH> Creates the RBF by extracting it from the AF file. The --gbsargument is required. If you do not specify the --rbf argument, the command defaults to <gbs_name.rbf . None, or any <CMD> --help Summarizes the <CMD> options. Typing packager --help gives a list of <CMD> values. Typing packager <CMD> --help provides detailed help for <CMD>"},{"location":"sw/fpga_tools/packager/packager/#examples","title":"Examples","text":"
                                            To generate an AF file, run:
                                             
                                            $ packager create-gbs --rbf=<RBF_PATH> --afu=<AFU_JSON_PATH> --gbs=<GBS_PATH>
                                             
                                            TIP: JSON files are very particular about syntax such as trailing commas. If you are getting errors, use jsonlint.com to validate that your JSON is formatted correctly. 
                                             
                                            To modify metadata in an existing AF, run the following command:
                                             
                                            $ packager modify-gbs --input-gbs=<PATH_TO_GBS_TO_BE_MODIFIED> --outputgbs=<NAME_FOR_NEW_GBS> --set-value <key>:<value>
                                             
                                            You can pass in a number of : pairs with --set-value to update values in an AF.  
                                            To print the metadata of an existing AF: 
                                             
                                            $ packager get-info --gbs=<GBS_PATH> 
                                             
                                            To extract the RBF from the AF:
                                             
                                            $ packager get-rbf --gbs=<GBS_PATH> --rbf=<NAME_FOR_RBF>
                                            "},{"location":"sw/fpga_tools/packager/packager/#accelerator-description-file","title":"Accelerator Description File","text":"
                                            The Accelerator Description File is a JSON file that describes the metadata associated with an AFU. The Open Progammable Accelerator Engine (OPAE) uses this metadata during reconfiguration. Here is an example file:
                                             
                                            {\n   \"version\": 1,\n   \"platform-name\": \"DCP\",\n   \"afu-image\": {\n      \"magic-no\": 488605312,\n      \"interface-uuid\": \"01234567-89AB-CDEF-0123-456789ABCDEF\",\n      \"power\": 0,\n      \"accelerator-clusters\": [{\n         \"name\": \"dma_test_afu\",\n         \"total-contexts\": 1,   \n         \"accelerator-type-uuid\": \"331DB30C-9885-41EA-9081-F88B8F655CAA\"\n      }\n      ]  \n   }\n}\n
                                             The packager stores these parameter values in the resultant AF. After reprogramming the AFU using partial reconfiguration (PR), the  software driver reconfigures the PLLs by writing the clock-frequency-high and clock-frequency-low values (if present) over the  PCIe\u00ae and CCI interfaces. 
                                             
                                            .. note:: 
                                            The JSON file format may change as the architecture evolves. Any changes to the current format trigger an update\nto the version number.  \n
                                             
                                            CATEGORY | NAME | TYPE | DESCRIPTION | MANDATORY ---------|------|------|-------------|:----------:| Per-AFU  | version | Integer | Version of the metadata format. | Yes Per-AFU  | magic-no (to be deprecated)| Integer | Magic no. Associated with the FPGA Interface Manager. | No Per-AFU  | platform-name | String | Name of the platform for which the metadata is intended. The field value is \u201cDCP\u201d for Intel  Acceleration Stack for FPGAs. | No Per-AFU  | interface-uuid | UUID | Interface id associated with the FPGA Interface Manager. | Yes Per-AFU  | power | Integer | Accelerator Function power consumption, in watts. Set to 0 for Intel Acceleration Stack for FPGAs platforms. | Yes Per-AFU  | clock-frequency-low | Float | Clock frequency for 1st PLL (Clock network)1 in MHz. | No Per-AFU  | clock-frequency-high | Float | Clock frequency for 2nd PLL (0 if absent) in MHz. | No Per-AFC Cluster | total-contexts | Integer | Number of AFCs in this cluster. Always be 1 in current architectures. | Yes Per-AFC Cluster | afc-type-uuid |  UUID | AFC type = AFU ID in current architectures. | Yes Per-AFC Cluster | name | string | AFC name = AFU name in current architectures. | Yes
                                             Date Intel Acceleration Stack Version Changes Made 2018.05.21 DCP 1.1 Beta (works with Quartus Prime Pro 17.1.1) Fixed typos."},{"location":"sw/fpga_tools/pci_device/pci_device/","title":"pci_device","text":""},{"location":"sw/fpga_tools/pci_device/pci_device/#synopsis","title":"SYNOPSIS","text":"
                                            pci_device [-h] [-E] device-filter [{aer,bind,plug,remove,rescan,topology,unbind,unplug,vf}]
                                            "},{"location":"sw/fpga_tools/pci_device/pci_device/#description","title":"DESCRIPTION","text":"
                                            pci_device is a tool to aid in common operations for managing PCIe devices and drivers.
                                            "},{"location":"sw/fpga_tools/pci_device/pci_device/#options","title":"OPTIONS","text":""},{"location":"sw/fpga_tools/pci_device/pci_device/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                            `device filter`\n\nPCIe address of a device or vendor/device ID pair.\nThe PCIe address follows the format of [segment:]bus:device.function\nwhile the vendor/device ID pair follows the format [vendor ID]:[device ID]\nwhere at least one of these must be present.\n\n`{aer,bind,plug,remove,rescan,topology,unbind,unplug,vf}`\n\naction to perform on device\n\n`aer`\nPerform AER (Advanced Error Reporting) operations.\nThe aer action has its own sub-commands which are listed below:\n\n* `dump` sub-command will print out the AER error counters as reported\n   by the sysfs files for the device.\n* `mask` can either print out the current AER mask bits or set them\n  * If `show` or `print` (or nothing) is given after the `mask`\n    command, it will show the current mask bits for AER.\nBy default output will be written in stdout but can be written to an\noutput file if `-o|--output FILENAME` argument is given.\n  * If `all` is given after the `mask` command, it will mask all bits\n    (by setting the values to 0xffffffff and 0xffffffff).\n  * If `off` is given after the `mask` command, it will unmask all\n    bits (by setting the values to 0x0 and 0x0).\n  * If two numbers are present after the `mask` command, those two\n    numbers will be used to set the mask bits.\nValues for setting the mask can also be read in from an input file if\n`-i|--input FILENAME` argument is given.\n\n_NOTE_: mask related operations require root privileges\n\n`bind`\n\nAssociate a device with its driver.\n\n`plug`\n\nRestore a device that was previously given to `pci_device <device> unplug`\n\n`remove`\n\nRemove the pci device from the pci bus\n\n`rescan`\n\nRescan the bus as identified by the bus component of the PCIe device address\n\n'topology`\n\nPrint the PCIe topology from the root port to the PCIe device.\nThis shows the PCIe tree rooted at the PCIe root port.\nEach line shows the the PCIe address, vendor ID, and device ID along with\nthe driver bound to the device. The indentation is used to show\nparent/child relationship of devices.\n\nThe line listing the target PCIe device as identified by the given PCIe\naddress will be highlighted in green while the endpoints will be\nhighlighted in cyan.\n\nThe example below shows the topology of an N3000 device with eight virtual\nfunctions created from one of the Ethernet controllers:\n\n```console\n[pci_address(0000:3a:00.0), pci_id(0x8086, 0x2030)] (pcieport)\n    [pci_address(0000:3b:00.0), pci_id(0x10b5, 0x8747)] (pcieport)\n        [pci_address(0000:3c:09.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:3f:00.0), pci_id(0x8086, 0x0b30)] (dfl-pci)\n        [pci_address(0000:3c:11.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:43:00.0), pci_id(0x8086, 0x0b32)] (no driver)\n    [pci_address(0000:3c:08.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:3d:02.0), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:00.1), pci_id(0x8086, 0x0d58)] (i40e)\n            [pci_address(0000:3d:02.7), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:02.5), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:02.3), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:02.1), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:00.0), pci_id(0x8086, 0x0d58)] (i40e)\n            [pci_address(0000:3d:02.6), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:02.4), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:02.2), pci_id(0x8086, 0x154c)] (iavf)\n        [pci_address(0000:3c:10.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:41:00.0), pci_id(0x8086, 0x0d58)] (i40e)\n            [pci_address(0000:41:00.1), pci_id(0x8086, 0x0d58)] (i40e)\n\n```\n\n`unbind`\n\nUnbind the driver bound to the device.\n\n`unplug`\n\nRemove device from PCI bus in anticipation of a RSU event by configuring its root port and associated endpoints.\n\n`vf`\n\nCreate/destroy VFs (virtual functions) by setting the number here.\nThe number given here will be written to sriov_numvfs sysfs file triggering\nthe PCIe subsystem to create/destroy VFs so that the current number of VFs\nwill be equal to the given number. If the number given is outside of the total VFs supported, an error message will be displayed to indicate this.\n
                                            "},{"location":"sw/fpga_tools/pci_device/pci_device/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                            `-h, --help`\n\nshow this help message and exit\n\n`-E, --other-endpoints`\n\nperform action on peer PCIe devices\n
                                            "},{"location":"sw/fpga_tools/pci_device/pci_device/#examples","title":"EXAMPLES","text":"
                                            pci_device 0000:3d:00.0 remove\npci_device 0000:3d:00.0 rescan\npci_device 3d:00.0 topology\npci_device :0b30 topology\npci_device :0b30 aer\npci_device :0b30 aer mask\npci_device :0b30 aer mask all\npci_device :0b30 aer mask -o mask.dat\npci_device :0b30 aer mask -i mask.dat\n
                                            "},{"location":"sw/fpga_tools/rsu/rsu/","title":"rsu","text":""},{"location":"sw/fpga_tools/rsu/rsu/#synopsis","title":"SYNOPSIS","text":"
                                            rsu [-h] [-d] {bmc,bmcimg,retimer,fpga,sdm,fpgadefault} [PCIE_ADDR]\n
                                            "},{"location":"sw/fpga_tools/rsu/rsu/#description","title":"DESCRIPTION","text":""},{"location":"sw/fpga_tools/rsu/rsu/#mode-1-rsu","title":"Mode 1: RSU","text":"
                                            rsu bmc --page=(user|factory) [PCIE_ADDR]\nrsu retimer [PCIE_ADDR]\nrsu fpga --page=(user1|user2|factory) [PCIE_ADDR]\nrsu sdm --type=(sr|pr|sr_cancel|pr_cancel) [PCIE_ADDR]\n
                                             
                                            Perform RSU (remote system update) operation on PAC device given its PCIe address. An RSU operation sends an instruction to the device to trigger a power cycle of the card only. This will force reconfiguration from flash for either BMC, Retimer, SDM, (on devices that support these) or the FPGA.
                                            "},{"location":"sw/fpga_tools/rsu/rsu/#mode-2-default-fpga-image","title":"Mode 2: Default FPGA Image","text":"
                                            rsu fpgadefault --page=(user1|user2|factory) --fallback=<csv> [PCIE_ADDR]\n
                                             
                                            Set the default FPGA boot sequence. The --page option determines the primary FPGA boot image. The --fallback option allows a comma-separated list of values to specify fallback images.
                                            "},{"location":"sw/fpga_tools/rsu/rsu/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                            {bmc,bmcimg,retimer,fpga,sdm,fpgadefault}
                                             
                                            type of RSU operation or set Default FPGA Image operation.
                                             
                                            PCIE_ADDR  PCIe address of device to do rsu (e.g. 04:00.0 or 0000:04:00.0) 
                                            "},{"location":"sw/fpga_tools/rsu/rsu/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                            -h, --help show this help message and exit
                                             
                                            -d, --debug log debug statements
                                             
                                            --force force rsu operation
                                            "},{"location":"sw/fpga_tools/rsu/rsu/#example","title":"EXAMPLE","text":"
                                            # rsu bmc --page=user 25:00.0\n
                                             
                                            Triggers a boot of the BMC image (user page) for the device with PCIe  address 25:00.0.
                                             
                                            # rsu bmc --page=factory 25:00.0\n
                                             
                                            Triggers a factory boot of the BMC image for the device with  PCIe address 25:00.0.
                                             
                                            # rsu fpga --page=user2 25:00.0\n
                                             
                                            Triggers a reconfiguration of the FPGA (user2 page) for the  device with PCIe address 25:00.0.
                                             
                                            # rsu --force fpga --page=user2 25:00.0\n
                                             
                                            Forces a reconfiguration of the FPGA (user2 page) for the  device with PCIe address 25:00.0. Default behavior is to not perform  the rsu operation if DPC (downstream port containment) is not supported  and AER (advanced error reporting) is also not supported. Using --force  changes this behavior to perform rsu operation regardless but may result  in a surprise removal of pci devices which may cause the Linux kernel  to panic.
                                             
                                            # rsu fpga --page=factory 25:00.0\n
                                             
                                            Triggers a factory reconfiguration of the FPGA for the device  with PCIe address 25:00.0.
                                             
                                            # rsu sdm --type=sr 25:00.0\n
                                             
                                            Triggers Static Region key programming for the device with  PCIE address 25:00.0.
                                             
                                            # rsu fpgadefault --page=factory --fallback=user1,user2 25:00.0\n
                                             
                                            Sets the FPGA boot sequence to factory with fallbacks user1, user2.
                                            "},{"location":"sw/fpga_tools/super-rsu/super-rsu/","title":"Super Remote System Update User Guide","text":"
                                            .. toctree::\n.. highlight:: c\n.. highlight:: console\n
                                            "},{"location":"sw/fpga_tools/super-rsu/super-rsu/#overview","title":"Overview","text":"
                                            Intel Programmable Acceleration Card (PAC) devices are comprised of multiple processors and controllers that execute firmware. Maintaining and updating these firmware images manually is error-prone and does not scale well within the Data Center. The solution described here is derived with the following goals in mind:
                                             
                                               
                                            • The ability to update one or more (possibly all) firwmare images with a single package.
                                              •  
                                            • The ability to complete all firmware updates within a stipulated time window.
                                              •  
                                            • The ability to update each PAC in the server, all servers in a Data Center, and multiple Data Centers remotely.
                                              •  
                                            • The ability to remotely initiate download of the package and its installation with a single command per server instance.
                                              •  
                                            • The ability to roll back firmware to a previous revision.
                                              •  
                                            "},{"location":"sw/fpga_tools/super-rsu/super-rsu/#implementation","title":"Implementation","text":"
                                            A single package containing firmware images for all programmable parts on a PAC is delivered as an RPM, eg opae-super-rsu-n3000.M.m.p-r.noarch.rpm. The RPM revision will sequentially increase with every update.
                                             
                                            Installing or upgrading the RPM invokes the complete update of all programmable parts on all PAC boards in the system.
                                             
                                            The standard RPM dependency framework ensures that correct versions of dependecy packages opae-intel-fpga-driver and fpga-tools-extra are installed on the system.
                                             
                                            Rolling back is achieved by uninstalling the current version and re-installing a previous version of the RPM.
                                             
                                            .. note:: 
                                            Note: once Secure Update is deployed, roll back restrictions shall be implemented to prevent\nrollback attacks.\n
                                             
                                            RPM management on remote systems is standard practice, requiring no new infrastructure/training.
                                            "},{"location":"sw/fpga_tools/super-rsu/super-rsu/#details","title":"Details","text":"
                                            The post-install hook of the opae-super-rsu-n3000 RPM is leveraged to call out to the super-rsu Python script to update all PAC boards. super-rsu uses the manifest file packaged within opae-super-rsu-n3000 to associate a firmware image with its version. Each of the firmware images contained in opae-super-rsu-n3000 is placed on the target system in /usr/share/opae/n3000.
                                            "},{"location":"sw/fpga_tools/super-rsu/super-rsu/#algorithm","title":"Algorithm","text":"
                                               
                                            • Acquire the current firmware versions of all programmable parts.
                                              •  
                                            • For each programmable image, if the installed version of firmware does not equal the version provided in the RPM manifest file, then update the firmware image, and set image_updated to True.
                                              •  
                                            • After all updates, if image_updated, then initiate a safe reboot of all boards in the system.
                                              •  
                                            • After safe reboot, verify that the reported firmware versions match those of the RPM manifest. If they do not match, then RPM installation exits with a failing status.
                                              •  
                                            • Run board self test. If the self test fails, then the RPM installation exits with a failing status.
                                              •  
                                            • If all of the above checks is successful, then RPM installation exits with a success status.
                                              •  
                                            "},{"location":"sw/fpga_tools/super-rsu/super-rsu/#dependencies","title":"Dependencies","text":"
                                               
                                            • The standard Python package for the distro (version 2.7).
                                              •  
                                            • The opae-intel-fpga-driver RPM. (version determined by opae-super-rsu-n3000)
                                              •  
                                            • The opae-tools-extra RPM. (version determined by opae-super-rsu-n3000)
                                              •  
                                            "},{"location":"sw/fpga_tools/userclk/userclk/","title":"userclk","text":""},{"location":"sw/fpga_tools/userclk/userclk/#synopsis","title":"SYNOPSIS","text":"
                                            userclk [-hv] [-S <segment>] [-B <bus>] [-D <device>] [-F <function>] [PCI_ADDR] [-H <User clock high frequency>] -L <User clock low frequency>]
                                            "},{"location":"sw/fpga_tools/userclk/userclk/#description","title":"DESCRIPTION","text":"
                                            userclk sets the frequency range for an AFU. 
                                            "},{"location":"sw/fpga_tools/userclk/userclk/#examples","title":"EXAMPLES","text":"
                                            ./userclk -B 0x5e -H 400 -L 200
                                             
                                            Sets AFU frequency.
                                            "},{"location":"sw/fpga_tools/userclk/userclk/#options","title":"OPTIONS","text":"
                                            -v,--version
                                             
                                            Prints version information and exits.
                                             
                                            -S,--segment 
                                             
                                            FPGA segment number.
                                             
                                            -B,--bus 
                                             
                                            FPGA Bus number.
                                             
                                            -D,--device 
                                             
                                            FPGA Device number.
                                             
                                            -F,--function 
                                             
                                            FPGA function number.
                                             
                                            -H,--freq-high 
                                             
                                            User clock high frequency. 
                                             
                                            -L,--freq-low 
                                             
                                            User clock low frequency. 
                                             Date Intel Acceleration Stack Version Changes Made 2018.05.21 DCP 1.1 Beta (works with Quartus Prime Pro 17.1.1) Fixed typos."},{"location":"sw/fpga_tools/vabtool/vabtool/","title":"vabtool","text":""},{"location":"sw/fpga_tools/vabtool/vabtool/#synopsis","title":"SYNOPSIS","text":"
                                            vabtool [-r RETRIES] [-d] [-y] [-v] <ACTION>
                                             
                                            Where ACTION is defined as one of the following:
                                             
                                            vabtool sr_key_provision PCIE_ADDRESS SR_RKH_FILE FPGA_IMG_FILE vabtool sr_status PCIE_ADDRESS vabtool pr_key_provision PCIE_ADDRESS PR_AUTH_CERT_FILE PR_RKH_FILE vabtool pr_status PCIE_ADDRESS vabtool sr_key_cancel PCIE_ADDRESS SR_RKH_CANCEL_FILE vabtool sr_cancel_status PCIE_ADDRESS vabtool pr_key_cancel PCIE_ADDRESS PR_RKH_CANCEL_FILE vabtool pr_cancel_status PCIE_ADDRESS
                                            "},{"location":"sw/fpga_tools/vabtool/vabtool/#description","title":"DESCRIPTION","text":"
                                            The vabtool command helps perform Vendor Authenticated Boot provisioning of Static Region and Partial Reconfiguration Region key hashes and helps perform SR and PR hash cancellation and status reporting.
                                            "},{"location":"sw/fpga_tools/vabtool/vabtool/#options","title":"OPTIONS","text":"
                                            -r RETRIES, --retries RETRIES
                                             
                                            Specifies the number of times a failed SR or PR key provision is to be\nretried. The default value for RETRIES is 3.\n
                                             
                                            -d, --dry-run
                                             
                                            Don't execute the actual fpgasupdate and rsu commands, but only print\nthe commands that would be executed during a normal run of the script.\n
                                             
                                            -y, --yes
                                             
                                            The tool will respond with an automatic Yes answer to all confirmation\nprompts posed by the sub-tools.\n
                                             
                                            -v, --version
                                             
                                            Display script version information and exit.\n
                                            "},{"location":"sw/fpga_tools/vabtool/vabtool/#examples","title":"EXAMPLES","text":"
                                            sudo vabtool -y sr_key_provision 0000:bc:00.0 my_sr_rkh.bin my_fpga.bin sudo vabtool sr_status 0000:bc:00.0 sudo vabtool -y pr_key_provision 0000:bc:00.0 pr_auth_cert.bin my_pr_rkh.bin sudo vabtool pr_status 0000:bc:00.0 sudo vabtool sr_key_cancel 0000:bc:00.0 my_sr_rhk_cancel.bin sudo vabtool sr_cancel_status 0000:bc:00.0 sudo vabtool pr_key_cancel 0000:bc:00.0 my_pr_rhk_cancel.bin sudo vabtool pr_cancel_status 0000:bc:00.0
                                            "},{"location":"sw/install_guide/installation_guide/","title":"OPAE Installation Guide","text":""},{"location":"sw/install_guide/installation_guide/#how-to-download-the-opae-sdk","title":"How to download the OPAE SDK","text":"
                                            OPAE SDK releases are available on GitHub. Source code for the OPAE DFL device driver for Linux is also available on GitHub.
                                            "},{"location":"sw/install_guide/installation_guide/#install-the-fedora","title":"Install the Fedora","text":"
                                            Download the Fedora  (x86_64 version) installation file in fedora, and install the Fedora  in yourserver. You can choose Fedora Workstation or Fedora server.
                                            "},{"location":"sw/install_guide/installation_guide/#build-the-kernel-and-dfl-drivers","title":"Build the kernel and DFL drivers","text":"
                                            For building the OPAE kernel and kernel driver, the kernel development environment is required. So before you build the kernel, you must install the required packages. Run the following commands:
                                             
                                            $ sudo dnf install gcc gcc-c++ make kernel-headers kernel-devel elfutils-libelf-devel ncurses-devel openssl-devel bison flex\n
                                             
                                            Download the OPAE upstream kernel tree from github, for example download from fpga-ofs-dev-5.15-lts branch. 
                                            $ git clone https://github.com/OPAE/linux-dfl.git -b fpga-ofs-dev-5.15-lts\n
                                             
                                            Configure the kernel. 
                                            $ cd linux-dfl\n$ cp /boot/config-`uname -r` .config\n$ cat configs/dfl-config >> .config\n$ echo 'CONFIG_LOCALVERSION=\"-dfl\"' >> .config\n$ echo 'CONFIG_LOCALVERSION_AUTO=y' >> .config\n$ sed -i -r 's/CONFIG_SYSTEM_TRUSTED_KEYS=.*/CONFIG_SYSTEM_TRUSTED_KEYS=\"\"/' .config\n$ sed -i '/^CONFIG_DEBUG_INFO_BTF/ s/./#&/' .config\n$ echo 'CONFIG_DEBUG_ATOMIC_SLEEP=y' >> .config\n$ make olddefconfig\n
                                             
                                            Compile and install the new kernel. 
                                            $ make -j $(nproc)\n$ sudo make modules_install -j $(nproc)\n$ sudo make install\n
                                             
                                            Build linux DFL Kernel instructions please also refer to: https://github.com/OPAE/linux-dfl/wiki/Build-the-linux-dfl-kernel
                                             
                                            When install finished, reboot your system. When the system login again, verify the kernel version is correct. For example: 
                                            [figo@localhost linux-dfl]$ uname -a\nLinux localhost.localdomain 5.15.lts-dfl-g73e16386cda0 #6 SMP Mon Jun 13 21:21:31 -04 2022 x86_64 x86_64 x86_64\n
                                             
                                            And also you can check the OPAE dfl drivers have auto-loaded. 
                                            [figo@localhost linux-dfl]$ lsmod | grep fpga\nifpga_sec_mgr          20480  1 intel_m10_bmc_secure\nfpga_region            20480  3 dfl_fme_region,dfl_fme,dfl\nfpga_bridge            24576  4 dfl_fme_region,fpga_region,dfl_fme,dfl_fme_br\nfpga_mgr               16384  4 dfl_fme_region,fpga_region,dfl_fme_mgr,dfl_fme\n[figo@localhost linux-dfl]$ lsmod | grep dfl\ndfl_eth_group          36864  0\ndfl_fme_region         20480  0\ndfl_emif               16384  0\ndfl_n3000_nios         20480  0\ndfl_fme_br             16384  0\ndfl_fme_mgr            20480  1\ndfl_fme                49152  0\ndfl_afu                36864  0\ndfl_pci                20480  0\ndfl                    40960  7 dfl_pci,dfl_fme,dfl_fme_br,dfl_eth_group,dfl_n3000_nios,dfl_afu,dfl_emif\nfpga_region            20480  3 dfl_fme_region,dfl_fme,dfl\nfpga_bridge            24576  4 dfl_fme_region,fpga_region,dfl_fme,dfl_fme_br\nfpga_mgr               16384  4 dfl_fme_region,fpga_region,dfl_fme_mgr,dfl_fme\n
                                            "},{"location":"sw/install_guide/installation_guide/#build-the-opae-sdk","title":"Build the OPAE-SDK","text":"
                                            Before you build the OPAE SDK, you must install the required packages. Run the following commands:
                                            "},{"location":"sw/install_guide/installation_guide/#rocky-linux-85","title":"Rocky Linux 8.5","text":"
                                            # dnf install -y 'dnf-command(config-manager)'\n# dnf config-manager --set-enabled powertools\n# dnf install -y epel-release\n# dnf check-update\n# dnf upgrade -y\n# dnf install -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml python3-pybind11 git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel rpm-build rpmdevtools pybind11-devel yaml-cpp-devel libudev-devel linuxptp numactl-devel\n# python3 -m pip install jsonschema virtualenv pyyaml\n
                                            "},{"location":"sw/install_guide/installation_guide/#fedora","title":"Fedora","text":"
                                            # dnf check-update\n# dnf upgrade -y\n# dnf install -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml python3-pybind11 git gcc g++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel libedit-devel rpm-build rpmdevtools pybind11-devel yaml-cpp-devel libudev-devel cli11-devel spdlog-devel linuxptp numactl-devel\n# pip3 install jsonschema virtualenv pyyaml\n
                                            "},{"location":"sw/install_guide/installation_guide/#ubuntu-2204","title":"Ubuntu 22.04","text":"
                                            # apt-get update\n# apt-get upgrade -y\n# apt-get install -y python3 python3-pip python3-dev git gcc g++ make cmake uuid-dev libjson-c-dev libhwloc-dev libtbb-dev libedit-dev libudev-dev linuxptp pandoc devscripts debhelper doxygen libnuma-dev\n# pip3 install jsonschema virtualenv pyyaml pybind11\n
                                            "},{"location":"sw/install_guide/installation_guide/#rhel-82","title":"RHEL 8.2","text":"
                                            Register and enable Red Hat subscription to install any packages on the system.
                                             
                                            # subscription-manager register --proxy=PROXY --username=USER --password=PASSWORD --auto-attach\n
                                             
                                            Set the RHEL version and install packages. Set proxy name and port number.
                                             
                                            # subscription-manager release --set=8.2 --proxy proxy-name.com:port number\n# subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms\n# dnf upgrade -y\n# dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n# dnf install -y python3 python3-pip python3-devel gdb vim git gcc gcc-c++ make cmake libuuid-devel rpm-build systemd-devel  nmap\n# dnf install -y python3-jsonschema json-c-devel tbb-devel rpmdevtools libcap-devel numactl-devel\n# dnf check-update || true\n# dnf install -y spdlog-devel cli11-devel python3-pyyaml python3-pybind11 hwloc-devel libedit-devel\n# python3 -m pip install --user jsonschema virtualenv pudb pyyaml\n
                                             
                                            Install the latest version of cmake on top of the outdated cmake package from the package manager.
                                             
                                            # cd cmake-3.25.1/\n# ./bootstrap --prefix=/usr\n# make\n# make install\n# which cmake\n/usr/bin/cmake\n
                                            "},{"location":"sw/install_guide/installation_guide/#create-opae-sdk-packages","title":"Create opae-sdk packages","text":"
                                            Download the OPAE-SDK source code from github. For example, download from Master branch.
                                             
                                            $ git clone https://github.com/OPAE/opae-sdk.git\n
                                             
                                            Compile and build the OPAE-SDK RPMs (Fedora, Rocky, RHEL 8.2). 
                                            $ cd opae-sdk/packaging/opae/rpm\n$ ./create fedora\n
                                             
                                            Note that if you find that your distribution has changed package names such that there is a conflict when building RPMs, you can install all of the build dependencies so that the SDK compiles and then build the RPMs in unrestricted mode: 
                                            $ cd opae-sdk/packaging/opae/rpm\n$ ./create unrestricted\n
                                             
                                            After a successful compile, there are 3 rpm packages generated (Fedora, Rocky, RHEL8.2). For example: 
                                            opae-2.1.0-1.fc34.x86_64.rpm\nopae-devel-2.1.0-1.fc34.x86_64.rpm\nopae-extra-tools-2.1.0-1.fc34.x86_64.rpm\n
                                             
                                            Compile and build the OPAE-SDK deb packages (Ubuntu 22.04). 
                                            $ cd opae-sdk/packaging/opae/deb\n$ ./create\n
                                             
                                            After a successful compile, there are 3 deb packages generated (Ubuntu 22.04). For example: 
                                            opae_2.1.1-1_amd64.deb  \nopae-devel_2.1.1-1_amd64.deb  \nopae-extra-tools_2.1.1-1_amd64.deb\n
                                            "},{"location":"sw/install_guide/installation_guide/#opae-sdk-installation-with-rpmdeb-packages","title":"OPAE SDK installation with rpm/deb packages","text":"
                                            The rpm packages generated in the previous step can be installed using these commands:
                                             
                                            $ sudo dnf install ./*.rpm\n
                                             
                                            The deb packages generated in the previous step can be installed using these commands:
                                             
                                            $ sudo dpkg -i  ./*.deb\n
                                             
                                            When you installed the rpms, you can run fpgainfo command to check the FPGA FME infomation. For example: 
                                            [figo@localhost install_guide]$ fpgainfo fme\nBoard Management Controller, MAX10 NIOS FW version: D.2.1.24\nBoard Management Controller, MAX10 Build version: D.2.0.7\n//****** FME ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:08:00.0\nDevice Id                        : 0x0B30\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x2300011001030F\nBitstream Version                : 0.2.3\nPr Interface Id                  : f3c99413-5081-4aad-bced-07eb84a6d0bb\nBoot Page                        : user\n
                                             
                                            To uninstall the OPAE rpms, you can use this commands 
                                            $ dnf list installed | grep opae\n$ sudo dnf remove opae*.x86_64\n
                                             
                                            To uninstall the OPAE deb, you can use this commands 
                                            $ dpkg -l  | grep opae\n$ dpkg -r opae-extra-tools:amd64\n$ dpkg -r opae-devel:amd64\n$ dpkg -r opae\n
                                            "},{"location":"sw/install_guide/installation_guide/#fpga-device-access-permissions","title":"FPGA Device Access Permissions","text":"
                                            Access to FPGA accelerators and devices is controlled using file access permissions on the Intel\u00ae FPGA device files, /dev/dfl-fme.* and /dev/dfl-port.*, as well as to the files reachable through /sys/class/fpga_region/.
                                             
                                            In order to allow regular (non-root) users to access accelerators, you need to grant them read and write permissions on /dev/dfl-port.* (with * denoting the respective socket, i.e. 0 or 1). E.g.:
                                             
                                            $ sudo chmod a+rw /dev/dfl-port.0\n
                                            "},{"location":"sw/install_guide/installation_guide/#memlock-limit","title":"Memlock limit","text":"
                                            Depending on the requirements of your application, you may also want to increase the maximum amount of memory a user process is allowed to lock. The exact way to do this depends on your Linux distribution.
                                             
                                            You can check the current memlock limit using
                                             
                                            $ ulimit -l\n
                                             
                                            A way to permanently remove the limit for locked memory for a regular user is to add the following lines to your /etc/security/limits.conf:
                                             
                                            user1    hard   memlock           unlimited\nuser1    soft   memlock           unlimited\n
                                             
                                            This removes the limit on locked memory for user user1. To remove it for all users, you can replace user1 with *:
                                             
                                            *    hard   memlock           unlimited\n*    soft   memlock           unlimited\n
                                             
                                            Note that settings in the /etc/security/limits.conf file don't apply to services.  To increase the locked memory limit for a service you need to modify the application's systemd service file and add the line:
                                             
                                            [Service]\nLimitMEMLOCK=infinity\n
                                            "},{"location":"sw/install_guide/installation_guide/#hugepage-settings","title":"Hugepage Settings","text":"
                                            Users need to configure system hugepages to reserve 2MB-hugepages or 1GB-hugepages. For example, the 'hello_fpga' sample requires several 2MB-hugepages. And the fpgadiag tool requires several 1GB-hugepages.
                                             
                                            The command below reserves 20 2M-hugepages:
                                             
                                            $ sudo sh -c 'echo 20 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages'\n
                                             
                                            The command below reserves 4 1GB-hugepages:
                                             
                                            $ sudo sh -c 'echo 4 > /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages'\n
                                             
                                            For x86_64 architecture processors, user can use following command to find out avaiable hugepage sizes:
                                             
                                            $ grep pse /proc/cpuinfo | uniq\nflags : ... pse ...\n
                                             
                                            If this commands returns a non-empty string, 2MB pages are supported.
                                             
                                            $ grep pse /proc/cpuinfo | uniq\nflags : ... pdpe1gb ...\n
                                             
                                            If this commands returns a non-empty string, 1GB pages are supported.
                                            "},{"location":"sw/pyopae/python_bindings/","title":"OPAE Python Bindings","text":"
                                            OPAE (Open Programmable Acceleration Engine) now includes Python bindings for interacting with FPGA resources. The OPAE Python API is built on top of the OPAE C++ Core API and its object model. Because of this, developing OPAE applications in Python is very similar to developing OPAE applications in C++ which significantly reduces the learning curve required to adapt to the Python API. While the object model remains the same, some static factory functions in the OPAE C++ Core API have been moved to module level methods in the OPAE Python API with the exception of the properties class. The goal of the OPAE Python API is to enable fast prototyping, test automation, infrastructure managment, and an easy to use framework for FPGA resource interactions that don\u2019t rely on software algorithms with a high runtime complexity.
                                             
                                            Currently, the only Python package that is part of OPAE is opae.fpga
                                            "},{"location":"sw/pyopae/python_bindings/#implementation","title":"Implementation","text":"
                                            The OPAE Python API is implemented by creating a Python extension using pybind11 <http://pybind11.readthedocs.io/en/stable>. This extension is created by using the pybind11 API which relies mostly on macros and compile time introspection to define the module initialization point as well as type converters between OPAE C++ Core types and OPAE Python types.
                                            "},{"location":"sw/pyopae/python_bindings/#benefits","title":"Benefits","text":"
                                            The major benefits of using pybind11 for developing the OPAE Python API include, but are not limited to, the following features of pybind11:
                                             
                                               
                                            • Uses C++ 11 standard library although it can use C++ 14 or C++17.
                                              •  
                                            • Automatic conversions of shared_ptr types
                                              •  
                                            • Built-in support for numpy and Eigen numerical libraries
                                              •  
                                            • Interoperable with the Python C API
                                              •  
                                            "},{"location":"sw/pyopae/python_bindings/#runtime-requirements","title":"Runtime Requirements","text":"
                                            Because opae.fpga is built on top of the opae-cxx-core API, it does require that the runtime libraries for both opae-cxx-core and opae-c be installed on the system (as well as any other libraries they depend on). Those libraries can be installed using the opae-libs package (from either RPM or DEB format - depending on your Linux distribution).
                                            "},{"location":"sw/pyopae/python_bindings/#installation","title":"Installation","text":""},{"location":"sw/pyopae/python_bindings/#python-wheels","title":"Python Wheels","text":"
                                            The preferred method of installation is to use a binary wheel package for your version of Python.
                                             
                                            The following table lists example names for different Python versions and platforms.
                                             
                                            | Python Version | Python ABI | Linux Platform | Package Name | |\u2014\u2014\u2014\u2014\u2014-|\u2014\u2014\u2014\u2014\u2014\u2013|\u2014\u2014\u2014\u2014\u2014-|\u2014\u2014\u2014\u2014\u2013| | 2.7 | CPython w/ UCS4 | x86_64 | opae.fpga.-cp27-cp27mu-linux_x86_64.whl | | 3.4 | CPython w/ UCS4 | x86_64 | opae.fpga.-cp34-cp34mu-linux_x86_64.whl | | 3.6 | CPython w/ UCS4 | x86_64 | opae.fpga.-cp36-cp36mu-linux_x86_64.whl |
                                             
                                            opae.fpga is currently not available in the Python Package Index but once it does become available, one should be able to install using pip by simply typing the following:
                                             
                                            > pip install --user opae.fpga\n
                                            "},{"location":"sw/pyopae/python_bindings/#installing-from-source","title":"Installing From Source","text":"
                                            In addition to the runtime libraries mentioned above, installing from source does require that the OPAE header files be installed as well as those header files for pybind11. The former can be installed with the opae-devel package and the latter can be installed by installing pybind11 Python module.
                                            "},{"location":"sw/pyopae/python_bindings/#example-installation","title":"Example Installation","text":"
                                            The following example shows how to build from source by installing the prerequisites before running the setup.py file.
                                             
                                            >sudo yum install opae-libs-<release>.x86_64.rpm\n>sudo yum install opae-devel-<release>.x86_64.rpm\n>pip install --user pybind11\n>pip install --user opae.fpga-<release>.tar.gz\n
                                             
                                            NOTE: The pip examples above use the --user flag to avoid requiring root permissions. Those packages will be installed in the user\u2019s site-packages directory found in the user\u2019s .local directory.
                                            "},{"location":"sw/pyopae/python_bindings/#example-scripts","title":"Example Scripts","text":"
                                            The following example is an implementation of the sample, hello_fpga.c, which is designed to configure the NLB0 diagnostic accelerator for a simple loopback.
                                             
                                            import time\nfrom opae import fpga\n\nNLB0 = \"d8424dc4-a4a3-c413-f89e-433683f9040b\"\nCTL = 0x138\nCFG = 0x140\nNUM_LINES = 0x130\nSRC_ADDR = 0x0120\nDST_ADDR = 0x0128\nDSM_ADDR = 0x0110\nDSM_STATUS = 0x40\n\ndef cl_align(addr):\n    return addr >> 6\n\ntokens = fpga.enumerate(type=fpga.ACCELERATOR, guid=NLB0)\nassert tokens, \"Could not enumerate accelerator: {}\".format(NlB0)\n\nwith fpga.open(tokens[0], fpga.OPEN_SHARED) as handle:\n    src = fpga.allocate_shared_buffer(handle, 4096)\n    dst = fpga.allocate_shared_buffer(handle, 4096)\n    dsm = fpga.allocate_shared_buffer(handle, 4096)\n    handle.write_csr32(CTL, 0)\n    handle.write_csr32(CTL, 1)\n    handle.write_csr64(DSM_ADDR, dsm.io_address())\n    handle.write_csr64(SRC_ADDR, cl_align(src.io_address())) # cacheline-aligned\n    handle.write_csr64(DST_ADDR, cl_align(dst.io_address())) # cacheline-aligned\n    handle.write_csr32(CFG, 0x42000)\n    handle.write_csr32(NUM_LINES, 4096/64)\n    handle.write_csr32(CTL, 3)\n    while dsm[DSM_STATUS] & 0x1 == 0:\n        time.sleep(0.001)\n    handle.write_csr32(CTL, 7)\n
                                             
                                            This example shows how one might reprogram (Partial Reconfiguration) an accelerator on a given bus, 0x5e, using a bitstream file, m0.gbs.
                                             
                                            from opae import fpga\n\nBUS = 0x5e\nGBS = 'm0.gbs'\ntokens = fpga.enumerate(type=fpga.DEVICE, bus=BUS)\nassert tokens, \"Could not enumerate device on bus: {}\".format(BUS)\nwith open(GBS, 'rb') as fd, fpga.open(tokens[0]) as device:\n    device.reconfigure(0, fd)\n
                                            "},{"location":"sw/tod/tod/","title":"Time of Day (ToD)","text":""},{"location":"sw/tod/tod/#synopsis","title":"SYNOPSIS","text":"
                                            The Intel FPGA ToD driver in the kernel space exposes ToD IP as PHC (PTP Hardware Clock) device to the Linux PTP (Precision Time Protocol) stack to synchronize the system clock to its ToD information. The phc2sys utility of Linux PTP stack is used to access ToD information and synchronize the system clock.
                                             
                                            Install the Linux PTP utilities: 
                                            # sudo yum install linuxptp\n
                                             
                                            phc_ctl and phc2sys utilities (linuxptp package) are used to control the PHC device and synchronize the system clock to its ToD information.
                                             
                                            phc_ctl: directly controls PHC device clock. 
                                            Usage: phc_ctl [options] <device> -- [command]\n\n    device         ethernet or ptp clock device\n\nOptions:\n    -l [num]       set the logging level to 'num'\n    -q             do not print messages to the syslog\n    -Q             do not print messages to stdout\n    -v             prints the software version and exits\n    -h             prints this message and exits\n\nCommands:\n specify commands with arguments. Can specify multiple commands to be executed in order. \n Seconds are read as double precision floating point values.\n\n    set  [seconds]  set PHC time (defaults to time on CLOCK_REALTIME)\n    get             get PHC time\n    adj  <seconds>  adjust PHC time by offset\n    freq [ppb]      adjust PHC frequency (default returns current offset)\n    cmp             compare PHC offset to CLOCK_REALTIME\n    caps            display device capabilities (default if no command given)\n    wait <seconds>  pause between commands\n                    This command may be useful for sanity checking whether the PHC clock is\n                    running as expected.\n                    The arguments specified in seconds are read as double precision floating point\n                    values, and will scale to nanoseconds. This means providing a value of 5.5 means 5\n                    and one half seconds. This allows specifying fairly precise values for time.\n
                                             
                                            phc2sys: synchronize two clocks. 
                                            Usage: phc2sys [options]\n\nAutomatic configuration:\n    -a             turn on autoconfiguration\n    -r             synchronize system (realtime) clock\n                   repeat -r to consider it also as a time source\n\nManual configuration:\n    -c [dev|name]  slave clock (CLOCK_REALTIME)\n    -d [dev]       master PPS device\n    -s [dev|name]  master clock\n    -O [offset]    slave-master time offset (0)\n    -w             wait for ptp4l\n\nOptions:\n    -f [file]      configuration file\n    -E [pi|linreg] clock servo (pi)\n    -P [kp]        proportional constant (0.7)\n    -I [ki]        integration constant (0.3)\n    -S [step]      step threshold (disabled)\n    -F [step]      step threshold only on start (0.00002)\n    -R [rate]      slave clock update rate in HZ (1.0)\n    -N [num]       number of master clock readings per update (5)\n    -L [limit]     sanity frequency limit in ppb (200000000)\n    -M [num]       NTP SHM segment number (0)\n    -u [num]       number of clock updates in summary stats (0)\n    -n [num]       domain number (0)\n    -x             apply leap seconds by servo instead of kernel\n    -z [path]      server address for UDS (/var/run/ptp4l)\n    -l [num]       set the logging level to 'num' (6)\n    -t [tag]       add tag to log messages\n    -m             print messages to stdout\n    -q             do not print messages to the syslog\n    -v             prints the software version and exits\n    -h             prints this message and exits\n
                                            "},{"location":"sw/tod/tod/#description","title":"DESCRIPTION","text":"
                                            The phc2sys utility is used to synchronize the system clock to the PTP Hardware Clock (PHC) or ToD clock. The phc_ctl utility is used to directly control PHC clock device.
                                            "},{"location":"sw/tod/tod/#configuring-the-ptp-service","title":"Configuring the PTP service","text":"
                                               
                                            1. Install the linuxptp package: 
                                              # sudo yum install linuxptp\n
                                              2.  
                                            2. Check PTP device is created successfully by the ToD driver. 
                                              3.  
                                             
                                            ToD driver registering as PHC device (clock_name: dfl_tod) to the Linux PTP stack and exposing to the Linux kernel to synchronize the system clock to its ToD information. 
                                            # cat /sys/class/ptp/ptp0/clock_name\ndfl_tod\n
                                             
                                               
                                            1. Configure phc2sys service on a system:
                                              2.  
                                             
                                            The phc2sys service is configured in the /etc/sysconfig/phc2sys configuration file. Define start-up option for phc2sys daemon in /etc/sysconfig/phc2sys. The master clock is /dev/ptp0 device and the slave clock is system clock/CLOCK_REALTIME: 
                                             OPTIONS=\"-s /dev/ptp0 -c CLOCK_REALTIME -r -O 0 -R 16\"\n
                                             
                                               
                                            1.  
                                              Start phc2sys service: 
                                              # service phc2sys start\n
                                               
                                              2.  
                                            2.  
                                              Stop phc2sys service: 
                                              # service phc2sys stop\n
                                               
                                              3.  
                                            "},{"location":"sw/tod/tod/#examples","title":"Examples","text":""},{"location":"sw/tod/tod/#using-phc_ctl-utility","title":"using phc_ctl utility","text":"
                                            Read the current clock time from the PHC clock device: 
                                            # sudo phc_ctl /dev/ptp0 get\n
                                             
                                            Set the PHC clock time to CLOCK_REALTIME: 
                                            # sudo phc_ctl /dev/ptp0 set\n
                                             
                                            Set PHC clock time to 0: 
                                            # sudo phc_ctl /dev/ptp0 set 0.0\n
                                             
                                            Set PHC clock time to 0 and wait for 10 sec and read the clock time: 
                                            # sudo phc_ctl /dev/ptp0 set 0.0 wait 10.0 get\n
                                             
                                            Set and compare PHC clock time to CLOCK_REALTIME: 
                                            # sudo phc_ctl /dev/ptp0 set cmp\n
                                             
                                            Read the PHC device capabilities: 
                                            # sudo phc_ctl /dev/ptp0 caps\n
                                            "},{"location":"sw/tod/tod/#using-phc2sys-utility","title":"using phc2sys utility","text":"
                                            To synchronize the system clock to the PHC clock: 
                                            # sudo phc2sys -s /dev/ptp0 -c CLOCK_REALTIME -r -O 0 -R 16 -m\nphc2sys[7896.789]: CLOCK_REALTIME phc offset  -1259509 s0 freq  -31462 delay   1338\nphc2sys[7896.852]: CLOCK_REALTIME phc offset  -1261498 s1 freq  -63144 delay   1328\nphc2sys[7896.914]: CLOCK_REALTIME phc offset       -15 s2 freq  -63159 delay   1328\nphc2sys[7896.977]: CLOCK_REALTIME phc offset       -19 s2 freq  -63167 delay   1327\nphc2sys[7897.039]: CLOCK_REALTIME phc offset       -35 s2 freq  -63189 delay   1328\nphc2sys[7897.102]: CLOCK_REALTIME phc offset       -37 s2 freq  -63201 delay   1331\nphc2sys[7897.165]: CLOCK_REALTIME phc offset       -30 s2 freq  -63205 delay   1328\nphc2sys[7897.227]: CLOCK_REALTIME phc offset       -50 s2 freq  -63234 delay   1331\nphc2sys[7897.290]: CLOCK_REALTIME phc offset       -50 s2 freq  -63249 delay   1329\nphc2sys[7897.353]: CLOCK_REALTIME phc offset       -62 s2 freq  -63276 delay   1334\nphc2sys[7897.415]: CLOCK_REALTIME phc offset       -53 s2 freq  -63286 delay   1335\nphc2sys[7897.478]: CLOCK_REALTIME phc offset       -46 s2 freq  -63295 delay   1325\nphc2sys[7897.541]: CLOCK_REALTIME phc offset       -57 s2 freq  -63320 delay   1334\nphc2sys[7897.603]: CLOCK_REALTIME phc offset       -39 s2 freq  -63319 delay   1327\nphc2sys[7897.666]: CLOCK_REALTIME phc offset       -31 s2 freq  -63323 delay   1328\nphc2sys[7897.728]: CLOCK_REALTIME phc offset       -48 s2 freq  -63349 delay   1327\nphc2sys[7897.791]: CLOCK_REALTIME phc offset       -42 s2 freq  -63357 delay   1323\nphc2sys[7897.854]: CLOCK_REALTIME phc offset       -41 s2 freq  -63369 delay   1324\nphc2sys[7897.916]: CLOCK_REALTIME phc offset       -44 s2 freq  -63384 delay   1328\nphc2sys[7897.979]: CLOCK_REALTIME phc offset       -13 s2 freq  -63366 delay   1327\nphc2sys[7898.042]: CLOCK_REALTIME phc offset       -16 s2 freq  -63373 delay   1327\nphc2sys[7898.104]: CLOCK_REALTIME phc offset       -19 s2 freq  -63381 delay   1328\nphc2sys[7898.167]: CLOCK_REALTIME phc offset       -16 s2 freq  -63384 delay   1327\nphc2sys[7898.229]: CLOCK_REALTIME phc offset         3 s2 freq  -63370 delay   1327\nphc2sys[7898.292]: CLOCK_REALTIME phc offset        16 s2 freq  -63356 delay   1325\nphc2sys[7898.355]: CLOCK_REALTIME phc offset        10 s2 freq  -63357 delay   1319\nphc2sys[7898.417]: CLOCK_REALTIME phc offset        23 s2 freq  -63341 delay   1327\nphc2sys[7898.480]: CLOCK_REALTIME phc offset        13 s2 freq  -63344 delay   1335\nphc2sys[7898.543]: CLOCK_REALTIME phc offset        23 s2 freq  -63330 delay   1323\nphc2sys[7898.605]: CLOCK_REALTIME phc offset        29 s2 freq  -63317 delay   1312\nphc2sys[7898.668]: CLOCK_REALTIME phc offset        22 s2 freq  -63315 delay   1324\nphc2sys[7898.730]: CLOCK_REALTIME phc offset        42 s2 freq  -63289 delay   1325\nphc2sys[7898.793]: CLOCK_REALTIME phc offset        29 s2 freq  -63289 delay   1333\nphc2sys[7898.856]: CLOCK_REALTIME phc offset        34 s2 freq  -63276 delay   1327\nphc2sys[7898.918]: CLOCK_REALTIME phc offset        21 s2 freq  -63278 delay   1331\nphc2sys[7898.981]: CLOCK_REALTIME phc offset        22 s2 freq  -63271 delay   1335\nphc2sys[7899.044]: CLOCK_REALTIME phc offset        30 s2 freq  -63256 delay   1327\nphc2sys[7899.106]: CLOCK_REALTIME phc offset        30 s2 freq  -63247 delay   1328\nphc2sys[7899.169]: CLOCK_REALTIME phc offset        37 s2 freq  -63231 delay   1333\nphc2sys[7899.232]: CLOCK_REALTIME phc offset        29 s2 freq  -63228 delay   1331\nphc2sys[7899.294]: CLOCK_REALTIME phc offset        24 s2 freq  -63225 delay   1330\n
                                            "},{"location":"opae-code/annotated/","title":"Class List","text":"
                                            Here are the classes, structs, unions and interfaces with brief descriptions:
                                             
                                               
                                            • struct _fpga_token_header Internal token type header. 
                                              •  
                                            • struct _opae_hash_map Hash map object. 
                                              •  
                                            • struct _opae_hash_map_item List link item. 
                                              •  
                                            • struct cache_line 
                                              •  
                                            • struct config 
                                              •  
                                            • struct fpga_error_info 
                                              •  
                                            • struct fpga_metric Metric struct. 
                                              •  
                                            • struct fpga_metric_info Metric info struct. 
                                              •  
                                            • struct fpga_version Semantic version. 
                                              •  
                                            • struct mem_alloc 
                                              •  
                                            • struct mem_link Provides an API for allocating/freeing a logical address space. 
                                              •  
                                            • struct metric_threshold 
                                              •  
                                            • union metric_value Metric value union. 
                                              •  
                                            • namespace opae 
                                                 
                                              • namespace fpga 
                                                   
                                                • namespace types 
                                                     
                                                  • class busy busy exception 
                                                    •  
                                                  • namespace detail 
                                                    •  
                                                  • class error An error object represents an error register for a resource. 
                                                    •  
                                                  • class event Wraps fpga event routines in OPAE C. 
                                                       
                                                    • struct type_t C++ struct that is interchangeable with fpga_event_type enum. 
                                                      •  
                                                     
                                                    •  
                                                  • class except Generic OPAE exception. 
                                                    •  
                                                  • class exception exception exception 
                                                    •  
                                                  • struct guid_t Representation of the guid member of a properties object. 
                                                    •  
                                                  • class handle An allocated accelerator resource. 
                                                    •  
                                                  • class invalid_param invalid_param exception 
                                                    •  
                                                  • class no_access no_access exception 
                                                    •  
                                                  • class no_daemon no_daemon exception 
                                                    •  
                                                  • class no_driver no_driver exception 
                                                    •  
                                                  • class no_memory no_memory exception 
                                                    •  
                                                  • class not_found not_found exception 
                                                    •  
                                                  • class not_supported not_supported exception 
                                                    •  
                                                  • class properties Wraps an OPAE fpga_properties object. 
                                                    •  
                                                  • struct pvalue Wraps OPAE properties defined in the OPAE C API by associating an fpga_properties reference with the getters and setters defined for a property. 
                                                    •  
                                                  • class reconf_error reconf_error exception 
                                                    •  
                                                  • class shared_buffer Host/AFU shared memory blocks. 
                                                    •  
                                                  • class src_location Identify a particular line in a source file. 
                                                    •  
                                                  • class sysobject Wraps the OPAE fpga_object primitive. 
                                                    •  
                                                  • class token Wraps the OPAE fpga_token primitive. 
                                                    •  
                                                  • class version 
                                                    •  
                                                   
                                                  •  
                                                 
                                                •  
                                               
                                              •  
                                            • struct opae_uio OPAE UIO device abstraction. 
                                              •  
                                            • struct opae_uio_device_region MMIO region info. 
                                              •  
                                            • struct opae_vfio OPAE VFIO device abstraction. 
                                              •  
                                            • struct opae_vfio_buffer System DMA buffer. 
                                              •  
                                            • struct opae_vfio_device VFIO device. 
                                              •  
                                            • struct opae_vfio_device_irq Interrupt info. 
                                              •  
                                            • struct opae_vfio_device_region MMIO region info. 
                                              •  
                                            • struct opae_vfio_group VFIO group. 
                                              •  
                                            • struct opae_vfio_iova_range IO Virtual Address Range. 
                                              •  
                                            • struct opae_vfio_sparse_info MMIO sparse-mappable region info. 
                                              •  
                                            • struct ras_inject_error 
                                              •  
                                            • namespace std 
                                              •  
                                            • struct threshold Threshold struct. 
                                              •  
                                            "},{"location":"opae-code/files/","title":"File List","text":"
                                            Here is a list of all files with brief descriptions:
                                             
                                               
                                            • dir docs 
                                                 
                                              • dir sw 
                                                   
                                                • dir include 
                                                     
                                                  • dir opae 
                                                       
                                                    • file access.h Functions to acquire, release, and reset OPAE FPGA resources. 
                                                      •  
                                                    • file buffer.h Functions for allocating and sharing system memory with an FPGA accelerator. 
                                                      •  
                                                    • dir cxx 
                                                         
                                                      • file core.h 
                                                        •  
                                                      • dir core 
                                                           
                                                        • file errors.h 
                                                          •  
                                                        • file events.h 
                                                          •  
                                                        • file except.h 
                                                          •  
                                                        • file handle.h 
                                                          •  
                                                        • file properties.h 
                                                          •  
                                                        • file pvalue.h 
                                                          •  
                                                        • file shared_buffer.h 
                                                          •  
                                                        • file sysobject.h 
                                                          •  
                                                        • file token.h 
                                                          •  
                                                        • file version.h 
                                                          •  
                                                         
                                                        •  
                                                       
                                                      •  
                                                    • file enum.h APIs for resource enumeration and managing tokens. 
                                                      •  
                                                    • file error.h Functions for reading and clearing errors in resources. 
                                                      •  
                                                    • file event.h Functions for registering events and managing the lifecycle for fpga_event_handle s. 
                                                      •  
                                                    • file fpga.h FPGA API. 
                                                      •  
                                                    • file hash_map.h A general-purpose hybrid array/list hash map implementation. 
                                                      •  
                                                    • file init.h Initialization routine. 
                                                      •  
                                                    • file log.h 
                                                      •  
                                                    • file manage.h Functions for managing FPGA configurations. 
                                                      •  
                                                    • file mem_alloc.h 
                                                      •  
                                                    • file metrics.h Functions for Discover/ Enumerates metrics and retrieves values. 
                                                      •  
                                                    • file mmio.h Functions for mapping and accessing MMIO space. 
                                                      •  
                                                    • file properties.h Functions for examining and manipulating fpga_properties objects. 
                                                      •  
                                                    • file sysobject.h Functions to read/write from system objects. 
                                                      •  
                                                    • file types.h Type definitions for FPGA API. 
                                                      •  
                                                    • file types_enum.h Definitions of enumerated types for the OPAE C API. 
                                                      •  
                                                    • file uio.h APIs to manage a PCIe device via UIO. 
                                                      •  
                                                    • file umsg.h FPGA UMsg API. 
                                                      •  
                                                    • file userclk.h Functions for setting and get afu user clock. 
                                                      •  
                                                    • file utils.h Utility functions and macros for the FPGA API. 
                                                      •  
                                                    • file version.h 
                                                      •  
                                                    • file vfio.h APIs to manage a PCIe device via vfio-pci. 
                                                      •  
                                                     
                                                    •  
                                                   
                                                  •  
                                                • dir samples 
                                                     
                                                  • dir hello_events 
                                                       
                                                    • file hello_events.c A code sample of using OPAE event API. 
                                                      •  
                                                     
                                                    •  
                                                  • dir hello_fpga 
                                                       
                                                    • file hello_fpga.c A code sample illustrates the basic usage of the OPAE C API. 
                                                      •  
                                                     
                                                    •  
                                                   
                                                  •  
                                                 
                                                •  
                                               
                                              •  
                                            "},{"location":"opae-code/struct__fpga__token__header/","title":"Struct _fpga_token_header","text":"
                                            ClassList > _fpga_token_header
                                             
                                            Internal token type header. More...
                                             
                                               
                                            • #include <types.h>
                                              •  
                                            "},{"location":"opae-code/struct__fpga__token__header/#public-attributes","title":"Public Attributes","text":"Type Name uint8_t bus uint8_t device uint16_t device_id uint8_t function fpga_guid guid fpga_interface interface uint64_t magic uint64_t object_id fpga_objtype objtype uint16_t segment uint16_t subsystem_device_id uint16_t subsystem_vendor_id uint16_t vendor_id"},{"location":"opae-code/struct__fpga__token__header/#detailed-description","title":"Detailed Description","text":"
                                            Each plugin (dfl: libxfpga.so, vfio: libopae-v.so) implements its own proprietary token type. This header must appear at offset zero within that structure.
                                             
                                            eg, see lib/plugins/xfpga/types_int.h:struct _fpga_token and lib/plugins/vfio/opae_vfio.h:struct _vfio_token. 
                                            "},{"location":"opae-code/struct__fpga__token__header/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/struct__fpga__token__header/#variable-bus","title":"variable bus","text":"
                                            uint8_t _fpga_token_header::bus;\n
                                            "},{"location":"opae-code/struct__fpga__token__header/#variable-device","title":"variable device","text":"
                                            uint8_t _fpga_token_header::device;\n
                                            "},{"location":"opae-code/struct__fpga__token__header/#variable-device_id","title":"variable device_id","text":"
                                            uint16_t _fpga_token_header::device_id;\n
                                            "},{"location":"opae-code/struct__fpga__token__header/#variable-function","title":"variable function","text":"
                                            uint8_t _fpga_token_header::function;\n
                                            "},{"location":"opae-code/struct__fpga__token__header/#variable-guid","title":"variable guid","text":"
                                            fpga_guid _fpga_token_header::guid;\n
                                            "},{"location":"opae-code/struct__fpga__token__header/#variable-interface","title":"variable interface","text":"
                                            fpga_interface _fpga_token_header::interface;\n
                                            "},{"location":"opae-code/struct__fpga__token__header/#variable-magic","title":"variable magic","text":"
                                            uint64_t _fpga_token_header::magic;\n
                                            "},{"location":"opae-code/struct__fpga__token__header/#variable-object_id","title":"variable object_id","text":"
                                            uint64_t _fpga_token_header::object_id;\n
                                            "},{"location":"opae-code/struct__fpga__token__header/#variable-objtype","title":"variable objtype","text":"
                                            fpga_objtype _fpga_token_header::objtype;\n
                                            "},{"location":"opae-code/struct__fpga__token__header/#variable-segment","title":"variable segment","text":"
                                            uint16_t _fpga_token_header::segment;\n
                                            "},{"location":"opae-code/struct__fpga__token__header/#variable-subsystem_device_id","title":"variable subsystem_device_id","text":"
                                            uint16_t _fpga_token_header::subsystem_device_id;\n
                                            "},{"location":"opae-code/struct__fpga__token__header/#variable-subsystem_vendor_id","title":"variable subsystem_vendor_id","text":"
                                            uint16_t _fpga_token_header::subsystem_vendor_id;\n
                                            "},{"location":"opae-code/struct__fpga__token__header/#variable-vendor_id","title":"variable vendor_id","text":"
                                            uint16_t _fpga_token_header::vendor_id;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                            "},{"location":"opae-code/struct__opae__hash__map/","title":"Struct _opae_hash_map","text":"
                                            ClassList > _opae_hash_map
                                             
                                            Hash map object. More...
                                             
                                               
                                            • #include <hash_map.h>
                                              •  
                                            "},{"location":"opae-code/struct__opae__hash__map/#public-attributes","title":"Public Attributes","text":"Type Name opae_hash_map_item ** buckets void * cleanup_context Optional second parameter to key_cleanup and value_cleanup. int flags uint32_t hash_seed void(* key_cleanup (optional) int(* key_compare (required) uint32_t(* key_hash uint32_t num_buckets void(* value_cleanup (optional)"},{"location":"opae-code/struct__opae__hash__map/#detailed-description","title":"Detailed Description","text":"
                                            This structure defines the internals of the hash map. Each of the parameters supplied to opae_hash_map_init() is stored in the structure. All parameters are required, except key_cleanup and value_cleanup, which may optionally be NULL. 
                                            "},{"location":"opae-code/struct__opae__hash__map/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/struct__opae__hash__map/#variable-buckets","title":"variable buckets","text":"
                                            opae_hash_map_item** _opae_hash_map::buckets;\n
                                            "},{"location":"opae-code/struct__opae__hash__map/#variable-cleanup_context","title":"variable cleanup_context","text":"
                                            void* _opae_hash_map::cleanup_context;\n
                                            "},{"location":"opae-code/struct__opae__hash__map/#variable-flags","title":"variable flags","text":"
                                            int _opae_hash_map::flags;\n
                                            "},{"location":"opae-code/struct__opae__hash__map/#variable-hash_seed","title":"variable hash_seed","text":"
                                            uint32_t _opae_hash_map::hash_seed;\n
                                            "},{"location":"opae-code/struct__opae__hash__map/#variable-key_cleanup","title":"variable key_cleanup","text":"
                                            void(* _opae_hash_map::key_cleanup) (void *key, void *context);\n
                                            "},{"location":"opae-code/struct__opae__hash__map/#variable-key_compare","title":"variable key_compare","text":"
                                            int(* _opae_hash_map::key_compare) (void *keya, void *keyb);\n
                                            "},{"location":"opae-code/struct__opae__hash__map/#variable-key_hash","title":"variable key_hash","text":"
                                            uint32_t(* _opae_hash_map::key_hash) (uint32_t num_buckets, uint32_t hash_seed, void *key);\n
                                            "},{"location":"opae-code/struct__opae__hash__map/#variable-num_buckets","title":"variable num_buckets","text":"
                                            uint32_t _opae_hash_map::num_buckets;\n
                                            "},{"location":"opae-code/struct__opae__hash__map/#variable-value_cleanup","title":"variable value_cleanup","text":"
                                            void(* _opae_hash_map::value_cleanup) (void *value, void *context);\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/hash_map.h
                                            "},{"location":"opae-code/struct__opae__hash__map__item/","title":"Struct _opae_hash_map_item","text":"
                                            ClassList > _opae_hash_map_item
                                             
                                            List link item. More...
                                             
                                               
                                            • #include <hash_map.h>
                                              •  
                                            "},{"location":"opae-code/struct__opae__hash__map__item/#public-attributes","title":"Public Attributes","text":"Type Name void * key struct _opae_hash_map_item * next void * value"},{"location":"opae-code/struct__opae__hash__map__item/#detailed-description","title":"Detailed Description","text":"
                                            This structure provides the association between key and value. When the supplied hash function for keys A and B returns the same bucket index, both A and B can co-exist on the same list rooted at the bucket index. 
                                            "},{"location":"opae-code/struct__opae__hash__map__item/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/struct__opae__hash__map__item/#variable-key","title":"variable key","text":"
                                            void* _opae_hash_map_item::key;\n
                                            "},{"location":"opae-code/struct__opae__hash__map__item/#variable-next","title":"variable next","text":"
                                            struct _opae_hash_map_item* _opae_hash_map_item::next;\n
                                            "},{"location":"opae-code/struct__opae__hash__map__item/#variable-value","title":"variable value","text":"
                                            void* _opae_hash_map_item::value;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/hash_map.h
                                            "},{"location":"opae-code/structcache__line/","title":"Struct cache_line","text":"
                                            ClassList > cache_line
                                            "},{"location":"opae-code/structcache__line/#public-attributes","title":"Public Attributes","text":"Type Name uint32_t uint"},{"location":"opae-code/structcache__line/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structcache__line/#variable-uint","title":"variable uint","text":"
                                            uint32_t cache_line::uint[16];\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/samples/hello_fpga/hello_fpga.c
                                            "},{"location":"opae-code/structconfig/","title":"Struct config","text":"
                                            ClassList > config
                                            "},{"location":"opae-code/structconfig/#public-attributes","title":"Public Attributes","text":"Type Name int open_flags int run_n3000"},{"location":"opae-code/structconfig/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structconfig/#variable-open_flags","title":"variable open_flags","text":"
                                            int config::open_flags;\n
                                            "},{"location":"opae-code/structconfig/#variable-run_n3000","title":"variable run_n3000","text":"
                                            int config::run_n3000;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/samples/hello_fpga/hello_fpga.c
                                            "},{"location":"opae-code/structfpga__error__info/","title":"Struct fpga_error_info","text":"
                                            ClassList > fpga_error_info
                                             
                                               
                                            • #include <types.h>
                                              •  
                                            "},{"location":"opae-code/structfpga__error__info/#public-attributes","title":"Public Attributes","text":"Type Name bool can_clear name of the error char name"},{"location":"opae-code/structfpga__error__info/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structfpga__error__info/#variable-can_clear","title":"variable can_clear","text":"
                                            bool fpga_error_info::can_clear;\n
                                            "},{"location":"opae-code/structfpga__error__info/#variable-name","title":"variable name","text":"
                                            char fpga_error_info::name[FPGA_ERROR_NAME_MAX];\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                            "},{"location":"opae-code/structfpga__metric/","title":"Struct fpga_metric","text":"
                                            ClassList > fpga_metric
                                             
                                            Metric struct. 
                                             
                                               
                                            • #include <types.h>
                                              •  
                                            "},{"location":"opae-code/structfpga__metric/#public-attributes","title":"Public Attributes","text":"Type Name bool isvalid uint64_t metric_num metric_value value"},{"location":"opae-code/structfpga__metric/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structfpga__metric/#variable-isvalid","title":"variable isvalid","text":"
                                            bool fpga_metric::isvalid;\n
                                            "},{"location":"opae-code/structfpga__metric/#variable-metric_num","title":"variable metric_num","text":"
                                            uint64_t fpga_metric::metric_num;\n
                                            "},{"location":"opae-code/structfpga__metric/#variable-value","title":"variable value","text":"
                                            metric_value fpga_metric::value;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                            "},{"location":"opae-code/structfpga__metric__info/","title":"Struct fpga_metric_info","text":"
                                            ClassList > fpga_metric_info
                                             
                                            Metric info struct. 
                                             
                                               
                                            • #include <types.h>
                                              •  
                                            "},{"location":"opae-code/structfpga__metric__info/#public-attributes","title":"Public Attributes","text":"Type Name char group_name enum fpga_metric_datatype metric_datatype fpga_guid metric_guid char metric_name uint64_t metric_num enum fpga_metric_type metric_type char metric_units char qualifier_name"},{"location":"opae-code/structfpga__metric__info/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structfpga__metric__info/#variable-group_name","title":"variable group_name","text":"
                                            char fpga_metric_info::group_name[FPGA_METRIC_STR_SIZE];\n
                                            "},{"location":"opae-code/structfpga__metric__info/#variable-metric_datatype","title":"variable metric_datatype","text":"
                                            enum fpga_metric_datatype fpga_metric_info::metric_datatype;\n
                                            "},{"location":"opae-code/structfpga__metric__info/#variable-metric_guid","title":"variable metric_guid","text":"
                                            fpga_guid fpga_metric_info::metric_guid;\n
                                            "},{"location":"opae-code/structfpga__metric__info/#variable-metric_name","title":"variable metric_name","text":"
                                            char fpga_metric_info::metric_name[FPGA_METRIC_STR_SIZE];\n
                                            "},{"location":"opae-code/structfpga__metric__info/#variable-metric_num","title":"variable metric_num","text":"
                                            uint64_t fpga_metric_info::metric_num;\n
                                            "},{"location":"opae-code/structfpga__metric__info/#variable-metric_type","title":"variable metric_type","text":"
                                            enum fpga_metric_type fpga_metric_info::metric_type;\n
                                            "},{"location":"opae-code/structfpga__metric__info/#variable-metric_units","title":"variable metric_units","text":"
                                            char fpga_metric_info::metric_units[FPGA_METRIC_STR_SIZE];\n
                                            "},{"location":"opae-code/structfpga__metric__info/#variable-qualifier_name","title":"variable qualifier_name","text":"
                                            char fpga_metric_info::qualifier_name[FPGA_METRIC_STR_SIZE];\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                            "},{"location":"opae-code/structfpga__version/","title":"Struct fpga_version","text":"
                                            ClassList > fpga_version
                                             
                                            Semantic version. More...
                                             
                                               
                                            • #include <types.h>
                                              •  
                                            "},{"location":"opae-code/structfpga__version/#public-attributes","title":"Public Attributes","text":"Type Name uint8_t major Major version. uint8_t minor Minor version. uint16_t patch Revision or patchlevel."},{"location":"opae-code/structfpga__version/#detailed-description","title":"Detailed Description","text":"
                                            Data structure for expressing version identifiers following the semantic versioning scheme. Used in various properties for tracking component versions. 
                                            "},{"location":"opae-code/structfpga__version/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structfpga__version/#variable-major","title":"variable major","text":"
                                            uint8_t fpga_version::major;\n
                                            "},{"location":"opae-code/structfpga__version/#variable-minor","title":"variable minor","text":"
                                            uint8_t fpga_version::minor;\n
                                            "},{"location":"opae-code/structfpga__version/#variable-patch","title":"variable patch","text":"
                                            uint16_t fpga_version::patch;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                            "},{"location":"opae-code/structmem__alloc/","title":"Struct mem_alloc","text":"
                                            ClassList > mem_alloc
                                             
                                               
                                            • #include <mem_alloc.h>
                                              •  
                                            "},{"location":"opae-code/structmem__alloc/#public-attributes","title":"Public Attributes","text":"Type Name struct mem_link allocated struct mem_link free"},{"location":"opae-code/structmem__alloc/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structmem__alloc/#variable-allocated","title":"variable allocated","text":"
                                            struct mem_link mem_alloc::allocated;\n
                                            "},{"location":"opae-code/structmem__alloc/#variable-free","title":"variable free","text":"
                                            struct mem_link mem_alloc::free;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/mem_alloc.h
                                            "},{"location":"opae-code/structmem__link/","title":"Struct mem_link","text":"
                                            ClassList > mem_link
                                             
                                            Provides an API for allocating/freeing a logical address space. More...
                                             
                                               
                                            • #include <mem_alloc.h>
                                              •  
                                            "},{"location":"opae-code/structmem__link/#public-attributes","title":"Public Attributes","text":"Type Name uint64_t address struct mem_link * next struct mem_link * prev uint64_t size"},{"location":"opae-code/structmem__link/#detailed-description","title":"Detailed Description","text":"
                                            There is no interaction with any OS memory allocation infrastructure, whether malloc, mmap, etc. The \"address ranges\" tracked by this allocator are arbitrary 64-bit integers. The allocator simply provides the bookeeping logic that ensures that a unique address with the appropriate size is returned for each allocation request, and that an allocation can be freed, ie released back to the available pool of logical address space for future allocations. The memory backing the allocator's internal data structures is managed by malloc()/free(). 
                                            "},{"location":"opae-code/structmem__link/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structmem__link/#variable-address","title":"variable address","text":"
                                            uint64_t mem_link::address;\n
                                            "},{"location":"opae-code/structmem__link/#variable-next","title":"variable next","text":"
                                            struct mem_link* mem_link::next;\n
                                            "},{"location":"opae-code/structmem__link/#variable-prev","title":"variable prev","text":"
                                            struct mem_link* mem_link::prev;\n
                                            "},{"location":"opae-code/structmem__link/#variable-size","title":"variable size","text":"
                                            uint64_t mem_link::size;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/mem_alloc.h
                                            "},{"location":"opae-code/structmetric__threshold/","title":"Struct metric_threshold","text":"
                                            ClassList > metric_threshold
                                             
                                               
                                            • #include <types.h>
                                              •  
                                            "},{"location":"opae-code/structmetric__threshold/#public-attributes","title":"Public Attributes","text":"Type Name threshold hysteresis threshold lower_c_threshold threshold lower_nc_threshold threshold lower_nr_threshold char metric_name threshold upper_c_threshold threshold upper_nc_threshold threshold upper_nr_threshold"},{"location":"opae-code/structmetric__threshold/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structmetric__threshold/#variable-hysteresis","title":"variable hysteresis","text":"
                                            threshold metric_threshold::hysteresis;\n
                                            "},{"location":"opae-code/structmetric__threshold/#variable-lower_c_threshold","title":"variable lower_c_threshold","text":"
                                            threshold metric_threshold::lower_c_threshold;\n
                                            "},{"location":"opae-code/structmetric__threshold/#variable-lower_nc_threshold","title":"variable lower_nc_threshold","text":"
                                            threshold metric_threshold::lower_nc_threshold;\n
                                            "},{"location":"opae-code/structmetric__threshold/#variable-lower_nr_threshold","title":"variable lower_nr_threshold","text":"
                                            threshold metric_threshold::lower_nr_threshold;\n
                                            "},{"location":"opae-code/structmetric__threshold/#variable-metric_name","title":"variable metric_name","text":"
                                            char metric_threshold::metric_name[FPGA_METRIC_STR_SIZE];\n
                                            "},{"location":"opae-code/structmetric__threshold/#variable-upper_c_threshold","title":"variable upper_c_threshold","text":"
                                            threshold metric_threshold::upper_c_threshold;\n
                                            "},{"location":"opae-code/structmetric__threshold/#variable-upper_nc_threshold","title":"variable upper_nc_threshold","text":"
                                            threshold metric_threshold::upper_nc_threshold;\n
                                            "},{"location":"opae-code/structmetric__threshold/#variable-upper_nr_threshold","title":"variable upper_nr_threshold","text":"
                                            threshold metric_threshold::upper_nr_threshold;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                            "},{"location":"opae-code/unionmetric__value/","title":"Union metric_value","text":"
                                            ClassList > metric_value
                                             
                                            Metric value union. 
                                             
                                               
                                            • #include <types.h>
                                              •  
                                            "},{"location":"opae-code/unionmetric__value/#public-attributes","title":"Public Attributes","text":"Type Name bool bvalue double dvalue float fvalue uint64_t ivalue"},{"location":"opae-code/unionmetric__value/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/unionmetric__value/#variable-bvalue","title":"variable bvalue","text":"
                                            bool metric_value::bvalue;\n
                                            "},{"location":"opae-code/unionmetric__value/#variable-dvalue","title":"variable dvalue","text":"
                                            double metric_value::dvalue;\n
                                            "},{"location":"opae-code/unionmetric__value/#variable-fvalue","title":"variable fvalue","text":"
                                            float metric_value::fvalue;\n
                                            "},{"location":"opae-code/unionmetric__value/#variable-ivalue","title":"variable ivalue","text":"
                                            uint64_t metric_value::ivalue;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                            "},{"location":"opae-code/namespaceopae/","title":"Namespace opae","text":"
                                            Namespace List > opae
                                            "},{"location":"opae-code/namespaceopae/#namespaces","title":"Namespaces","text":"Type Name namespace fpga 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/errors.h
                                            "},{"location":"opae-code/namespaceopae_1_1fpga/","title":"Namespace opae::fpga","text":"
                                            Namespace List > opae > fpga
                                            "},{"location":"opae-code/namespaceopae_1_1fpga/#namespaces","title":"Namespaces","text":"Type Name namespace types 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/errors.h
                                            "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types/","title":"Namespace opae::fpga::types","text":"
                                            Namespace List > opae > fpga > types
                                            "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types/#namespaces","title":"Namespaces","text":"Type Name namespace detail"},{"location":"opae-code/namespaceopae_1_1fpga_1_1types/#classes","title":"Classes","text":"Type Name class busy busy exception class error An error object represents an error register for a resource. class event Wraps fpga event routines in OPAE C. class except Generic OPAE exception. class exception exception exception struct guid_t Representation of the guid member of a properties object. class handle An allocated accelerator resource. class invalid_param invalid_param exception class no_access no_access exception class no_daemon no_daemon exception class no_driver no_driver exception class no_memory no_memory exception class not_found not_found exception class not_supported not_supported exception class properties Wraps an OPAE fpga_properties object. struct pvalue <typename T>Wraps OPAE properties defined in the OPAE C API by associating an fpga_properties reference with the getters and setters defined for a property. class reconf_error reconf_error exception class shared_buffer Host/AFU shared memory blocks. class src_location Identify a particular line in a source file. class sysobject Wraps the OPAE fpga_object primitive. class token Wraps the OPAE fpga_token primitive. class version 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/errors.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/","title":"Class opae::fpga::types::busy","text":"
                                            ClassList > opae > fpga > types > busy
                                             
                                            busy exception More...
                                             
                                               
                                            • #include <except.h>
                                              •  
                                             
                                            Inherits the following classes: opae::fpga::types::except
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#public-functions","title":"Public Functions","text":"Type Name busy (src_location loc) noexceptbusy constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#detailed-description","title":"Detailed Description","text":"
                                            busy tracks the source line of origin for exceptions thrown when the error code FPGA_BUSY is returned from a call to an OPAE C API function 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#function-busy","title":"function busy","text":"
                                            busy constructor 
                                            inline opae::fpga::types::busy::busy (\n    src_location loc\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • loc Location where the exception was constructed. 
                                              •  
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/","title":"Namespace opae::fpga::types::detail","text":"
                                            Namespace List > opae > fpga > types > detail
                                            "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-types","title":"Public Types","text":"Type Name typedef bool(* exception_fn typedef function pointer that returns bool if result is FPGA_OK"},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-static-attributes","title":"Public Static Attributes","text":"Type Name exception_fn opae_exceptions   = = { is_ok<opae::fpga::types::invalid_param>, is_ok<opae::fpga::types::busy>, is_ok<opae::fpga::types::exception>, is_ok<opae::fpga::types::not_found>, is_ok<opae::fpga::types::no_memory>, is_ok<opae::fpga::types::not_supported>, is_ok<opae::fpga::types::no_driver>, is_ok<opae::fpga::types::no_daemon>, is_ok<opae::fpga::types::no_access>, is_ok<opae::fpga::types::reconf_error>}"},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-functions","title":"Public Functions","text":"Type Name constexpr bool is_ok (fpga_result result, const opae::fpga::types::src_location & loc) is_ok is a template function that throws an excpetion of its template argument type if the result code is not FPGA_OK."},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-static-functions","title":"Public Static Functions","text":"Type Name void assert_fpga_ok (fpga_result result, const opae::fpga::types::src_location & loc)"},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#typedef-exception_fn","title":"typedef exception_fn","text":"
                                            typedef bool(* opae::fpga::types::detail::exception_fn) (fpga_result, const opae::fpga::types::src_location &loc);\n
                                            "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-static-attributes-documentation","title":"Public Static Attributes Documentation","text":""},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#variable-opae_exceptions","title":"variable opae_exceptions","text":"
                                            exception_fn opae::fpga::types::detail::opae_exceptions[12];\n
                                            "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#function-is_ok","title":"function is_ok","text":"
                                            is_ok is a template function that throws an excpetion of its template argument type if the result code is not FPGA_OK. 
                                            template<typename T typename T>\nconstexpr bool opae::fpga::types::detail::is_ok (\n    fpga_result result,\n    const opae::fpga::types::src_location & loc\n) \n
                                             
                                            Otherwise it returns true. 
                                            "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#function-assert_fpga_ok","title":"function assert_fpga_ok","text":"
                                            static inline void opae::fpga::types::detail::assert_fpga_ok (\n    fpga_result result,\n    const opae::fpga::types::src_location & loc\n) \n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/","title":"Class opae::fpga::types::error","text":"
                                            ClassList > opae > fpga > types > error
                                             
                                            An error object represents an error register for a resource. More...
                                             
                                               
                                            • #include <errors.h>
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< error > ptr_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#public-functions","title":"Public Functions","text":"Type Name fpga_error_info c_type () constGet the C data structure. bool can_clear () Indicates whether an error register can be cleared. error (const error & e) = delete std::string name () Get the error register name. error & operator= (const error & e) = delete uint64_t read_value () Read the raw value contained in the associated error register. ~error ()"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#public-static-functions","title":"Public Static Functions","text":"Type Name error::ptr_t get (token::ptr_t tok, uint32_t num) Factory function for creating an error object."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#detailed-description","title":"Detailed Description","text":"
                                            This is used to read out the raw value in the register. No parsing is done by this class. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                            typedef std::shared_ptr<error> opae::fpga::types::error::ptr_t;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-c_type","title":"function c_type","text":"
                                            Get the C data structure. 
                                            inline fpga_error_info opae::fpga::types::error::c_type () const\n
                                             
                                            Returns:
                                             
                                            The fpga_error_info that contains the name and the can_clear boolean. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-can_clear","title":"function can_clear","text":"
                                            Indicates whether an error register can be cleared. 
                                            inline bool opae::fpga::types::error::can_clear () \n
                                             
                                            Returns:
                                             
                                            A boolean value indicating if the error register can be cleared. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-error-12","title":"function error [\u00bd]","text":"
                                            opae::fpga::types::error::error (\n    const error & e\n) = delete\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-name","title":"function name","text":"
                                            Get the error register name. 
                                            inline std::string opae::fpga::types::error::name () \n
                                             
                                            Returns:
                                             
                                            A std::string object set to the error name. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-operator","title":"function operator=","text":"
                                            error & opae::fpga::types::error::operator= (\n    const error & e\n) = delete\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-read_value","title":"function read_value","text":"
                                            Read the raw value contained in the associated error register. 
                                            uint64_t opae::fpga::types::error::read_value () \n
                                             
                                            Returns:
                                             
                                            A 64-bit value (unparsed) read from the error register 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-error","title":"function ~error","text":"
                                            inline opae::fpga::types::error::~error () \n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-get","title":"function get","text":"
                                            Factory function for creating an error object. 
                                            static error::ptr_t opae::fpga::types::error::get (\n    token::ptr_t tok,\n    uint32_t num\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • tok The token object representing a resource. 
                                              •  
                                            • num The index of the error register. This must be lower than the num_errors property of the resource.
                                              •  
                                             
                                            Returns:
                                             
                                            A shared_ptr containing the error object 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/errors.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/","title":"Class opae::fpga::types::event","text":"
                                            ClassList > opae > fpga > types > event
                                             
                                            Wraps fpga event routines in OPAE C. 
                                             
                                               
                                            • #include <events.h>
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#classes","title":"Classes","text":"Type Name struct type_t C++ struct that is interchangeable with fpga_event_type enum."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< event > ptr_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#public-functions","title":"Public Functions","text":"Type Name fpga_event_handle get () Get the fpga_event_handle contained in this object. operator fpga_event_handle () Coversion operator for converting to fpga_event_handle objects. int os_object () constGet OS Object from the event object. virtual ~event () Destroy event and associated resources."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#public-static-functions","title":"Public Static Functions","text":"Type Name event::ptr_t register_event (handle::ptr_t h, event::type_t t, int flags=0) Factory function to create event objects."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                            typedef std::shared_ptr<event> opae::fpga::types::event::ptr_t;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#function-get","title":"function get","text":"
                                            Get the fpga_event_handle contained in this object. 
                                            inline fpga_event_handle opae::fpga::types::event::get () \n
                                             
                                            Returns:
                                             
                                            The fpga_event_handle contained in this object 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#function-operator-fpga_event_handle","title":"function operator fpga_event_handle","text":"
                                            Coversion operator for converting to fpga_event_handle objects. 
                                            opae::fpga::types::event::operator fpga_event_handle () \n
                                             
                                            Returns:
                                             
                                            The fpga_event_handle contained in this object 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#function-os_object","title":"function os_object","text":"
                                            Get OS Object from the event object. 
                                            int opae::fpga::types::event::os_object () const\n
                                             
                                            Get an OS specific object from the event which can be used to subscribe for events. On Linux, the object corresponds to a file descriptor that can be used with select/poll/epoll calls.
                                             
                                            Returns:
                                             
                                            An integer object representing the OS object 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#function-event","title":"function ~event","text":"
                                            virtual opae::fpga::types::event::~event () \n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#function-register_event","title":"function register_event","text":"
                                            Factory function to create event objects. 
                                            static event::ptr_t opae::fpga::types::event::register_event (\n    handle::ptr_t h,\n    event::type_t t,\n    int flags=0\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • h A shared ptr of a resource handle 
                                              •  
                                            • t The resource type 
                                              •  
                                            • flags Event registration flags passed on to fpgaRegisterEvent
                                              •  
                                             
                                            Returns:
                                             
                                            A shared ptr to an event object 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/events.h
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/","title":"Struct opae::fpga::types::event::type_t","text":"
                                            ClassList > opae > fpga > types > event > type_t
                                             
                                            C++ struct that is interchangeable with fpga_event_type enum. 
                                             
                                               
                                            • #include <events.h>
                                              •  
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#public-static-attributes","title":"Public Static Attributes","text":"Type Name constexpr fpga_event_type error   = = FPGA_EVENT_ERROR constexpr fpga_event_type interrupt   = = FPGA_EVENT_INTERRUPT constexpr fpga_event_type power_thermal   = = FPGA_EVENT_POWER_THERMAL"},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#public-functions","title":"Public Functions","text":"Type Name operator fpga_event_type ()  type_t (fpga_event_type c_type)"},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#public-static-attributes-documentation","title":"Public Static Attributes Documentation","text":""},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#variable-error","title":"variable error","text":"
                                            constexpr fpga_event_type opae::fpga::types::event::type_t::error;\n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#variable-interrupt","title":"variable interrupt","text":"
                                            constexpr fpga_event_type opae::fpga::types::event::type_t::interrupt;\n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#variable-power_thermal","title":"variable power_thermal","text":"
                                            constexpr fpga_event_type opae::fpga::types::event::type_t::power_thermal;\n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#function-operator-fpga_event_type","title":"function operator fpga_event_type","text":"
                                            inline opae::fpga::types::event::type_t::operator fpga_event_type () \n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#function-type_t","title":"function type_t","text":"
                                            inline opae::fpga::types::event::type_t::type_t (\n    fpga_event_type c_type\n) \n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/events.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/","title":"Class opae::fpga::types::except","text":"
                                            ClassList > opae > fpga > types > except
                                             
                                            Generic OPAE exception. More...
                                             
                                               
                                            • #include <except.h>
                                              •  
                                             
                                            Inherits the following classes: std::exception
                                             
                                            Inherited by the following classes: opae::fpga::types::busy,  opae::fpga::types::exception,  opae::fpga::types::invalid_param,  opae::fpga::types::no_access,  opae::fpga::types::no_daemon,  opae::fpga::types::no_driver,  opae::fpga::types::no_memory,  opae::fpga::types::not_found,  opae::fpga::types::not_supported,  opae::fpga::types::reconf_error
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#public-static-attributes","title":"Public Static Attributes","text":"Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#public-functions","title":"Public Functions","text":"Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#protected-attributes","title":"Protected Attributes","text":"Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#detailed-description","title":"Detailed Description","text":"
                                            An except tracks the source line of origin and an optional fpga_result. If no fpga_result is given, then FPGA_EXCEPTION is used. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#public-static-attributes-documentation","title":"Public Static Attributes Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#variable-max_except","title":"variable MAX_EXCEPT","text":"
                                            const std::size_t opae::fpga::types::except::MAX_EXCEPT;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#function-except-13","title":"function except [\u2153]","text":"
                                            except constructor The fpga_result value is FPGA_EXCEPTION. 
                                            opae::fpga::types::except::except (\n    src_location loc\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • loc Location where the exception was constructed. 
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#function-except-23","title":"function except [\u2154]","text":"
                                            except constructor 
                                            opae::fpga::types::except::except (\n    fpga_result res,\n    src_location loc\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • res The fpga_result value associated with this exception. 
                                              •  
                                            • loc Location where the exception was constructed. 
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#function-except-33","title":"function except [3/3]","text":"
                                            except constructor 
                                            opae::fpga::types::except::except (\n    fpga_result res,\n    const char * msg,\n    src_location loc\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • res The fpga_result value associated with this exception. 
                                              •  
                                            • msg The error message as a string 
                                              •  
                                            • loc Location where the exception was constructed. 
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#function-operator-fpga_result","title":"function operator fpga_result","text":"
                                            inline opae::fpga::types::except::operator fpga_result () noexcept const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#function-what","title":"function what","text":"
                                            virtual const char * opae::fpga::types::except::what () noexcept override const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#protected-attributes-documentation","title":"Protected Attributes Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#variable-buf_","title":"variable buf_","text":"
                                            char opae::fpga::types::except::buf_[MAX_EXCEPT];\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#variable-loc_","title":"variable loc_","text":"
                                            src_location opae::fpga::types::except::loc_;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#variable-msg_","title":"variable msg_","text":"
                                            const char* opae::fpga::types::except::msg_;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#variable-res_","title":"variable res_","text":"
                                            fpga_result opae::fpga::types::except::res_;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/","title":"Class opae::fpga::types::exception","text":"
                                            ClassList > opae > fpga > types > exception
                                             
                                            exception exception More...
                                             
                                               
                                            • #include <except.h>
                                              •  
                                             
                                            Inherits the following classes: opae::fpga::types::except
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#public-functions","title":"Public Functions","text":"Type Name exception (src_location loc) noexceptexception constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#detailed-description","title":"Detailed Description","text":"
                                            exception tracks the source line of origin for exceptions thrown when the error code FPGA_EXCEPTION is returned from a call to an OPAE C API function 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#function-exception","title":"function exception","text":"
                                            exception constructor 
                                            inline opae::fpga::types::exception::exception (\n    src_location loc\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • loc Location where the exception was constructed. 
                                              •  
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/","title":"Struct opae::fpga::types::guid_t","text":"
                                            ClassList > opae > fpga > types > guid_t
                                             
                                            Representation of the guid member of a properties object. 
                                             
                                               
                                            • #include <pvalue.h>
                                              •  
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#public-functions","title":"Public Functions","text":"Type Name const uint8_t * c_type () constReturn a raw pointer to the guid. guid_t (fpga_properties * p) Construct the guid_t given its containing fpga_properties. void invalidate () Invalidate the cached local copy of the guid. bool is_set () constTracks whether the cached local copy of the guid is valid. operator uint8_t * () Return a raw pointer to the guid. guid_t & operator= (fpga_guid g) Assign from fpga_guid Sets the guid field of the associated properties object using the OPAE properties API. bool operator== (const fpga_guid & g) Compare contents with an fpga_guid. void parse (const char * str) Convert a string representation of a guid to binary. void update () Update the local cached copy of the guid."},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-c_type","title":"function c_type","text":"
                                            inline const uint8_t * opae::fpga::types::guid_t::c_type () const\n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-guid_t","title":"function guid_t","text":"
                                            inline opae::fpga::types::guid_t::guid_t (\n    fpga_properties * p\n) \n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-invalidate","title":"function invalidate","text":"
                                            inline void opae::fpga::types::guid_t::invalidate () \n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-is_set","title":"function is_set","text":"
                                            inline bool opae::fpga::types::guid_t::is_set () const\n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-operator-uint8_t","title":"function operator uint8_t *","text":"
                                            Return a raw pointer to the guid. 
                                            inline opae::fpga::types::guid_t::operator uint8_t * () \n
                                             
                                            Return value:
                                             
                                               
                                            • nullptr if the guid could not be queried. 
                                              •  
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-operator","title":"function operator=","text":"
                                            Assign from fpga_guid Sets the guid field of the associated properties object using the OPAE properties API. 
                                            inline guid_t & opae::fpga::types::guid_t::operator= (\n    fpga_guid g\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • g The given fpga_guid. 
                                              •  
                                             
                                            Returns:
                                             
                                            a reference to this guid_t. 
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-operator_1","title":"function operator==","text":"
                                            Compare contents with an fpga_guid. 
                                            inline bool opae::fpga::types::guid_t::operator== (\n    const fpga_guid & g\n) \n
                                             
                                            Return value:
                                             
                                               
                                            • The result of memcmp of the two objects. 
                                              •  
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-parse","title":"function parse","text":"
                                            Convert a string representation of a guid to binary. 
                                            inline void opae::fpga::types::guid_t::parse (\n    const char * str\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • str The guid string. 
                                              •  
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-update","title":"function update","text":"
                                            inline void opae::fpga::types::guid_t::update () \n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#friends-documentation","title":"Friends Documentation","text":""},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#friend-operator","title":"friend operator<<","text":"
                                            inline std::ostream & opae::fpga::types::guid_t::operator<< (\n    std::ostream & ostr,\n    const guid_t & g\n) \n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/pvalue.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/","title":"Class opae::fpga::types::handle","text":"
                                            ClassList > opae > fpga > types > handle
                                             
                                            An allocated accelerator resource. More...
                                             
                                               
                                            • #include <handle.h>
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< handle > ptr_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#public-functions","title":"Public Functions","text":"Type Name fpga_handle c_type () constRetrieve the underlying OPAE handle. fpga_result close () Close an accelerator resource (if opened) token::ptr_t get_token () constRetrieve the token corresponding to this handle object. handle (const handle &) = delete uint8_t * mmio_ptr (uint64_t offset, uint32_t csr_space=0) constRetrieve a pointer to the MMIO region. operator fpga_handle () constRetrieve the underlying OPAE handle. handle & operator= (const handle &) = delete uint32_t read_csr32 (uint64_t offset, uint32_t csr_space=0) constRead 32 bits from a CSR belonging to a resource associated with a handle. uint64_t read_csr64 (uint64_t offset, uint32_t csr_space=0) constRead 64 bits from a CSR belonging to a resource associated with a handle. void reconfigure (uint32_t slot, const uint8_t * bitstream, size_t size, int flags) Load a bitstream into an FPGA slot. virtual void reset () Reset the accelerator identified by this handle. void write_csr32 (uint64_t offset, uint32_t value, uint32_t csr_space=0) Write 32 bit to a CSR belonging to a resource associated with a handle. void write_csr512 (uint64_t offset, const void * value, uint32_t csr_space=0) Write 512 bits to a CSR belonging to a resource associated with a handle. void write_csr64 (uint64_t offset, uint64_t value, uint32_t csr_space=0) Write 64 bits to a CSR belonging to a resource associated with a handle. virtual ~handle ()"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#public-static-functions","title":"Public Static Functions","text":"Type Name handle::ptr_t open (fpga_token token, int flags) Open an accelerator resource, given a raw fpga_token. handle::ptr_t open (token::ptr_t token, int flags) Open an accelerator resource, given a token object."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#detailed-description","title":"Detailed Description","text":"
                                            Represents an accelerator resource that has been allocated by OPAE. Depending on the type of resource, its register space may be read/written using a handle object. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                            typedef std::shared_ptr<handle> opae::fpga::types::handle::ptr_t;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-c_type","title":"function c_type","text":"
                                            inline fpga_handle opae::fpga::types::handle::c_type () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-close","title":"function close","text":"
                                            Close an accelerator resource (if opened) 
                                            fpga_result opae::fpga::types::handle::close () \n
                                             
                                            Returns:
                                             
                                            fpga_result indication the result of closing the handle or FPGA_EXCEPTION if handle is not opened
                                             
                                            Note:
                                             
                                            This is available for explicitly closing a handle. The destructor for handle will call close. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-get_token","title":"function get_token","text":"
                                            token::ptr_t opae::fpga::types::handle::get_token () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-handle-12","title":"function handle [\u00bd]","text":"
                                            opae::fpga::types::handle::handle (\n    const handle &\n) = delete\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-mmio_ptr","title":"function mmio_ptr","text":"
                                            Retrieve a pointer to the MMIO region. 
                                            uint8_t * opae::fpga::types::handle::mmio_ptr (\n    uint64_t offset,\n    uint32_t csr_space=0\n) const\n
                                             
                                            Parameters:
                                             
                                               
                                            • offset The byte offset to add to MMIO base. 
                                              •  
                                            • csr_space The desired CSR space. Default is 0. 
                                              •  
                                             
                                            Returns:
                                             
                                            MMIO base + offset 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-operator-fpga_handle","title":"function operator fpga_handle","text":"
                                            inline opae::fpga::types::handle::operator fpga_handle () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-operator","title":"function operator=","text":"
                                            handle & opae::fpga::types::handle::operator= (\n    const handle &\n) = delete\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-read_csr32","title":"function read_csr32","text":"
                                            Read 32 bits from a CSR belonging to a resource associated with a handle. 
                                            uint32_t opae::fpga::types::handle::read_csr32 (\n    uint64_t offset,\n    uint32_t csr_space=0\n) const\n
                                             
                                            Parameters:
                                             
                                               
                                            • offset The register offset 
                                              •  
                                            • csr_space The CSR space to read from. Default is 0.
                                              •  
                                             
                                            Returns:
                                             
                                            The 32-bit value read from the CSR 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-read_csr64","title":"function read_csr64","text":"
                                            Read 64 bits from a CSR belonging to a resource associated with a handle. 
                                            uint64_t opae::fpga::types::handle::read_csr64 (\n    uint64_t offset,\n    uint32_t csr_space=0\n) const\n
                                             
                                            Parameters:
                                             
                                               
                                            • offset The register offset 
                                              •  
                                            • csr_space The CSR space to read from. Default is 0.
                                              •  
                                             
                                            Returns:
                                             
                                            The 64-bit value read from the CSR 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-reconfigure","title":"function reconfigure","text":"
                                            Load a bitstream into an FPGA slot. 
                                            void opae::fpga::types::handle::reconfigure (\n    uint32_t slot,\n    const uint8_t * bitstream,\n    size_t size,\n    int flags\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • slot The slot number to program 
                                              •  
                                            • bitstream The bitstream binary data 
                                              •  
                                            • size The size of the bitstream 
                                              •  
                                            • flags Flags that control behavior of reconfiguration. Value of 0 indicates no flags. FPGA_RECONF_FORCE indicates that the bitstream is programmed into the slot without checking if the resource is currently in use.
                                              •  
                                             
                                            Exception:
                                             
                                               
                                            • invalid_param if the handle is not an FPGA device handle or if the other parameters are not valid. 
                                              •  
                                            • exception if an internal error is encountered. 
                                              •  
                                            • busy if the accelerator for the given slot is in use. 
                                              •  
                                            • reconf_error if errors are reported by the driver (CRC or protocol errors). 
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-reset","title":"function reset","text":"
                                            virtual void opae::fpga::types::handle::reset () \n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-write_csr32","title":"function write_csr32","text":"
                                            Write 32 bit to a CSR belonging to a resource associated with a handle. 
                                            void opae::fpga::types::handle::write_csr32 (\n    uint64_t offset,\n    uint32_t value,\n    uint32_t csr_space=0\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • offset The register offset. 
                                              •  
                                            • value The 32-bit value to write to the register. 
                                              •  
                                            • csr_space The CSR space to read from. Default is 0. 
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-write_csr512","title":"function write_csr512","text":"
                                            Write 512 bits to a CSR belonging to a resource associated with a handle. 
                                            void opae::fpga::types::handle::write_csr512 (\n    uint64_t offset,\n    const void * value,\n    uint32_t csr_space=0\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • offset The register offset. 
                                              •  
                                            • value Pointer to the 512-bit value to write to the register. 
                                              •  
                                            • csr_space The CSR space to read from. Default is 0. 
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-write_csr64","title":"function write_csr64","text":"
                                            Write 64 bits to a CSR belonging to a resource associated with a handle. 
                                            void opae::fpga::types::handle::write_csr64 (\n    uint64_t offset,\n    uint64_t value,\n    uint32_t csr_space=0\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • offset The register offset. 
                                              •  
                                            • value The 64-bit value to write to the register. 
                                              •  
                                            • csr_space The CSR space to read from. Default is 0. 
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-handle","title":"function ~handle","text":"
                                            virtual opae::fpga::types::handle::~handle () \n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-open-12","title":"function open [\u00bd]","text":"
                                            Open an accelerator resource, given a raw fpga_token. 
                                            static handle::ptr_t opae::fpga::types::handle::open (\n    fpga_token token,\n    int flags\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • token A token describing the accelerator resource to be allocated.
                                              •  
                                            • flags The flags parameter to fpgaOpen().
                                              •  
                                             
                                            Returns:
                                             
                                            pointer to the mmio base + offset for the given csr space 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-open-22","title":"function open [2/2]","text":"
                                            Open an accelerator resource, given a token object. 
                                            static handle::ptr_t opae::fpga::types::handle::open (\n    token::ptr_t token,\n    int flags\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • token A token object describing the accelerator resource to be allocated.
                                              •  
                                            • flags The flags parameter to fpgaOpen().
                                              •  
                                             
                                            Returns:
                                             
                                            shared ptr to a handle object 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/handle.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/","title":"Class opae::fpga::types::invalid_param","text":"
                                            ClassList > opae > fpga > types > invalid_param
                                             
                                            invalid_param exceptionMore...
                                             
                                               
                                            • #include <except.h>
                                              •  
                                             
                                            Inherits the following classes: opae::fpga::types::except
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#public-functions","title":"Public Functions","text":"Type Name invalid_param (src_location loc) noexceptinvalid_param constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#detailed-description","title":"Detailed Description","text":"
                                            invalid_param tracks the source line of origin for exceptions thrown when the error code FPGA_INVALID_PARAM is returned from a call to an OPAE C API function 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#function-invalid_param","title":"function invalid_param","text":"
                                            invalid_param constructor 
                                            inline opae::fpga::types::invalid_param::invalid_param (\n    src_location loc\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • loc Location where the exception was constructed. 
                                              •  
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/","title":"Class opae::fpga::types::no_access","text":"
                                            ClassList > opae > fpga > types > no_access
                                             
                                            no_access exceptionMore...
                                             
                                               
                                            • #include <except.h>
                                              •  
                                             
                                            Inherits the following classes: opae::fpga::types::except
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#public-functions","title":"Public Functions","text":"Type Name no_access (src_location loc) noexceptno_access constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#detailed-description","title":"Detailed Description","text":"
                                            no_access tracks the source line of origin for exceptions thrown when the error code FPGA_NO_ACCESS is returned from a call to an OPAE C API function 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#function-no_access","title":"function no_access","text":"
                                            no_access constructor 
                                            inline opae::fpga::types::no_access::no_access (\n    src_location loc\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • loc Location where the exception was constructed. 
                                              •  
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/","title":"Class opae::fpga::types::no_daemon","text":"
                                            ClassList > opae > fpga > types > no_daemon
                                             
                                            no_daemon exceptionMore...
                                             
                                               
                                            • #include <except.h>
                                              •  
                                             
                                            Inherits the following classes: opae::fpga::types::except
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#public-functions","title":"Public Functions","text":"Type Name no_daemon (src_location loc) noexceptno_daemon constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#detailed-description","title":"Detailed Description","text":"
                                            no_daemon tracks the source line of origin for exceptions thrown when the error code FPGA_NO_DAEMON is returned from a call to an OPAE C API function 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#function-no_daemon","title":"function no_daemon","text":"
                                            no_daemon constructor 
                                            inline opae::fpga::types::no_daemon::no_daemon (\n    src_location loc\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • loc Location where the exception was constructed. 
                                              •  
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/","title":"Class opae::fpga::types::no_driver","text":"
                                            ClassList > opae > fpga > types > no_driver
                                             
                                            no_driver exceptionMore...
                                             
                                               
                                            • #include <except.h>
                                              •  
                                             
                                            Inherits the following classes: opae::fpga::types::except
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#public-functions","title":"Public Functions","text":"Type Name no_driver (src_location loc) noexceptno_driver constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#detailed-description","title":"Detailed Description","text":"
                                            no_driver tracks the source line of origin for exceptions thrown when the error code FPGA_NO_DRIVER is returned from a call to an OPAE C API function 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#function-no_driver","title":"function no_driver","text":"
                                            no_driver constructor 
                                            inline opae::fpga::types::no_driver::no_driver (\n    src_location loc\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • loc Location where the exception was constructed. 
                                              •  
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/","title":"Class opae::fpga::types::no_memory","text":"
                                            ClassList > opae > fpga > types > no_memory
                                             
                                            no_memory exceptionMore...
                                             
                                               
                                            • #include <except.h>
                                              •  
                                             
                                            Inherits the following classes: opae::fpga::types::except
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#public-functions","title":"Public Functions","text":"Type Name no_memory (src_location loc) noexceptno_memory constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#detailed-description","title":"Detailed Description","text":"
                                            no_memory tracks the source line of origin for exceptions thrown when the error code FPGA_NO_MEMORY is returned from a call to an OPAE C API function 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#function-no_memory","title":"function no_memory","text":"
                                            no_memory constructor 
                                            inline opae::fpga::types::no_memory::no_memory (\n    src_location loc\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • loc Location where the exception was constructed. 
                                              •  
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/","title":"Class opae::fpga::types::not_found","text":"
                                            ClassList > opae > fpga > types > not_found
                                             
                                            not_found exceptionMore...
                                             
                                               
                                            • #include <except.h>
                                              •  
                                             
                                            Inherits the following classes: opae::fpga::types::except
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#public-functions","title":"Public Functions","text":"Type Name not_found (src_location loc) noexceptnot_found constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#detailed-description","title":"Detailed Description","text":"
                                            not_found tracks the source line of origin for exceptions thrown when the error code FPGA_NOT_FOUND is returned from a call to an OPAE C API function 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#function-not_found","title":"function not_found","text":"
                                            not_found constructor 
                                            inline opae::fpga::types::not_found::not_found (\n    src_location loc\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • loc Location where the exception was constructed. 
                                              •  
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/","title":"Class opae::fpga::types::not_supported","text":"
                                            ClassList > opae > fpga > types > not_supported
                                             
                                            not_supported exceptionMore...
                                             
                                               
                                            • #include <except.h>
                                              •  
                                             
                                            Inherits the following classes: opae::fpga::types::except
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#public-functions","title":"Public Functions","text":"Type Name not_supported (src_location loc) noexceptnot_supported constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#detailed-description","title":"Detailed Description","text":"
                                            not_supported tracks the source line of origin for exceptions thrown when the error code FPGA_NOT_SUPPORTED is returned from a call to an OPAE C API function 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#function-not_supported","title":"function not_supported","text":"
                                            not_supported constructor 
                                            inline opae::fpga::types::not_supported::not_supported (\n    src_location loc\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • loc Location where the exception was constructed. 
                                              •  
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/","title":"Class opae::fpga::types::properties","text":"
                                            ClassList > opae > fpga > types > properties
                                             
                                            Wraps an OPAE fpga_properties object. More...
                                             
                                               
                                            • #include <properties.h>
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< properties > ptr_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-attributes","title":"Public Attributes","text":"Type Name pvalue< fpga_accelerator_state > accelerator_state pvalue< uint64_t > bbs_id pvalue< fpga_version > bbs_version pvalue< uint8_t > bus pvalue< uint64_t > capabilities pvalue< uint8_t > device pvalue< uint16_t > device_id pvalue< uint8_t > function guid_t guid pvalue< fpga_interface > interface pvalue< uint64_t > local_memory_size pvalue< char * > model pvalue< uint32_t > num_errors pvalue< uint32_t > num_interrupts pvalue< uint32_t > num_mmio pvalue< uint32_t > num_slots pvalue< uint64_t > object_id pvalue< fpga_token > parent pvalue< uint16_t > segment pvalue< uint8_t > socket_id pvalue< uint16_t > subsystem_device_id pvalue< uint16_t > subsystem_vendor_id pvalue< fpga_objtype > type pvalue< uint16_t > vendor_id"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-static-attributes","title":"Public Static Attributes","text":"Type Name const std::vector< properties::ptr_t > none An empty vector of properties."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-functions","title":"Public Functions","text":"Type Name fpga_properties c_type () constGet the underlying fpga_properties object. operator fpga_properties () constGet the underlying fpga_properties object. properties & operator= (const properties & p) = delete properties (const properties & p) = delete ~properties ()"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-static-functions","title":"Public Static Functions","text":"Type Name properties::ptr_t get () Create a new properties object. properties::ptr_t get (fpga_guid guid_in) Create a new properties object from a guid. properties::ptr_t get (fpga_objtype objtype) Create a new properties object from an fpga_objtype. properties::ptr_t get (std::shared_ptr< token > t) Retrieve the properties for a given token object. properties::ptr_t get (fpga_token t) Retrieve the properties for a given fpga_token. properties::ptr_t get (std::shared_ptr< handle > h) Retrieve the properties for a given handle object."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#detailed-description","title":"Detailed Description","text":"
                                            properties are information describing an accelerator resource that is identified by its token. The properties are used during enumeration to narrow the search for an accelerator resource, and after enumeration to provide the configuration of that resource. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                            typedef std::shared_ptr<properties> opae::fpga::types::properties::ptr_t;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-accelerator_state","title":"variable accelerator_state","text":"
                                            pvalue<fpga_accelerator_state> opae::fpga::types::properties::accelerator_state;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-bbs_id","title":"variable bbs_id","text":"
                                            pvalue<uint64_t> opae::fpga::types::properties::bbs_id;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-bbs_version","title":"variable bbs_version","text":"
                                            pvalue<fpga_version> opae::fpga::types::properties::bbs_version;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-bus","title":"variable bus","text":"
                                            pvalue<uint8_t> opae::fpga::types::properties::bus;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-capabilities","title":"variable capabilities","text":"
                                            pvalue<uint64_t> opae::fpga::types::properties::capabilities;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-device","title":"variable device","text":"
                                            pvalue<uint8_t> opae::fpga::types::properties::device;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-device_id","title":"variable device_id","text":"
                                            pvalue<uint16_t> opae::fpga::types::properties::device_id;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-function","title":"variable function","text":"
                                            pvalue<uint8_t> opae::fpga::types::properties::function;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-guid","title":"variable guid","text":"
                                            guid_t opae::fpga::types::properties::guid;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-interface","title":"variable interface","text":"
                                            pvalue<fpga_interface> opae::fpga::types::properties::interface;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-local_memory_size","title":"variable local_memory_size","text":"
                                            pvalue<uint64_t> opae::fpga::types::properties::local_memory_size;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-model","title":"variable model","text":"
                                            pvalue<char *> opae::fpga::types::properties::model;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-num_errors","title":"variable num_errors","text":"
                                            pvalue<uint32_t> opae::fpga::types::properties::num_errors;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-num_interrupts","title":"variable num_interrupts","text":"
                                            pvalue<uint32_t> opae::fpga::types::properties::num_interrupts;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-num_mmio","title":"variable num_mmio","text":"
                                            pvalue<uint32_t> opae::fpga::types::properties::num_mmio;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-num_slots","title":"variable num_slots","text":"
                                            pvalue<uint32_t> opae::fpga::types::properties::num_slots;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-object_id","title":"variable object_id","text":"
                                            pvalue<uint64_t> opae::fpga::types::properties::object_id;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-parent","title":"variable parent","text":"
                                            pvalue<fpga_token> opae::fpga::types::properties::parent;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-segment","title":"variable segment","text":"
                                            pvalue<uint16_t> opae::fpga::types::properties::segment;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-socket_id","title":"variable socket_id","text":"
                                            pvalue<uint8_t> opae::fpga::types::properties::socket_id;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-subsystem_device_id","title":"variable subsystem_device_id","text":"
                                            pvalue<uint16_t> opae::fpga::types::properties::subsystem_device_id;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-subsystem_vendor_id","title":"variable subsystem_vendor_id","text":"
                                            pvalue<uint16_t> opae::fpga::types::properties::subsystem_vendor_id;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-type","title":"variable type","text":"
                                            pvalue<fpga_objtype> opae::fpga::types::properties::type;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-vendor_id","title":"variable vendor_id","text":"
                                            pvalue<uint16_t> opae::fpga::types::properties::vendor_id;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-static-attributes-documentation","title":"Public Static Attributes Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-none","title":"variable none","text":"
                                            An empty vector of properties. 
                                            const std::vector<properties::ptr_t> opae::fpga::types::properties::none;\n
                                             
                                            Useful for enumerating based on a \"match all\" criteria. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-c_type","title":"function c_type","text":"
                                            inline fpga_properties opae::fpga::types::properties::c_type () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-operator-fpga_properties","title":"function operator fpga_properties","text":"
                                            inline opae::fpga::types::properties::operator fpga_properties () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-operator","title":"function operator=","text":"
                                            properties & opae::fpga::types::properties::operator= (\n    const properties & p\n) = delete\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-properties-12","title":"function properties [\u00bd]","text":"
                                            opae::fpga::types::properties::properties (\n    const properties & p\n) = delete\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-properties","title":"function ~properties","text":"
                                            opae::fpga::types::properties::~properties () \n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-get-16","title":"function get [\u2159]","text":"
                                            Create a new properties object. 
                                            static properties::ptr_t opae::fpga::types::properties::get () \n
                                             
                                            Returns:
                                             
                                            A properties smart pointer. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-get-26","title":"function get [2/6]","text":"
                                            Create a new properties object from a guid. 
                                            static properties::ptr_t opae::fpga::types::properties::get (\n    fpga_guid guid_in\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • guid_in A guid to set in the properties 
                                              •  
                                             
                                            Returns:
                                             
                                            A properties smart pointer with its guid initialized to guid_in 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-get-36","title":"function get [3/6]","text":"
                                            Create a new properties object from an fpga_objtype. 
                                            static properties::ptr_t opae::fpga::types::properties::get (\n    fpga_objtype objtype\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • objtype An object type to set in the properties 
                                              •  
                                             
                                            Returns:
                                             
                                            A properties smart pointer with its object type set to objtype. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-get-46","title":"function get [4/6]","text":"
                                            Retrieve the properties for a given token object. 
                                            static properties::ptr_t opae::fpga::types::properties::get (\n    std::shared_ptr< token > t\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • t A token identifying the accelerator resource. 
                                              •  
                                             
                                            Returns:
                                             
                                            A properties smart pointer for the given token. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-get-56","title":"function get [\u215a]","text":"
                                            Retrieve the properties for a given fpga_token. 
                                            static properties::ptr_t opae::fpga::types::properties::get (\n    fpga_token t\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • t An fpga_token identifying the accelerator resource. 
                                              •  
                                             
                                            Returns:
                                             
                                            A properties smart pointer for the given fpga_token. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-get-66","title":"function get [6/6]","text":"
                                            Retrieve the properties for a given handle object. 
                                            static properties::ptr_t opae::fpga::types::properties::get (\n    std::shared_ptr< handle > h\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • h A handle identifying the accelerator resource. 
                                              •  
                                             
                                            Returns:
                                             
                                            A properties smart pointer for the given handle. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/properties.h
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/","title":"Struct opae::fpga::types::pvalue","text":"
                                            template <typename T typename T>
                                             
                                            ClassList > opae > fpga > types > pvalue
                                             
                                            Wraps OPAE properties defined in the OPAE C API by associating an fpga_properties reference with the getters and setters defined for a property.More...
                                             
                                               
                                            • #include <pvalue.h>
                                              •  
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#public-types","title":"Public Types","text":"Type Name typedef std::conditional< std::is_same< T, char * >::value, typename std::string, T >::type copy_t Define the type of our copy variable For char* types use std::string as the copy. typedef std::conditional< std::is_same< T, char * >::value, fpga_result(*)(fpga_properties, T), fpga_result(*)(fpga_properties, T *)>::type getter_t Define getter function as getter_t For char* types, do not use T* as the second argument but instead use T. typedef fpga_result(* setter_t Define the setter function as setter_t."},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#public-functions","title":"Public Functions","text":"Type Name fpga_result get_value (T & value) const void invalidate () Invalidate the cached local copy of the pvalue. bool is_set () constTracks whether the cached local copy of the pvalue is valid. operator copy_t () Implicit converter operator - calls the wrapped getter. pvalue< T > & operator= (const T & v) Overload of = operator that calls the wrapped setter. bool operator== (const T & other) Compare a property for equality with a value. pvalue ()  pvalue (fpga_properties * p, getter_t g, setter_t s) pvalue contructor that takes in a reference to fpga_properties and corresponding accessor methods for a property void update ()  void update () Template specialization of char* type property updater."},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#detailed-description","title":"Detailed Description","text":"
                                            Template parameters:
                                             
                                               
                                            • T The type of the property value being wrapped 
                                              •  
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#typedef-copy_t","title":"typedef copy_t","text":"
                                            typedef std::conditional<std::is_same<T, char *>::value, typename std::string, T>::type opae::fpga::types::pvalue< T >::copy_t;\n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#typedef-getter_t","title":"typedef getter_t","text":"
                                            typedef std::conditional< std::is_same<T, char *>::value, fpga_result (*)(fpga_properties, T), fpga_result (*)(fpga_properties, T *)>::type opae::fpga::types::pvalue< T >::getter_t;\n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#typedef-setter_t","title":"typedef setter_t","text":"
                                            typedef fpga_result(* opae::fpga::types::pvalue< T >::setter_t) (fpga_properties, T);\n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-get_value","title":"function get_value","text":"
                                            inline fpga_result opae::fpga::types::pvalue::get_value (\n    T & value\n) const\n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-invalidate","title":"function invalidate","text":"
                                            inline void opae::fpga::types::pvalue::invalidate () \n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-is_set","title":"function is_set","text":"
                                            inline bool opae::fpga::types::pvalue::is_set () const\n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-operator-copy_t","title":"function operator copy_t","text":"
                                            Implicit converter operator - calls the wrapped getter. 
                                            inline opae::fpga::types::pvalue::operator copy_t () \n
                                             
                                            Returns:
                                             
                                            The property value after calling the getter or a default value of the value type 
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-operator","title":"function operator=","text":"
                                            Overload of = operator that calls the wrapped setter. 
                                            inline pvalue < T > & opae::fpga::types::pvalue::operator= (\n    const T & v\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • v The value to set
                                              •  
                                             
                                            Returns:
                                             
                                            A reference to itself 
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-operator_1","title":"function operator==","text":"
                                            Compare a property for equality with a value. 
                                            inline bool opae::fpga::types::pvalue::operator== (\n    const T & other\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • other The value being compared to
                                              •  
                                             
                                            Returns:
                                             
                                            Whether or not the property is equal to the value 
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-pvalue-12","title":"function pvalue [\u00bd]","text":"
                                            inline opae::fpga::types::pvalue::pvalue () \n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-pvalue-22","title":"function pvalue [2/2]","text":"
                                            pvalue contructor that takes in a reference to fpga_properties and corresponding accessor methods for a property 
                                            inline opae::fpga::types::pvalue::pvalue (\n    fpga_properties * p,\n    getter_t g,\n    setter_t s\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • p A reference to an fpga_properties 
                                              •  
                                            • g The getter function 
                                              •  
                                            • s The setter function 
                                              •  
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-update-12","title":"function update [\u00bd]","text":"
                                            inline void opae::fpga::types::pvalue::update () \n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-update-22","title":"function update [2/2]","text":"
                                            Template specialization of char* type property updater. 
                                            inline void opae::fpga::types::pvalue::update () \n
                                             
                                            Returns:
                                             
                                            The result of the property getter function. 
                                             
                                                ## Friends Documentation\n
                                            "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#friend-operator","title":"friend operator<<","text":"
                                            Stream overalod operator. 
                                            inline std::ostream & opae::fpga::types::pvalue::operator<< (\n    std::ostream & ostr,\n    const pvalue < T > & p\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • ostr The output stream 
                                              •  
                                            • p A reference to a pvalue<T> object
                                              •  
                                             
                                            Returns:
                                             
                                            The stream operator after streaming the property value 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/pvalue.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/","title":"Class opae::fpga::types::reconf_error","text":"
                                            ClassList > opae > fpga > types > reconf_error
                                             
                                            reconf_error exceptionMore...
                                             
                                               
                                            • #include <except.h>
                                              •  
                                             
                                            Inherits the following classes: opae::fpga::types::except
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#public-functions","title":"Public Functions","text":"Type Name reconf_error (src_location loc) noexceptreconf_error constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                            See opae::fpga::types::except
                                             Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#detailed-description","title":"Detailed Description","text":"
                                            reconf_error tracks the source line of origin for exceptions thrown when the error code FPGA_RECONF_ERROR is returned from a call to an OPAE C API function 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#function-reconf_error","title":"function reconf_error","text":"
                                            reconf_error constructor 
                                            inline opae::fpga::types::reconf_error::reconf_error (\n    src_location loc\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • loc Location where the exception was constructed. 
                                              •  
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/","title":"Class opae::fpga::types::shared_buffer","text":"
                                            ClassList > opae > fpga > types > shared_buffer
                                             
                                            Host/AFU shared memory blocks. More...
                                             
                                               
                                            • #include <shared_buffer.h>
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< shared_buffer > ptr_t typedef std::size_t size_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#public-functions","title":"Public Functions","text":"Type Name uint8_t * c_type () constRetrieve the virtual address of the buffer base. int compare (ptr_t other, size_t len) constCompare this shared_buffer (the first len bytes) to that held in other, using memcmp(). void fill (int c) Write c to each byte location in the buffer. uint64_t io_address () constRetrieve the address of the buffer suitable for programming into the accelerator device. shared_buffer & operator= (const shared_buffer &) = delete handle::ptr_t owner () constRetrieve the handle smart pointer associated with this buffer. T read (size_t offset) constRead a T-sized block of memory at the given location. void release () Disassociate the shared_buffer object from the resource used to create it. shared_buffer (const shared_buffer &) = delete size_t size () constRetrieve the length of the buffer in bytes. void write (const T & value, size_t offset) Write a T-sized block of memory to the given location. uint64_t wsid () constRetrieve the underlying buffer's workspace id. virtual ~shared_buffer () shared_buffer destructor."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#public-static-functions","title":"Public Static Functions","text":"Type Name shared_buffer::ptr_t allocate (handle::ptr_t handle, size_t len, bool read_only=false) shared_buffer factory method - allocate ashared_buffer . shared_buffer::ptr_t attach (handle::ptr_t handle, uint8_t * base, size_t len, bool read_only=false) Attach a pre-allocated buffer to a shared_buffer object."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#protected-attributes","title":"Protected Attributes","text":"Type Name handle::ptr_t handle_ uint64_t io_address_ size_t len_ uint8_t * virt_ uint64_t wsid_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#protected-functions","title":"Protected Functions","text":"Type Name shared_buffer (handle::ptr_t handle, size_t len, uint8_t * virt, uint64_t wsid, uint64_t io_address)"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#detailed-description","title":"Detailed Description","text":"
                                            shared_buffer abstracts a memory block that may be shared between the host cpu and an accelerator. The block may be allocated by the shared_buffer class itself (see allocate), or it may be allocated elsewhere and then attached to a shared_buffer object via attach. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                            typedef std::shared_ptr<shared_buffer> opae::fpga::types::shared_buffer::ptr_t;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#typedef-size_t","title":"typedef size_t","text":"
                                            typedef std::size_t opae::fpga::types::shared_buffer::size_t;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-c_type","title":"function c_type","text":"
                                            Retrieve the virtual address of the buffer base. 
                                            inline uint8_t * opae::fpga::types::shared_buffer::c_type () const\n
                                             
                                            Note:
                                             
                                            Instances of a shared buffer can only be created using either 'allocate' or 'attach' static factory function. Because these functions return a shared pointer (std::shared_ptr) to the instance, references to an instance are counted automatically by design of the shared_ptr class. Calling 'c_type()' function is provided to get access to the raw data but isn't used in tracking its reference count. Assigning this to a variable should be done in limited scopes as this variable can be defined in an outer scope and may outlive the shared_buffer object. Once the reference count in the shared_ptr reaches zero, the shared_buffer object will be released and deallocated, turning any variables assigned from a call to 'c_type()' into dangling pointers. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-compare","title":"function compare","text":"
                                            int opae::fpga::types::shared_buffer::compare (\n    ptr_t other,\n    size_t len\n) const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-fill","title":"function fill","text":"
                                            void opae::fpga::types::shared_buffer::fill (\n    int c\n) \n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-io_address","title":"function io_address","text":"
                                            inline uint64_t opae::fpga::types::shared_buffer::io_address () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-operator","title":"function operator=","text":"
                                            shared_buffer & opae::fpga::types::shared_buffer::operator= (\n    const shared_buffer &\n) = delete\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-owner","title":"function owner","text":"
                                            inline handle::ptr_t opae::fpga::types::shared_buffer::owner () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-read","title":"function read","text":"
                                            Read a T-sized block of memory at the given location. 
                                            template<typename T typename T>\ninline T opae::fpga::types::shared_buffer::read (\n    size_t offset\n) const\n
                                             
                                            Parameters:
                                             
                                               
                                            • offset The byte offset from the start of the buffer. 
                                              •  
                                             
                                            Returns:
                                             
                                            A T from buffer base + offset. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-release","title":"function release","text":"
                                            Disassociate the shared_buffer object from the resource used to create it. 
                                            void opae::fpga::types::shared_buffer::release () \n
                                             
                                            If the buffer was allocated using the allocate function then the buffer is freed. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-shared_buffer-12","title":"function shared_buffer [\u00bd]","text":"
                                            opae::fpga::types::shared_buffer::shared_buffer (\n    const shared_buffer &\n) = delete\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-size","title":"function size","text":"
                                            inline size_t opae::fpga::types::shared_buffer::size () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-write","title":"function write","text":"
                                            Write a T-sized block of memory to the given location. 
                                            template<typename T typename T>\ninline void opae::fpga::types::shared_buffer::write (\n    const T & value,\n    size_t offset\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • value The value to write. 
                                              •  
                                            • offset The byte offset from the start of the buffer. 
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-wsid","title":"function wsid","text":"
                                            inline uint64_t opae::fpga::types::shared_buffer::wsid () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-shared_buffer","title":"function ~shared_buffer","text":"
                                            virtual opae::fpga::types::shared_buffer::~shared_buffer () \n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-allocate","title":"function allocate","text":"
                                            shared_buffer factory method - allocate ashared_buffer . 
                                            static shared_buffer::ptr_t opae::fpga::types::shared_buffer::allocate (\n    handle::ptr_t handle,\n    size_t len,\n    bool read_only=false\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • handle The handle used to allocate the buffer. 
                                              •  
                                            • len The length in bytes of the requested buffer. 
                                              •  
                                             
                                            Returns:
                                             
                                            A valid shared_buffer smart pointer on success, or an empty smart pointer on failure. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-attach","title":"function attach","text":"
                                            Attach a pre-allocated buffer to a shared_buffer object. 
                                            static shared_buffer::ptr_t opae::fpga::types::shared_buffer::attach (\n    handle::ptr_t handle,\n    uint8_t * base,\n    size_t len,\n    bool read_only=false\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • handle The handle used to attach the buffer. 
                                              •  
                                            • base The base of the pre-allocated memory. 
                                              •  
                                            • len The size of the pre-allocated memory, which must be a multiple of the page size. 
                                              •  
                                             
                                            Returns:
                                             
                                            A valid shared_buffer smart pointer on success, or an empty smart pointer on failure. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#protected-attributes-documentation","title":"Protected Attributes Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#variable-handle_","title":"variable handle_","text":"
                                            handle::ptr_t opae::fpga::types::shared_buffer::handle_;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#variable-io_address_","title":"variable io_address_","text":"
                                            uint64_t opae::fpga::types::shared_buffer::io_address_;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#variable-len_","title":"variable len_","text":"
                                            size_t opae::fpga::types::shared_buffer::len_;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#variable-virt_","title":"variable virt_","text":"
                                            uint8_t* opae::fpga::types::shared_buffer::virt_;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#variable-wsid_","title":"variable wsid_","text":"
                                            uint64_t opae::fpga::types::shared_buffer::wsid_;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#protected-functions-documentation","title":"Protected Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-shared_buffer-22","title":"function shared_buffer [2/2]","text":"
                                            opae::fpga::types::shared_buffer::shared_buffer (\n    handle::ptr_t handle,\n    size_t len,\n    uint8_t * virt,\n    uint64_t wsid,\n    uint64_t io_address\n) \n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/shared_buffer.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/","title":"Class opae::fpga::types::src_location","text":"
                                            ClassList > opae > fpga > types > src_location
                                             
                                            Identify a particular line in a source file. 
                                             
                                               
                                            • #include <except.h>
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#public-functions","title":"Public Functions","text":"Type Name const char * file () noexcept constRetrieve the file name component of the location. const char * fn () noexcept constRetrieve the function name component of the location. int line () noexcept constRetrieve the line number component of the location. src_location & operator= (const src_location & other) noexcept src_location (const char * file, const char * fn, int line) noexceptsrc_location constructor src_location (const src_location & other) noexcept"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#function-file","title":"function file","text":"
                                            const char * opae::fpga::types::src_location::file () noexcept const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#function-fn","title":"function fn","text":"
                                            inline const char * opae::fpga::types::src_location::fn () noexcept const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#function-line","title":"function line","text":"
                                            inline int opae::fpga::types::src_location::line () noexcept const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#function-operator","title":"function operator=","text":"
                                            src_location & opae::fpga::types::src_location::operator= (\n    const src_location & other\n) noexcept\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#function-src_location-12","title":"function src_location [\u00bd]","text":"
                                            src_location constructor 
                                            opae::fpga::types::src_location::src_location (\n    const char * file,\n    const char * fn,\n    int line\n) noexcept\n
                                             
                                            Parameters:
                                             
                                               
                                            • file The source file name, typically FILE. 
                                              •  
                                            • fn The current function, typically func. 
                                              •  
                                            • line The current line number, typically LINE. 
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#function-src_location-22","title":"function src_location [2/2]","text":"
                                            opae::fpga::types::src_location::src_location (\n    const src_location & other\n) noexcept\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/","title":"Class opae::fpga::types::sysobject","text":"
                                            ClassList > opae > fpga > types > sysobject
                                             
                                            Wraps the OPAE fpga_object primitive. More...
                                             
                                               
                                            • #include <sysobject.h>
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< sysobject > ptr_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#public-functions","title":"Public Functions","text":"Type Name std::vector< uint8_t > bytes (int flags=0) constGet all raw bytes from the object. std::vector< uint8_t > bytes (uint32_t offset, uint32_t size, int flags=0) constGet a subset of raw bytes from the object. fpga_object c_type () constRetrieve the underlying fpga_object primitive. sysobject::ptr_t get (const std::string & name, int flags=0) Get a sysobject from an object. sysobject::ptr_t get (int index) Get a sysobject from a container object. operator fpga_object () constRetrieve the underlying fpga_object primitive. sysobject & operator= (const sysobject & o) = delete uint64_t read64 (int flags=0) constRead a 64-bit value from an FPGA object. uint32_t size () constGet the size (in bytes) of the object. sysobject () = delete sysobject (const sysobject & o) = delete enum fpga_sysobject_type type () constGet the object type (attribute or container) void write64 (uint64_t value, int flags=0) constWrite 64-bit value to an FPGA object. virtual ~sysobject ()"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#public-static-functions","title":"Public Static Functions","text":"Type Name sysobject::ptr_t get (token::ptr_t t, const std::string & name, int flags=0) Get a sysobject from a token. sysobject::ptr_t get (handle::ptr_t h, const std::string & name, int flags=0) Get a sysobject from a handle."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#detailed-description","title":"Detailed Description","text":"
                                            sysobject's are created from a call to fpgaTokenGetObject, fpgaHandleGetObject, or fpgaObjectGetObject 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                            typedef std::shared_ptr<sysobject> opae::fpga::types::sysobject::ptr_t;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-bytes-12","title":"function bytes [\u00bd]","text":"
                                            Get all raw bytes from the object. 
                                            std::vector< uint8_t > opae::fpga::types::sysobject::bytes (\n    int flags=0\n) const\n
                                             
                                            Parameters:
                                             
                                               
                                            • flags Flags that control how object is read If FPGA_OBJECT_SYNC is used then object will update its buffered copy before retrieving the data.
                                              •  
                                             
                                            Returns:
                                             
                                            A vector of all bytes in the object. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-bytes-22","title":"function bytes [2/2]","text":"
                                            Get a subset of raw bytes from the object. 
                                            std::vector< uint8_t > opae::fpga::types::sysobject::bytes (\n    uint32_t offset,\n    uint32_t size,\n    int flags=0\n) const\n
                                             
                                            Parameters:
                                             
                                               
                                            • offset The bytes offset for the start of the returned vector. 
                                              •  
                                            • size The number of bytes for the returned vector. 
                                              •  
                                            • flags Flags that control how object is read If FPGA_OBJECT_SYNC is used then object will update its buffered copy before retrieving the data.
                                              •  
                                             
                                            Returns:
                                             
                                            A vector of size bytes in the object starting at offset. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-c_type","title":"function c_type","text":"
                                            inline fpga_object opae::fpga::types::sysobject::c_type () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-get-14","title":"function get [\u00bc]","text":"
                                            Get a sysobject from an object. 
                                            sysobject::ptr_t opae::fpga::types::sysobject::get (\n    const std::string & name,\n    int flags=0\n) \n
                                             
                                            This will be read-write if its parent was created from a handle..
                                             
                                            Parameters:
                                             
                                               
                                            • name An identifier representing an object belonging to this object. 
                                              •  
                                            • flags Control behavior of object identification and creation. FPGA_OBJECT_GLOB is used to indicate that the name should be treated as a globbing expression. FPGA_OBJECT_RECURSE_ONE indicates that subobjects be created for objects one level down from the object identified by name. FPGA_OBJECT_RECURSE_ALL indicates that subobjects be created for all objects. Flags are defaulted to 0 meaning no flags.
                                              •  
                                             
                                            Returns:
                                             
                                            A shared_ptr to a sysobject instance. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-get-24","title":"function get [2/4]","text":"
                                            Get a sysobject from a container object. 
                                            sysobject::ptr_t opae::fpga::types::sysobject::get (\n    int index\n) \n
                                             
                                            This will be read-write if its parent was created from a handle..
                                             
                                            Parameters:
                                             
                                               
                                            • index An index number to get.
                                              •  
                                             
                                            Returns:
                                             
                                            A shared_ptr to a sysobject instance. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-operator-fpga_object","title":"function operator fpga_object","text":"
                                            inline opae::fpga::types::sysobject::operator fpga_object () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-operator","title":"function operator=","text":"
                                            sysobject & opae::fpga::types::sysobject::operator= (\n    const sysobject & o\n) = delete\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-read64","title":"function read64","text":"
                                            Read a 64-bit value from an FPGA object. 
                                            uint64_t opae::fpga::types::sysobject::read64 (\n    int flags=0\n) const\n
                                             
                                            The value is assumed to be in string format and will be parsed. See flags below for changing that behavior.
                                             
                                            Parameters:
                                             
                                               
                                            • flags Flags that control how the object is read If FPGA_OBJECT_SYNC is used then object will update its buffered copy before retrieving the data. If FPGA_OBJECT_RAW is used, then the data will be read as raw bytes into the uint64_t pointer variable. Flags are defaulted to 0 meaning no flags.
                                              •  
                                             
                                            Returns:
                                             
                                            A 64-bit value from the object. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-size","title":"function size","text":"
                                            Get the size (in bytes) of the object. 
                                            uint32_t opae::fpga::types::sysobject::size () const\n
                                             
                                            Returns:
                                             
                                            The number of bytes that the object occupies in memory. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-sysobject-13","title":"function sysobject [\u2153]","text":"
                                            opae::fpga::types::sysobject::sysobject () = delete\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-sysobject-23","title":"function sysobject [\u2154]","text":"
                                            opae::fpga::types::sysobject::sysobject (\n    const sysobject & o\n) = delete\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-type","title":"function type","text":"
                                            enum fpga_sysobject_type opae::fpga::types::sysobject::type () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-write64","title":"function write64","text":"
                                            Write 64-bit value to an FPGA object. 
                                            void opae::fpga::types::sysobject::write64 (\n    uint64_t value,\n    int flags=0\n) const\n
                                             
                                            The value will be converted to string before writing. See flags below for changing that behavior.
                                             
                                            Parameters:
                                             
                                               
                                            • value The value to write to the object. 
                                              •  
                                            • flags Flags that control how the object is written If FPGA_OBJECT_RAW is used, then the value will be written as raw bytes. Flags are defaulted to 0 meaning no flags.
                                              •  
                                             
                                            Note:
                                             
                                            This operation will force a sync operation to update its cached buffer 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-sysobject","title":"function ~sysobject","text":"
                                            virtual opae::fpga::types::sysobject::~sysobject () \n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-get-34","title":"function get [\u00be]","text":"
                                            Get a sysobject from a token. 
                                            static sysobject::ptr_t opae::fpga::types::sysobject::get (\n    token::ptr_t t,\n    const std::string & name,\n    int flags=0\n) \n
                                             
                                            This will be read-only.
                                             
                                            Parameters:
                                             
                                               
                                            • t Token object representing a resource. 
                                              •  
                                            • name An identifier representing an object belonging to a resource represented by the token. 
                                              •  
                                            • flags Control behavior of object identification and creation. FPGA_OBJECT_GLOB is used to indicate that the name should be treated as a globbing expression. FPGA_OBJECT_RECURSE_ONE indicates that subobjects be created for objects one level down from the object identified by name. FPGA_OBJECT_RECURSE_ALL indicates that subobjects be created for all objects below the current object identified by name.
                                              •  
                                             
                                            Returns:
                                             
                                            A shared_ptr to a sysobject instance. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-get-44","title":"function get [4/4]","text":"
                                            Get a sysobject from a handle. 
                                            static sysobject::ptr_t opae::fpga::types::sysobject::get (\n    handle::ptr_t h,\n    const std::string & name,\n    int flags=0\n) \n
                                             
                                            This will be read-write.
                                             
                                            Parameters:
                                             
                                               
                                            • h Handle object representing an open resource. 
                                              •  
                                            • name An identifier representing an object belonging to a resource represented by the handle. 
                                              •  
                                            • flags Control behavior of object identification and creation. FPGA_OBJECT_GLOB is used to indicate that the name should be treated as a globbing expression. FPGA_OBJECT_RECURSE_ONE indicates that subobjects be created for objects one level down from the object identified by name. FPGA_OBJECT_RECURSE_ALL indicates that subobjects be created for all objects below the current object identified by name.
                                              •  
                                             
                                            Returns:
                                             
                                            A shared_ptr to a sysobject instance. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/sysobject.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/","title":"Class opae::fpga::types::token","text":"
                                            ClassList > opae > fpga > types > token
                                             
                                            Wraps the OPAE fpga_token primitive. More...
                                             
                                               
                                            • #include <token.h>
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< token > ptr_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#public-functions","title":"Public Functions","text":"Type Name fpga_token c_type () constRetrieve the underlying fpga_token primitive. ptr_t get_parent () constRetrieve the parent token, or an empty pointer if there is none. operator fpga_token () constRetrieve the underlying fpga_token primitive. ~token ()"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#public-static-functions","title":"Public Static Functions","text":"Type Name std::vector< token::ptr_t > enumerate (const std::vector< properties::ptr_t > & props) Obtain a vector of token smart pointers for given search criteria."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#detailed-description","title":"Detailed Description","text":"
                                            token's are created from an enumeration operation that uses properties describing an accelerator resource as search criteria. 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                            typedef std::shared_ptr<token> opae::fpga::types::token::ptr_t;\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#function-c_type","title":"function c_type","text":"
                                            inline fpga_token opae::fpga::types::token::c_type () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#function-get_parent","title":"function get_parent","text":"
                                            ptr_t opae::fpga::types::token::get_parent () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#function-operator-fpga_token","title":"function operator fpga_token","text":"
                                            inline opae::fpga::types::token::operator fpga_token () const\n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#function-token","title":"function ~token","text":"
                                            opae::fpga::types::token::~token () \n
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#function-enumerate","title":"function enumerate","text":"
                                            Obtain a vector of token smart pointers for given search criteria. 
                                            static std::vector< token::ptr_t > opae::fpga::types::token::enumerate (\n    const std::vector< properties::ptr_t > & props\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • props The search criteria. 
                                              •  
                                             
                                            Returns:
                                             
                                            A set of known tokens that match the search. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/token.h
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1version/","title":"Class opae::fpga::types::version","text":"
                                            ClassList > opae > fpga > types > version
                                             
                                               
                                            • #include <version.h>
                                              •  
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1version/#public-static-functions","title":"Public Static Functions","text":"Type Name std::string as_string () Get the package version information as a string. fpga_version as_struct () Get the package version information as a struct. std::string build () Get the package build information as a string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1version/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1version/#function-as_string","title":"function as_string","text":"
                                            Get the package version information as a string. 
                                            static std::string opae::fpga::types::version::as_string () \n
                                             
                                            Returns:
                                             
                                            The package version as an std::string object 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1version/#function-as_struct","title":"function as_struct","text":"
                                            Get the package version information as a struct. 
                                            static fpga_version opae::fpga::types::version::as_struct () \n
                                             
                                            Returns:
                                             
                                            The package version as an fpga_version struct 
                                            "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1version/#function-build","title":"function build","text":"
                                            Get the package build information as a string. 
                                            static std::string opae::fpga::types::version::build () \n
                                             
                                            Returns:
                                             
                                            The package build as an std::string object 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/version.h
                                            "},{"location":"opae-code/structopae__uio/","title":"Struct opae_uio","text":"
                                            ClassList > opae_uio
                                             
                                            OPAE UIO device abstraction. More...
                                             
                                               
                                            • #include <uio.h>
                                              •  
                                            "},{"location":"opae-code/structopae__uio/#public-attributes","title":"Public Attributes","text":"Type Name int device_fd char device_path struct opae_uio_device_region * regions"},{"location":"opae-code/structopae__uio/#detailed-description","title":"Detailed Description","text":"
                                            This structure is used to interact with the OPAE UIO API. Each UIO device has a file descriptor that is used to mmap its regions into user address space. Each device also has one or more MMIO regions. 
                                            "},{"location":"opae-code/structopae__uio/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__uio/#variable-device_fd","title":"variable device_fd","text":"
                                            int opae_uio::device_fd;\n
                                            "},{"location":"opae-code/structopae__uio/#variable-device_path","title":"variable device_path","text":"
                                            char opae_uio::device_path[OPAE_UIO_PATH_MAX];\n
                                            "},{"location":"opae-code/structopae__uio/#variable-regions","title":"variable regions","text":"
                                            struct opae_uio_device_region* opae_uio::regions;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/uio.h
                                            "},{"location":"opae-code/structopae__uio__device__region/","title":"Struct opae_uio_device_region","text":"
                                            ClassList > opae_uio_device_region
                                             
                                            MMIO region info. More...
                                             
                                               
                                            • #include <uio.h>
                                              •  
                                            "},{"location":"opae-code/structopae__uio__device__region/#public-attributes","title":"Public Attributes","text":"Type Name struct opae_uio_device_region * next uint32_t region_index size_t region_page_offset uint8_t * region_ptr size_t region_size"},{"location":"opae-code/structopae__uio__device__region/#detailed-description","title":"Detailed Description","text":"
                                            Describes a range of mappable MMIO. 
                                            "},{"location":"opae-code/structopae__uio__device__region/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__uio__device__region/#variable-next","title":"variable next","text":"
                                            struct opae_uio_device_region* opae_uio_device_region::next;\n
                                            "},{"location":"opae-code/structopae__uio__device__region/#variable-region_index","title":"variable region_index","text":"
                                            uint32_t opae_uio_device_region::region_index;\n
                                            "},{"location":"opae-code/structopae__uio__device__region/#variable-region_page_offset","title":"variable region_page_offset","text":"
                                            size_t opae_uio_device_region::region_page_offset;\n
                                            "},{"location":"opae-code/structopae__uio__device__region/#variable-region_ptr","title":"variable region_ptr","text":"
                                            uint8_t* opae_uio_device_region::region_ptr;\n
                                            "},{"location":"opae-code/structopae__uio__device__region/#variable-region_size","title":"variable region_size","text":"
                                            size_t opae_uio_device_region::region_size;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/uio.h
                                            "},{"location":"opae-code/structopae__vfio/","title":"Struct opae_vfio","text":"
                                            ClassList > opae_vfio
                                             
                                            OPAE VFIO device abstraction. More...
                                             
                                               
                                            • #include <vfio.h>
                                              •  
                                            "},{"location":"opae-code/structopae__vfio/#public-attributes","title":"Public Attributes","text":"Type Name opae_hash_map cont_buffers Map of allocated DMA buffers. char * cont_device \"/dev/vfio/vfio\" int cont_fd Container file descriptor. char * cont_pciaddr PCIe address, eg 0000:00:00.0. struct opae_vfio_iova_range * cont_ranges List of IOVA ranges. struct opae_vfio_device device The VFIO device. struct opae_vfio_group group The VFIO device group. struct mem_alloc iova_alloc Allocator for IOVA space. pthread_mutex_t lock For thread safety."},{"location":"opae-code/structopae__vfio/#detailed-description","title":"Detailed Description","text":"
                                            This structure is used to interact with the OPAE VFIO API. It tracks data related to the VFIO container, group, and device. A mutex is provided for thread safety. 
                                            "},{"location":"opae-code/structopae__vfio/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio/#variable-cont_buffers","title":"variable cont_buffers","text":"
                                            opae_hash_map opae_vfio::cont_buffers;\n
                                            "},{"location":"opae-code/structopae__vfio/#variable-cont_device","title":"variable cont_device","text":"
                                            char* opae_vfio::cont_device;\n
                                            "},{"location":"opae-code/structopae__vfio/#variable-cont_fd","title":"variable cont_fd","text":"
                                            int opae_vfio::cont_fd;\n
                                            "},{"location":"opae-code/structopae__vfio/#variable-cont_pciaddr","title":"variable cont_pciaddr","text":"
                                            char* opae_vfio::cont_pciaddr;\n
                                            "},{"location":"opae-code/structopae__vfio/#variable-cont_ranges","title":"variable cont_ranges","text":"
                                            struct opae_vfio_iova_range* opae_vfio::cont_ranges;\n
                                            "},{"location":"opae-code/structopae__vfio/#variable-device","title":"variable device","text":"
                                            struct opae_vfio_device opae_vfio::device;\n
                                            "},{"location":"opae-code/structopae__vfio/#variable-group","title":"variable group","text":"
                                            struct opae_vfio_group opae_vfio::group;\n
                                            "},{"location":"opae-code/structopae__vfio/#variable-iova_alloc","title":"variable iova_alloc","text":"
                                            struct mem_alloc opae_vfio::iova_alloc;\n
                                            "},{"location":"opae-code/structopae__vfio/#variable-lock","title":"variable lock","text":"
                                            pthread_mutex_t opae_vfio::lock;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                            "},{"location":"opae-code/structopae__vfio__buffer/","title":"Struct opae_vfio_buffer","text":"
                                            ClassList > opae_vfio_buffer
                                             
                                            System DMA buffer. More...
                                             
                                               
                                            • #include <vfio.h>
                                              •  
                                            "},{"location":"opae-code/structopae__vfio__buffer/#public-attributes","title":"Public Attributes","text":"Type Name uint64_t buffer_iova Buffer IOVA address. uint8_t * buffer_ptr Buffer virtual address. size_t buffer_size Buffer size. int flags See opae_vfio_buffer_flags."},{"location":"opae-code/structopae__vfio__buffer/#detailed-description","title":"Detailed Description","text":"
                                            Describes a system memory address space that is capable of DMA. 
                                            "},{"location":"opae-code/structopae__vfio__buffer/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__buffer/#variable-buffer_iova","title":"variable buffer_iova","text":"
                                            uint64_t opae_vfio_buffer::buffer_iova;\n
                                            "},{"location":"opae-code/structopae__vfio__buffer/#variable-buffer_ptr","title":"variable buffer_ptr","text":"
                                            uint8_t* opae_vfio_buffer::buffer_ptr;\n
                                            "},{"location":"opae-code/structopae__vfio__buffer/#variable-buffer_size","title":"variable buffer_size","text":"
                                            size_t opae_vfio_buffer::buffer_size;\n
                                            "},{"location":"opae-code/structopae__vfio__buffer/#variable-flags","title":"variable flags","text":"
                                            int opae_vfio_buffer::flags;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                            "},{"location":"opae-code/structopae__vfio__device/","title":"Struct opae_vfio_device","text":"
                                            ClassList > opae_vfio_device
                                             
                                            VFIO device. More...
                                             
                                               
                                            • #include <vfio.h>
                                              •  
                                            "},{"location":"opae-code/structopae__vfio__device/#public-attributes","title":"Public Attributes","text":"Type Name uint64_t device_config_offset Offset of PCIe config space. int device_fd Device file descriptor. uint32_t device_num_irqs IRQ index count. uint32_t device_num_regions Total MMIO region count. struct opae_vfio_device_irq * irqs IRQ list pointer. struct opae_vfio_device_region * regions Region list pointer."},{"location":"opae-code/structopae__vfio__device/#detailed-description","title":"Detailed Description","text":"
                                            Each VFIO device has a file descriptor that is used to query information about the device MMIO regions and config space. 
                                            "},{"location":"opae-code/structopae__vfio__device/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__device/#variable-device_config_offset","title":"variable device_config_offset","text":"
                                            uint64_t opae_vfio_device::device_config_offset;\n
                                            "},{"location":"opae-code/structopae__vfio__device/#variable-device_fd","title":"variable device_fd","text":"
                                            int opae_vfio_device::device_fd;\n
                                            "},{"location":"opae-code/structopae__vfio__device/#variable-device_num_irqs","title":"variable device_num_irqs","text":"
                                            uint32_t opae_vfio_device::device_num_irqs;\n
                                            "},{"location":"opae-code/structopae__vfio__device/#variable-device_num_regions","title":"variable device_num_regions","text":"
                                            uint32_t opae_vfio_device::device_num_regions;\n
                                            "},{"location":"opae-code/structopae__vfio__device/#variable-irqs","title":"variable irqs","text":"
                                            struct opae_vfio_device_irq* opae_vfio_device::irqs;\n
                                            "},{"location":"opae-code/structopae__vfio__device/#variable-regions","title":"variable regions","text":"
                                            struct opae_vfio_device_region* opae_vfio_device::regions;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                            "},{"location":"opae-code/structopae__vfio__device__irq/","title":"Struct opae_vfio_device_irq","text":"
                                            ClassList > opae_vfio_device_irq
                                             
                                            Interrupt info. More...
                                             
                                               
                                            • #include <vfio.h>
                                              •  
                                            "},{"location":"opae-code/structopae__vfio__device__irq/#public-attributes","title":"Public Attributes","text":"Type Name uint32_t count Number of IRQs at this index. int32_t * event_fds Event file descriptors. uint32_t flags Flags. uint32_t index The IRQ index. int32_t * masks IRQ masks. struct opae_vfio_device_irq * next Pointer to next in list."},{"location":"opae-code/structopae__vfio__device__irq/#detailed-description","title":"Detailed Description","text":"
                                            Describes an interrupt capability. 
                                            "},{"location":"opae-code/structopae__vfio__device__irq/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__device__irq/#variable-count","title":"variable count","text":"
                                            uint32_t opae_vfio_device_irq::count;\n
                                            "},{"location":"opae-code/structopae__vfio__device__irq/#variable-event_fds","title":"variable event_fds","text":"
                                            int32_t* opae_vfio_device_irq::event_fds;\n
                                            "},{"location":"opae-code/structopae__vfio__device__irq/#variable-flags","title":"variable flags","text":"
                                            Flags. 
                                            uint32_t opae_vfio_device_irq::flags;\n
                                             
                                            See struct vfio_irq_info. 
                                            "},{"location":"opae-code/structopae__vfio__device__irq/#variable-index","title":"variable index","text":"
                                            uint32_t opae_vfio_device_irq::index;\n
                                            "},{"location":"opae-code/structopae__vfio__device__irq/#variable-masks","title":"variable masks","text":"
                                            int32_t* opae_vfio_device_irq::masks;\n
                                            "},{"location":"opae-code/structopae__vfio__device__irq/#variable-next","title":"variable next","text":"
                                            struct opae_vfio_device_irq* opae_vfio_device_irq::next;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                            "},{"location":"opae-code/structopae__vfio__device__region/","title":"Struct opae_vfio_device_region","text":"
                                            ClassList > opae_vfio_device_region
                                             
                                            MMIO region info. More...
                                             
                                               
                                            • #include <vfio.h>
                                              •  
                                            "},{"location":"opae-code/structopae__vfio__device__region/#public-attributes","title":"Public Attributes","text":"Type Name struct opae_vfio_device_region * next Pointer to next in list. uint32_t region_index Region index, from 0. uint8_t * region_ptr Virtual address of region. size_t region_size Size of region. struct opae_vfio_sparse_info * region_sparse For sparse regions."},{"location":"opae-code/structopae__vfio__device__region/#detailed-description","title":"Detailed Description","text":"
                                            Describes a range of mappable MMIO. 
                                            "},{"location":"opae-code/structopae__vfio__device__region/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__device__region/#variable-next","title":"variable next","text":"
                                            struct opae_vfio_device_region* opae_vfio_device_region::next;\n
                                            "},{"location":"opae-code/structopae__vfio__device__region/#variable-region_index","title":"variable region_index","text":"
                                            uint32_t opae_vfio_device_region::region_index;\n
                                            "},{"location":"opae-code/structopae__vfio__device__region/#variable-region_ptr","title":"variable region_ptr","text":"
                                            uint8_t* opae_vfio_device_region::region_ptr;\n
                                            "},{"location":"opae-code/structopae__vfio__device__region/#variable-region_size","title":"variable region_size","text":"
                                            size_t opae_vfio_device_region::region_size;\n
                                            "},{"location":"opae-code/structopae__vfio__device__region/#variable-region_sparse","title":"variable region_sparse","text":"
                                            struct opae_vfio_sparse_info* opae_vfio_device_region::region_sparse;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                            "},{"location":"opae-code/structopae__vfio__group/","title":"Struct opae_vfio_group","text":"
                                            ClassList > opae_vfio_group
                                             
                                            VFIO group. More...
                                             
                                               
                                            • #include <vfio.h>
                                              •  
                                            "},{"location":"opae-code/structopae__vfio__group/#public-attributes","title":"Public Attributes","text":"Type Name char * group_device Full path to the group device. int group_fd File descriptor for the group device."},{"location":"opae-code/structopae__vfio__group/#detailed-description","title":"Detailed Description","text":"
                                            Each device managed by vfio-pci belongs to a VFIO group rooted at /dev/vfio/N, where N is the group number. 
                                            "},{"location":"opae-code/structopae__vfio__group/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__group/#variable-group_device","title":"variable group_device","text":"
                                            char* opae_vfio_group::group_device;\n
                                            "},{"location":"opae-code/structopae__vfio__group/#variable-group_fd","title":"variable group_fd","text":"
                                            int opae_vfio_group::group_fd;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                            "},{"location":"opae-code/structopae__vfio__iova__range/","title":"Struct opae_vfio_iova_range","text":"
                                            ClassList > opae_vfio_iova_range
                                             
                                            IO Virtual Address Range. More...
                                             
                                               
                                            • #include <vfio.h>
                                              •  
                                            "},{"location":"opae-code/structopae__vfio__iova__range/#public-attributes","title":"Public Attributes","text":"Type Name uint64_t end End of this range of offsets. struct opae_vfio_iova_range * next Pointer to next in list. uint64_t start Start of this range of offsets."},{"location":"opae-code/structopae__vfio__iova__range/#detailed-description","title":"Detailed Description","text":"
                                            A range of allocatable IOVA offsets. Used for mapping DMA buffers. 
                                            "},{"location":"opae-code/structopae__vfio__iova__range/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__iova__range/#variable-end","title":"variable end","text":"
                                            uint64_t opae_vfio_iova_range::end;\n
                                            "},{"location":"opae-code/structopae__vfio__iova__range/#variable-next","title":"variable next","text":"
                                            struct opae_vfio_iova_range* opae_vfio_iova_range::next;\n
                                            "},{"location":"opae-code/structopae__vfio__iova__range/#variable-start","title":"variable start","text":"
                                            uint64_t opae_vfio_iova_range::start;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                            "},{"location":"opae-code/structopae__vfio__sparse__info/","title":"Struct opae_vfio_sparse_info","text":"
                                            ClassList > opae_vfio_sparse_info
                                             
                                            MMIO sparse-mappable region info. More...
                                             
                                               
                                            • #include <vfio.h>
                                              •  
                                            "},{"location":"opae-code/structopae__vfio__sparse__info/#public-attributes","title":"Public Attributes","text":"Type Name uint32_t index Region index, from 0. struct opae_vfio_sparse_info * next Pointer to next in list. uint32_t offset Offset of sparse region. uint8_t * ptr Virtual address of sparse region. uint32_t size Size of sparse region."},{"location":"opae-code/structopae__vfio__sparse__info/#detailed-description","title":"Detailed Description","text":"
                                            Describes a range of sparse-mappable MMIO, for MMIO ranges that have non-contiguous addresses. 
                                            "},{"location":"opae-code/structopae__vfio__sparse__info/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__sparse__info/#variable-index","title":"variable index","text":"
                                            uint32_t opae_vfio_sparse_info::index;\n
                                            "},{"location":"opae-code/structopae__vfio__sparse__info/#variable-next","title":"variable next","text":"
                                            struct opae_vfio_sparse_info* opae_vfio_sparse_info::next;\n
                                            "},{"location":"opae-code/structopae__vfio__sparse__info/#variable-offset","title":"variable offset","text":"
                                            uint32_t opae_vfio_sparse_info::offset;\n
                                            "},{"location":"opae-code/structopae__vfio__sparse__info/#variable-ptr","title":"variable ptr","text":"
                                            uint8_t* opae_vfio_sparse_info::ptr;\n
                                            "},{"location":"opae-code/structopae__vfio__sparse__info/#variable-size","title":"variable size","text":"
                                            uint32_t opae_vfio_sparse_info::size;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                            "},{"location":"opae-code/structras__inject__error/","title":"Struct ras_inject_error","text":"
                                            ClassList > ras_inject_error
                                            "},{"location":"opae-code/structras__inject__error/#public-attributes","title":"Public Attributes","text":"Type Name union ras_inject_error::@0 @1 uint64_t catastrophicr_error uint64_t csr uint64_t fatal_error uint64_t nonfatal_error uint64_t rsvd"},{"location":"opae-code/structras__inject__error/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structras__inject__error/#variable-1","title":"variable @1","text":"
                                            union ras_inject_error::@0 ras_inject_error::@1;\n
                                            "},{"location":"opae-code/structras__inject__error/#variable-catastrophicr_error","title":"variable catastrophicr_error","text":"
                                            uint64_t ras_inject_error::catastrophicr_error;\n
                                            "},{"location":"opae-code/structras__inject__error/#variable-csr","title":"variable csr","text":"
                                            uint64_t ras_inject_error::csr;\n
                                            "},{"location":"opae-code/structras__inject__error/#variable-fatal_error","title":"variable fatal_error","text":"
                                            uint64_t ras_inject_error::fatal_error;\n
                                            "},{"location":"opae-code/structras__inject__error/#variable-nonfatal_error","title":"variable nonfatal_error","text":"
                                            uint64_t ras_inject_error::nonfatal_error;\n
                                            "},{"location":"opae-code/structras__inject__error/#variable-rsvd","title":"variable rsvd","text":"
                                            uint64_t ras_inject_error::rsvd;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/samples/hello_events/hello_events.c
                                            "},{"location":"opae-code/namespacestd/","title":"Namespace std","text":"
                                            Namespace List > std
                                             
                                            The documentation for this class was generated from the following file [generated]
                                            "},{"location":"opae-code/structthreshold/","title":"Struct threshold","text":"
                                            ClassList > threshold
                                             
                                            Threshold struct. 
                                             
                                               
                                            • #include <types.h>
                                              •  
                                            "},{"location":"opae-code/structthreshold/#public-attributes","title":"Public Attributes","text":"Type Name uint32_t is_valid char threshold_name double value"},{"location":"opae-code/structthreshold/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structthreshold/#variable-is_valid","title":"variable is_valid","text":"
                                            uint32_t threshold::is_valid;\n
                                            "},{"location":"opae-code/structthreshold/#variable-threshold_name","title":"variable threshold_name","text":"
                                            char threshold::threshold_name[FPGA_METRIC_STR_SIZE];\n
                                            "},{"location":"opae-code/structthreshold/#variable-value","title":"variable value","text":"
                                            double threshold::value;\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                            "},{"location":"opae-code/dir_49e56c817e5e54854c35e136979f97ca/","title":"Dir docs","text":"
                                            FileList > docs
                                            "},{"location":"opae-code/dir_49e56c817e5e54854c35e136979f97ca/#directories","title":"Directories","text":"Type Name dir sw 
                                            The documentation for this class was generated from the following file docs/
                                            "},{"location":"opae-code/dir_55721a669a8e0900d975c02921addb49/","title":"Dir docs/sw","text":"
                                            FileList > docs > sw
                                            "},{"location":"opae-code/dir_55721a669a8e0900d975c02921addb49/#directories","title":"Directories","text":"Type Name dir include dir samples 
                                            The documentation for this class was generated from the following file docs/sw/
                                            "},{"location":"opae-code/dir_97b4588afba69bf89bbe554642ac6431/","title":"Dir docs/sw/include","text":"
                                            FileList > docs > sw > include
                                            "},{"location":"opae-code/dir_97b4588afba69bf89bbe554642ac6431/#directories","title":"Directories","text":"Type Name dir opae 
                                            The documentation for this class was generated from the following file docs/sw/include/
                                            "},{"location":"opae-code/dir_ade97cd9199f278c0723672dd8647ba4/","title":"Dir docs/sw/include/opae","text":"
                                            FileList > docs > sw > include > opae
                                            "},{"location":"opae-code/dir_ade97cd9199f278c0723672dd8647ba4/#files","title":"Files","text":"Type Name file access.h Functions to acquire, release, and reset OPAE FPGA resources. file buffer.h Functions for allocating and sharing system memory with an FPGA accelerator. file enum.h APIs for resource enumeration and managing tokens. file error.h Functions for reading and clearing errors in resources. file event.h Functions for registering events and managing the lifecycle for fpga_event_handle s. file fpga.h FPGA API. file hash_map.h A general-purpose hybrid array/list hash map implementation. file init.h Initialization routine. file log.h file manage.h Functions for managing FPGA configurations. file mem_alloc.h file metrics.h Functions for Discover/ Enumerates metrics and retrieves values. file mmio.h Functions for mapping and accessing MMIO space. file properties.h Functions for examining and manipulating fpga_properties objects. file sysobject.h Functions to read/write from system objects. file types.h Type definitions for FPGA API. file types_enum.h Definitions of enumerated types for the OPAE C API. file uio.h APIs to manage a PCIe device via UIO. file umsg.h FPGA UMsg API. file userclk.h Functions for setting and get afu user clock. file utils.h Utility functions and macros for the FPGA API. file version.h file vfio.h APIs to manage a PCIe device via vfio-pci."},{"location":"opae-code/dir_ade97cd9199f278c0723672dd8647ba4/#directories","title":"Directories","text":"Type Name dir cxx 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/
                                            "},{"location":"opae-code/access_8h/","title":"File access.h","text":"
                                            FileList > docs > sw > include > opae > access.h
                                             
                                            Go to the source code of this file.
                                             
                                            Functions to acquire, release, and reset OPAE FPGA resources. 
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/access_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaClose (fpga_handle handle) Close a previously opened FPGA object. fpga_result fpgaOpen (fpga_token token, fpga_handle * handle, int flags) Open an FPGA object. fpga_result fpgaReset (fpga_handle handle) Reset an FPGA object."},{"location":"opae-code/access_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/access_8h/#function-fpgaclose","title":"function fpgaClose","text":"
                                            Close a previously opened FPGA object. 
                                            fpga_result fpgaClose (\n    fpga_handle handle\n) \n
                                             
                                            Relinquishes ownership of a previously fpgaOpen()ed resource. This enables others to acquire ownership if the resource was opened exclusively. Also deallocates / unmaps MMIO and UMsg memory areas.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened FPGA object 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if handle does not refer to an acquired resource, or if handle is NULL. FPGA_EXCEPTION if an internal error occurred while accessing the handle. 
                                            "},{"location":"opae-code/access_8h/#function-fpgaopen","title":"function fpgaOpen","text":"
                                            Open an FPGA object. 
                                            fpga_result fpgaOpen (\n    fpga_token token,\n    fpga_handle * handle,\n    int flags\n) \n
                                             
                                            Acquires ownership of the FPGA resource referred to by 'token'.
                                             
                                            Most often this will be used to open an accelerator object to directly interact with an accelerator function, or to open an FPGA object to perform management functions.
                                             
                                            Parameters:
                                             
                                               
                                            • token Pointer to token identifying resource to acquire ownership of 
                                              •  
                                            • handle Pointer to preallocated memory to place a handle in. This handle will be used in subsequent API calls. 
                                              •  
                                            • flags One of the following flags:
                                                 
                                              • FPGA_OPEN_SHARED allows the resource to be opened multiple times (not supported in ASE) Shared resources (including buffers) are released when all associated handles have been closed (either explicitly with fpgaClose() or by process termination). 
                                                •  
                                               
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_NOT_FOUND if the resource for 'token' could not be found. FPGA_INVALID_PARAM if 'token' does not refer to a resource that can be opened, or if either argument is NULL or invalid. FPGA_EXCEPTION if an internal exception occurred while creating the handle. FPGA_NO_DRIVER if the driver is not loaded. FPGA_BUSY if trying to open a resource that has already been opened in exclusive mode. FPGA_NO_ACCESS if the current process' privileges are not sufficient to open the resource. 
                                            "},{"location":"opae-code/access_8h/#function-fpgareset","title":"function fpgaReset","text":"
                                            Reset an FPGA object. 
                                            fpga_result fpgaReset (\n    fpga_handle handle\n) \n
                                             
                                            Performs an accelerator reset.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened FPGA object 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if handle does not refer to an acquired resource or to a resource that cannot be reset. FPGA_EXCEPTION if an internal error occurred while trying to access the handle or resetting the resource. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/access.h
                                            "},{"location":"opae-code/access_8h_source/","title":"File access.h","text":"
                                            File List > docs > sw > include > opae > access.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_ACCESS_H__\n#define __FPGA_ACCESS_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaOpen(fpga_token token, fpga_handle *handle,\n             int flags);\n\nfpga_result fpgaClose(fpga_handle handle);\n\nfpga_result fpgaReset(fpga_handle handle);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_ACCESS_H__\n
                                            "},{"location":"opae-code/buffer_8h/","title":"File buffer.h","text":"
                                            FileList > docs > sw > include > opae > buffer.h
                                             
                                            Go to the source code of this file.
                                             
                                            Functions for allocating and sharing system memory with an FPGA accelerator. More...
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/buffer_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaGetIOAddress (fpga_handle handle, uint64_t wsid, uint64_t * ioaddr) Retrieve base IO address for buffer. fpga_result fpgaPrepareBuffer (fpga_handle handle, uint64_t len, void ** buf_addr, uint64_t * wsid, int flags) Prepare a shared memory buffer. fpga_result fpgaReleaseBuffer (fpga_handle handle, uint64_t wsid) Release a shared memory buffer."},{"location":"opae-code/buffer_8h/#detailed-description","title":"Detailed Description","text":"
                                            To share memory between a software application and an FPGA accelerator, these functions set up system components (e.g. an IOMMU) to allow accelerator access to a provided memory region.
                                             
                                            There are a number of restrictions on what memory can be shared, depending on platform capabilities. Usually, FPGA accelerators to not have access to virtual address mappings of the CPU, so they can only access physical addresses. To support this, the OPAE C library on Linux uses hugepages to allocate large, contiguous pages of physical memory that can be shared with an accelerator. It also supports sharing memory that has already been allocated by an application, as long as that memory satisfies the requirements of being physically contigous and page-aligned. 
                                            "},{"location":"opae-code/buffer_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/buffer_8h/#function-fpgagetioaddress","title":"function fpgaGetIOAddress","text":"
                                            Retrieve base IO address for buffer. 
                                            fpga_result fpgaGetIOAddress (\n    fpga_handle handle,\n    uint64_t wsid,\n    uint64_t * ioaddr\n) \n
                                             
                                            This function is used to acquire the physical base address (on some platforms called IO Virtual Address or IOVA) for a shared buffer identified by wsid.
                                             
                                            Note:
                                             
                                            This function will disappear once the APIs for secure sharing of buffer addresses is implemented.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource 
                                              •  
                                            • wsid Buffer handle / workspace ID referring to the buffer for which the IO address is requested 
                                              •  
                                            • ioaddr Pointer to memory where the IO address will be returned 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if invalid parameters were provided, or if the parameter combination is not valid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. FPGA_NOT_FOUND if wsid does not refer to a previously shared buffer. 
                                            "},{"location":"opae-code/buffer_8h/#function-fpgapreparebuffer","title":"function fpgaPrepareBuffer","text":"
                                            Prepare a shared memory buffer. 
                                            fpga_result fpgaPrepareBuffer (\n    fpga_handle handle,\n    uint64_t len,\n    void ** buf_addr,\n    uint64_t * wsid,\n    int flags\n) \n
                                             
                                            Prepares a memory buffer for shared access between an accelerator and the calling process. This may either include allocation of physical memory, or preparation of already allocated memory for sharing. The latter case is indicated by supplying the FPGA_BUF_PREALLOCATED flag.
                                             
                                            This function will ask the driver to pin the indicated memory (make it non-swappable), and program the IOMMU to allow access from the accelerator. If the buffer was not pre-allocated (flag FPGA_BUF_PREALLOCATED), the function will also allocate physical memory of the requested size and map the memory into the caller's process' virtual address space. It returns in 'wsid' an fpga_buffer object that can be used to program address registers in the accelerator for shared access to the memory.
                                             
                                            When using FPGA_BUF_PREALLOCATED, the input len must be a non-zero multiple of the page size, else the function returns FPGA_INVALID_PARAM. When not using FPGA_BUF_PREALLOCATED, the input len is rounded up to the nearest multiple of page size.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource 
                                              •  
                                            • len Length of the buffer to allocate/prepare in bytes 
                                              •  
                                            • buf_addr Virtual address of buffer. Contents may be NULL (OS will choose mapping) or non-NULL (OS will take contents as a hint for the virtual address). 
                                              •  
                                            • wsid Handle to the allocated/prepared buffer to be used with other functions 
                                              •  
                                            • flags Flags. FPGA_BUF_PREALLOCATED indicates that memory pointed at in '*buf_addr' is already allocated an mapped into virtual memory. FPGA_BUF_READ_ONLY pins pages with only read access from the FPGA. 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_NO_MEMORY if the requested memory could not be allocated. FPGA_INVALID_PARAM if invalid parameters were provided, or if the parameter combination is not valid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle.
                                             
                                            Note:
                                             
                                            As a special case, when FPGA_BUF_PREALLOCATED is present in flags, if len == 0 and buf_addr == NULL, then the function returns FPGA_OK if pre-allocated buffers are supported. In this case, a return value other than FPGA_OK indicates that pre-allocated buffers are not supported. 
                                            "},{"location":"opae-code/buffer_8h/#function-fpgareleasebuffer","title":"function fpgaReleaseBuffer","text":"
                                            Release a shared memory buffer. 
                                            fpga_result fpgaReleaseBuffer (\n    fpga_handle handle,\n    uint64_t wsid\n) \n
                                             
                                            Releases a previously prepared shared buffer. If the buffer was allocated using fpgaPrepareBuffer (FPGA_BUF_PREALLOCATED was not specified), this call will deallocate/free that memory. Otherwise, it will only be returned to it's previous state (pinned/unpinned, cached/non-cached).
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource 
                                              •  
                                            • wsid Handle to the allocated/prepared buffer 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if invalid parameters were provided, or if the parameter combination is not valid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/buffer.h
                                            "},{"location":"opae-code/buffer_8h_source/","title":"File buffer.h","text":"
                                            File List > docs > sw > include > opae > buffer.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_BUFFER_H__\n#define __FPGA_BUFFER_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaPrepareBuffer(fpga_handle handle,\n                  uint64_t len,\n                  void **buf_addr, uint64_t *wsid, int flags);\n\nfpga_result fpgaReleaseBuffer(fpga_handle handle, uint64_t wsid);\n\nfpga_result fpgaGetIOAddress(fpga_handle handle, uint64_t wsid,\n                 uint64_t *ioaddr);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_BUFFER_H__\n
                                            "},{"location":"opae-code/dir_3731a7e5669218b938d292e51b4e531c/","title":"Dir docs/sw/include/opae/cxx","text":"
                                            FileList > cxx
                                            "},{"location":"opae-code/dir_3731a7e5669218b938d292e51b4e531c/#files","title":"Files","text":"Type Name file core.h"},{"location":"opae-code/dir_3731a7e5669218b938d292e51b4e531c/#directories","title":"Directories","text":"Type Name dir core 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/
                                            "},{"location":"opae-code/core_8h/","title":"File core.h","text":"
                                            FileList > cxx > core.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <opae/cxx/core/errors.h>
                                              •  
                                            • #include <opae/cxx/core/events.h>
                                              •  
                                            • #include <opae/cxx/core/except.h>
                                              •  
                                            • #include <opae/cxx/core/handle.h>
                                              •  
                                            • #include <opae/cxx/core/properties.h>
                                              •  
                                            • #include <opae/cxx/core/pvalue.h>
                                              •  
                                            • #include <opae/cxx/core/shared_buffer.h>
                                              •  
                                            • #include <opae/cxx/core/token.h>
                                              •  
                                            • #include <opae/cxx/core/version.h>
                                              •  
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core.h
                                            "},{"location":"opae-code/core_8h_source/","title":"File core.h","text":"
                                            File List > cxx > core.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2020, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/cxx/core/errors.h>\n#include <opae/cxx/core/events.h>\n#include <opae/cxx/core/except.h>\n#include <opae/cxx/core/handle.h>\n#include <opae/cxx/core/properties.h>\n#include <opae/cxx/core/pvalue.h>\n#include <opae/cxx/core/shared_buffer.h>\n#include <opae/cxx/core/token.h>\n#include <opae/cxx/core/version.h>\n
                                            "},{"location":"opae-code/dir_23b1b9d7ef54caa3fa7bb54d9bc2d47a/","title":"Dir docs/sw/include/opae/cxx/core","text":"
                                            FileList > core
                                            "},{"location":"opae-code/dir_23b1b9d7ef54caa3fa7bb54d9bc2d47a/#files","title":"Files","text":"Type Name file errors.h file events.h file except.h file handle.h file properties.h file pvalue.h file shared_buffer.h file sysobject.h file token.h file version.h 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/
                                            "},{"location":"opae-code/errors_8h/","title":"File errors.h","text":"
                                            FileList > core > errors.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <opae/cxx/core/token.h>
                                              •  
                                            • #include <opae/types_enum.h>
                                              •  
                                            • #include <memory>
                                              •  
                                            "},{"location":"opae-code/errors_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/errors_8h/#classes","title":"Classes","text":"Type Name class error An error object represents an error register for a resource. 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/errors.h
                                            "},{"location":"opae-code/errors_8h_source/","title":"File errors.h","text":"
                                            File List > core > errors.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n\n#include <opae/cxx/core/token.h>\n#include <opae/types_enum.h>\n\n#include <memory>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass error {\n public:\n  typedef std::shared_ptr<error> ptr_t;\n\n  error(const error &e) = delete;\n\n  error &operator=(const error &e) = delete;\n\n  static error::ptr_t get(token::ptr_t tok, uint32_t num);\n\n  std::string name() { return error_info_.name; }\n\n  bool can_clear() { return error_info_.can_clear; }\n\n  uint64_t read_value();\n\n  ~error() {}\n\n  fpga_error_info c_type() const { return error_info_; }\n\n private:\n  error(token::ptr_t token, uint32_t num);\n  token::ptr_t token_;\n  fpga_error_info error_info_;\n  uint32_t error_num_;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                            "},{"location":"opae-code/events_8h/","title":"File events.h","text":"
                                            FileList > core > events.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <opae/cxx/core/handle.h>
                                              •  
                                            • #include <opae/types_enum.h>
                                              •  
                                            • #include <memory>
                                              •  
                                            "},{"location":"opae-code/events_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/events_8h/#classes","title":"Classes","text":"Type Name class event Wraps fpga event routines in OPAE C. struct type_t C++ struct that is interchangeable with fpga_event_type enum. 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/events.h
                                            "},{"location":"opae-code/events_8h_source/","title":"File events.h","text":"
                                            File List > core > events.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n\n#include <opae/cxx/core/handle.h>\n#include <opae/types_enum.h>\n\n#include <memory>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass event {\n public:\n  typedef std::shared_ptr<event> ptr_t;\n\n  virtual ~event();\n\n  struct type_t {\n    type_t(fpga_event_type c_type) : type_(c_type) {}\n\n    operator fpga_event_type() { return type_; }\n\n    static constexpr fpga_event_type interrupt = FPGA_EVENT_INTERRUPT;\n    static constexpr fpga_event_type error = FPGA_EVENT_ERROR;\n    static constexpr fpga_event_type power_thermal = FPGA_EVENT_POWER_THERMAL;\n\n   private:\n    fpga_event_type type_;\n  };\n\n  fpga_event_handle get() { return event_handle_; }\n\n  operator fpga_event_handle();\n\n  static event::ptr_t register_event(handle::ptr_t h, event::type_t t,\n                                     int flags = 0);\n\n  int os_object() const;\n\n private:\n  event(handle::ptr_t h, event::type_t t, fpga_event_handle event_h);\n  handle::ptr_t handle_;\n  event::type_t type_;\n  fpga_event_handle event_handle_;\n  int os_object_;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                            "},{"location":"opae-code/except_8h/","title":"File except.h","text":"
                                            FileList > core > except.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <opae/types_enum.h>
                                              •  
                                            • #include <cstdint>
                                              •  
                                            • #include <exception>
                                              •  
                                            "},{"location":"opae-code/except_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types namespace detail"},{"location":"opae-code/except_8h/#classes","title":"Classes","text":"Type Name class busy busy exception class except Generic OPAE exception. class exception exception exception class invalid_param invalid_param exception class no_access no_access exception class no_daemon no_daemon exception class no_driver no_driver exception class no_memory no_memory exception class not_found not_found exception class not_supported not_supported exception class reconf_error reconf_error exception class src_location Identify a particular line in a source file."},{"location":"opae-code/except_8h/#macros","title":"Macros","text":"Type Name define ASSERT_FPGA_OK \u00ae Macro to check of result is FPGA_OK If not, throw exception that corresponds to the result code. define OPAECXX_HERE opae::fpga::types::src_location(__FILE__, __func__, __LINE__)Construct a src_location object for the current source line."},{"location":"opae-code/except_8h/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/except_8h/#define-assert_fpga_ok","title":"define ASSERT_FPGA_OK","text":"
                                            #define ASSERT_FPGA_OK (\n    r\n) opae::fpga::types::detail::assert_fpga_ok( \\\n      r, opae::fpga::types::src_location (__FILE__, __func__, __LINE__));\n
                                            "},{"location":"opae-code/except_8h/#define-opaecxx_here","title":"define OPAECXX_HERE","text":"
                                            #define OPAECXX_HERE opae::fpga::types::src_location (__FILE__, __func__, __LINE__)\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                            "},{"location":"opae-code/except_8h_source/","title":"File except.h","text":"
                                            File List > core > except.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/types_enum.h>\n\n#include <cstdint>\n#include <exception>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass src_location {\n public:\n  src_location(const char *file, const char *fn, int line) noexcept;\n\n  src_location(const src_location &other) noexcept;\n\n  src_location &operator=(const src_location &other) noexcept;\n\n  const char *file() const noexcept;\n\n  const char *fn() const noexcept { return fn_; }\n\n  int line() const noexcept { return line_; }\n\n private:\n  const char *file_;\n  const char *fn_;\n  int line_;\n};\n\n#define OPAECXX_HERE \\\n  opae::fpga::types::src_location(__FILE__, __func__, __LINE__)\n\nclass except : public std::exception {\n public:\n  static const std::size_t MAX_EXCEPT = 256;\n\n  except(src_location loc) noexcept;\n\n  except(fpga_result res, src_location loc) noexcept;\n\n  except(fpga_result res, const char *msg, src_location loc) noexcept;\n\n  virtual const char *what() const noexcept override;\n\n  operator fpga_result() const noexcept { return res_; }\n\n protected:\n  fpga_result res_;\n  const char *msg_;\n  src_location loc_;\n  mutable char buf_[MAX_EXCEPT];\n};\n\nclass invalid_param : public except {\n public:\n  invalid_param(src_location loc) noexcept\n      : except(FPGA_INVALID_PARAM, \"failed with return code FPGA_INVALID_PARAM\",\n               loc) {}\n};\n\nclass busy : public except {\n public:\n  busy(src_location loc) noexcept\n      : except(FPGA_BUSY, \"failed with return code FPGA_BUSY\", loc) {}\n};\n\nclass exception : public except {\n public:\n  exception(src_location loc) noexcept\n      : except(FPGA_EXCEPTION, \"failed with return code FPGA_EXCEPTION\", loc) {}\n};\n\nclass not_found : public except {\n public:\n  not_found(src_location loc) noexcept\n      : except(FPGA_NOT_FOUND, \"failed with return code FPGA_NOT_FOUND\", loc) {}\n};\n\nclass no_memory : public except {\n public:\n  no_memory(src_location loc) noexcept\n      : except(FPGA_NO_MEMORY, \"failed with return code FPGA_NO_MEMORY\", loc) {}\n};\n\nclass not_supported : public except {\n public:\n  not_supported(src_location loc) noexcept\n      : except(FPGA_NOT_SUPPORTED, \"failed with return code FPGA_NOT_SUPPORTED\",\n               loc) {}\n};\n\nclass no_driver : public except {\n public:\n  no_driver(src_location loc) noexcept\n      : except(FPGA_NO_DRIVER, \"failed with return code FPGA_NO_DRIVER\", loc) {}\n};\n\nclass no_daemon : public except {\n public:\n  no_daemon(src_location loc) noexcept\n      : except(FPGA_NO_DAEMON, \"failed with return code FPGA_NO_DAEMON\", loc) {}\n};\n\nclass no_access : public except {\n public:\n  no_access(src_location loc) noexcept\n      : except(FPGA_NO_ACCESS, \"failed with return code FPGA_NO_ACCESS\", loc) {}\n};\n\nclass reconf_error : public except {\n public:\n  reconf_error(src_location loc) noexcept\n      : except(FPGA_RECONF_ERROR, \"failed with return code FPGA_RECONF_ERROR\",\n               loc) {}\n};\n\nnamespace detail {\n\ntypedef bool (*exception_fn)(fpga_result,\n                             const opae::fpga::types::src_location &loc);\n\ntemplate <typename T>\nconstexpr bool is_ok(fpga_result result,\n                     const opae::fpga::types::src_location &loc) {\n  return result == FPGA_OK ? true : throw T(loc);\n}\n\nstatic exception_fn opae_exceptions[12] = {\n    is_ok<opae::fpga::types::invalid_param>,\n    is_ok<opae::fpga::types::busy>,\n    is_ok<opae::fpga::types::exception>,\n    is_ok<opae::fpga::types::not_found>,\n    is_ok<opae::fpga::types::no_memory>,\n    is_ok<opae::fpga::types::not_supported>,\n    is_ok<opae::fpga::types::no_driver>,\n    is_ok<opae::fpga::types::no_daemon>,\n    is_ok<opae::fpga::types::no_access>,\n    is_ok<opae::fpga::types::reconf_error>};\n\nstatic inline void assert_fpga_ok(fpga_result result,\n                                  const opae::fpga::types::src_location &loc) {\n  if (result > FPGA_OK && result <= FPGA_RECONF_ERROR)\n    // our exception table above starts at invalid_param with index 0\n    // but FPGA_INVALID_PARAM is actually enum 1 - let's account for that\n    opae_exceptions[result - 1](result, loc);\n}\n\n}  // end of namespace detail\n\n#define ASSERT_FPGA_OK(r)                    \\\n  opae::fpga::types::detail::assert_fpga_ok( \\\n      r, opae::fpga::types::src_location(__FILE__, __func__, __LINE__));\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                            "},{"location":"opae-code/handle_8h/","title":"File handle.h","text":"
                                            FileList > core > handle.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <opae/cxx/core/token.h>
                                              •  
                                            • #include <opae/enum.h>
                                              •  
                                            • #include <opae/types.h>
                                              •  
                                            • #include <memory>
                                              •  
                                            • #include <vector>
                                              •  
                                            "},{"location":"opae-code/handle_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/handle_8h/#classes","title":"Classes","text":"Type Name class handle An allocated accelerator resource. 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/handle.h
                                            "},{"location":"opae-code/handle_8h_source/","title":"File handle.h","text":"
                                            File List > core > handle.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018-2021, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/cxx/core/token.h>\n#include <opae/enum.h>\n#include <opae/types.h>\n\n#include <memory>\n#include <vector>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass handle {\n public:\n  typedef std::shared_ptr<handle> ptr_t;\n\n  handle(const handle &) = delete;\n  handle &operator=(const handle &) = delete;\n\n  virtual ~handle();\n\n  fpga_handle c_type() const { return handle_; }\n\n  operator fpga_handle() const { return handle_; }\n\n  void reconfigure(uint32_t slot, const uint8_t *bitstream, size_t size,\n                   int flags);\n\n  uint32_t read_csr32(uint64_t offset, uint32_t csr_space = 0) const;\n\n  void write_csr32(uint64_t offset, uint32_t value, uint32_t csr_space = 0);\n\n  uint64_t read_csr64(uint64_t offset, uint32_t csr_space = 0) const;\n\n  void write_csr64(uint64_t offset, uint64_t value, uint32_t csr_space = 0);\n\n  void write_csr512(uint64_t offset, const void *value, uint32_t csr_space = 0);\n\n  uint8_t *mmio_ptr(uint64_t offset, uint32_t csr_space = 0) const;\n\n  static handle::ptr_t open(fpga_token token, int flags);\n\n  static handle::ptr_t open(token::ptr_t token, int flags);\n\n  virtual void reset();\n\n  fpga_result close();\n\n  token::ptr_t get_token() const;\n\n private:\n  handle(fpga_handle h);\n\n  fpga_handle handle_;\n  fpga_token token_;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                            "},{"location":"opae-code/cxx_2core_2properties_8h/","title":"File properties.h","text":"
                                            FileList > core > properties.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <opae/cxx/core/pvalue.h>
                                              •  
                                            • #include <opae/properties.h>
                                              •  
                                            • #include <iostream>
                                              •  
                                            • #include <map>
                                              •  
                                            • #include <memory>
                                              •  
                                            • #include <vector>
                                              •  
                                            "},{"location":"opae-code/cxx_2core_2properties_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/cxx_2core_2properties_8h/#classes","title":"Classes","text":"Type Name class properties Wraps an OPAE fpga_properties object. 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/properties.h
                                            "},{"location":"opae-code/cxx_2core_2properties_8h_source/","title":"File properties.h","text":"
                                            File List > core > properties.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018-2022, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/cxx/core/pvalue.h>\n#include <opae/properties.h>\n\n#include <iostream>\n#include <map>\n#include <memory>\n#include <vector>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass token;\nclass handle;\nclass properties {\n public:\n  typedef std::shared_ptr<properties> ptr_t;\n\n  const static std::vector<properties::ptr_t> none;\n\n  properties(const properties &p) = delete;\n\n  properties &operator=(const properties &p) = delete;\n\n  ~properties();\n\n  fpga_properties c_type() const { return props_; }\n\n  operator fpga_properties() const { return props_; }\n\n  static properties::ptr_t get();\n\n  static properties::ptr_t get(fpga_guid guid_in);\n\n  static properties::ptr_t get(fpga_objtype objtype);\n\n  static properties::ptr_t get(std::shared_ptr<token> t);\n\n  static properties::ptr_t get(fpga_token t);\n\n  static properties::ptr_t get(std::shared_ptr<handle> h);\n\n private:\n  properties(bool alloc_props = true);\n  fpga_properties props_;\n\n public:\n  pvalue<fpga_objtype> type;\n  pvalue<uint32_t> num_errors;\n  pvalue<uint16_t> segment;\n  pvalue<uint8_t> bus;\n  pvalue<uint8_t> device;\n  pvalue<uint8_t> function;\n  pvalue<uint8_t> socket_id;\n  pvalue<uint32_t> num_slots;\n  pvalue<uint64_t> bbs_id;\n  pvalue<fpga_version> bbs_version;\n  pvalue<uint16_t> vendor_id;\n  pvalue<uint16_t> device_id;\n  pvalue<uint16_t> subsystem_vendor_id;\n  pvalue<uint16_t> subsystem_device_id;\n  pvalue<char *> model;\n  pvalue<uint64_t> local_memory_size;\n  pvalue<uint64_t> capabilities;\n  pvalue<uint32_t> num_mmio;\n  pvalue<uint32_t> num_interrupts;\n  pvalue<fpga_accelerator_state> accelerator_state;\n  pvalue<uint64_t> object_id;\n  pvalue<fpga_token> parent;\n  pvalue<fpga_interface> interface;\n  guid_t guid;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                            "},{"location":"opae-code/pvalue_8h/","title":"File pvalue.h","text":"
                                            FileList > core > pvalue.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <opae/cxx/core/except.h>
                                              •  
                                            • #include <opae/properties.h>
                                              •  
                                            • #include <opae/utils.h>
                                              •  
                                            • #include <uuid/uuid.h>
                                              •  
                                            • #include <algorithm>
                                              •  
                                            • #include <array>
                                              •  
                                            • #include <cstring>
                                              •  
                                            • #include <iostream>
                                              •  
                                            • #include <type_traits>
                                              •  
                                            "},{"location":"opae-code/pvalue_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/pvalue_8h/#classes","title":"Classes","text":"Type Name struct guid_t Representation of the guid member of a properties object. struct pvalue <typename T>Wraps OPAE properties defined in the OPAE C API by associating an fpga_properties reference with the getters and setters defined for a property. 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/pvalue.h
                                            "},{"location":"opae-code/pvalue_8h_source/","title":"File pvalue.h","text":"
                                            File List > core > pvalue.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018-2020, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/cxx/core/except.h>\n#include <opae/properties.h>\n#include <opae/utils.h>\n#include <uuid/uuid.h>\n\n#include <algorithm>\n#include <array>\n#include <cstring>\n#include <iostream>\n#include <type_traits>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nstruct guid_t {\n  guid_t(fpga_properties *p) : props_(p), is_set_(false) {}\n\n  void update() {\n    fpga_result res = fpgaPropertiesGetGUID(\n        *props_, reinterpret_cast<fpga_guid *>(data_.data()));\n    ASSERT_FPGA_OK(res);\n    is_set_ = true;\n  }\n\n  operator uint8_t *() {\n    update();\n    return data_.data();\n  }\n\n  const uint8_t *c_type() const { return data_.data(); }\n\n  guid_t &operator=(fpga_guid g) {\n    is_set_ = false;\n    ASSERT_FPGA_OK(fpgaPropertiesSetGUID(*props_, g));\n    is_set_ = true;\n    uint8_t *begin = &g[0];\n    uint8_t *end = begin + sizeof(fpga_guid);\n    std::copy(begin, end, data_.begin());\n    return *this;\n  }\n\n  bool operator==(const fpga_guid &g) {\n    return is_set() && (0 == std::memcmp(data_.data(), g, sizeof(fpga_guid)));\n  }\n\n  void parse(const char *str) {\n    is_set_ = false;\n    if (0 != uuid_parse(str, data_.data())) {\n      throw except(OPAECXX_HERE);\n    }\n    ASSERT_FPGA_OK(fpgaPropertiesSetGUID(*props_, data_.data()));\n    is_set_ = true;\n  }\n\n  friend std::ostream &operator<<(std::ostream &ostr, const guid_t &g) {\n    fpga_properties props = *g.props_;\n    fpga_guid guid_value;\n    fpga_result res;\n    if ((res = fpgaPropertiesGetGUID(props, &guid_value)) == FPGA_OK) {\n      char guid_str[84];\n      uuid_unparse(guid_value, guid_str);\n      ostr << guid_str;\n    } else if (FPGA_NOT_FOUND == res) {\n      std::cerr << \"[guid_t::<<] GUID property not set\\n\";\n    } else {\n      ASSERT_FPGA_OK(res);\n    }\n    return ostr;\n  }\n\n  bool is_set() const { return is_set_; }\n\n  void invalidate() { is_set_ = false; }\n\n private:\n  fpga_properties *props_;\n  bool is_set_;\n  std::array<uint8_t, 16> data_;\n};\n\ntemplate <typename T>\nstruct pvalue {\n  typedef typename std::conditional<\n      std::is_same<T, char *>::value, fpga_result (*)(fpga_properties, T),\n      fpga_result (*)(fpga_properties, T *)>::type getter_t;\n\n  typedef fpga_result (*setter_t)(fpga_properties, T);\n\n  typedef typename std::conditional<std::is_same<T, char *>::value,\n                                    typename std::string, T>::type copy_t;\n\n  pvalue() : props_(0), is_set_(false), get_(nullptr), set_(nullptr), copy_() {}\n\n  pvalue(fpga_properties *p, getter_t g, setter_t s)\n      : props_(p), is_set_(false), get_(g), set_(s), copy_() {}\n\n  pvalue<T> &operator=(const T &v) {\n    is_set_ = false;\n    ASSERT_FPGA_OK(set_(*props_, v));\n    is_set_ = true;\n    copy_ = v;\n    return *this;\n  }\n\n  bool operator==(const T &other) { return is_set() && (copy_ == other); }\n\n  void update() {\n    ASSERT_FPGA_OK(get_(*props_, &copy_));\n    is_set_ = true;\n  }\n\n  operator copy_t() {\n    update();\n    return copy_;\n  }\n\n  // TODO: Remove this once all properties are tested\n  fpga_result get_value(T &value) const { return get_(*props_, &value); }\n\n  friend std::ostream &operator<<(std::ostream &ostr, const pvalue<T> &p) {\n    T value;\n    fpga_properties props = *p.props_;\n    fpga_result res;\n    if ((res = p.get_(props, &value)) == FPGA_OK) {\n      ostr << +(value);\n    } else if (FPGA_NOT_FOUND == res) {\n      std::cerr << \"property getter returned (\" << res << \") \"\n                << fpgaErrStr(res);\n    } else {\n      ASSERT_FPGA_OK(res);\n    }\n    return ostr;\n  }\n\n  bool is_set() const { return is_set_; }\n\n  void invalidate() { is_set_ = false; }\n\n private:\n  fpga_properties *props_;\n  bool is_set_;\n  getter_t get_;\n  setter_t set_;\n  copy_t copy_;\n};\n\ntemplate <>\ninline void pvalue<char *>::update() {\n  char buf[256];\n  ASSERT_FPGA_OK(get_(*props_, buf));\n  copy_.assign(buf);\n  is_set_ = true;\n}\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                            "},{"location":"opae-code/shared__buffer_8h/","title":"File shared_buffer.h","text":"
                                            FileList > core > shared_buffer.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <opae/buffer.h>
                                              •  
                                            • #include <opae/cxx/core/except.h>
                                              •  
                                            • #include <opae/cxx/core/handle.h>
                                              •  
                                            • #include <chrono>
                                              •  
                                            • #include <cstdint>
                                              •  
                                            • #include <initializer_list>
                                              •  
                                            • #include <memory>
                                              •  
                                            • #include <thread>
                                              •  
                                            • #include <vector>
                                              •  
                                            "},{"location":"opae-code/shared__buffer_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/shared__buffer_8h/#classes","title":"Classes","text":"Type Name class shared_buffer Host/AFU shared memory blocks. 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/shared_buffer.h
                                            "},{"location":"opae-code/shared__buffer_8h_source/","title":"File shared_buffer.h","text":"
                                            File List > core > shared_buffer.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/buffer.h>\n#include <opae/cxx/core/except.h>\n#include <opae/cxx/core/handle.h>\n\n#include <chrono>\n#include <cstdint>\n#include <initializer_list>\n#include <memory>\n#include <thread>\n#include <vector>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass shared_buffer {\n public:\n  typedef std::size_t size_t;\n  typedef std::shared_ptr<shared_buffer> ptr_t;\n\n  shared_buffer(const shared_buffer &) = delete;\n  shared_buffer &operator=(const shared_buffer &) = delete;\n\n  virtual ~shared_buffer();\n\n  static shared_buffer::ptr_t allocate(handle::ptr_t handle, size_t len,\n                                       bool read_only = false);\n\n  static shared_buffer::ptr_t attach(handle::ptr_t handle, uint8_t *base,\n                                     size_t len, bool read_only = false);\n\n  void release();\n\n  volatile uint8_t *c_type() const { return virt_; }\n\n  handle::ptr_t owner() const { return handle_; }\n\n  size_t size() const { return len_; }\n\n  uint64_t wsid() const { return wsid_; }\n\n  uint64_t io_address() const { return io_address_; }\n\n  void fill(int c);\n\n  int compare(ptr_t other, size_t len) const;\n\n  template <typename T>\n  T read(size_t offset) const {\n    if ((offset < len_) && (virt_ != nullptr)) {\n      return *reinterpret_cast<T *>(virt_ + offset);\n    } else if (offset >= len_) {\n      throw except(OPAECXX_HERE);\n    } else {\n      throw except(OPAECXX_HERE);\n    }\n    return T();\n  }\n\n  template <typename T>\n  void write(const T &value, size_t offset) {\n    if ((offset < len_) && (virt_ != nullptr)) {\n      *reinterpret_cast<T *>(virt_ + offset) = value;\n    } else if (offset >= len_) {\n      throw except(OPAECXX_HERE);\n    } else {\n      throw except(OPAECXX_HERE);\n    }\n  }\n\n protected:\n  shared_buffer(handle::ptr_t handle, size_t len, uint8_t *virt, uint64_t wsid,\n                uint64_t io_address);\n\n  handle::ptr_t handle_;\n  size_t len_;\n  uint8_t *virt_;\n  uint64_t wsid_;\n  uint64_t io_address_;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                            "},{"location":"opae-code/cxx_2core_2sysobject_8h/","title":"File sysobject.h","text":"
                                            FileList > core > sysobject.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <opae/cxx/core/handle.h>
                                              •  
                                            • #include <opae/cxx/core/token.h>
                                              •  
                                            • #include <opae/types.h>
                                              •  
                                            • #include <memory>
                                              •  
                                            • #include <vector>
                                              •  
                                            "},{"location":"opae-code/cxx_2core_2sysobject_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/cxx_2core_2sysobject_8h/#classes","title":"Classes","text":"Type Name class sysobject Wraps the OPAE fpga_object primitive. 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/sysobject.h
                                            "},{"location":"opae-code/cxx_2core_2sysobject_8h_source/","title":"File sysobject.h","text":"
                                            File List > core > sysobject.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/cxx/core/handle.h>\n#include <opae/cxx/core/token.h>\n#include <opae/types.h>\n\n#include <memory>\n#include <vector>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass sysobject {\n public:\n  typedef std::shared_ptr<sysobject> ptr_t;\n\n  sysobject() = delete;\n\n  sysobject(const sysobject &o) = delete;\n\n  sysobject &operator=(const sysobject &o) = delete;\n\n  static sysobject::ptr_t get(token::ptr_t t, const std::string &name,\n                              int flags = 0);\n\n  static sysobject::ptr_t get(handle::ptr_t h, const std::string &name,\n                              int flags = 0);\n\n  sysobject::ptr_t get(const std::string &name, int flags = 0);\n\n  sysobject::ptr_t get(int index);\n\n  virtual ~sysobject();\n\n  uint32_t size() const;\n\n  uint64_t read64(int flags = 0) const;\n\n  void write64(uint64_t value, int flags = 0) const;\n\n  std::vector<uint8_t> bytes(int flags = 0) const;\n\n  std::vector<uint8_t> bytes(uint32_t offset, uint32_t size,\n                             int flags = 0) const;\n\n  enum fpga_sysobject_type type() const;\n\n  fpga_object c_type() const { return sysobject_; }\n\n  operator fpga_object() const { return sysobject_; }\n\n private:\n  sysobject(fpga_object sysobj, token::ptr_t token, handle::ptr_t hnd);\n  fpga_object sysobject_;\n  token::ptr_t token_;\n  handle::ptr_t handle_;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                            "},{"location":"opae-code/token_8h/","title":"File token.h","text":"
                                            FileList > core > token.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <opae/access.h>
                                              •  
                                            • #include <opae/cxx/core/properties.h>
                                              •  
                                            • #include <opae/enum.h>
                                              •  
                                            • #include <opae/types.h>
                                              •  
                                            • #include <memory>
                                              •  
                                            • #include <vector>
                                              •  
                                            "},{"location":"opae-code/token_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/token_8h/#classes","title":"Classes","text":"Type Name class token Wraps the OPAE fpga_token primitive. 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/token.h
                                            "},{"location":"opae-code/token_8h_source/","title":"File token.h","text":"
                                            File List > core > token.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018-2021, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/access.h>\n#include <opae/cxx/core/properties.h>\n#include <opae/enum.h>\n#include <opae/types.h>\n\n#include <memory>\n#include <vector>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass token {\n public:\n  typedef std::shared_ptr<token> ptr_t;\n\n  static std::vector<token::ptr_t> enumerate(\n      const std::vector<properties::ptr_t>& props);\n\n  ~token();\n\n  fpga_token c_type() const { return token_; }\n\n  operator fpga_token() const { return token_; }\n\n  ptr_t get_parent() const;\n\n private:\n  token(fpga_token tok);\n\n  fpga_token token_;\n\n  friend class handle;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                            "},{"location":"opae-code/cxx_2core_2version_8h/","title":"File version.h","text":"
                                            FileList > core > version.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            • #include <string>
                                              •  
                                            "},{"location":"opae-code/cxx_2core_2version_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/cxx_2core_2version_8h/#classes","title":"Classes","text":"Type Name class version 
                                            The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/version.h
                                            "},{"location":"opae-code/cxx_2core_2version_8h_source/","title":"File version.h","text":"
                                            File List > core > version.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/types.h>\n\n#include <string>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass version {\n public:\n  static fpga_version as_struct();\n\n  static std::string as_string();\n\n  static std::string build();\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                            "},{"location":"opae-code/enum_8h/","title":"File enum.h","text":"
                                            FileList > docs > sw > include > opae > enum.h
                                             
                                            Go to the source code of this file.
                                             
                                            APIs for resource enumeration and managing tokens. More...
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/enum_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaCloneToken (fpga_token src, fpga_token * dst) Clone a fpga_token object. fpga_result fpgaDestroyToken (fpga_token * token) Destroy a Token. fpga_result fpgaEnumerate (const fpga_properties * filters, uint32_t num_filters, fpga_token * tokens, uint32_t max_tokens, uint32_t * num_matches) Enumerate FPGA resources present in the system."},{"location":"opae-code/enum_8h/#detailed-description","title":"Detailed Description","text":"
                                            These APIs are the first step for any application using OPAE to discover resources that are present on the system. They allow selective enumeration (i.e. getting a list of resources that match a given list of criteria) and methods to manage the lifecycle of tokens generated by fpgaEnumerate(). 
                                            "},{"location":"opae-code/enum_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/enum_8h/#function-fpgaclonetoken","title":"function fpgaCloneToken","text":"
                                            Clone a fpga_token object. 
                                            fpga_result fpgaCloneToken (\n    fpga_token src,\n    fpga_token * dst\n) \n
                                             
                                            Creates a copy of an fpga_token object.
                                             
                                            Note:
                                             
                                            This call creates a new token object and allocates memory for it. It is the responsibility of the using application to free this memory after use by calling fpgaDestroyToken() for the cloned token.
                                             
                                            Parameters:
                                             
                                               
                                            • src fpga_token object to copy 
                                              •  
                                            • dst New fpga_token object cloned from 'src' 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success 
                                            "},{"location":"opae-code/enum_8h/#function-fpgadestroytoken","title":"function fpgaDestroyToken","text":"
                                            Destroy a Token. 
                                            fpga_result fpgaDestroyToken (\n    fpga_token * token\n) \n
                                             
                                            This function destroys a token created by fpgaEnumerate() and frees the associated memory.
                                             
                                            Note:
                                             
                                            fpgaDestroyToken() requires the address of an fpga_token as previously created by fpgaEnumerate() or fpgaCloneToken(). Passing any other value results in undefined behavior.
                                             
                                            Parameters:
                                             
                                               
                                            • token fpga_token to destroy 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success 
                                            "},{"location":"opae-code/enum_8h/#function-fpgaenumerate","title":"function fpgaEnumerate","text":"
                                            Enumerate FPGA resources present in the system. 
                                            fpga_result fpgaEnumerate (\n    const fpga_properties * filters,\n    uint32_t num_filters,\n    fpga_token * tokens,\n    uint32_t max_tokens,\n    uint32_t * num_matches\n) \n
                                             
                                            This call allows the user to query the system for FPGA resources that match a certain set of criteria, e.g. all accelerators that are assigned to a host interface and available, all FPGAs of a specific type, etc.
                                             
                                            fpgaEnumerate() will create a number of fpga_tokens to represent the matching resources and populate the array tokens with these tokens. The max_tokens argument can be used to limit the number of tokens allocated/returned by fpgaEnumerate(); i.e., the number of tokens in the returned tokens array will be either max_tokens or num_matches (the number of resources matching the filter), whichever is smaller. Use fpgaDestroyToken() to destroy tokens that are no longer needed.
                                             
                                            To query the number of matches for a particular set of filters (e.g. to allocate a tokens array of the appropriate size), call fpgaEnumerate() with the parameter tokens set to NULL; this will only return the number of matches in num_matches.
                                             
                                            Note:
                                             
                                            fpgaEnumerate() will allocate memory for the created tokens returned in tokens. It is the responsibility of the using application to free this memory after use by calling fpgaDestroyToken() for each of the returned tokens.
                                             
                                            Parameters:
                                             
                                               
                                            • filters Array of fpga_properties objects describing the properties of the objects that should be returned. A resource is considered matching if its properties match any one of the supplied filters. To match all FPGA resources, pass an empty filters object (one without any filter criteria set) or pass a NULL filters parameter with num_filters set to 0. 
                                              •  
                                            • num_filters Number of entries in the filters array, or 0 to match all FPGA resources when filters is NULL. 
                                              •  
                                            • tokens Pointer to an array of fpga_token variables to be populated. If NULL is supplied, fpgaEnumerate() will not create any tokens, but it will return the number of possible matches in num_match. 
                                              •  
                                            • max_tokens Maximum number of tokens that fpgaEnumerate() shall return (length of tokens array). There may be more or fewer matches than this number; num_matches is set to the number of actual matches. 
                                              •  
                                            • num_matches Number of resources matching the filter criteria. This number can be higher than the number of tokens returned in the tokens array (depending on the value of max_tokens). 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if invalid pointers or objects are passed into the function. FPGA_NO_DRIVER if OPAE can't find the respective enumeration data structures usually provided by the driver. FPGA_NO_MEMORY if there was not enough memory to create tokens. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/enum.h
                                            "},{"location":"opae-code/enum_8h_source/","title":"File enum.h","text":"
                                            File List > docs > sw > include > opae > enum.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_ENUM_H__\n#define __FPGA_ENUM_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaEnumerate(const fpga_properties *filters,\n              uint32_t num_filters, fpga_token *tokens,\n              uint32_t max_tokens, uint32_t *num_matches);\n\nfpga_result fpgaCloneToken(fpga_token src, fpga_token *dst);\n\nfpga_result fpgaDestroyToken(fpga_token *token);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_ENUM_H__\n
                                            "},{"location":"opae-code/error_8h/","title":"File error.h","text":"
                                            FileList > docs > sw > include > opae > error.h
                                             
                                            Go to the source code of this file.
                                             
                                            Functions for reading and clearing errors in resources. More...
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/error_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaClearAllErrors (fpga_token token) Clear all error registers of a particular resource. fpga_result fpgaClearError (fpga_token token, uint32_t error_num) Clear error register. fpga_result fpgaGetErrorInfo (fpga_token token, uint32_t error_num, struct fpga_error_info * error_info) Get information about a particular error register. fpga_result fpgaReadError (fpga_token token, uint32_t error_num, uint64_t * value) Read error value."},{"location":"opae-code/error_8h/#detailed-description","title":"Detailed Description","text":"
                                            Many FPGA resources have the ability to track the occurrence of errors. This file provides functions to retrieve information about errors within resources. 
                                            "},{"location":"opae-code/error_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/error_8h/#function-fpgaclearallerrors","title":"function fpgaClearAllErrors","text":"
                                            Clear all error registers of a particular resource. 
                                            fpga_result fpgaClearAllErrors (\n    fpga_token token\n) \n
                                             
                                            This function will clear all error registers of the resource referenced by token, observing the necessary order of clearing errors, if any.
                                             
                                            Parameters:
                                             
                                               
                                            • token Token to accelerator resource to query 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the token, and FPGA_BUSY if error could not be cleared. 
                                            "},{"location":"opae-code/error_8h/#function-fpgaclearerror","title":"function fpgaClearError","text":"
                                            Clear error register. 
                                            fpga_result fpgaClearError (\n    fpga_token token,\n    uint32_t error_num\n) \n
                                             
                                            This function will clear the error register error_num of the resource referenced by token.
                                             
                                            Parameters:
                                             
                                               
                                            • token Token to accelerator resource to query 
                                              •  
                                            • error_num Number of error register to clear 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the token, and FPGA_BUSY if error could not be cleared. 
                                            "},{"location":"opae-code/error_8h/#function-fpgageterrorinfo","title":"function fpgaGetErrorInfo","text":"
                                            Get information about a particular error register. 
                                            fpga_result fpgaGetErrorInfo (\n    fpga_token token,\n    uint32_t error_num,\n    struct fpga_error_info * error_info\n) \n
                                             
                                            This function will populate a fpga_error_info struct with information about error number error_num of the resource referenced by token.
                                             
                                            Parameters:
                                             
                                               
                                            • token Token to accelerator resource to query 
                                              •  
                                            • error_num Error register to retrieve information about 
                                              •  
                                            • error_info Pointer to memory to store information into 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the token. 
                                            "},{"location":"opae-code/error_8h/#function-fpgareaderror","title":"function fpgaReadError","text":"
                                            Read error value. 
                                            fpga_result fpgaReadError (\n    fpga_token token,\n    uint32_t error_num,\n    uint64_t * value\n) \n
                                             
                                            This function will read the value of error register error_num of the resource referenced by token into the memory location pointed to by value.
                                             
                                            Parameters:
                                             
                                               
                                            • token Token to accelerator resource to query 
                                              •  
                                            • error_num Number of error register to read 
                                              •  
                                            • value Pointer to memory to store error value into (64 bit) 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the token. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/error.h
                                            "},{"location":"opae-code/error_8h_source/","title":"File error.h","text":"
                                            File List > docs > sw > include > opae > error.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_ERROR_H__\n#define __FPGA_ERROR_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n\nfpga_result fpgaReadError(fpga_token token, uint32_t error_num, uint64_t *value);\n\nfpga_result fpgaClearError(fpga_token token, uint32_t error_num);\n\nfpga_result fpgaClearAllErrors(fpga_token token);\n\nfpga_result fpgaGetErrorInfo(fpga_token token,\n                 uint32_t error_num,\n                 struct fpga_error_info *error_info);\n\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_ERROR_H__\n
                                            "},{"location":"opae-code/event_8h/","title":"File event.h","text":"
                                            FileList > docs > sw > include > opae > event.h
                                             
                                            Go to the source code of this file.
                                             
                                            Functions for registering events and managing the lifecycle for fpga_event_handle s.More...
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/event_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaCreateEventHandle (fpga_event_handle * event_handle) Initialize an event_handle. fpga_result fpgaDestroyEventHandle (fpga_event_handle * event_handle) Destroy an event_handle. fpga_result fpgaGetOSObjectFromEventHandle (const fpga_event_handle eh, int * fd) Get OS object from event handle. fpga_result fpgaRegisterEvent (fpga_handle handle, fpga_event_type event_type, fpga_event_handle event_handle, uint32_t flags) Register an FPGA event. fpga_result fpgaUnregisterEvent (fpga_handle handle, fpga_event_type event_type, fpga_event_handle event_handle) Unregister an FPGA event."},{"location":"opae-code/event_8h/#detailed-description","title":"Detailed Description","text":"
                                            OPAE provides an interface to asynchronous events that can be generated by different FPGA resources. The event API provides functions to register for these events; associated with every event a process has registered for is an fpga_event_handle, which encapsulates the OS-specific data structure for event objects. On Linux, an fpga_event_handle can be used as a file descriptor and passed to select(), poll(), epoll() and similar functions to wait for asynchronous events. 
                                            "},{"location":"opae-code/event_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/event_8h/#function-fpgacreateeventhandle","title":"function fpgaCreateEventHandle","text":"
                                            Initialize an event_handle. 
                                            fpga_result fpgaCreateEventHandle (\n    fpga_event_handle * event_handle\n) \n
                                             
                                            Platform independent way to initialize an event_handle used for notifications from the driver to application. For Linux, this function creates an eventfd and returns the eventfd file descriptor in *event_handle.
                                             
                                            Parameters:
                                             
                                               
                                            • event_handle Pointer to event handle variable.
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if event_handle is NULL. FPGA_NOT_SUPPORTED if platform does not support events. 
                                            "},{"location":"opae-code/event_8h/#function-fpgadestroyeventhandle","title":"function fpgaDestroyEventHandle","text":"
                                            Destroy an event_handle. 
                                            fpga_result fpgaDestroyEventHandle (\n    fpga_event_handle * event_handle\n) \n
                                             
                                            Destroy handle and free resources. On Linux this corresponds to closing the file descriptor pointed to by handle
                                             
                                            Note:
                                             
                                            fpgaDestroyEventHandle() requires the address of an event_handle as created by fpgaCreateEventHandle(). Passing any other value results in undefined behavior.
                                             
                                            Parameters:
                                             
                                               
                                            • event_handle Pointer to handle to be destroyed
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if event_handle is NULL. 
                                            "},{"location":"opae-code/event_8h/#function-fpgagetosobjectfromeventhandle","title":"function fpgaGetOSObjectFromEventHandle","text":"
                                            Get OS object from event handle. 
                                            fpga_result fpgaGetOSObjectFromEventHandle (\n    const fpga_event_handle eh,\n    int * fd\n) \n
                                             
                                            Check validity of event handle, and get the OS object used to subscribe and unsubscribe to events. On Linux, the object corresponds to a file descriptor.
                                             
                                            Parameters:
                                             
                                               
                                            • eh Event handle to get the descriptor value from 
                                              •  
                                            • fd integer to store the descriptor value
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if event_handle is invalid. 
                                            "},{"location":"opae-code/event_8h/#function-fpgaregisterevent","title":"function fpgaRegisterEvent","text":"
                                            Register an FPGA event. 
                                            fpga_result fpgaRegisterEvent (\n    fpga_handle handle,\n    fpga_event_type event_type,\n    fpga_event_handle event_handle,\n    uint32_t flags\n) \n
                                             
                                            This function tells the driver that the caller is interested in notification for the event specified by the type and flags pair.
                                             
                                            The event_handle points to an OS specific mechanism for event notification. An event_handle is associated with only a single event.
                                             
                                            In case of user interrupts, the flags parameter will be used to specify the vector ID. The value of the flags parameter indicates the vector ID, no bit encoding is used.
                                             
                                            Todo
                                             
                                            define if calling fpgaRegisterEvent multiple times with the same event_handle is an error condition or if it is silently ignored.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened FPGA resource. 
                                              •  
                                            • event_type Type of event 
                                              •  
                                            • event_handle Handle to previously opened resource for event notification. 
                                              •  
                                            • flags Optional argument for specifying additional information about event. For example irq number for interrupt events. 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if handle does not refer to a resource supporting the requested event, or if event_handle is not valid. FPGA_EXCEPTION if an internal exception occurred while accessing the handle or the event_handle. On Linux: FPGA_NO_DAEMON if the driver does not support the requested event and there is no FPGA Daemon (fpgad) running to proxy it. 
                                            "},{"location":"opae-code/event_8h/#function-fpgaunregisterevent","title":"function fpgaUnregisterEvent","text":"
                                            Unregister an FPGA event. 
                                            fpga_result fpgaUnregisterEvent (\n    fpga_handle handle,\n    fpga_event_type event_type,\n    fpga_event_handle event_handle\n) \n
                                             
                                            This function tells the driver that the caller is no longer interested in notification for the event associated with the event_handle
                                             
                                            The event_handle points to an OS specific mechanism for event notification. An event_handle is associated with only a single event.
                                             
                                            Todo
                                             
                                            define if calling fpgaUnregisterEvent multiple times with the same event_handle is an error condition or if it is silently ignored.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened FPGA resource. 
                                              •  
                                            • event_type Type of event to unregister. 
                                              •  
                                            • event_handle Handle to previously registered resource for event notification. 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if handle does not refer to a resource supporting the requested event, or if event_handle is not valid. FPGA_EXCEPTION if an internal error occurred accessing the handle or the event_handle. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/event.h
                                            "},{"location":"opae-code/event_8h_source/","title":"File event.h","text":"
                                            File List > docs > sw > include > opae > event.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_EVENT_H__\n#define __FPGA_EVENT_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaCreateEventHandle(fpga_event_handle *event_handle);\n\nfpga_result fpgaDestroyEventHandle(fpga_event_handle *event_handle);\n\n\nfpga_result fpgaGetOSObjectFromEventHandle(const fpga_event_handle eh, int *fd);\n\nfpga_result fpgaRegisterEvent(fpga_handle handle,\n                  fpga_event_type event_type,\n                  fpga_event_handle event_handle,\n                  uint32_t flags);\n\nfpga_result fpgaUnregisterEvent(fpga_handle handle,\n                         fpga_event_type event_type,\n                         fpga_event_handle event_handle);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_EVENT_H__\n
                                            "},{"location":"opae-code/fpga_8h/","title":"File fpga.h","text":"
                                            FileList > docs > sw > include > opae > fpga.h
                                             
                                            Go to the source code of this file.
                                             
                                            FPGA API. More...
                                             
                                               
                                            • #include <opae/log.h>
                                              •  
                                            • #include <opae/init.h>
                                              •  
                                            • #include <opae/types.h>
                                              •  
                                            • #include <opae/access.h>
                                              •  
                                            • #include <opae/buffer.h>
                                              •  
                                            • #include <opae/enum.h>
                                              •  
                                            • #include <opae/event.h>
                                              •  
                                            • #include <opae/manage.h>
                                              •  
                                            • #include <opae/mmio.h>
                                              •  
                                            • #include <opae/properties.h>
                                              •  
                                            • #include <opae/umsg.h>
                                              •  
                                            • #include <opae/utils.h>
                                              •  
                                            • #include <opae/error.h>
                                              •  
                                            • #include <opae/version.h>
                                              •  
                                            • #include <opae/sysobject.h>
                                              •  
                                            • #include <opae/userclk.h>
                                              •  
                                            • #include <opae/metrics.h>
                                              •  
                                            "},{"location":"opae-code/fpga_8h/#detailed-description","title":"Detailed Description","text":"
                                            This conveniently includes all APIs that a part of the OPAE release (base and extensions). 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/fpga.h
                                            "},{"location":"opae-code/fpga_8h_source/","title":"File fpga.h","text":"
                                            File List > docs > sw > include > opae > fpga.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_FPGA_H__\n#define __FPGA_FPGA_H__\n\n#include <opae/log.h>\n#include <opae/init.h>\n#include <opae/types.h>\n#include <opae/access.h>\n#include <opae/buffer.h>\n#include <opae/enum.h>\n#include <opae/event.h>\n#include <opae/manage.h>\n#include <opae/mmio.h>\n#include <opae/properties.h>\n#include <opae/umsg.h>\n#include <opae/utils.h>\n#include <opae/error.h>\n#include <opae/version.h>\n#include <opae/sysobject.h>\n#include <opae/userclk.h>\n#include <opae/metrics.h>\n\n#endif // __FPGA_FPGA_H__\n
                                            "},{"location":"opae-code/hash__map_8h/","title":"File hash_map.h","text":"
                                            FileList > docs > sw > include > opae > hash_map.h
                                             
                                            Go to the source code of this file.
                                             
                                            A general-purpose hybrid array/list hash map implementation. More...
                                             
                                               
                                            • #include <stdint.h>
                                              •  
                                            • #include <stdbool.h>
                                              •  
                                            • #include <opae/types_enum.h>
                                              •  
                                            "},{"location":"opae-code/hash__map_8h/#classes","title":"Classes","text":"Type Name struct _opae_hash_map Hash map object. struct _opae_hash_map_item List link item."},{"location":"opae-code/hash__map_8h/#public-types","title":"Public Types","text":"Type Name enum _opae_hash_map_flags Flags used to initialize a hash map. typedef struct _opae_hash_map opae_hash_map Hash map object. typedef enum _opae_hash_map_flags opae_hash_map_flags Flags used to initialize a hash map. typedef struct _opae_hash_map_item opae_hash_map_item List link item."},{"location":"opae-code/hash__map_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result opae_hash_map_add (opae_hash_map * hm, void * key, void * value) Map a key to a value. fpga_result opae_hash_map_destroy (opae_hash_map * hm) Tear down a hash map. fpga_result opae_hash_map_find (opae_hash_map * hm, void * key, void ** value) Retrieve the value for a given key. fpga_result opae_hash_map_init (opae_hash_map * hm, uint32_t num_buckets, uint32_t hash_seed, int flags, uint32_t(*)(uint32_t num_buckets, uint32_t hash_seed, void *key) key_hash, int(*)(void *keya, void *keyb) key_compare, void(*)(void *key, void *context) key_cleanup, void(*)(void *value, void *context) value_cleanup) Initialize a hash map. bool opae_hash_map_is_empty (opae_hash_map * hm) Determine whether a hash map is empty. fpga_result opae_hash_map_remove (opae_hash_map * hm, void * key) Remove a key/value association. int opae_u64_key_compare (void * keya, void * keyb) Convenience key comparison function for 64-bit values. uint32_t opae_u64_key_hash (uint32_t num_buckets, uint32_t hash_seed, void * key) Convenience hash function for arbitrary pointers/64-bit values."},{"location":"opae-code/hash__map_8h/#detailed-description","title":"Detailed Description","text":"
                                            Presents a generic interface for mapping key objects to value objects. Both keys and values may be arbitrary data structures. The user supplies the means by which the hash of values is generated and by which the keys are compared to each other. 
                                            "},{"location":"opae-code/hash__map_8h/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/hash__map_8h/#enum-_opae_hash_map_flags","title":"enum _opae_hash_map_flags","text":"
                                            Flags used to initialize a hash map. 
                                            enum _opae_hash_map_flags {\n    OPAE_HASH_MAP_UNIQUE_KEYSPACE = (1u << 0)\n};\n
                                             
                                            OPAE_HASH_MAP_UNIQUE_KEYSPACE says that the user provides a guarantee that the key space is truly unique. In other words, when the provided hash function for keys A and B returns the same bucket index, the key comparison function when comparing A and B will never return a result saying that the keys are equal in value. This is helpful in situations where the key space is guaranteed to produce unique values, for example a memory allocator. When the key space is guaranteed to be unique, opae_hash_map_add() can implement a small performance improvement. 
                                            "},{"location":"opae-code/hash__map_8h/#typedef-opae_hash_map","title":"typedef opae_hash_map","text":"
                                            Hash map object. 
                                            typedef struct _opae_hash_map opae_hash_map;\n
                                             
                                            This structure defines the internals of the hash map. Each of the parameters supplied to opae_hash_map_init() is stored in the structure. All parameters are required, except key_cleanup and value_cleanup, which may optionally be NULL. 
                                            "},{"location":"opae-code/hash__map_8h/#typedef-opae_hash_map_flags","title":"typedef opae_hash_map_flags","text":"
                                            Flags used to initialize a hash map. 
                                            typedef enum _opae_hash_map_flags opae_hash_map_flags;\n
                                             
                                            OPAE_HASH_MAP_UNIQUE_KEYSPACE says that the user provides a guarantee that the key space is truly unique. In other words, when the provided hash function for keys A and B returns the same bucket index, the key comparison function when comparing A and B will never return a result saying that the keys are equal in value. This is helpful in situations where the key space is guaranteed to produce unique values, for example a memory allocator. When the key space is guaranteed to be unique, opae_hash_map_add() can implement a small performance improvement. 
                                            "},{"location":"opae-code/hash__map_8h/#typedef-opae_hash_map_item","title":"typedef opae_hash_map_item","text":"
                                            List link item. 
                                            typedef struct _opae_hash_map_item opae_hash_map_item;\n
                                             
                                            This structure provides the association between key and value. When the supplied hash function for keys A and B returns the same bucket index, both A and B can co-exist on the same list rooted at the bucket index. 
                                            "},{"location":"opae-code/hash__map_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/hash__map_8h/#function-opae_hash_map_add","title":"function opae_hash_map_add","text":"
                                            Map a key to a value. 
                                            fpga_result opae_hash_map_add (\n    opae_hash_map * hm,\n    void * key,\n    void * value\n) \n
                                             
                                            Inserts a mapping from key to value in the given hash map object. Subsequent calls to opae_hash_map_find() that are given the key will retrieve the value.
                                             
                                            Parameters:
                                             
                                               
                                            • hm A pointer to the storage for the hash map object. 
                                              •  
                                            • key The hash map key. 
                                              •  
                                            • value The hash map value. 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success, FPGA_INVALID_PARAM if hm is NULL, FPGA_NO_MEMORY if malloc() fails when allocating the list item, or FPGA_INVALID_PARAM if the key hash produced by key_hash is out of bounds. 
                                            "},{"location":"opae-code/hash__map_8h/#function-opae_hash_map_destroy","title":"function opae_hash_map_destroy","text":"
                                            Tear down a hash map. 
                                            fpga_result opae_hash_map_destroy (\n    opae_hash_map * hm\n) \n
                                             
                                            Given a hash map that was previously initialized by opae_hash_map_init(), destroy the hash map, releasing all keys, values, and the bucket array.
                                             
                                            Parameters:
                                             
                                               
                                            • hm A pointer to the storage for the hash map object. 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success or FPGA_INVALID_PARAM is hm is NULL. 
                                            "},{"location":"opae-code/hash__map_8h/#function-opae_hash_map_find","title":"function opae_hash_map_find","text":"
                                            Retrieve the value for a given key. 
                                            fpga_result opae_hash_map_find (\n    opae_hash_map * hm,\n    void * key,\n    void ** value\n) \n
                                             
                                            Given a key that was previously passed to opae_hash_map_add(), retrieve its associated value.
                                             
                                            Parameters:
                                             
                                               
                                            • hm A pointer to the storage for the hash map object. 
                                              •  
                                            • key The hash map key. 
                                              •  
                                            • value A pointer to receive the hash map value. 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success, FPGA_INVALID_PARAM if hm is NULL or if the key hash produced by key_hash is out of bounds, or FPGA_NOT_FOUND if the given key was not found in the hash map. 
                                            "},{"location":"opae-code/hash__map_8h/#function-opae_hash_map_init","title":"function opae_hash_map_init","text":"
                                            Initialize a hash map. 
                                            fpga_result opae_hash_map_init (\n    opae_hash_map * hm,\n    uint32_t num_buckets,\n    uint32_t hash_seed,\n    int flags,\n    uint32_t(*)(uint32_t num_buckets, uint32_t hash_seed, void *key) key_hash,\n    int(*)(void *keya, void *keyb) key_compare,\n    void(*)(void *key, void *context) key_cleanup,\n    void(*)(void *value, void *context) value_cleanup\n) \n
                                             
                                            Populates the hash map data structure and allocates the buckets array.
                                             
                                            Parameters:
                                             
                                               
                                            • hm A pointer to the storage for the hash map object. 
                                              •  
                                            • num_buckets The desired size of the buckets array. Each array entry may be empty (NULL), or may contain a list of opae_hash_map_item structures for which the given key_hash function returned the same key hash value. 
                                              •  
                                            • hash_seed A seed value used during key hash computation. This value will be the hash_seed parameter to the key hash function. 
                                              •  
                                            • flags Initialization flags. See opae_hash_map_flags. 
                                              •  
                                            • key_hash A pointer to a function that produces the hash value, given the number of buckets, the hash seed, and the key. Valid values are between 0 and num_buckets - 1, inclusively. 
                                              •  
                                            • key_compare A pointer to a function that compares two keys. The return value is similar to that of strcmp(), where a negative value means that keya < keyb, 0 means that keya == keyb, and a positive values means that keya > keyb. 
                                              •  
                                            • key_cleanup A pointer to a function that is called when a key is being removed from the map. This function is optional and may be NULL. When supplied, the function is responsible for freeing any resources allocated when the key was created. 
                                              •  
                                            • value_cleanup A pointer to a function that is called when a value is being removed from the map. This function is optional and may be NULL. When supplied, the function is responsible for freeing any resources allocated when the value was created. 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success, FPGA_INVALID_PARAM if any of the required parameters are NULL, or FPGA_NO_MEMORY if the bucket array could not be allocated. 
                                            "},{"location":"opae-code/hash__map_8h/#function-opae_hash_map_is_empty","title":"function opae_hash_map_is_empty","text":"
                                            Determine whether a hash map is empty. 
                                            bool opae_hash_map_is_empty (\n    opae_hash_map * hm\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • hm A pointer to the storage for the hash map object. 
                                              •  
                                             
                                            Returns:
                                             
                                            true if there are no key/value mappings present, false otherwise. 
                                            "},{"location":"opae-code/hash__map_8h/#function-opae_hash_map_remove","title":"function opae_hash_map_remove","text":"
                                            Remove a key/value association. 
                                            fpga_result opae_hash_map_remove (\n    opae_hash_map * hm,\n    void * key\n) \n
                                             
                                            Given a key that was previously passed to opae_hash_map_add(), remove the key and its associated value, calling the cleanup functions as needed.
                                             
                                            Parameters:
                                             
                                               
                                            • hm A pointer to the storage for the hash map object. 
                                              •  
                                            • key The hash map key. 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success, FPGA_INVALID_PARAM when hm is NULL or when the key hash produced by key_hash is out of bounds, or FPGA_NOT_FOUND if the key is not found in the hash map. 
                                            "},{"location":"opae-code/hash__map_8h/#function-opae_u64_key_compare","title":"function opae_u64_key_compare","text":"
                                            Convenience key comparison function for 64-bit values. 
                                            int opae_u64_key_compare (\n    void * keya,\n    void * keyb\n) \n
                                             
                                            Simply converts the key pointers to uint64_t's and performs unsigned integer comparison. 
                                            "},{"location":"opae-code/hash__map_8h/#function-opae_u64_key_hash","title":"function opae_u64_key_hash","text":"
                                            Convenience hash function for arbitrary pointers/64-bit values. 
                                            uint32_t opae_u64_key_hash (\n    uint32_t num_buckets,\n    uint32_t hash_seed,\n    void * key\n) \n
                                             
                                            Simply converts the key to a uint64_t and then performs the modulus operation with the configured num_buckets. hash_seed is unused. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/hash_map.h
                                            "},{"location":"opae-code/hash__map_8h_source/","title":"File hash_map.h","text":"
                                            File List > docs > sw > include > opae > hash_map.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2022-2023, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __OPAE_HASH_MAP_H__\n#define __OPAE_HASH_MAP_H__\n#include <stdint.h>\n#include <stdbool.h>\n#include <opae/types_enum.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif // __cplusplus\n\ntypedef enum _opae_hash_map_flags {\n    OPAE_HASH_MAP_UNIQUE_KEYSPACE = (1u << 0)\n} opae_hash_map_flags;\n\ntypedef struct _opae_hash_map_item {\n    void *key;\n    void *value;\n    struct _opae_hash_map_item *next;\n} opae_hash_map_item;\n\ntypedef struct _opae_hash_map {\n    uint32_t num_buckets;\n    uint32_t hash_seed;\n    opae_hash_map_item **buckets;\n    int flags;\n    void *cleanup_context; \n    uint32_t (*key_hash)(uint32_t num_buckets,     \n                 uint32_t hash_seed,\n                 void *key);\n    int (*key_compare)(void *keya, void *keyb);    \n    void (*key_cleanup)(void *key, void *context);     \n    void (*value_cleanup)(void *value, void *context); \n} opae_hash_map;\n\nfpga_result opae_hash_map_init(opae_hash_map *hm,\n                   uint32_t num_buckets,\n                   uint32_t hash_seed,\n                   int flags,\n                   uint32_t (*key_hash)(uint32_t num_buckets,\n                            uint32_t hash_seed,\n                            void *key),\n                   int (*key_compare)(void *keya, void *keyb),\n                   void (*key_cleanup)(void *key, void *context),\n                   void (*value_cleanup)(void *value, void *context));\n\nfpga_result opae_hash_map_add(opae_hash_map *hm,\n                  void *key,\n                  void *value);\n\nfpga_result opae_hash_map_find(opae_hash_map *hm,\n                   void *key,\n                   void **value);\n\nfpga_result opae_hash_map_remove(opae_hash_map *hm,\n                 void *key);\n\nfpga_result opae_hash_map_destroy(opae_hash_map *hm);\n\nbool opae_hash_map_is_empty(opae_hash_map *hm);\n\nuint32_t opae_u64_key_hash(uint32_t num_buckets,\n               uint32_t hash_seed,\n               void *key);\n\nint opae_u64_key_compare(void *keya, void *keyb);\n\n#ifdef __cplusplus\n}\n#endif // __cplusplus\n\n#endif // __OPAE_HASH_MAP_H__\n
                                            "},{"location":"opae-code/init_8h/","title":"File init.h","text":"
                                            FileList > docs > sw > include > opae > init.h
                                             
                                            Go to the source code of this file.
                                             
                                            Initialization routine. 
                                             
                                               
                                            • #include <opae/types_enum.h>
                                              •  
                                            "},{"location":"opae-code/init_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaFinalize (void) Finalize the OPAE library. fpga_result fpgaInitialize (const char * config_file) Initialize the OPAE library."},{"location":"opae-code/init_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/init_8h/#function-fpgafinalize","title":"function fpgaFinalize","text":"
                                            Finalize the OPAE library. 
                                            fpga_result fpgaFinalize (\n    void\n) \n
                                             
                                            Returns:
                                             
                                            Whether OPAE finalized successfully. 
                                            "},{"location":"opae-code/init_8h/#function-fpgainitialize","title":"function fpgaInitialize","text":"
                                            Initialize the OPAE library. 
                                            fpga_result fpgaInitialize (\n    const char * config_file\n) \n
                                             
                                            Initialize OPAE using the given configuration file path, or perform default initialization if config_file is NULL.
                                             
                                            Parameters:
                                             
                                               
                                            • config_file Path to OPAE configuration file. 
                                              •  
                                             
                                            Returns:
                                             
                                            Whether OPAE initialized successfully. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/init.h
                                            "},{"location":"opae-code/init_8h_source/","title":"File init.h","text":"
                                            File List > docs > sw > include > opae > init.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_INIT_H__\n#define __FPGA_INIT_H__\n\n#include <opae/types_enum.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaInitialize(const char *config_file);\n\nfpga_result fpgaFinalize(void);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_INIT_H__\n
                                            "},{"location":"opae-code/log_8h/","title":"File log.h","text":"
                                            FileList > docs > sw > include > opae > log.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <stdint.h>
                                              •  
                                            • #include <stdlib.h>
                                              •  
                                            • #include <string.h>
                                              •  
                                            • #include <errno.h>
                                              •  
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/log_8h/#public-types","title":"Public Types","text":"Type Name enum opae_loglevel"},{"location":"opae-code/log_8h/#public-functions","title":"Public Functions","text":"Type Name void opae_print (int loglevel, const char * fmt, ...)"},{"location":"opae-code/log_8h/#macros","title":"Macros","text":"Type Name define OPAE_DBG (format, ...) {  } define OPAE_DEFAULT_LOGLEVEL  OPAE_LOG_ERROR define OPAE_ERR (format, ...)  define OPAE_MSG (format, ...)  define __SHORT_FILE__"},{"location":"opae-code/log_8h/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/log_8h/#enum-opae_loglevel","title":"enum opae_loglevel","text":"
                                            enum opae_loglevel {\n    OPAE_LOG_ERROR = 0,\n    OPAE_LOG_MESSAGE,\n    OPAE_LOG_DEBUG\n};\n
                                            "},{"location":"opae-code/log_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/log_8h/#function-opae_print","title":"function opae_print","text":"
                                            void opae_print (\n    int loglevel,\n    const char * fmt,\n    ...\n) \n
                                            "},{"location":"opae-code/log_8h/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/log_8h/#define-opae_dbg","title":"define OPAE_DBG","text":"
                                            #define OPAE_DBG (\n    format,\n    ...\n) { }\n
                                            "},{"location":"opae-code/log_8h/#define-opae_default_loglevel","title":"define OPAE_DEFAULT_LOGLEVEL","text":"
                                            #define OPAE_DEFAULT_LOGLEVEL OPAE_LOG_ERROR\n
                                            "},{"location":"opae-code/log_8h/#define-opae_err","title":"define OPAE_ERR","text":"
                                            #define OPAE_ERR (\n    format,\n    ...\n) opae_print ( OPAE_LOG_ERROR ,                                \\\n    \"%s:%u:%s() **ERROR** : \" format \"\\n\",                    \\\n    __SHORT_FILE__, __LINE__, __func__, ##__VA_ARGS__)\n
                                            "},{"location":"opae-code/log_8h/#define-opae_msg","title":"define OPAE_MSG","text":"
                                            #define OPAE_MSG (\n    format,\n    ...\n) opae_print ( OPAE_LOG_MESSAGE , \"%s:%u:%s() : \" format \"\\n\", \\\n    __SHORT_FILE__, __LINE__, __func__, ##__VA_ARGS__)\n
                                            "},{"location":"opae-code/log_8h/#define-__short_file__","title":"define __SHORT_FILE__","text":"
                                            #define __SHORT_FILE__ ({                                                     \\\n    const char *file = __FILE__;                           \\\n    const char *p = file;                                  \\\n    while (*p)                                             \\\n        ++p;                                           \\\n    while ((p > file) && ('/' != *p) && ('\\\\' != *p))      \\\n        --p;                                           \\\n    if (p > file)                                          \\\n        ++p;                                           \\\n    p;                                                     \\\n    })\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/log.h
                                            "},{"location":"opae-code/log_8h_source/","title":"File log.h","text":"
                                            File List > docs > sw > include > opae > log.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018-2021, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __OPAE_LOG_H__\n#define __OPAE_LOG_H__\n\n#include <stdint.h>\n#include <stdlib.h>\n#include <string.h>\n#include <errno.h>\n\n#include <opae/types.h>\n\n/*\n* Convenience macros for printing messages and errors.\n*/\n#ifdef __SHORT_FILE__\n#undef __SHORT_FILE__\n#endif // __SHORT_FILE__\n#define __SHORT_FILE__                                         \\\n    ({                                                     \\\n    const char *file = __FILE__;                           \\\n    const char *p = file;                                  \\\n    while (*p)                                             \\\n        ++p;                                           \\\n    while ((p > file) && ('/' != *p) && ('\\\\' != *p))      \\\n        --p;                                           \\\n    if (p > file)                                          \\\n        ++p;                                           \\\n    p;                                                     \\\n    })\n\n#ifdef OPAE_MSG\n#undef OPAE_MSG\n#endif // OPAE_MSG\n#define OPAE_MSG(format, ...)                                     \\\n    opae_print(OPAE_LOG_MESSAGE, \"%s:%u:%s() : \" format \"\\n\", \\\n    __SHORT_FILE__, __LINE__, __func__, ##__VA_ARGS__)\n\n#ifdef OPAE_ERR\n#undef OPAE_ERR\n#endif // OPAE_ERR\n#define OPAE_ERR(format, ...)                                     \\\n    opae_print(OPAE_LOG_ERROR,                                \\\n    \"%s:%u:%s() **ERROR** : \" format \"\\n\",                    \\\n    __SHORT_FILE__, __LINE__, __func__, ##__VA_ARGS__)\n\n#ifdef OPAE_DBG\n#undef OPAE_DBG\n#endif // OPAE_DBG\n#ifdef LIBOPAE_DEBUG\n#define OPAE_DBG(format, ...)                                    \\\n    opae_print(OPAE_LOG_DEBUG,                               \\\n    \"%s:%u:%s() *DEBUG* : \" format \"\\n\",                     \\\n    __SHORT_FILE__, __LINE__, __func__, ##__VA_ARGS__)\n#else\n#define OPAE_DBG(format, ...)                                    \\\n{   }\n#endif // LIBOPAE_DEBUG\n\n/*\n* Logging functions\n*/\nenum opae_loglevel {\n    OPAE_LOG_ERROR = 0, /* critical errors (always print) */\n    OPAE_LOG_MESSAGE,   /* information (i.e. explain return code */\n    OPAE_LOG_DEBUG      /* debugging (also needs #define DEBUG 1) */\n};\n\n#define OPAE_DEFAULT_LOGLEVEL OPAE_LOG_ERROR\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif // __cplusplus\n\nvoid opae_print(int loglevel, const char *fmt, ...);\n\n#ifdef __cplusplus\n}\n#endif // __cplusplus\n\n#endif // __OPAE_LOG_H__\n
                                            "},{"location":"opae-code/manage_8h/","title":"File manage.h","text":"
                                            FileList > docs > sw > include > opae > manage.h
                                             
                                            Go to the source code of this file.
                                             
                                            Functions for managing FPGA configurations. More...
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/manage_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaAssignPortToInterface (fpga_handle fpga, uint32_t interface_num, uint32_t slot_num, int flags) Assign Port to a host interface. fpga_result fpgaAssignToInterface (fpga_handle fpga, fpga_token accelerator, uint32_t host_interface, int flags) Assign an accelerator to a host interface. fpga_result fpgaReconfigureSlot (fpga_handle fpga, uint32_t slot, const uint8_t * bitstream, size_t bitstream_len, int flags) Reconfigure a slot. fpga_result fpgaReleaseFromInterface (fpga_handle fpga, fpga_token accelerator) Unassign a previously assigned accelerator."},{"location":"opae-code/manage_8h/#detailed-description","title":"Detailed Description","text":"
                                            FPGA accelerators can be reprogrammed at run time by providing new partial bitstreams (\"green bitstreams\"). This file defines API functions for programming green bitstreams as well as for assigning accelerators to host interfaces for more complex deployment setups, such as virtualized systems. 
                                            "},{"location":"opae-code/manage_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/manage_8h/#function-fpgaassignporttointerface","title":"function fpgaAssignPortToInterface","text":"
                                            Assign Port to a host interface. 
                                            fpga_result fpgaAssignPortToInterface (\n    fpga_handle fpga,\n    uint32_t interface_num,\n    uint32_t slot_num,\n    int flags\n) \n
                                             
                                            This function assign Port to a host interface for subsequent use. Only Port that have been assigned to a host interface can be opened by fpgaOpen().
                                             
                                            Parameters:
                                             
                                               
                                            • fpga Handle to an FPGA object previously opened that both the host interface and the slot belong to 
                                              •  
                                            • interface_num Host interface number 
                                              •  
                                            • slot_num Slot number 
                                              •  
                                            • flags Flags (to be defined) 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success FPGA_INVALID_PARAM if input parameter combination is not valid. FPGA_EXCEPTION if an exception occcurred accessing the fpga handle. FPGA_NOT_SUPPORTED if driver does not support assignment. 
                                            "},{"location":"opae-code/manage_8h/#function-fpgaassigntointerface","title":"function fpgaAssignToInterface","text":"
                                            Assign an accelerator to a host interface. 
                                            fpga_result fpgaAssignToInterface (\n    fpga_handle fpga,\n    fpga_token accelerator,\n    uint32_t host_interface,\n    int flags\n) \n
                                             
                                            This function assigns an accelerator to a host interface for subsequent use. Only accelerators that have been assigned to a host interface can be opened by fpgaOpen().
                                             
                                            Note:
                                             
                                            This function is currently not supported.
                                             
                                            Parameters:
                                             
                                               
                                            • fpga Handle to an FPGA object previously opened that both the host interface and the accelerator belong to 
                                              •  
                                            • accelerator accelerator to assign 
                                              •  
                                            • host_interface Host interface to assign accelerator to 
                                              •  
                                            • flags Flags (to be defined) 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success 
                                            "},{"location":"opae-code/manage_8h/#function-fpgareconfigureslot","title":"function fpgaReconfigureSlot","text":"
                                            Reconfigure a slot. 
                                            fpga_result fpgaReconfigureSlot (\n    fpga_handle fpga,\n    uint32_t slot,\n    const uint8_t * bitstream,\n    size_t bitstream_len,\n    int flags\n) \n
                                             
                                            Sends a green bitstream file to an FPGA to reconfigure a specific slot. This call, if successful, will overwrite the currently programmed AFU in that slot with the AFU in the provided bitstream.
                                             
                                            As part of the reconfiguration flow, all accelerators associated with this slot will be unassigned and reset.
                                             
                                            Parameters:
                                             
                                               
                                            • fpga Handle to an FPGA object previously opened 
                                              •  
                                            • slot Token identifying the slot to reconfigure 
                                              •  
                                            • bitstream Pointer to memory holding the bitstream 
                                              •  
                                            • bitstream_len Length of the bitstream in bytes 
                                              •  
                                            • flags Flags that control behavior of reconfiguration. Value of 0 indicates no flags. FPGA_RECONF_FORCE indicates that the bitstream is programmed into the slot without checking if the resource is currently in use. 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if the provided parameters are not valid. FPGA_EXCEPTION if an internal error occurred accessing the handle or while sending the bitstream data to the driver. FPGA_BUSY if the accelerator for the given slot is in use. FPGA_RECONF_ERROR on errors reported by the driver (such as CRC or protocol errors).
                                             
                                            Note:
                                             
                                            By default, fpgaReconfigureSlot will not allow reconfiguring a slot with an accelerator in use. Add the flag FPGA_RECONF_FORCE to force reconfiguration without checking for accelerators in use. 
                                            "},{"location":"opae-code/manage_8h/#function-fpgareleasefrominterface","title":"function fpgaReleaseFromInterface","text":"
                                            Unassign a previously assigned accelerator. 
                                            fpga_result fpgaReleaseFromInterface (\n    fpga_handle fpga,\n    fpga_token accelerator\n) \n
                                             
                                            This function removes the assignment of an accelerator to an host interface (e.g. to be later assigned to a different host interface). As a consequence, the accelerator referred to by token 'accelerator' will be reset during the course of this function.
                                             
                                            Note:
                                             
                                            This function is currently not supported.
                                             
                                            Parameters:
                                             
                                               
                                            • fpga Handle to an FPGA object previously opened that both the host interface and the accelerator belong to 
                                              •  
                                            • accelerator accelerator to unassign/release 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/manage.h
                                            "},{"location":"opae-code/manage_8h_source/","title":"File manage.h","text":"
                                            File List > docs > sw > include > opae > manage.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_MANAGE_H__\n#define __FPGA_MANAGE_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaAssignPortToInterface(fpga_handle fpga,\n                    uint32_t interface_num,\n                    uint32_t slot_num,\n                    int flags);\n\nfpga_result fpgaAssignToInterface(fpga_handle fpga,\n                  fpga_token accelerator,\n                  uint32_t host_interface,\n                  int flags);\n\nfpga_result fpgaReleaseFromInterface(fpga_handle fpga,\n                     fpga_token accelerator);\n\nfpga_result fpgaReconfigureSlot(fpga_handle fpga,\n                uint32_t slot,\n                const uint8_t *bitstream,\n                size_t bitstream_len, int flags);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_MANAGE_H__\n
                                            "},{"location":"opae-code/mem__alloc_8h/","title":"File mem_alloc.h","text":"
                                            FileList > docs > sw > include > opae > mem_alloc.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <stdint.h>
                                              •  
                                            "},{"location":"opae-code/mem__alloc_8h/#classes","title":"Classes","text":"Type Name struct mem_alloc struct mem_link Provides an API for allocating/freeing a logical address space."},{"location":"opae-code/mem__alloc_8h/#public-functions","title":"Public Functions","text":"Type Name int mem_alloc_add_free (struct mem_alloc * m, uint64_t address, uint64_t size) Add a memory region to an allocator. void mem_alloc_destroy (struct mem_alloc * m) Destroy a memory allocator object. int mem_alloc_get (struct mem_alloc * m, uint64_t * address, uint64_t size) Allocate memory. void mem_alloc_init (struct mem_alloc * m) Initialize a memory allocator object. int mem_alloc_put (struct mem_alloc * m, uint64_t address) Free memory."},{"location":"opae-code/mem__alloc_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/mem__alloc_8h/#function-mem_alloc_add_free","title":"function mem_alloc_add_free","text":"
                                            Add a memory region to an allocator. 
                                            int mem_alloc_add_free (\n    struct mem_alloc * m,\n    uint64_t address,\n    uint64_t size\n) \n
                                             
                                            The memory region is added to the allocatable space and is immediately ready for allocation.
                                             
                                            Parameters:
                                             
                                               
                                            • m The memory allocator object. 
                                              •  
                                            • address The beginning address of the memory region. 
                                              •  
                                            • size The size of the memory region. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error. Zero on success.
                                             
                                            Example  struct mem_alloc m;
                                             
                                            mem_alloc_init(&m);
                                             
                                            if (mem_alloc_add_free(&m, 0x4000, 4096)) {   // handle error }
                                            "},{"location":"opae-code/mem__alloc_8h/#function-mem_alloc_destroy","title":"function mem_alloc_destroy","text":"
                                            Destroy a memory allocator object. 
                                            void mem_alloc_destroy (\n    struct mem_alloc * m\n) \n
                                             
                                            Frees all of the allocator's internal resources.
                                             
                                            Parameters:
                                             
                                               
                                            • m The address of the memory allocator to destroy. 
                                              •  
                                            "},{"location":"opae-code/mem__alloc_8h/#function-mem_alloc_get","title":"function mem_alloc_get","text":"
                                            Allocate memory. 
                                            int mem_alloc_get (\n    struct mem_alloc * m,\n    uint64_t * address,\n    uint64_t size\n) \n
                                             
                                            Retrieve an available memory address for a free block that is at least size bytes.
                                             
                                            Parameters:
                                             
                                               
                                            • m The memory allocator object. 
                                              •  
                                            • address The retrieved address for the allocation. 
                                              •  
                                            • size The request size in bytes. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error. Zero on success.
                                             
                                            Example  struct mem_alloc m; uint64_t addr = 0;
                                             
                                            mem_alloc_init(&m);
                                             
                                            if (mem_alloc_add_free(&m, 0x4000, 4096)) {   // handle error }
                                             
                                            ...
                                             
                                            if (mem_alloc_get(&m, &addr, 4096)) {   // handle allocation error }
                                            "},{"location":"opae-code/mem__alloc_8h/#function-mem_alloc_init","title":"function mem_alloc_init","text":"
                                            Initialize a memory allocator object. 
                                            void mem_alloc_init (\n    struct mem_alloc * m\n) \n
                                             
                                            After the call, the allocator is initialized but \"empty\". To add allocatable memory regions, further initialize the allocator with mem_alloc_add_free().
                                             
                                            Parameters:
                                             
                                               
                                            • m The address of the memory allocator to initialize. 
                                              •  
                                            "},{"location":"opae-code/mem__alloc_8h/#function-mem_alloc_put","title":"function mem_alloc_put","text":"
                                            Free memory. 
                                            int mem_alloc_put (\n    struct mem_alloc * m,\n    uint64_t address\n) \n
                                             
                                            Release a previously-allocated memory block.
                                             
                                            Parameters:
                                             
                                               
                                            • m The memory allocator object. 
                                              •  
                                            • address The address to free. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error. Zero on success.
                                             
                                            Example  struct mem_alloc m; uint64_t addr = 0;
                                             
                                            mem_alloc_init(&m);
                                             
                                            if (mem_alloc_add_free(&m, 0x4000, 4096)) {   // handle error }
                                             
                                            ...
                                             
                                            if (mem_alloc_get(&m, &addr, 4096)) {   // handle allocation error }
                                             
                                            ...
                                             
                                            if (mem_alloc_put(&m, addr)) {   // handle free error }
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/mem_alloc.h
                                            "},{"location":"opae-code/mem__alloc_8h_source/","title":"File mem_alloc.h","text":"
                                            File List > docs > sw > include > opae > mem_alloc.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2020, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#ifndef __OPAE_MEM_ALLOC_H__\n#define __OPAE_MEM_ALLOC_H__\n\n#include <stdint.h>\n\nstruct mem_link {\n    uint64_t address;\n    uint64_t size;\n    struct mem_link *prev;\n    struct mem_link *next;\n};\n\nstruct mem_alloc {\n    struct mem_link free;\n    struct mem_link allocated;\n};\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif // __cplusplus\n\nvoid mem_alloc_init(struct mem_alloc *m);\n\nvoid mem_alloc_destroy(struct mem_alloc *m);\n\nint mem_alloc_add_free(struct mem_alloc *m,\n               uint64_t address,\n               uint64_t size);\n\nint mem_alloc_get(struct mem_alloc *m,\n          uint64_t *address,\n          uint64_t size);\n\nint mem_alloc_put(struct mem_alloc *m,\n          uint64_t address);\n\n#ifdef __cplusplus\n}\n#endif // __cplusplus\n\n#endif // __OPAE_MEM_ALLOC_H__\n
                                            "},{"location":"opae-code/metrics_8h/","title":"File metrics.h","text":"
                                            FileList > docs > sw > include > opae > metrics.h
                                             
                                            Go to the source code of this file.
                                             
                                            Functions for Discover/ Enumerates metrics and retrieves values. 
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/metrics_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaGetMetricsByIndex (fpga_handle handle, uint64_t * metric_num, uint64_t num_metric_indexes, fpga_metric * metrics) Retrieve metrics values by index. fpga_result fpgaGetMetricsByName (fpga_handle handle, char ** metrics_names, uint64_t num_metric_names, fpga_metric * metrics) Retrieve metric values by names. fpga_result fpgaGetMetricsInfo (fpga_handle handle, fpga_metric_info * metric_info, uint64_t * num_metrics) Retrieve metrics information. fpga_result fpgaGetMetricsThresholdInfo (fpga_handle handle, struct metric_threshold * metric_thresholds, uint32_t * num_thresholds) Retrieve metrics / sendor threshold information and values. fpga_result fpgaGetNumMetrics (fpga_handle handle, uint64_t * num_metrics) Enumerates number of metrics."},{"location":"opae-code/metrics_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/metrics_8h/#function-fpgagetmetricsbyindex","title":"function fpgaGetMetricsByIndex","text":"
                                            Retrieve metrics values by index. 
                                            fpga_result fpgaGetMetricsByIndex (\n    fpga_handle handle,\n    uint64_t * metric_num,\n    uint64_t num_metric_indexes,\n    fpga_metric * metrics\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened fpga resource 
                                              •  
                                            • metric_num Pointer to array of metric index user allocates metric array 
                                              •  
                                            • num_metric_indexes Size of metric array 
                                              •  
                                            • metrics pointer to array of metric struct
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_NOT_FOUND if the Metrics are not found. FPGA_NO_MEMORY if there was not enough memory to enumerates metrics. 
                                            "},{"location":"opae-code/metrics_8h/#function-fpgagetmetricsbyname","title":"function fpgaGetMetricsByName","text":"
                                            Retrieve metric values by names. 
                                            fpga_result fpgaGetMetricsByName (\n    fpga_handle handle,\n    char ** metrics_names,\n    uint64_t num_metric_names,\n    fpga_metric * metrics\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened fpga resource 
                                              •  
                                            • metrics_names Pointer to array of metrics name user allocates metrics name array 
                                              •  
                                            • num_metric_names Size of metric name array 
                                              •  
                                            • metrics Pointer to array of metric struct
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_NOT_FOUND if the Metrics are not found 
                                            "},{"location":"opae-code/metrics_8h/#function-fpgagetmetricsinfo","title":"function fpgaGetMetricsInfo","text":"
                                            Retrieve metrics information. 
                                            fpga_result fpgaGetMetricsInfo (\n    fpga_handle handle,\n    fpga_metric_info * metric_info,\n    uint64_t * num_metrics\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened fpga resource 
                                              •  
                                            • metric_info Pointer to array of metric info struct user allocates metrics info array
                                              •  
                                            • num_metrics Size of metric info array
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_NOT_FOUND if the Metrics are not found. FPGA_NO_MEMORY if there was not enough memory to enumerates metrics. 
                                            "},{"location":"opae-code/metrics_8h/#function-fpgagetmetricsthresholdinfo","title":"function fpgaGetMetricsThresholdInfo","text":"
                                            Retrieve metrics / sendor threshold information and values. 
                                            fpga_result fpgaGetMetricsThresholdInfo (\n    fpga_handle handle,\n    struct metric_threshold * metric_thresholds,\n    uint32_t * num_thresholds\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened fpga resource 
                                              •  
                                            • metrics_threshold pointer to array of metric thresholds user allocates threshold array memory Number of thresholds returns enumerated thresholds if user pass NULL metrics_thresholds 
                                              •  
                                            • num_thresholds number of thresholds
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_NOT_FOUND if the Metrics are not found. FPGA_NO_MEMORY if there was not enough memory to enumerates metrics. 
                                            "},{"location":"opae-code/metrics_8h/#function-fpgagetnummetrics","title":"function fpgaGetNumMetrics","text":"
                                            Enumerates number of metrics. 
                                            fpga_result fpgaGetNumMetrics (\n    fpga_handle handle,\n    uint64_t * num_metrics\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened fpga resource 
                                              •  
                                            • num_metrics Number of metrics are discovered in fpga resource
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_NOT_FOUND if the Metrics are not discovered 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/metrics.h
                                            "},{"location":"opae-code/metrics_8h_source/","title":"File metrics.h","text":"
                                            File List > docs > sw > include > opae > metrics.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_METRICS_H__\n#define __FPGA_METRICS_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaGetNumMetrics(fpga_handle handle,\n                uint64_t *num_metrics);\n\nfpga_result fpgaGetMetricsInfo(fpga_handle handle,\n                fpga_metric_info *metric_info,\n                uint64_t *num_metrics);\n\nfpga_result fpgaGetMetricsByIndex(fpga_handle handle,\n                uint64_t *metric_num,\n                uint64_t num_metric_indexes,\n                fpga_metric *metrics);\n\nfpga_result fpgaGetMetricsByName(fpga_handle handle,\n                char **metrics_names,\n                uint64_t num_metric_names,\n                fpga_metric *metrics);\n\n\nfpga_result fpgaGetMetricsThresholdInfo(fpga_handle handle,\n                struct metric_threshold *metric_thresholds,\n                uint32_t *num_thresholds);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_METRICS_H__\n
                                            "},{"location":"opae-code/mmio_8h/","title":"File mmio.h","text":"
                                            FileList > docs > sw > include > opae > mmio.h
                                             
                                            Go to the source code of this file.
                                             
                                            Functions for mapping and accessing MMIO space. More...
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/mmio_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaMapMMIO (fpga_handle handle, uint32_t mmio_num, uint64_t ** mmio_ptr) Map MMIO space. fpga_result fpgaReadMMIO32 (fpga_handle handle, uint32_t mmio_num, uint64_t offset, uint32_t * value) Read 32 bit value from MMIO space. fpga_result fpgaReadMMIO64 (fpga_handle handle, uint32_t mmio_num, uint64_t offset, uint64_t * value) Read 64 bit value from MMIO space. fpga_result fpgaUnmapMMIO (fpga_handle handle, uint32_t mmio_num) Unmap MMIO space. fpga_result fpgaWriteMMIO32 (fpga_handle handle, uint32_t mmio_num, uint64_t offset, uint32_t value) Write 32 bit value to MMIO space. fpga_result fpgaWriteMMIO512 (fpga_handle handle, uint32_t mmio_num, uint64_t offset, const void * value) Write 512 bit value to MMIO space. fpga_result fpgaWriteMMIO64 (fpga_handle handle, uint32_t mmio_num, uint64_t offset, uint64_t value) Write 64 bit value to MMIO space."},{"location":"opae-code/mmio_8h/#detailed-description","title":"Detailed Description","text":"
                                            Most FPGA accelerators provide access to control registers through memory-mappable address spaces, commonly referred to as \"MMIO spaces\". This file provides functions to map, unmap, read, and write MMIO spaces.
                                             
                                            Note that an accelerator may have multiple MMIO spaces, denoted by the mmio_num argument of the APIs below. The meaning and properties of each MMIO space are up to the accelerator designer. 
                                            "},{"location":"opae-code/mmio_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/mmio_8h/#function-fpgamapmmio","title":"function fpgaMapMMIO","text":"
                                            Map MMIO space. 
                                            fpga_result fpgaMapMMIO (\n    fpga_handle handle,\n    uint32_t mmio_num,\n    uint64_t ** mmio_ptr\n) \n
                                             
                                            This function will return a pointer to the specified MMIO space of the target object in process virtual memory, if supported by the target. Some MMIO spaces may be restricted to privileged processes, depending on the used handle and type.
                                             
                                            After mapping the respective MMIO space, you can access it through direct pointer operations (observing supported access sizes and alignments of the target platform and accelerator).
                                             
                                            Note:
                                             
                                            Some targets (such as the ASE simulator) do not support memory-mapping of IO register spaces and will not return a pointer to an actually mapped space. Instead, they will return FPGA_NOT_SUPPORTED. Usually, these platforms still allow the application to issue MMIO operations using fpgaReadMMIO32(), fpgaWriteMMIO32(), fpgeReadMMIO64(), and fpgaWriteMMIO64().
                                             
                                            If the caller passes in NULL for mmio_ptr, no mapping will be performed, and no virtual address will be returned, though the call will return FPGA_OK. This implies that all accesses will be performed through fpgaReadMMIO32(), fpgaWriteMMIO32(), fpgeReadMMIO64(), and fpgaWriteMMIO64(). This is the only supported case for ASE.
                                             
                                            The number of available MMIO spaces can be retrieved through the num_mmio property (fpgaPropertyGetNumMMIO()).
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened resource 
                                              •  
                                            • mmio_num Number of MMIO space to access 
                                              •  
                                            • mmio_ptr Pointer to memory where a pointer to the MMIO space will be returned. May be NULL, in which case no pointer is returned. Returned address may be NULL if underlying platform does not support memory mapping for register access. 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. FPGA_NO_ACCESS if the process' permissions are not sufficient to map the requested MMIO space. FPGA_NOT_SUPPORTED if platform does not support memory mapped IO. 
                                            "},{"location":"opae-code/mmio_8h/#function-fpgareadmmio32","title":"function fpgaReadMMIO32","text":"
                                            Read 32 bit value from MMIO space. 
                                            fpga_result fpgaReadMMIO32 (\n    fpga_handle handle,\n    uint32_t mmio_num,\n    uint64_t offset,\n    uint32_t * value\n) \n
                                             
                                            This function will read from MMIO space of the target object at a specified offset.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource 
                                              •  
                                            • mmio_num Number of MMIO space to access 
                                              •  
                                            • offset Byte offset into MMIO space 
                                              •  
                                            • value Pointer to memory where read value is returned (32 bit) 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                            "},{"location":"opae-code/mmio_8h/#function-fpgareadmmio64","title":"function fpgaReadMMIO64","text":"
                                            Read 64 bit value from MMIO space. 
                                            fpga_result fpgaReadMMIO64 (\n    fpga_handle handle,\n    uint32_t mmio_num,\n    uint64_t offset,\n    uint64_t * value\n) \n
                                             
                                            This function will read from MMIO space of the target object at a specified offset.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource 
                                              •  
                                            • mmio_num Number of MMIO space to access 
                                              •  
                                            • offset Byte offset into MMIO space 
                                              •  
                                            • value Pointer to memory where read value is returned (64 bit) 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                            "},{"location":"opae-code/mmio_8h/#function-fpgaunmapmmio","title":"function fpgaUnmapMMIO","text":"
                                            Unmap MMIO space. 
                                            fpga_result fpgaUnmapMMIO (\n    fpga_handle handle,\n    uint32_t mmio_num\n) \n
                                             
                                            This function will unmap a previously mapped MMIO space of the target object, rendering any pointers to it invalid.
                                             
                                            Note:
                                             
                                            This call is only supported by hardware targets, not by ASE simulation.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened resource 
                                              •  
                                            • mmio_num Number of MMIO space to access 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                            "},{"location":"opae-code/mmio_8h/#function-fpgawritemmio32","title":"function fpgaWriteMMIO32","text":"
                                            Write 32 bit value to MMIO space. 
                                            fpga_result fpgaWriteMMIO32 (\n    fpga_handle handle,\n    uint32_t mmio_num,\n    uint64_t offset,\n    uint32_t value\n) \n
                                             
                                            This function will write to MMIO space of the target object at a specified offset.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource 
                                              •  
                                            • mmio_num Number of MMIO space to access 
                                              •  
                                            • offset Byte offset into MMIO space 
                                              •  
                                            • value Value to write (32 bit) 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                            "},{"location":"opae-code/mmio_8h/#function-fpgawritemmio512","title":"function fpgaWriteMMIO512","text":"
                                            Write 512 bit value to MMIO space. 
                                            fpga_result fpgaWriteMMIO512 (\n    fpga_handle handle,\n    uint32_t mmio_num,\n    uint64_t offset,\n    const void * value\n) \n
                                             
                                            512 bit MMIO writes may not be supported on all platforms.
                                             
                                            This function will write to MMIO space of the target object at a specified offset.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource 
                                              •  
                                            • mmio_num Number of MMIO space to access 
                                              •  
                                            • offset Byte offset into MMIO space 
                                              •  
                                            • value Pointer to memory holding value to write (512 bits) 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                            "},{"location":"opae-code/mmio_8h/#function-fpgawritemmio64","title":"function fpgaWriteMMIO64","text":"
                                            Write 64 bit value to MMIO space. 
                                            fpga_result fpgaWriteMMIO64 (\n    fpga_handle handle,\n    uint32_t mmio_num,\n    uint64_t offset,\n    uint64_t value\n) \n
                                             
                                            This function will write to MMIO space of the target object at a specified offset.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource 
                                              •  
                                            • mmio_num Number of MMIO space to access 
                                              •  
                                            • offset Byte offset into MMIO space 
                                              •  
                                            • value Value to write (64 bit) 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/mmio.h
                                            "},{"location":"opae-code/mmio_8h_source/","title":"File mmio.h","text":"
                                            File List > docs > sw > include > opae > mmio.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_MMIO_H__\n#define __FPGA_MMIO_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaWriteMMIO64(fpga_handle handle,\n                uint32_t mmio_num, uint64_t offset,\n                uint64_t value);\n\nfpga_result fpgaReadMMIO64(fpga_handle handle,\n               uint32_t mmio_num,\n               uint64_t offset, uint64_t *value);\n\nfpga_result fpgaWriteMMIO32(fpga_handle handle,\n                uint32_t mmio_num, uint64_t offset,\n                uint32_t value);\n\nfpga_result fpgaReadMMIO32(fpga_handle handle,\n               uint32_t mmio_num,\n               uint64_t offset, uint32_t *value);\n\nfpga_result fpgaWriteMMIO512(fpga_handle handle,\n                uint32_t mmio_num, uint64_t offset,\n                const void *value);\n\nfpga_result fpgaMapMMIO(fpga_handle handle,\n            uint32_t mmio_num, uint64_t **mmio_ptr);\n\nfpga_result fpgaUnmapMMIO(fpga_handle handle,\n              uint32_t mmio_num);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_MMIO_H__\n
                                            "},{"location":"opae-code/properties_8h/","title":"File properties.h","text":"
                                            FileList > docs > sw > include > opae > properties.h
                                             
                                            Go to the source code of this file.
                                             
                                            Functions for examining and manipulating fpga_properties objects.More...
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/properties_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaClearProperties (fpga_properties prop) Clear a fpga_properties object. fpga_result fpgaCloneProperties (fpga_properties src, fpga_properties * dst) Clone a fpga_properties object. fpga_result fpgaDestroyProperties (fpga_properties * prop) Destroy a fpga_properties object. fpga_result fpgaGetProperties (fpga_token token, fpga_properties * prop) Create a fpga_properties object. fpga_result fpgaGetPropertiesFromHandle (fpga_handle handle, fpga_properties * prop) Create a fpga_properties object. fpga_result fpgaPropertiesGetAcceleratorState (const fpga_properties prop, fpga_accelerator_state * state) Get the state of a accelerator resource property. fpga_result fpgaPropertiesGetBBSID (const fpga_properties prop, uint64_t * bbs_id) Get the BBS ID of an FPGA resource property. fpga_result fpgaPropertiesGetBBSVersion (const fpga_properties prop, fpga_version * bbs_version) Get the BBS Version of an FPGA resource property. fpga_result fpgaPropertiesGetBus (const fpga_properties prop, uint8_t * bus) Get the PCI bus number of a resource. fpga_result fpgaPropertiesGetCapabilities (const fpga_properties prop, uint64_t * capabilities) Get the capabilities FPGA resource property. fpga_result fpgaPropertiesGetDevice (const fpga_properties prop, uint8_t * device) Get the PCI device number of a resource. fpga_result fpgaPropertiesGetDeviceID (const fpga_properties prop, uint16_t * device_id) Get the device id of the resource. fpga_result fpgaPropertiesGetFunction (const fpga_properties prop, uint8_t * function) Get the PCI function number of a resource. fpga_result fpgaPropertiesGetGUID (const fpga_properties prop, fpga_guid * guid) Get the GUID of a resource. fpga_result fpgaPropertiesGetInterface (const fpga_properties prop, fpga_interface * interface) Get the OPAE plugin interface implemented by a resource. fpga_result fpgaPropertiesGetLocalMemorySize (const fpga_properties prop, uint64_t * lms) Get the local memory size of an FPGA resource property. fpga_result fpgaPropertiesGetModel (const fpga_properties prop, char * model) Get the model of an FPGA resource property. fpga_result fpgaPropertiesGetNumErrors (const fpga_properties prop, uint32_t * num_errors) Get the number of errors that can be reported by a resource. fpga_result fpgaPropertiesGetNumInterrupts (const fpga_properties prop, uint32_t * num_interrupts) Get the number of interrupts. fpga_result fpgaPropertiesGetNumMMIO (const fpga_properties prop, uint32_t * mmio_spaces) Get the number of mmio spaces. fpga_result fpgaPropertiesGetNumSlots (const fpga_properties prop, uint32_t * num_slots) Get the number of slots of an FPGA resource property. fpga_result fpgaPropertiesGetObjectID (const fpga_properties prop, uint64_t * object_id) Get the object ID of a resource. fpga_result fpgaPropertiesGetObjectType (const fpga_properties prop, fpga_objtype * objtype) Get the object type of a resource. fpga_result fpgaPropertiesGetParent (const fpga_properties prop, fpga_token * parent) Get the token of the parent object. fpga_result fpgaPropertiesGetSegment (const fpga_properties prop, uint16_t * segment) Get the PCI segment number of a resource. fpga_result fpgaPropertiesGetSocketID (const fpga_properties prop, uint8_t * socket_id) Get the socket id of a resource. fpga_result fpgaPropertiesGetSubsystemDeviceID (const fpga_properties prop, uint16_t * subsystem_device_id) Get the subsystem device id of an FPGA resource property. fpga_result fpgaPropertiesGetSubsystemVendorID (const fpga_properties prop, uint16_t * subsystem_vendor_id) Get the subsystem vendor id of an FPGA resource property. fpga_result fpgaPropertiesGetVendorID (const fpga_properties prop, uint16_t * vendor_id) Get the vendor id of an FPGA resource property. fpga_result fpgaPropertiesSetAcceleratorState (fpga_properties prop, fpga_accelerator_state state) Set the state of an accelerator resource property. fpga_result fpgaPropertiesSetBBSID (fpga_properties prop, uint64_t bbs_id) Set the BBS ID of an FPGA resource property. fpga_result fpgaPropertiesSetBBSVersion (fpga_properties prop, fpga_version version) Set the BBS Version of an FPGA resource property. fpga_result fpgaPropertiesSetBus (fpga_properties prop, uint8_t bus) Set the PCI bus number of a resource. fpga_result fpgaPropertiesSetCapabilities (fpga_properties prop, uint64_t capabilities) Set the capabilities of an FPGA resource property. fpga_result fpgaPropertiesSetDevice (fpga_properties prop, uint8_t device) Set the PCI device number of a resource. fpga_result fpgaPropertiesSetDeviceID (fpga_properties prop, uint16_t device_id) Set the device id of the resource. fpga_result fpgaPropertiesSetFunction (fpga_properties prop, uint8_t function) Set the PCI function number of a resource. fpga_result fpgaPropertiesSetGUID (fpga_properties prop, fpga_guid guid) Set the GUID of a resource. fpga_result fpgaPropertiesSetInterface (const fpga_properties prop, fpga_interface interface) Set the OPAE plugin interface implemented by a resource. fpga_result fpgaPropertiesSetLocalMemorySize (fpga_properties prop, uint64_t lms) Set the local memory size of an FPGA resource property. fpga_result fpgaPropertiesSetModel (fpga_properties prop, char * model) Set the model of an FPGA resource property. fpga_result fpgaPropertiesSetNumErrors (const fpga_properties prop, uint32_t num_errors) Set the number of error registers. fpga_result fpgaPropertiesSetNumInterrupts (fpga_properties prop, uint32_t num_interrupts) Set the number of interrupts. fpga_result fpgaPropertiesSetNumMMIO (fpga_properties prop, uint32_t mmio_spaces) Set the number of mmio spaces. fpga_result fpgaPropertiesSetNumSlots (fpga_properties prop, uint32_t num_slots) Set the number of slots of an FPGA resource property. fpga_result fpgaPropertiesSetObjectID (const fpga_properties prop, uint64_t object_id) Set the object ID of a resource. fpga_result fpgaPropertiesSetObjectType (fpga_properties prop, fpga_objtype objtype) Set the object type of a resource. fpga_result fpgaPropertiesSetParent (fpga_properties prop, fpga_token parent) Set the token of the parent object. fpga_result fpgaPropertiesSetSegment (fpga_properties prop, uint16_t segment) Set the PCI segment number of a resource. fpga_result fpgaPropertiesSetSocketID (fpga_properties prop, uint8_t socket_id) Set the socket id of the resource. fpga_result fpgaPropertiesSetSubsystemDeviceID (fpga_properties prop, uint16_t subsystem_device_id) Set the subsystem device id of an FPGA resource property. fpga_result fpgaPropertiesSetSubsystemVendorID (fpga_properties prop, uint16_t subsystem_vendor_id) Set the subsystem vendor id of an FPGA resource property. fpga_result fpgaPropertiesSetVendorID (fpga_properties prop, uint16_t vendor_id) Set the vendor id of an FPGA resource property. fpga_result fpgaUpdateProperties (fpga_token token, fpga_properties prop) Update a fpga_properties object."},{"location":"opae-code/properties_8h/#detailed-description","title":"Detailed Description","text":"
                                            In OPAE, fpga_properties objects are used both for obtaining information about resources and for selectively enumerating resources based on their properties. This file provides accessor functions (get/set) to allow reading and writing individual items of an fpga_properties object. Generally, not all object types supported by OPAE carry all properties. If you call a property accessor method on a fpga_properties object that does not support this particular property, it will return FPGA_INVALID_PARAM.
                                            "},{"location":"opae-code/properties_8h/#accessor-return-values","title":"Accessor Return Values","text":"
                                            In addition to the return values specified in the documentation below, all accessor functions return FPGA_OK on success, FPGA_INVALID_PARAM if you pass NULL or invalid parameters (i.e. non-initialized properties objects), FPGA_EXCEPTION if an internal exception occurred trying to access the properties object, FPGA_NOT_FOUND if the requested property is not part of the supplied properties object. 
                                            "},{"location":"opae-code/properties_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/properties_8h/#function-fpgaclearproperties","title":"function fpgaClearProperties","text":"
                                            Clear a fpga_properties object. 
                                            fpga_result fpgaClearProperties (\n    fpga_properties prop\n) \n
                                             
                                            Sets all fields of the properties object pointed at by 'prop' to 'don't care', which implies that the fpga_properties object would match all FPGA resources if used for an fpgaEnumerate() query. The matching criteria can be further refined by using fpgaSet* functions on the properties object.
                                             
                                            Instead of creating a new fpga_properties object every time, this function can be used to re-use fpga_properties objects from previous queries.
                                             
                                            Parameters:
                                             
                                               
                                            • prop fpga_properties object to clear 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if prop is not a valid object. FPGA_EXCEPTION if an * internal exception occured when trying to access prop. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgacloneproperties","title":"function fpgaCloneProperties","text":"
                                            Clone a fpga_properties object. 
                                            fpga_result fpgaCloneProperties (\n    fpga_properties src,\n    fpga_properties * dst\n) \n
                                             
                                            Creates a copy of an fpga_properties object.
                                             
                                            Note:
                                             
                                            This call creates a new properties object and allocates memory for it. Both the 'src' and the newly created 'dst' objects will eventually need to be destroyed using fpgaDestroyProperties().
                                             
                                            Parameters:
                                             
                                               
                                            • src fpga_properties object to copy 
                                              •  
                                            • dst New fpga_properties object cloned from 'src' 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if src is not a valid object, or if dst is NULL. FPGA_NO_MEMORY if there was not enough memory to allocate an fpga_properties object for dst. FPGA_EXCEPTION if an internal exception occurred either accessing src or updating dst. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgadestroyproperties","title":"function fpgaDestroyProperties","text":"
                                            Destroy a fpga_properties object. 
                                            fpga_result fpgaDestroyProperties (\n    fpga_properties * prop\n) \n
                                             
                                            Destroys an existing fpga_properties object that the caller has previously created using fpgaGetProperties() or fpgaCloneProperties().
                                             
                                            Note:
                                             
                                            fpgaDestroyProperties() requires the address of an fpga_properties object, similar to fpgaGetPropertiesFromHandle(), fpgaGetProperties(), and fpgaCloneProperties(). Passing any other value results in undefined behavior.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Pointer to the fpga_properties object to destroy 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM is prop is not a valid object. FPGA_EXCEPTION if an internal exception occurrred while trying to access prop. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgagetproperties","title":"function fpgaGetProperties","text":"
                                            Create a fpga_properties object. 
                                            fpga_result fpgaGetProperties (\n    fpga_token token,\n    fpga_properties * prop\n) \n
                                             
                                            Initializes the memory pointed at by prop to represent a properties object, and populates it with the properties of the resource referred to by token. Individual properties can then be queried using fpgaPropertiesGet*() accessor functions.
                                             
                                            If token is NULL, an \"empty\" properties object is created to be used as a filter for fpgaEnumerate(). All individual fields are set to dont care`, which implies that the fpga_properties object would match all FPGA resources if used for an fpgaEnumerate() query. The matching criteria can be further refined by using fpgaSet* functions on the properties object, or the object can be populated with the actual properties of a resource by using fpgaUpdateProperties().
                                             
                                            Note:
                                             
                                            fpgaGetProperties() will allocate memory for the created properties object returned in prop. It is the responsibility of the caller to free this memory after use by calling fpgaDestroyProperties().
                                             
                                            Parameters:
                                             
                                               
                                            • token Token to get properties for. Can be NULL, which will create an empty properties object to be used as a filter for fpgaEnumerate(). 
                                              •  
                                            • prop Pointer to a variable of type fpga_properties 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_NO_MEMORY if no memory could be allocated to create the fpga_properties object. FPGA_EXCEPTION if an exception happend while initializing the fpga_properties object. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgagetpropertiesfromhandle","title":"function fpgaGetPropertiesFromHandle","text":"
                                            Create a fpga_properties object. 
                                            fpga_result fpgaGetPropertiesFromHandle (\n    fpga_handle handle,\n    fpga_properties * prop\n) \n
                                             
                                            Initializes the memory pointed at by prop to represent a properties object, and populates it with the properties of the resource referred to by handle. Individual properties can then be queried using fpgaPropertiesGet*() accessor functions.
                                             
                                            Note:
                                             
                                            fpgaGetPropertiesFromHandle() will allocate memory for the created properties object returned in prop. It is the responsibility of the caller to free this memory after use by calling fpgaDestroyProperties().
                                             
                                            Parameters:
                                             
                                               
                                            • handle Open handle to get properties for. 
                                              •  
                                            • prop Pointer to a variable of type fpga_properties 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_NO_MEMORY if no memory could be allocated to create the fpga_properties object. FPGA_EXCEPTION if an exception happend while initializing the fpga_properties object. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetacceleratorstate","title":"function fpgaPropertiesGetAcceleratorState","text":"
                                            Get the state of a accelerator resource property. 
                                            fpga_result fpgaPropertiesGetAcceleratorState (\n    const fpga_properties prop,\n    fpga_accelerator_state * state\n) \n
                                             
                                            Returns the accelerator state of a accelerator.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query - must be of type FPGA_ACCELERATOR 
                                              •  
                                            • state Pointer to a accelerator state variable of the accelerator 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_ACCELERATOR. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetbbsid","title":"function fpgaPropertiesGetBBSID","text":"
                                            Get the BBS ID of an FPGA resource property. 
                                            fpga_result fpgaPropertiesGetBBSID (\n    const fpga_properties prop,\n    uint64_t * bbs_id\n) \n
                                             
                                            Returns the blue bitstream id of an FPGA.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query - must be of type FPGA_DEVICE 
                                              •  
                                            • bbs_id Pointer to a bbs id variable of the FPGA 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetbbsversion","title":"function fpgaPropertiesGetBBSVersion","text":"
                                            Get the BBS Version of an FPGA resource property. 
                                            fpga_result fpgaPropertiesGetBBSVersion (\n    const fpga_properties prop,\n    fpga_version * bbs_version\n) \n
                                             
                                            Returns the blue bitstream version of an FPGA.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query - must be of type FPGA_DEVICE 
                                              •  
                                            • bbs_version Pointer to a bbs version variable of the FPGA 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetbus","title":"function fpgaPropertiesGetBus","text":"
                                            Get the PCI bus number of a resource. 
                                            fpga_result fpgaPropertiesGetBus (\n    const fpga_properties prop,\n    uint8_t * bus\n) \n
                                             
                                            Returns the bus number the queried resource.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • bus Pointer to a PCI bus variable of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetcapabilities","title":"function fpgaPropertiesGetCapabilities","text":"
                                            Get the capabilities FPGA resource property. 
                                            fpga_result fpgaPropertiesGetCapabilities (\n    const fpga_properties prop,\n    uint64_t * capabilities\n) \n
                                             
                                            Returns the capabilities of an FPGA. Capabilities is a bitfield value
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query - must be of type FPGA_DEVICE 
                                              •  
                                            • capabilities Pointer to a capabilities variable of the FPGA 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                             
                                            Note:
                                             
                                            This API is not currently supported. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetdevice","title":"function fpgaPropertiesGetDevice","text":"
                                            Get the PCI device number of a resource. 
                                            fpga_result fpgaPropertiesGetDevice (\n    const fpga_properties prop,\n    uint8_t * device\n) \n
                                             
                                            Returns the device number the queried resource.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • device Pointer to a PCI device variable of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetdeviceid","title":"function fpgaPropertiesGetDeviceID","text":"
                                            Get the device id of the resource. 
                                            fpga_result fpgaPropertiesGetDeviceID (\n    const fpga_properties prop,\n    uint16_t * device_id\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • device_id Pointer to a device id variable of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetfunction","title":"function fpgaPropertiesGetFunction","text":"
                                            Get the PCI function number of a resource. 
                                            fpga_result fpgaPropertiesGetFunction (\n    const fpga_properties prop,\n    uint8_t * function\n) \n
                                             
                                            Returns the function number the queried resource.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • function Pointer to PCI function variable of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetguid","title":"function fpgaPropertiesGetGUID","text":"
                                            Get the GUID of a resource. 
                                            fpga_result fpgaPropertiesGetGUID (\n    const fpga_properties prop,\n    fpga_guid * guid\n) \n
                                             
                                            Returns the GUID of an FPGA or accelerator object.
                                             
                                            For an accelerator, the GUID uniquely identifies a specific accelerator context type, i.e. different accelerators will have different GUIDs. For an FPGA, the GUID is used to identify a certain instance of an FPGA, e.g. to determine whether a given bitstream would be compatible.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • guid Pointer to a GUID of the slot variable 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetinterface","title":"function fpgaPropertiesGetInterface","text":"
                                            Get the OPAE plugin interface implemented by a resource. 
                                            fpga_result fpgaPropertiesGetInterface (\n    const fpga_properties prop,\n    fpga_interface * interface\n) \n
                                             
                                            Returns the plugin interface enumerator.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • interface Pointer to an fpga_interface location to store the interface in 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetlocalmemorysize","title":"function fpgaPropertiesGetLocalMemorySize","text":"
                                            Get the local memory size of an FPGA resource property. 
                                            fpga_result fpgaPropertiesGetLocalMemorySize (\n    const fpga_properties prop,\n    uint64_t * lms\n) \n
                                             
                                            Returns the local memory size of an FPGA.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query - must be of type FPGA_DEVICE 
                                              •  
                                            • lms Pointer to a memory size variable of the FPGA 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                             
                                            Note:
                                             
                                            This API is not currently supported. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetmodel","title":"function fpgaPropertiesGetModel","text":"
                                            Get the model of an FPGA resource property. 
                                            fpga_result fpgaPropertiesGetModel (\n    const fpga_properties prop,\n    char * model\n) \n
                                             
                                            Returns the model of an FPGA.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query - must be of type FPGA_DEVICE 
                                              •  
                                            • model Model of the FPGA resource (string of minimum FPGA_MODEL_LENGTH length 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                             
                                            Note:
                                             
                                            This API is not currently supported. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetnumerrors","title":"function fpgaPropertiesGetNumErrors","text":"
                                            Get the number of errors that can be reported by a resource. 
                                            fpga_result fpgaPropertiesGetNumErrors (\n    const fpga_properties prop,\n    uint32_t * num_errors\n) \n
                                             
                                            Returns the number of error registers understood by a resource.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • num_errors Pointer to a 32 bit memory location to store the number of supported errors in 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetnuminterrupts","title":"function fpgaPropertiesGetNumInterrupts","text":"
                                            Get the number of interrupts. 
                                            fpga_result fpgaPropertiesGetNumInterrupts (\n    const fpga_properties prop,\n    uint32_t * num_interrupts\n) \n
                                             
                                            Returns the number of interrupts of an accelerator properties structure.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query - must be of type FPGA_ACCELERATOR 
                                              •  
                                            • num_interrupts Pointer to a variable for number of interrupts 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_ACCELERATOR. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetnummmio","title":"function fpgaPropertiesGetNumMMIO","text":"
                                            Get the number of mmio spaces. 
                                            fpga_result fpgaPropertiesGetNumMMIO (\n    const fpga_properties prop,\n    uint32_t * mmio_spaces\n) \n
                                             
                                            Returns the number of mmio spaces of an AFU properties structure.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query - must be of type FPGA_ACCELERATOR 
                                              •  
                                            • mmio_spaces Pointer to a variable for number of mmio spaces 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_ACCELERATOR. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetnumslots","title":"function fpgaPropertiesGetNumSlots","text":"
                                            Get the number of slots of an FPGA resource property. 
                                            fpga_result fpgaPropertiesGetNumSlots (\n    const fpga_properties prop,\n    uint32_t * num_slots\n) \n
                                             
                                            Returns the number of slots present in an FPGA.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query - must be of type FPGA_DEVICE 
                                              •  
                                            • num_slots Pointer to number of slots variable of the FPGA 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetobjectid","title":"function fpgaPropertiesGetObjectID","text":"
                                            Get the object ID of a resource. 
                                            fpga_result fpgaPropertiesGetObjectID (\n    const fpga_properties prop,\n    uint64_t * object_id\n) \n
                                             
                                            Returns the object ID of a resource. The object ID is a 64 bit identifier that is unique within a single node or system. It represents a similar concept as the token, but can be used across processes (e.g. passed on the command line).
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • object_id Pointer to a 64bit memory location to store the object ID in 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetobjecttype","title":"function fpgaPropertiesGetObjectType","text":"
                                            Get the object type of a resource. 
                                            fpga_result fpgaPropertiesGetObjectType (\n    const fpga_properties prop,\n    fpga_objtype * objtype\n) \n
                                             
                                            Returns the object type of the queried resource.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • objtype Pointer to an object type variable of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetparent","title":"function fpgaPropertiesGetParent","text":"
                                            Get the token of the parent object. 
                                            fpga_result fpgaPropertiesGetParent (\n    const fpga_properties prop,\n    fpga_token * parent\n) \n
                                             
                                            Returns the token of the parent of the queried resource in '*parent'.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • parent Pointer to a token variable of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_NOT_FOUND if resource does not have a parent (e.g. an FPGA_DEVICE resource does not have parents). Also see \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetsegment","title":"function fpgaPropertiesGetSegment","text":"
                                            Get the PCI segment number of a resource. 
                                            fpga_result fpgaPropertiesGetSegment (\n    const fpga_properties prop,\n    uint16_t * segment\n) \n
                                             
                                            Returns the segment number of the queried resource.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • segment Pointer to a PCI segment variable of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetsocketid","title":"function fpgaPropertiesGetSocketID","text":"
                                            Get the socket id of a resource. 
                                            fpga_result fpgaPropertiesGetSocketID (\n    const fpga_properties prop,\n    uint8_t * socket_id\n) \n
                                             
                                            Returns the socket id of the queried resource.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • socket_id Pointer to a socket id variable of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetsubsystemdeviceid","title":"function fpgaPropertiesGetSubsystemDeviceID","text":"
                                            Get the subsystem device id of an FPGA resource property. 
                                            fpga_result fpgaPropertiesGetSubsystemDeviceID (\n    const fpga_properties prop,\n    uint16_t * subsystem_device_id\n) \n
                                             
                                            Returns the subsystem device id of an FPGA.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • subsystem_device_id Pointer to a device id variable of the FPGA 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetsubsystemvendorid","title":"function fpgaPropertiesGetSubsystemVendorID","text":"
                                            Get the subsystem vendor id of an FPGA resource property. 
                                            fpga_result fpgaPropertiesGetSubsystemVendorID (\n    const fpga_properties prop,\n    uint16_t * subsystem_vendor_id\n) \n
                                             
                                            Returns the subsystem vendor id of an FPGA.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • subsystem_vendor_id Pointer to a vendor id variable of the FPGA 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetvendorid","title":"function fpgaPropertiesGetVendorID","text":"
                                            Get the vendor id of an FPGA resource property. 
                                            fpga_result fpgaPropertiesGetVendorID (\n    const fpga_properties prop,\n    uint16_t * vendor_id\n) \n
                                             
                                            Returns the vendor id of an FPGA.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query - must be of type FPGA_DEVICE 
                                              •  
                                            • vendor_id Pointer to a vendor id variable of the FPGA 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                             
                                            Note:
                                             
                                            This API is not currently supported. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetacceleratorstate","title":"function fpgaPropertiesSetAcceleratorState","text":"
                                            Set the state of an accelerator resource property. 
                                            fpga_result fpgaPropertiesSetAcceleratorState (\n    fpga_properties prop,\n    fpga_accelerator_state state\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify - must be of type FPGA_ACCELERATOR 
                                              •  
                                            • state accelerator state of the accelerator resource 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_ACCELERATOR. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetbbsid","title":"function fpgaPropertiesSetBBSID","text":"
                                            Set the BBS ID of an FPGA resource property. 
                                            fpga_result fpgaPropertiesSetBBSID (\n    fpga_properties prop,\n    uint64_t bbs_id\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify - must be of type FPGA_DEVICE 
                                              •  
                                            • bbs_id Blue bitstream id of the FPGA resource 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetbbsversion","title":"function fpgaPropertiesSetBBSVersion","text":"
                                            Set the BBS Version of an FPGA resource property. 
                                            fpga_result fpgaPropertiesSetBBSVersion (\n    fpga_properties prop,\n    fpga_version version\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify - must be of type FPGA_DEVICE 
                                              •  
                                            • version Blue bitstream version of the FPGA resource 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetbus","title":"function fpgaPropertiesSetBus","text":"
                                            Set the PCI bus number of a resource. 
                                            fpga_result fpgaPropertiesSetBus (\n    fpga_properties prop,\n    uint8_t bus\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify 
                                              •  
                                            • bus PCI bus number of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetcapabilities","title":"function fpgaPropertiesSetCapabilities","text":"
                                            Set the capabilities of an FPGA resource property. 
                                            fpga_result fpgaPropertiesSetCapabilities (\n    fpga_properties prop,\n    uint64_t capabilities\n) \n
                                             
                                            Capabilities is a bitfield value
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify - must be of type FPGA_DEVICE 
                                              •  
                                            • capabilities Capabilities of the FPGA resource 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                             
                                            Note:
                                             
                                            This API is not currently supported. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetdevice","title":"function fpgaPropertiesSetDevice","text":"
                                            Set the PCI device number of a resource. 
                                            fpga_result fpgaPropertiesSetDevice (\n    fpga_properties prop,\n    uint8_t device\n) \n
                                             
                                            Enforces the limitation on the number of devices as specified in the PCI spec.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify 
                                              •  
                                            • device PCI device number of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetdeviceid","title":"function fpgaPropertiesSetDeviceID","text":"
                                            Set the device id of the resource. 
                                            fpga_result fpgaPropertiesSetDeviceID (\n    fpga_properties prop,\n    uint16_t device_id\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify 
                                              •  
                                            • device_id Device id of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetfunction","title":"function fpgaPropertiesSetFunction","text":"
                                            Set the PCI function number of a resource. 
                                            fpga_result fpgaPropertiesSetFunction (\n    fpga_properties prop,\n    uint8_t function\n) \n
                                             
                                            Enforces the limitation on the number of functions as specified in the PCI spec.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify 
                                              •  
                                            • function PCI function number of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetguid","title":"function fpgaPropertiesSetGUID","text":"
                                            Set the GUID of a resource. 
                                            fpga_result fpgaPropertiesSetGUID (\n    fpga_properties prop,\n    fpga_guid guid\n) \n
                                             
                                            Sets the GUID of an FPGA or accelerator object.
                                             
                                            For an accelerator, the GUID uniquely identifies a specific accelerator context type, i.e. different accelerators will have different GUIDs. For an FPGA, the GUID is used to identify a certain instance of an FPGA, e.g. to determine whether a given bitstream would be compatible.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify 
                                              •  
                                            • guid Pointer to a GUID of the slot variable 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetinterface","title":"function fpgaPropertiesSetInterface","text":"
                                            Set the OPAE plugin interface implemented by a resource. 
                                            fpga_result fpgaPropertiesSetInterface (\n    const fpga_properties prop,\n    fpga_interface interface\n) \n
                                             
                                            Set the plugin interface enumerator.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • interface The interface enumerator to set 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetlocalmemorysize","title":"function fpgaPropertiesSetLocalMemorySize","text":"
                                            Set the local memory size of an FPGA resource property. 
                                            fpga_result fpgaPropertiesSetLocalMemorySize (\n    fpga_properties prop,\n    uint64_t lms\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify - must be of type FPGA_DEVICE 
                                              •  
                                            • lms Local memory size of the FPGA resource 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                             
                                            Note:
                                             
                                            This API is not currently supported. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetmodel","title":"function fpgaPropertiesSetModel","text":"
                                            Set the model of an FPGA resource property. 
                                            fpga_result fpgaPropertiesSetModel (\n    fpga_properties prop,\n    char * model\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify - must be of type FPGA_DEVICE 
                                              •  
                                            • model Model of the FPGA resource (string of maximum FPGA_MODEL_LENGTH length 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                             
                                            Note:
                                             
                                            This API is not currently supported. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetnumerrors","title":"function fpgaPropertiesSetNumErrors","text":"
                                            Set the number of error registers. 
                                            fpga_result fpgaPropertiesSetNumErrors (\n    const fpga_properties prop,\n    uint32_t num_errors\n) \n
                                             
                                            Set the number of error registers understood by a resource to enumerate.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • num_errors Number of errors 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetnuminterrupts","title":"function fpgaPropertiesSetNumInterrupts","text":"
                                            Set the number of interrupts. 
                                            fpga_result fpgaPropertiesSetNumInterrupts (\n    fpga_properties prop,\n    uint32_t num_interrupts\n) \n
                                             
                                            Sets the number of interrupts of an accelerator properties structure.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify - must be of type FPGA_ACCELERATOR 
                                              •  
                                            • num_interrupts Number of interrupts of the accelerator 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_ACCELERATOR. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetnummmio","title":"function fpgaPropertiesSetNumMMIO","text":"
                                            Set the number of mmio spaces. 
                                            fpga_result fpgaPropertiesSetNumMMIO (\n    fpga_properties prop,\n    uint32_t mmio_spaces\n) \n
                                             
                                            Sets the number of mmio spaces of an AFU properties structure.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify - must be of type FPGA_ACCELERATOR 
                                              •  
                                            • mmio_spaces Number of MMIO spaces of the accelerator 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_ACCELERATOR. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetnumslots","title":"function fpgaPropertiesSetNumSlots","text":"
                                            Set the number of slots of an FPGA resource property. 
                                            fpga_result fpgaPropertiesSetNumSlots (\n    fpga_properties prop,\n    uint32_t num_slots\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify - must be of type FPGA_DEVICE 
                                              •  
                                            • num_slots Number of slots of the FPGA 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetobjectid","title":"function fpgaPropertiesSetObjectID","text":"
                                            Set the object ID of a resource. 
                                            fpga_result fpgaPropertiesSetObjectID (\n    const fpga_properties prop,\n    uint64_t object_id\n) \n
                                             
                                            Sets the object ID of a resource. The object ID is a 64 bit identifier that is unique within a single node or system. It represents a similar concept as the token, but can be used across processes (e.g. passed on the command line).
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to query 
                                              •  
                                            • object_id A 64bit value to use as the object ID 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetobjecttype","title":"function fpgaPropertiesSetObjectType","text":"
                                            Set the object type of a resource. 
                                            fpga_result fpgaPropertiesSetObjectType (\n    fpga_properties prop,\n    fpga_objtype objtype\n) \n
                                             
                                            Sets the object type of the resource. * Currently supported object types are FPGA_DEVICE and FPGA_ACCELERATOR.
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify 
                                              •  
                                            • objtype Object type of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetparent","title":"function fpgaPropertiesSetParent","text":"
                                            Set the token of the parent object. 
                                            fpga_result fpgaPropertiesSetParent (\n    fpga_properties prop,\n    fpga_token parent\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify 
                                              •  
                                            • parent Pointer to a token variable of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetsegment","title":"function fpgaPropertiesSetSegment","text":"
                                            Set the PCI segment number of a resource. 
                                            fpga_result fpgaPropertiesSetSegment (\n    fpga_properties prop,\n    uint16_t segment\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify 
                                              •  
                                            • segment PCI segment number of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetsocketid","title":"function fpgaPropertiesSetSocketID","text":"
                                            Set the socket id of the resource. 
                                            fpga_result fpgaPropertiesSetSocketID (\n    fpga_properties prop,\n    uint8_t socket_id\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify 
                                              •  
                                            • socket_id Socket id of the resource 'prop' is associated with 
                                              •  
                                             
                                            Returns:
                                             
                                            See \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetsubsystemdeviceid","title":"function fpgaPropertiesSetSubsystemDeviceID","text":"
                                            Set the subsystem device id of an FPGA resource property. 
                                            fpga_result fpgaPropertiesSetSubsystemDeviceID (\n    fpga_properties prop,\n    uint16_t subsystem_device_id\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify 
                                              •  
                                            • subsystem_device_id Subsystem Device id of the FPGA resource 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetsubsystemvendorid","title":"function fpgaPropertiesSetSubsystemVendorID","text":"
                                            Set the subsystem vendor id of an FPGA resource property. 
                                            fpga_result fpgaPropertiesSetSubsystemVendorID (\n    fpga_properties prop,\n    uint16_t subsystem_vendor_id\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify 
                                              •  
                                            • subsystem_vendor_id Subsystem Vendor id of the FPGA resource 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. See also \"Accessor Return Values\" in properties.h. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetvendorid","title":"function fpgaPropertiesSetVendorID","text":"
                                            Set the vendor id of an FPGA resource property. 
                                            fpga_result fpgaPropertiesSetVendorID (\n    fpga_properties prop,\n    uint16_t vendor_id\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • prop Properties object to modify - must be of type FPGA_DEVICE 
                                              •  
                                            • vendor_id Vendor id of the FPGA resource 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                             
                                            Note:
                                             
                                            This API is not currently supported. 
                                            "},{"location":"opae-code/properties_8h/#function-fpgaupdateproperties","title":"function fpgaUpdateProperties","text":"
                                            Update a fpga_properties object. 
                                            fpga_result fpgaUpdateProperties (\n    fpga_token token,\n    fpga_properties prop\n) \n
                                             
                                            Populates the properties object 'prop' with properties of the resource referred to by 'token'. Unlike fpgaGetProperties(), this call will not create a new properties object or allocate memory for it, but use a previously created properties object.
                                             
                                            Parameters:
                                             
                                               
                                            • token Token to retrieve properties for 
                                              •  
                                            • prop fpga_properties object to update 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if token or prop are not valid objects. FPGA_NOT_FOUND if the resource referred to by token was not found. FPGA_NO_DRIVER if not driver is loaded. FPGA_EXCEPTION if an internal exception occured when trying to update prop. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/properties.h
                                            "},{"location":"opae-code/properties_8h_source/","title":"File properties.h","text":"
                                            File List > docs > sw > include > opae > properties.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017-2021, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_PROPERTIES_H__\n#define __FPGA_PROPERTIES_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaGetPropertiesFromHandle(fpga_handle handle, fpga_properties *prop);\n\nfpga_result fpgaGetProperties(fpga_token token, fpga_properties *prop);\n\nfpga_result fpgaUpdateProperties(fpga_token token, fpga_properties prop);\n\nfpga_result fpgaClearProperties(fpga_properties prop);\n\nfpga_result fpgaCloneProperties(fpga_properties src, fpga_properties *dst);\n\nfpga_result fpgaDestroyProperties(fpga_properties *prop);\n\nfpga_result fpgaPropertiesGetParent(const fpga_properties prop,\n                    fpga_token *parent);\n\nfpga_result fpgaPropertiesSetParent(fpga_properties prop,\n                    fpga_token parent);\nfpga_result fpgaPropertiesGetObjectType(const fpga_properties prop,\n                    fpga_objtype *objtype);\n\nfpga_result fpgaPropertiesSetObjectType(fpga_properties prop,\n                    fpga_objtype objtype);\nfpga_result fpgaPropertiesGetSegment(const fpga_properties prop, uint16_t *segment);\n\nfpga_result fpgaPropertiesSetSegment(fpga_properties prop, uint16_t segment);\n\nfpga_result fpgaPropertiesGetBus(const fpga_properties prop, uint8_t *bus);\n\nfpga_result fpgaPropertiesSetBus(fpga_properties prop, uint8_t bus);\n\nfpga_result fpgaPropertiesGetDevice(const fpga_properties prop,\n                    uint8_t *device);\n\nfpga_result fpgaPropertiesSetDevice(fpga_properties prop,\n                    uint8_t device);\n\nfpga_result fpgaPropertiesGetFunction(const fpga_properties prop,\n                      uint8_t *function);\n\nfpga_result fpgaPropertiesSetFunction(fpga_properties prop,\n                      uint8_t function);\n\nfpga_result fpgaPropertiesGetSocketID(const fpga_properties prop,\n                      uint8_t *socket_id);\n\nfpga_result fpgaPropertiesSetSocketID(fpga_properties prop,\n                      uint8_t socket_id);\n\nfpga_result fpgaPropertiesGetDeviceID(const fpga_properties prop,\n                      uint16_t *device_id);\n\nfpga_result fpgaPropertiesSetDeviceID(fpga_properties prop,\n                      uint16_t device_id);\n\nfpga_result fpgaPropertiesGetNumSlots(const fpga_properties prop,\n                      uint32_t *num_slots);\n\nfpga_result fpgaPropertiesSetNumSlots(fpga_properties prop,\n                      uint32_t num_slots);\n\nfpga_result fpgaPropertiesGetBBSID(const fpga_properties prop,\n                   uint64_t *bbs_id);\n\n\nfpga_result fpgaPropertiesSetBBSID(fpga_properties prop,\n                   uint64_t bbs_id);\n\n\nfpga_result fpgaPropertiesGetBBSVersion(const fpga_properties prop,\n                    fpga_version *bbs_version);\n\nfpga_result fpgaPropertiesSetBBSVersion(fpga_properties prop,\n                    fpga_version version);\n\n\nfpga_result fpgaPropertiesGetVendorID(const fpga_properties prop,\n                      uint16_t *vendor_id);\n\n\nfpga_result fpgaPropertiesSetVendorID(fpga_properties prop,\n                      uint16_t vendor_id);\n\nfpga_result fpgaPropertiesGetModel(const fpga_properties prop,\n                   char *model);\n\n\nfpga_result fpgaPropertiesSetModel(fpga_properties prop,\n                   char *model);\n\n\nfpga_result fpgaPropertiesGetLocalMemorySize(const fpga_properties prop,\n                         uint64_t *lms);\n\n\nfpga_result fpgaPropertiesSetLocalMemorySize(fpga_properties prop,\n                         uint64_t lms);\n\nfpga_result fpgaPropertiesGetCapabilities(const fpga_properties prop,\n                      uint64_t *capabilities);\n\n\nfpga_result fpgaPropertiesSetCapabilities(fpga_properties prop,\n                      uint64_t capabilities);\n\nfpga_result fpgaPropertiesGetGUID(const fpga_properties prop,\n                  fpga_guid *guid);\n\nfpga_result fpgaPropertiesSetGUID(fpga_properties prop, fpga_guid guid);\n\nfpga_result fpgaPropertiesGetNumMMIO(const fpga_properties prop,\n                     uint32_t *mmio_spaces);\n\nfpga_result fpgaPropertiesSetNumMMIO(fpga_properties prop,\n                     uint32_t mmio_spaces);\n\nfpga_result fpgaPropertiesGetNumInterrupts(const fpga_properties prop,\n                       uint32_t *num_interrupts);\n\nfpga_result fpgaPropertiesSetNumInterrupts(fpga_properties prop,\n                       uint32_t num_interrupts);\n\nfpga_result fpgaPropertiesGetAcceleratorState(const fpga_properties prop,\n                          fpga_accelerator_state *state);\n\n\nfpga_result fpgaPropertiesSetAcceleratorState(fpga_properties prop,\n                          fpga_accelerator_state state);\n\nfpga_result fpgaPropertiesGetObjectID(const fpga_properties prop,\n                        uint64_t *object_id);\n\n\nfpga_result fpgaPropertiesSetObjectID(const fpga_properties prop,\n                        uint64_t object_id);\n\n\nfpga_result fpgaPropertiesGetNumErrors(const fpga_properties prop,\n                       uint32_t *num_errors);\n\n\nfpga_result fpgaPropertiesSetNumErrors(const fpga_properties prop,\n                       uint32_t num_errors);\n\nfpga_result fpgaPropertiesGetInterface(const fpga_properties prop,\n                       fpga_interface *interface);\n\nfpga_result fpgaPropertiesSetInterface(const fpga_properties prop,\n                       fpga_interface interface);\n\nfpga_result fpgaPropertiesGetSubsystemVendorID(const fpga_properties prop,\n                           uint16_t *subsystem_vendor_id);\n\n\nfpga_result fpgaPropertiesSetSubsystemVendorID(fpga_properties prop,\n                           uint16_t subsystem_vendor_id);\n\nfpga_result fpgaPropertiesGetSubsystemDeviceID(const fpga_properties prop,\n                           uint16_t *subsystem_device_id);\n\n\nfpga_result fpgaPropertiesSetSubsystemDeviceID(fpga_properties prop,\n                           uint16_t subsystem_device_id);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_PROPERTIES_H__\n
                                            "},{"location":"opae-code/sysobject_8h/","title":"File sysobject.h","text":"
                                            FileList > docs > sw > include > opae > sysobject.h
                                             
                                            Go to the source code of this file.
                                             
                                            Functions to read/write from system objects. More...
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/sysobject_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaDestroyObject (fpga_object * obj) Free memory used for the fpga_object data structure. fpga_result fpgaHandleGetObject (fpga_handle handle, const char * name, fpga_object * object, int flags) Create an fpga_object data structure from a handle. fpga_result fpgaObjectGetObject (fpga_object parent, const char * name, fpga_object * object, int flags) Create an fpga_object data structure from a parent object. fpga_result fpgaObjectGetObjectAt (fpga_object parent, size_t idx, fpga_object * object) Create an fpga_object data structure from a parent object using a given index. fpga_result fpgaObjectGetSize (fpga_object obj, uint32_t * value, int flags) Retrieve the size of the object. fpga_result fpgaObjectGetType (fpga_object obj, enum fpga_sysobject_type * type) Get the sysobject type (container or attribute) fpga_result fpgaObjectRead (fpga_object obj, uint8_t * buffer, size_t offset, size_t len, int flags) Read bytes from an FPGA object. fpga_result fpgaObjectRead64 (fpga_object obj, uint64_t * value, int flags) Read a 64-bit value from an FPGA object. fpga_result fpgaObjectWrite64 (fpga_object obj, uint64_t value, int flags) Write 64-bit value to an FPGA object. fpga_result fpgaTokenGetObject (fpga_token token, const char * name, fpga_object * object, int flags) Create an fpga_object data structures."},{"location":"opae-code/sysobject_8h/#detailed-description","title":"Detailed Description","text":"
                                            On Linux systems with the OPAE kernel driver, this is used to access sysfs nodes created by the driver. 
                                            "},{"location":"opae-code/sysobject_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/sysobject_8h/#function-fpgadestroyobject","title":"function fpgaDestroyObject","text":"
                                            Free memory used for the fpga_object data structure. 
                                            fpga_result fpgaDestroyObject (\n    fpga_object * obj\n) \n
                                             
                                            Note:
                                             
                                            fpgaDestroyObject() requires the address of an fpga_object as created by fpgaTokenGetObject(), fpgaHandleGetObject(), or fpgaObjectGetObject(). Passing any other value results in undefind behavior.
                                             
                                            Parameters:
                                             
                                               
                                            • obj Pointer to the fpga_object instance to destroy
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success, FPGA_INVALID_PARAM if the object is NULL, FPGA_EXCEPTION if an internal error is encountered. 
                                            "},{"location":"opae-code/sysobject_8h/#function-fpgahandlegetobject","title":"function fpgaHandleGetObject","text":"
                                            Create an fpga_object data structure from a handle. 
                                            fpga_result fpgaHandleGetObject (\n    fpga_handle handle,\n    const char * name,\n    fpga_object * object,\n    int flags\n) \n
                                             
                                            An fpga_object is a handle to an FPGA resource which can be an attribute, register, or container. This object has read/write access..
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle identifying a resource (accelerator or device) 
                                              •  
                                            • name A key identifying an object belonging to a resource. 
                                              •  
                                            • object Pointer to memory to store the object in 
                                              •  
                                            • flags Control behavior of object identification and creation FPGA_OBJECT_GLOB is used to indicate that the name should be treated as a globbing expression. FPGA_OBJECT_RECURSE_ONE indicates that subobjects be created for objects one level down from the object identified by name. FPGA_OBJECT_RECURSE_ALL indicates that subobjects be created for all objects below the current object identified by name.
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_NOT_FOUND if an object cannot be found with the given key. FPGA_NOT_SUPPORTED if this function is not supported by the current implementation of this API.
                                             
                                            Note:
                                             
                                            Names that begin with '.' or '/' or contain '..' are not allowed and result in FPGA_INVALID_PARAM being returned 
                                            "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectgetobject","title":"function fpgaObjectGetObject","text":"
                                            Create an fpga_object data structure from a parent object. 
                                            fpga_result fpgaObjectGetObject (\n    fpga_object parent,\n    const char * name,\n    fpga_object * object,\n    int flags\n) \n
                                             
                                            An fpga_object is a handle to an FPGA resource which can be an attribute, register, or container. If the parent object was created with a handle, then the new object will inherit the handle allowing it to have read-write access to the object data.
                                             
                                            Parameters:
                                             
                                               
                                            • parent A parent container fpga_object. 
                                              •  
                                            • name A key identifying a sub-object of the parent container. 
                                              •  
                                            • object Pointer to memory to store the object in. 
                                              •  
                                            • flags Control behavior of object identification and creation. FPGA_OBJECT_GLOB is used to indicate that the name should be treated as a globbing expression. FPGA_OBJECT_RECURSE_ONE indicates that subobjects be created for objects one level down from the object identified by name. FPGA_OBJECT_RECURSE_ALL indicates that subobjects be created for all objects below the current object identified by name.
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid - this includes a parent object that is not a container object. FPGA_NOT_FOUND if an object cannot be found with the given key. FPGA_NOT_SUPPORTED if this function is not supported by the current implementation of this API.
                                             
                                            Note:
                                             
                                            Names that begin with '.' or '/' or contain '..' are not allowed and result in FPGA_INVALID_PARAM being returned 
                                            "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectgetobjectat","title":"function fpgaObjectGetObjectAt","text":"
                                            Create an fpga_object data structure from a parent object using a given index. 
                                            fpga_result fpgaObjectGetObjectAt (\n    fpga_object parent,\n    size_t idx,\n    fpga_object * object\n) \n
                                             
                                            An fpga_object is a handle to an FPGA resource which can be an attribute, register, or container. If the parent object was created with a handle, then the new object will inherit the handle allowing it to have read-write access to the object data.
                                             
                                            Parameters:
                                             
                                               
                                            • parent A parent container 'fpga_object' 
                                              •  
                                            • idx A positive index less than the size reported by the parent. 
                                              •  
                                            • object Pointer to memory to store the object in.
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid - this includes a parent object that is not a container object. FPGA_NOT_FOUND if an object cannot be found with the given key. FPGA_NOT_SUPPORTED if this function is not supported by the current implementation of this API. 
                                            "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectgetsize","title":"function fpgaObjectGetSize","text":"
                                            Retrieve the size of the object. 
                                            fpga_result fpgaObjectGetSize (\n    fpga_object obj,\n    uint32_t * value,\n    int flags\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • obj An fpga_object instance. 
                                              •  
                                            • value Pointer to variable to store size in. 
                                              •  
                                            • flags Flags that control how the object is read If FPGA_OBJECT_SYNC is used then object will update its buffered copy before retrieving the size.
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of supplied parameters is invalid. FPGA_EXCEPTION if error occurred. 
                                            "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectgettype","title":"function fpgaObjectGetType","text":"
                                            Get the sysobject type (container or attribute) 
                                            fpga_result fpgaObjectGetType (\n    fpga_object obj,\n    enum fpga_sysobject_type * type\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • obj An fpga_object instance 
                                              •  
                                            • type The type of object (FPGA_OBJECT_CONTAINER or FPGA_OBJECT_ATTRIBUTE)
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success, FPGA_INVALID_PARAM if any of the supplied parameters are null or invalid 
                                            "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectread","title":"function fpgaObjectRead","text":"
                                            Read bytes from an FPGA object. 
                                            fpga_result fpgaObjectRead (\n    fpga_object obj,\n    uint8_t * buffer,\n    size_t offset,\n    size_t len,\n    int flags\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • obj An fpga_object instance. 
                                              •  
                                            • buffer Pointer to a buffer to read bytes into. 
                                              •  
                                            • offset Byte offset relative to objects internal buffer where to begin reading bytes from. 
                                              •  
                                            • len The length, in bytes, to read from the object. 
                                              •  
                                            • flags Flags that control how object is read If FPGA_OBJECT_SYNC is used then object will update its buffered copy before retrieving the data.
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success, FPGA_INVALID_PARAM if any of the supplied parameters is invalid 
                                            "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectread64","title":"function fpgaObjectRead64","text":"
                                            Read a 64-bit value from an FPGA object. 
                                            fpga_result fpgaObjectRead64 (\n    fpga_object obj,\n    uint64_t * value,\n    int flags\n) \n
                                             
                                            The value is assumed to be in string format and will be parsed. See flags below for changing that behavior.
                                             
                                            Parameters:
                                             
                                               
                                            • obj An fpga_object instance 
                                              •  
                                            • value Pointer to a 64-bit variable to store the value in 
                                              •  
                                            • flags Flags that control how the object is read If FPGA_OBJECT_SYNC is used then object will update its buffered copy before retrieving the data. If FPGA_OBJECT_RAW is used, then the data will be read as raw bytes into the uint64_t pointer variable.
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success, FPGA_INVALID_PARAM if any of the supplied parameters is invalid 
                                            "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectwrite64","title":"function fpgaObjectWrite64","text":"
                                            Write 64-bit value to an FPGA object. 
                                            fpga_result fpgaObjectWrite64 (\n    fpga_object obj,\n    uint64_t value,\n    int flags\n) \n
                                             
                                            The value will be converted to string before writing. See flags below for changing that behavior.
                                             
                                            Parameters:
                                             
                                               
                                            • obj An fpga_object instance. 
                                              •  
                                            • value The value to write to the object 
                                              •  
                                            • flags Flags that control how the object is written If FPGA_OBJECT_RAW is used, then the value will be written as raw bytes.
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success, FPGA_INVALID_PARAM if any of the supplied parameters is invalid
                                             
                                            Note:
                                             
                                            The object must have been created using a handle to a resource. 
                                            "},{"location":"opae-code/sysobject_8h/#function-fpgatokengetobject","title":"function fpgaTokenGetObject","text":"
                                            Create an fpga_object data structures. 
                                            fpga_result fpgaTokenGetObject (\n    fpga_token token,\n    const char * name,\n    fpga_object * object,\n    int flags\n) \n
                                             
                                            An fpga_object is a handle to an FPGA resource which can be an attribute, register or a container. This object is read-only.
                                             
                                            Parameters:
                                             
                                               
                                            • token Token identifying a resource (accelerator or device) 
                                              •  
                                            • name A key identifying an object belonging to a resource. 
                                              •  
                                            • object Pointer to memory to store the object in 
                                              •  
                                            • flags Control behavior of object identification and creation. FPGA_OBJECT_GLOB is used to indicate that the name should be treated as a globbing expression. FPGA_OBJECT_RECURSE_ONE indicates that subobjects be created for objects one level down from the object identified by name. FPGA_OBJECT_RECURSE_ALL indicates that subobjects be created for all objects below the current object identified by name.
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_NOT_FOUND if an object cannot be found with the given key. FPGA_NOT_SUPPORTED if this function is not supported by the current implementation of this API.
                                             
                                            Note:
                                             
                                            Names that begin with '.' or '/' or contain '..' are not allowed and result in FPGA_INVALID_PARAM being returned 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/sysobject.h
                                            "},{"location":"opae-code/sysobject_8h_source/","title":"File sysobject.h","text":"
                                            File List > docs > sw > include > opae > sysobject.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017-2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_SYSOBJECT_H__\n#define __FPGA_SYSOBJECT_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaTokenGetObject(fpga_token token, const char *name,\n                   fpga_object *object, int flags);\n\nfpga_result fpgaHandleGetObject(fpga_handle handle, const char *name,\n                fpga_object *object, int flags);\n\nfpga_result fpgaObjectGetObject(fpga_object parent, const char *name,\n                fpga_object *object, int flags);\n\nfpga_result fpgaObjectGetObjectAt(fpga_object parent, size_t idx,\n                  fpga_object *object);\nfpga_result fpgaObjectGetType(fpga_object obj, enum fpga_sysobject_type *type);\n\nfpga_result fpgaDestroyObject(fpga_object *obj);\n\nfpga_result fpgaObjectGetSize(fpga_object obj, uint32_t *value, int flags);\n\nfpga_result fpgaObjectRead(fpga_object obj, uint8_t *buffer, size_t offset,\n               size_t len, int flags);\n\nfpga_result fpgaObjectRead64(fpga_object obj, uint64_t *value, int flags);\n\nfpga_result fpgaObjectWrite64(fpga_object obj, uint64_t value, int flags);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif /* !__FPGA_SYSOBJECT_H__ */\n
                                            "},{"location":"opae-code/types_8h/","title":"File types.h","text":"
                                            FileList > docs > sw > include > opae > types.h
                                             
                                            Go to the source code of this file.
                                             
                                            Type definitions for FPGA API. More...
                                             
                                               
                                            • #include <stdint.h>
                                              •  
                                            • #include <stddef.h>
                                              •  
                                            • #include <stdbool.h>
                                              •  
                                            • #include <opae/types_enum.h>
                                              •  
                                            "},{"location":"opae-code/types_8h/#classes","title":"Classes","text":"Type Name struct _fpga_token_header Internal token type header. struct fpga_error_info struct fpga_metric Metric struct. struct fpga_metric_info Metric info struct. struct fpga_version Semantic version. struct metric_threshold struct threshold Threshold struct."},{"location":"opae-code/types_8h/#public-types","title":"Public Types","text":"Type Name typedef void * fpga_event_handle Handle to an event object. typedef uint8_t fpga_guid Globally unique identifier (GUID) typedef void * fpga_handle Handle to an FPGA resource. typedef struct fpga_metric fpga_metric Metric struct. typedef struct fpga_metric_info fpga_metric_info Metric info struct. typedef void * fpga_object Object pertaining to an FPGA resource as identified by a unique name. typedef void * fpga_properties Object for expressing FPGA resource properties. typedef void * fpga_token Token for referencing FPGA resources. typedef struct _fpga_token_header fpga_token_header Internal token type header. typedef struct metric_threshold metric_threshold union metric_value Metric value union. typedef struct threshold threshold Threshold struct."},{"location":"opae-code/types_8h/#macros","title":"Macros","text":"Type Name define FPGA_ERROR_NAME_MAX  64Information about an error register. define FPGA_METRIC_STR_SIZE  256FPGA Metric string size. define fpga_is_parent_child (__parent_hdr, __child_hdr) Determine token parent/child relationship."},{"location":"opae-code/types_8h/#detailed-description","title":"Detailed Description","text":"
                                            OPAE uses the three opaque types fpga_properties, fpga_token, and fpga_handle to create a hierarchy of objects that can be used to enumerate, reference, acquire, and query FPGA resources. This object model is designed to be extensible to account for different FPGA architectures and platforms.
                                            "},{"location":"opae-code/types_8h/#initialization","title":"Initialization","text":"
                                            OPAEs management of the opaque types fpga_properties, fpga_token, and fpga_handle relies on the proper initialization of variables of these types. In other words, before doing anything with a variable of one of these opaque types, you need to first initialize them.
                                             
                                            The respective functions that initialize opaque types are:
                                             
                                               
                                            • fpgaGetProperties() and fpgaCloneProperties() for fpga_properties
                                              •  
                                            • fpgaEnumerate() and fpgaCloneToken() for fpga_token
                                              •  
                                            • fpgaOpen() for fpga_handle
                                              •  
                                             
                                            This should intuitively make sense - fpgaGetProperties() creates fpga_properties objects, fpgaEnumerate() creates fpga_token objects, fpgaOpen() creates fpga_handle objects, and fpgaCloneProperties() and fpgaCloneToken() clone (create) fpga_properties and fpga_token objects, respectively.
                                             
                                            Since these opaque types are interpreted as pointers (they are typedef'd to a void *), passing an uninitialized opaque type into any function except the respective initailzation function will result in undefined behaviour, because OPAE will try to follow an invalid pointer. Undefined behaviour in this case may include an unexpected error code, or an application crash. 
                                            "},{"location":"opae-code/types_8h/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/types_8h/#typedef-fpga_event_handle","title":"typedef fpga_event_handle","text":"
                                            Handle to an event object. 
                                            typedef void* fpga_event_handle;\n
                                             
                                            OPAE provides an interface to asynchronous events that can be generated by different FPGA resources. The event API provides functions to register for these events; associated with every event a process has registered for is an fpga_event_handle, which encapsulates the OS-specific data structure for event objects.
                                             
                                            After use, fpga_event_handle objects should be destroyed using fpgaDestroyEventHandle() to free backing memory used by the fpga_event_handle object. 
                                            "},{"location":"opae-code/types_8h/#typedef-fpga_guid","title":"typedef fpga_guid","text":"
                                            Globally unique identifier (GUID) 
                                            typedef uint8_t fpga_guid[16];\n
                                             
                                            GUIDs are used widely within OPAE for helping identify FPGA resources. For example, every FPGA resource has a guid property, which can be (and in the case of FPGA_ACCELERATOR resource primarily is) used for enumerating a resource of a specific type.
                                             
                                            fpga_guid is compatible with libuuid's uuid_t, so users can use libuuid functions like uuid_parse() to create and work with GUIDs. 
                                            "},{"location":"opae-code/types_8h/#typedef-fpga_handle","title":"typedef fpga_handle","text":"
                                            Handle to an FPGA resource. 
                                            typedef void* fpga_handle;\n
                                             
                                            A valid fpga_handle object, as populated by fpgaOpen(), denotes ownership of an FPGA resource. Note that ownership can be exclusive or shared, depending on the flags used in fpgaOpen(). Ownership can be released by calling fpgaClose(), which will render the underlying handle invalid.
                                             
                                            Many OPAE C API functions require a valid token (which is synonymous with ownership of the resource). 
                                            "},{"location":"opae-code/types_8h/#typedef-fpga_metric","title":"typedef fpga_metric","text":"
                                            typedef struct fpga_metric fpga_metric;\n
                                            "},{"location":"opae-code/types_8h/#typedef-fpga_metric_info","title":"typedef fpga_metric_info","text":"
                                            typedef struct fpga_metric_info fpga_metric_info;\n
                                            "},{"location":"opae-code/types_8h/#typedef-fpga_object","title":"typedef fpga_object","text":"
                                            Object pertaining to an FPGA resource as identified by a unique name. 
                                            typedef void* fpga_object;\n
                                             
                                            An fpga_object represents either a device attribute or a container of attributes. Similar to filesystems, a '/' may be used to seperate objects in an object hierarchy. Once on object is acquired, it may be used to read or write data in a resource attribute or to query sub-objects if the object is a container object. The data in an object is buffered and will be kept around until the object is destroyed. Additionally, the data in an attribute can by synchronized from the owning resource using the FPGA_OBJECT_SYNC flag during read operations. The name identifying the object is unique with respect to the resource that owns it. A parent resource may be identified by an fpga_token object, by an fpga_handle object, or another fpga_object object. If a handle object is used when opening the object, then the object is opened with read-write access. Otherwise, the object is read-only. 
                                            "},{"location":"opae-code/types_8h/#typedef-fpga_properties","title":"typedef fpga_properties","text":"
                                            Object for expressing FPGA resource properties. 
                                            typedef void* fpga_properties;\n
                                             
                                            fpga_properties objects encapsulate all enumerable information about an FPGA resources. They can be used for two purposes: selective enumeration (discovery) and querying information about existing resources.
                                             
                                            For selective enumeration, usually an empty fpga_properties object is created (using fpgaGetProperties()) and then populated with the desired criteria for enumeration. An array of fpga_properties can then be passed to fpgaEnumerate(), which will return a list of fpga_token objects matching these criteria.
                                             
                                            For querying properties of existing FPGA resources, fpgaGetProperties() can also take an fpga_token and will return an fpga_properties object populated with information about the resource referenced by that token.
                                             
                                            After use, fpga_properties objects should be destroyed using fpga_destroyProperties() to free backing memory used by the fpga_properties object. 
                                            "},{"location":"opae-code/types_8h/#typedef-fpga_token","title":"typedef fpga_token","text":"
                                            Token for referencing FPGA resources. 
                                            typedef void* fpga_token;\n
                                             
                                            An fpga_token serves as a reference to a specific FPGA resource present in the system. Holding an fpga_token does not constitute ownership of the FPGA resource - it merely allows the user to query further information about a resource, or to use fpgaOpen() to acquire ownership.
                                             
                                            fpga_tokens are usually returned by fpgaEnumerate() or fpgaPropertiesGetParent(), and used by fpgaOpen() to acquire ownership and yield a handle to the resource. Some API calls also take fpga_tokens as arguments if they don't require ownership of the resource in question. 
                                            "},{"location":"opae-code/types_8h/#typedef-fpga_token_header","title":"typedef fpga_token_header","text":"
                                            Internal token type header. 
                                            typedef struct _fpga_token_header fpga_token_header;\n
                                             
                                            Each plugin (dfl: libxfpga.so, vfio: libopae-v.so) implements its own proprietary token type. This header must appear at offset zero within that structure.
                                             
                                            eg, see lib/plugins/xfpga/types_int.h:struct _fpga_token and lib/plugins/vfio/opae_vfio.h:struct _vfio_token. 
                                            "},{"location":"opae-code/types_8h/#typedef-metric_threshold","title":"typedef metric_threshold","text":"
                                            typedef struct metric_threshold metric_threshold;\n
                                            "},{"location":"opae-code/types_8h/#union-metric_value","title":"union metric_value","text":""},{"location":"opae-code/types_8h/#typedef-threshold","title":"typedef threshold","text":"
                                            typedef struct threshold threshold;\n
                                            "},{"location":"opae-code/types_8h/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/types_8h/#define-fpga_error_name_max","title":"define FPGA_ERROR_NAME_MAX","text":"
                                            Information about an error register. 
                                            #define FPGA_ERROR_NAME_MAX 64\n
                                             
                                            This data structure captures information about an error register exposed by an accelerator resource. The error API provides functions to retrieve these information structures from a particular resource. 
                                            "},{"location":"opae-code/types_8h/#define-fpga_metric_str_size","title":"define FPGA_METRIC_STR_SIZE","text":"
                                            #define FPGA_METRIC_STR_SIZE 256\n
                                            "},{"location":"opae-code/types_8h/#define-fpga_is_parent_child","title":"define fpga_is_parent_child","text":"
                                            Determine token parent/child relationship. 
                                            #define fpga_is_parent_child (\n    __parent_hdr,\n    __child_hdr\n) (((__parent_hdr)->objtype == FPGA_DEVICE ) && \\\n ((__child_hdr)->objtype == FPGA_ACCELERATOR ) && \\\n ((__parent_hdr)->segment == (__child_hdr)->segment) && \\\n ((__parent_hdr)->bus == (__child_hdr)->bus) && \\\n ((__parent_hdr)->device == (__child_hdr)->device))\n
                                             
                                            Given pointers to two fpga_token_header structs, determine whether the first is the parent of the second. A parent will have objtype == FPGA_DEVICE. A child will have objtype == FPGA_ACCELERATOR. The PCIe address of the two headers will match in all but the function fields. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                            "},{"location":"opae-code/types_8h_source/","title":"File types.h","text":"
                                            File List > docs > sw > include > opae > types.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018-2022, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_TYPES_H__\n#define __FPGA_TYPES_H__\n\n#include <stdint.h>\n#include <stddef.h>\n#include <stdbool.h>\n#include <opae/types_enum.h>\n\ntypedef void *fpga_properties;\n\ntypedef void *fpga_token;\n\ntypedef void *fpga_handle;\n\ntypedef uint8_t fpga_guid[16];\n\ntypedef struct {\n    uint8_t major;        \n    uint8_t minor;        \n    uint16_t patch;       \n} fpga_version;\n\ntypedef void *fpga_event_handle;\n\n#define FPGA_ERROR_NAME_MAX 64\nstruct fpga_error_info {\n    char name[FPGA_ERROR_NAME_MAX];   \n    bool can_clear;                   \n};\n\ntypedef void *fpga_object;\n\n#define FPGA_METRIC_STR_SIZE   256\ntypedef union {\n    uint64_t   ivalue;  // Metric integer value\n    double     dvalue;  // Metric double value\n    float      fvalue;  // Metric float value\n    bool       bvalue;  // Metric bool value\n} metric_value;\n\n\ntypedef struct fpga_metric_info {\n    uint64_t metric_num;                         // Metric index num\n    fpga_guid metric_guid;                       // Metric guid\n    char qualifier_name[FPGA_METRIC_STR_SIZE];   // Metric full name\n    char group_name[FPGA_METRIC_STR_SIZE];       // Metric group name\n    char metric_name[FPGA_METRIC_STR_SIZE];      // Metric name\n    char metric_units[FPGA_METRIC_STR_SIZE];     // Metric units\n    enum fpga_metric_datatype metric_datatype;   // Metric data type\n    enum fpga_metric_type metric_type;           // Metric group type\n} fpga_metric_info;\n\ntypedef struct fpga_metric {\n    uint64_t metric_num;    // Metric index num\n    metric_value value;     // Metric value\n    bool isvalid;           // Metric value is valid\n} fpga_metric;\n\n\ntypedef struct threshold {\n    char threshold_name[FPGA_METRIC_STR_SIZE]; // Threshold name\n    uint32_t is_valid;                         // Threshold is valid\n    double value;                              // Threshold value\n} threshold;\n\ntypedef struct metric_threshold {\n    char metric_name[FPGA_METRIC_STR_SIZE];        // Metric Threshold name\n    threshold upper_nr_threshold;                  // Upper Non-Recoverable Threshold\n    threshold upper_c_threshold;                   // Upper Critical Threshold\n    threshold upper_nc_threshold;                  // Upper Non-Critical Threshold\n    threshold lower_nr_threshold;                  // Lower Non-Recoverable Threshold\n    threshold lower_c_threshold;                   // Lower Critical Threshold\n    threshold lower_nc_threshold;                  // Lower Non-Critical Threshold\n    threshold hysteresis;                          // Hysteresis\n} metric_threshold;\n\ntypedef struct _fpga_token_header {\n    uint64_t magic;\n    uint16_t vendor_id;\n    uint16_t device_id;\n    uint16_t segment;\n    uint8_t bus;\n    uint8_t device;\n    uint8_t function;\n    fpga_interface interface;\n    fpga_objtype objtype;\n    uint64_t object_id;\n    fpga_guid guid;\n    uint16_t subsystem_vendor_id;\n    uint16_t subsystem_device_id;\n} fpga_token_header;\n\n#define fpga_is_parent_child(__parent_hdr, __child_hdr) \\\n(((__parent_hdr)->objtype == FPGA_DEVICE) && \\\n ((__child_hdr)->objtype == FPGA_ACCELERATOR) && \\\n ((__parent_hdr)->segment == (__child_hdr)->segment) && \\\n ((__parent_hdr)->bus == (__child_hdr)->bus) && \\\n ((__parent_hdr)->device == (__child_hdr)->device))\n\n#endif // __FPGA_TYPES_H__\n
                                            "},{"location":"opae-code/types__enum_8h/","title":"File types_enum.h","text":"
                                            FileList > docs > sw > include > opae > types_enum.h
                                             
                                            Go to the source code of this file.
                                             
                                            Definitions of enumerated types for the OPAE C API. More...
                                            "},{"location":"opae-code/types__enum_8h/#public-types","title":"Public Types","text":"Type Name enum fpga_accelerator_state accelerator state enum fpga_buffer_flags Buffer flags. enum fpga_event_type FPGA events. enum fpga_interface OPAE plugin interface. enum fpga_metric_datatype Metrics data type. enum fpga_metric_type fpga metrics types opae defines power,thermal, performance counter and afu metric types enum fpga_objtype OPAE FPGA resources (objects) enum fpga_open_flags Open flags. enum fpga_reconf_flags Reconfiguration flags. enum fpga_result OPAE C API function return codes. enum fpga_sysobject_flags enum fpga_sysobject_type"},{"location":"opae-code/types__enum_8h/#detailed-description","title":"Detailed Description","text":"
                                            This file defines return and error codes, event and object types, states, and flags as used or reported by OPAE C API functions. 
                                            "},{"location":"opae-code/types__enum_8h/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/types__enum_8h/#enum-fpga_accelerator_state","title":"enum fpga_accelerator_state","text":"
                                            enum fpga_accelerator_state {\n    FPGA_ACCELERATOR_ASSIGNED = 0,\n    FPGA_ACCELERATOR_UNASSIGNED\n};\n
                                            "},{"location":"opae-code/types__enum_8h/#enum-fpga_buffer_flags","title":"enum fpga_buffer_flags","text":"
                                            Buffer flags. 
                                            enum fpga_buffer_flags {\n    FPGA_BUF_PREALLOCATED = (1u << 0),\n    FPGA_BUF_QUIET = (1u << 1),\n    FPGA_BUF_READ_ONLY = (1u << 2)\n};\n
                                             
                                            These flags can be passed to the fpgaPrepareBuffer() function. 
                                            "},{"location":"opae-code/types__enum_8h/#enum-fpga_event_type","title":"enum fpga_event_type","text":"
                                            FPGA events. 
                                            enum fpga_event_type {\n    FPGA_EVENT_INTERRUPT = 0,\n    FPGA_EVENT_ERROR,\n    FPGA_EVENT_POWER_THERMAL\n};\n
                                             
                                            OPAE currently defines the following event types that applications can register for. Note that not all FPGA resources and target platforms may support all event types. 
                                            "},{"location":"opae-code/types__enum_8h/#enum-fpga_interface","title":"enum fpga_interface","text":"
                                            OPAE plugin interface. 
                                            enum fpga_interface {\n    FPGA_IFC_DFL = 0,\n    FPGA_IFC_VFIO,\n    FPGA_IFC_SIM_DFL,\n    FPGA_IFC_SIM_VFIO,\n    FPGA_IFC_UIO\n};\n
                                             
                                            These are the supported plugin interfaces. 
                                            "},{"location":"opae-code/types__enum_8h/#enum-fpga_metric_datatype","title":"enum fpga_metric_datatype","text":"
                                            enum fpga_metric_datatype {\n    FPGA_METRIC_DATATYPE_INT,\n    FPGA_METRIC_DATATYPE_FLOAT,\n    FPGA_METRIC_DATATYPE_DOUBLE,\n    FPGA_METRIC_DATATYPE_BOOL,\n    FPGA_METRIC_DATATYPE_UNKNOWN\n};\n
                                            "},{"location":"opae-code/types__enum_8h/#enum-fpga_metric_type","title":"enum fpga_metric_type","text":"
                                            enum fpga_metric_type {\n    FPGA_METRIC_TYPE_POWER,\n    FPGA_METRIC_TYPE_THERMAL,\n    FPGA_METRIC_TYPE_PERFORMANCE_CTR,\n    FPGA_METRIC_TYPE_AFU,\n    FPGA_METRIC_TYPE_UNKNOWN\n};\n
                                            "},{"location":"opae-code/types__enum_8h/#enum-fpga_objtype","title":"enum fpga_objtype","text":"
                                            OPAE FPGA resources (objects) 
                                            enum fpga_objtype {\n    FPGA_DEVICE = 0,\n    FPGA_ACCELERATOR\n};\n
                                             
                                            These are the FPGA resources currently supported by the OPAE object model. 
                                            "},{"location":"opae-code/types__enum_8h/#enum-fpga_open_flags","title":"enum fpga_open_flags","text":"
                                            Open flags. 
                                            enum fpga_open_flags {\n    FPGA_OPEN_SHARED = (1u << 0)\n};\n
                                             
                                            These flags can be passed to the fpgaOpen() function. 
                                            "},{"location":"opae-code/types__enum_8h/#enum-fpga_reconf_flags","title":"enum fpga_reconf_flags","text":"
                                            Reconfiguration flags. 
                                            enum fpga_reconf_flags {\n    FPGA_RECONF_FORCE = (1u << 0),\n    FPGA_RECONF_SKIP_USRCLK = (1u << 1)\n};\n
                                             
                                            These flags can be passed to the fpgaReconfigureSlot() function. 
                                            "},{"location":"opae-code/types__enum_8h/#enum-fpga_result","title":"enum fpga_result","text":"
                                            OPAE C API function return codes. 
                                            enum fpga_result {\n    FPGA_OK = 0,\n    FPGA_INVALID_PARAM,\n    FPGA_BUSY,\n    FPGA_EXCEPTION,\n    FPGA_NOT_FOUND,\n    FPGA_NO_MEMORY,\n    FPGA_NOT_SUPPORTED,\n    FPGA_NO_DRIVER,\n    FPGA_NO_DAEMON,\n    FPGA_NO_ACCESS,\n    FPGA_RECONF_ERROR\n};\n
                                             
                                            Every public API function exported by the OPAE C library will return one of these codes. Usually, FPGA_OK denotes successful completion of the requested operation, while any return code other than FPGA_OK indicates an error or other deviation from the expected behavior. Users of the OPAE C API should always check the return codes of the APIs they call, and not use output parameters of functions that did not execute successfully.
                                             
                                            The fpgaErrStr() function converts error codes into printable messages.
                                             
                                            OPAE also has a logging mechanism that allows a developer to get more information about why a particular call failed with a specific message. If enabled, any function that returns an error code different from FPGA_OK will also print out a message with further details. This mechanism can be enabled by setting the environment variable LIBOPAE_LOG to 1 before running the respective application. 
                                            "},{"location":"opae-code/types__enum_8h/#enum-fpga_sysobject_flags","title":"enum fpga_sysobject_flags","text":"
                                            enum fpga_sysobject_flags {\n    FPGA_OBJECT_SYNC = (1u << 0),\n    FPGA_OBJECT_GLOB = (1u << 1),\n    FPGA_OBJECT_RAW =\n        (1u << 2),\n    FPGA_OBJECT_RECURSE_ONE =\n        (1u\n         << 3),\n    FPGA_OBJECT_RECURSE_ALL =\n        (1u\n         << 4)\n};\n
                                            "},{"location":"opae-code/types__enum_8h/#enum-fpga_sysobject_type","title":"enum fpga_sysobject_type","text":"
                                            enum fpga_sysobject_type {\n    FPGA_OBJECT_CONTAINER =\n        (1u << 0),\n    FPGA_OBJECT_ATTRIBUTE =\n        (1u << 1)\n};\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/types_enum.h
                                            "},{"location":"opae-code/types__enum_8h_source/","title":"File types_enum.h","text":"
                                            File List > docs > sw > include > opae > types_enum.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017-2022, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_TYPES_ENUM_H__\n#define __FPGA_TYPES_ENUM_H__\n\ntypedef enum {\n    FPGA_OK = 0,         \n    FPGA_INVALID_PARAM,  \n    FPGA_BUSY,           \n    FPGA_EXCEPTION,      \n    FPGA_NOT_FOUND,      \n    FPGA_NO_MEMORY,      \n    FPGA_NOT_SUPPORTED,  \n    FPGA_NO_DRIVER,      \n    FPGA_NO_DAEMON,      \n    FPGA_NO_ACCESS,      \n    FPGA_RECONF_ERROR    \n} fpga_result;\n\ntypedef enum {\n    FPGA_EVENT_INTERRUPT = 0,   \n    FPGA_EVENT_ERROR,           \n    FPGA_EVENT_POWER_THERMAL    \n} fpga_event_type;\n\n/* TODO: consider adding lifecycle events in the future\n * to help with orchestration.  Need a complete specification\n * before including them in the API.  Proposed events:\n *  FPGA_EVENT_APPEAR\n *  FPGA_EVENT_DISAPPEAR\n *  FPGA_EVENT_CHANGE\n */\n\ntypedef enum {\n    FPGA_ACCELERATOR_ASSIGNED = 0,\n    FPGA_ACCELERATOR_UNASSIGNED\n} fpga_accelerator_state;\n\ntypedef enum {\n    FPGA_DEVICE = 0,\n    FPGA_ACCELERATOR\n} fpga_objtype;\n\ntypedef enum {\n    FPGA_IFC_DFL = 0,\n    FPGA_IFC_VFIO,\n    FPGA_IFC_SIM_DFL,\n    FPGA_IFC_SIM_VFIO,\n    FPGA_IFC_UIO,\n} fpga_interface;\n\nenum fpga_buffer_flags {\n    FPGA_BUF_PREALLOCATED = (1u << 0), \n    FPGA_BUF_QUIET = (1u << 1),        \n    FPGA_BUF_READ_ONLY = (1u << 2)     \n};\n\nenum fpga_open_flags {\n    FPGA_OPEN_SHARED = (1u << 0)\n};\n\nenum fpga_reconf_flags {\n    FPGA_RECONF_FORCE = (1u << 0),\n    FPGA_RECONF_SKIP_USRCLK = (1u << 1)\n};\n\nenum fpga_sysobject_flags {\n    FPGA_OBJECT_SYNC = (1u << 0), \n    FPGA_OBJECT_GLOB = (1u << 1), \n    FPGA_OBJECT_RAW =\n        (1u << 2), \n    FPGA_OBJECT_RECURSE_ONE =\n        (1u\n         << 3), \n    FPGA_OBJECT_RECURSE_ALL =\n        (1u\n         << 4) \n};\n\nenum fpga_sysobject_type {\n    FPGA_OBJECT_CONTAINER =\n        (1u << 0), \n    FPGA_OBJECT_ATTRIBUTE =\n        (1u << 1) \n};\n\nenum fpga_metric_type {\n    FPGA_METRIC_TYPE_POWER,             // Metric power\n    FPGA_METRIC_TYPE_THERMAL,           // Metric Thermal\n    FPGA_METRIC_TYPE_PERFORMANCE_CTR,   // Metric Performance counter\n    FPGA_METRIC_TYPE_AFU,               // Metric AFU\n    FPGA_METRIC_TYPE_UNKNOWN            // Unknown\n};\n\nenum fpga_metric_datatype {\n    FPGA_METRIC_DATATYPE_INT,       // Metric datatype integer\n    FPGA_METRIC_DATATYPE_FLOAT,     // Metric datatype float\n    FPGA_METRIC_DATATYPE_DOUBLE,    // Metric datatype double\n    FPGA_METRIC_DATATYPE_BOOL,      // Metric datatype bool\n    FPGA_METRIC_DATATYPE_UNKNOWN    // Metric datatype unknown\n};\n\n#endif // __FPGA_TYPES_ENUM_H__\n
                                            "},{"location":"opae-code/uio_8h/","title":"File uio.h","text":"
                                            FileList > docs > sw > include > opae > uio.h
                                             
                                            Go to the source code of this file.
                                             
                                            APIs to manage a PCIe device via UIO. More...
                                             
                                               
                                            • #include <stdio.h>
                                              •  
                                            • #include <stdint.h>
                                              •  
                                            "},{"location":"opae-code/uio_8h/#classes","title":"Classes","text":"Type Name struct opae_uio OPAE UIO device abstraction. struct opae_uio_device_region MMIO region info."},{"location":"opae-code/uio_8h/#public-functions","title":"Public Functions","text":"Type Name void opae_uio_close (struct opae_uio * u) Release and close a UIO device. int opae_uio_open (struct opae_uio * u, const char * dfl_device) Open and populate a UIO device. int opae_uio_region_get (struct opae_uio * u, uint32_t index, uint8_t ** ptr, size_t * size) Query device MMIO region."},{"location":"opae-code/uio_8h/#macros","title":"Macros","text":"Type Name define OPAE_UIO_PATH_MAX  256"},{"location":"opae-code/uio_8h/#detailed-description","title":"Detailed Description","text":"
                                            Presents a simple interface for interacting with a DFL device that is bound to its UIO driver. See https://kernel.org/doc/html/v4.14/driver-api/uio-howto.html for a description of UIO.
                                             
                                            Provides APIs for opening/closing the device and for querying info about the MMIO regions of the device. 
                                            "},{"location":"opae-code/uio_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/uio_8h/#function-opae_uio_close","title":"function opae_uio_close","text":"
                                            Release and close a UIO device. 
                                            void opae_uio_close (\n    struct opae_uio * u\n) \n
                                             
                                            The given device pointer must have been previously initialized by opae_uio_open. Releases all data structures.
                                             
                                            Parameters:
                                             
                                               
                                            • u Storage for the device info. May be stack-resident.
                                              •  
                                             
                                            Example  struct opae_uio u;
                                             
                                            if (opae_uio_open(&u, \"dfl_dev.10\")) {   // handle error } else {   // interact with the device   ...   // free the device   opae_uio_close(&u); }
                                             
                                            Example  $ sudo opaeuio -r dfl_dev.10
                                            "},{"location":"opae-code/uio_8h/#function-opae_uio_open","title":"function opae_uio_open","text":"
                                            Open and populate a UIO device. 
                                            int opae_uio_open (\n    struct opae_uio * u,\n    const char * dfl_device\n) \n
                                             
                                            Opens the Device Feature List device corresponding to the device name given in dfl_device, eg \"dfl_dev.10\". The device must be bound to the dfl-uio-pdev driver prior to opening it. The data structures corresponding to the MMIO regions are initialized.
                                             
                                            Parameters:
                                             
                                               
                                            • u Storage for the device. May be stack-resident. 
                                              •  
                                            • dfl_device The name of the desired DFL device. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error. Zero on success.
                                             
                                            Example  $ sudo opaeuio -i -u lab -g lab dfl_dev.10
                                             
                                            Example  struct opae_uio u;
                                             
                                            if (opae_uio_open(&u, \"dfl_dev.10\")) {   // handle error }
                                            "},{"location":"opae-code/uio_8h/#function-opae_uio_region_get","title":"function opae_uio_region_get","text":"
                                            Query device MMIO region. 
                                            int opae_uio_region_get (\n    struct opae_uio * u,\n    uint32_t index,\n    uint8_t ** ptr,\n    size_t * size\n) \n
                                             
                                            Retrieves info describing the MMIO region with the given region index. The device structure u must have been previously opened by a call to opae_uio_open.
                                             
                                            Parameters:
                                             
                                               
                                            • u The open OPAE UIO device. 
                                              •  
                                            • index The zero-based index of the desired MMIO region. 
                                              •  
                                            • ptr Optional pointer to receive the virtual address for the region. Pass NULL to ignore. 
                                              •  
                                            • size Optional pointer to receive the size of the MMIO region. Pass NULL to ignore. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error (including index out-of-range). Zero on success.
                                             
                                            Example  struct opae_uio u;
                                             
                                            uint8_t *virt = NULL; size_t size = 0;
                                             
                                            if (opae_uio_open(&u, \"dfl_dev.10\")) {   // handle error } else {   opae_uio_region_get(&u, 0, &virt, &size); }
                                            "},{"location":"opae-code/uio_8h/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/uio_8h/#define-opae_uio_path_max","title":"define OPAE_UIO_PATH_MAX","text":"
                                            #define OPAE_UIO_PATH_MAX 256\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/uio.h
                                            "},{"location":"opae-code/uio_8h_source/","title":"File uio.h","text":"
                                            File List > docs > sw > include > opae > uio.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2020, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __OPAE_UIO_H__\n#define __OPAE_UIO_H__\n\n#include <stdio.h>\n#include <stdint.h>\n\n#define OPAE_UIO_PATH_MAX 256\n\nstruct opae_uio_device_region {\n    uint32_t region_index;\n    uint8_t *region_ptr;\n    size_t region_page_offset;\n    size_t region_size;\n    struct opae_uio_device_region *next;\n};\n\nstruct opae_uio {\n    char device_path[OPAE_UIO_PATH_MAX];\n    int device_fd;\n    struct opae_uio_device_region *regions;\n};\n\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nint opae_uio_open(struct opae_uio *u,\n          const char *dfl_device);\n\nint opae_uio_region_get(struct opae_uio *u,\n            uint32_t index,\n            uint8_t **ptr,\n            size_t *size);\n\nvoid opae_uio_close(struct opae_uio *u);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __OPAE_UIO_H__\n
                                            "},{"location":"opae-code/umsg_8h/","title":"File umsg.h","text":"
                                            FileList > docs > sw > include > opae > umsg.h
                                             
                                            Go to the source code of this file.
                                             
                                            FPGA UMsg API. 
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/umsg_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaGetNumUmsg (fpga_handle handle, uint64_t * value) Get number of Umsgs. fpga_result fpgaGetUmsgPtr (fpga_handle handle, uint64_t ** umsg_ptr) Access UMsg memory directly. fpga_result fpgaSetUmsgAttributes (fpga_handle handle, uint64_t value) Sets Umsg hint. fpga_result fpgaTriggerUmsg (fpga_handle handle, uint64_t value) Trigger Umsg."},{"location":"opae-code/umsg_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/umsg_8h/#function-fpgagetnumumsg","title":"function fpgaGetNumUmsg","text":"
                                            Get number of Umsgs. 
                                            fpga_result fpgaGetNumUmsg (\n    fpga_handle handle,\n    uint64_t * value\n) \n
                                             
                                            Retuns number of umsg supported by AFU.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource 
                                              •  
                                            • value Returns number of UMsgs 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if input parameter combination is not valid. FPGA_EXCEPTION if input parameter fpga handle is not valid. 
                                            "},{"location":"opae-code/umsg_8h/#function-fpgagetumsgptr","title":"function fpgaGetUmsgPtr","text":"
                                            Access UMsg memory directly. 
                                            fpga_result fpgaGetUmsgPtr (\n    fpga_handle handle,\n    uint64_t ** umsg_ptr\n) \n
                                             
                                            This function will return a pointer to the memory allocated for low latency accelerator notifications (UMsgs).
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource 
                                              •  
                                            • umsg_ptr Pointer to memory where a pointer to the virtual address space will be returned 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if input parameter combination is not valid. FPGA_EXCEPTION if input parameter fpga handle is not valid. FPGA_NO_MEMORY if memory allocation fails or system doesn't configure huge pages. 
                                            "},{"location":"opae-code/umsg_8h/#function-fpgasetumsgattributes","title":"function fpgaSetUmsgAttributes","text":"
                                            Sets Umsg hint. 
                                            fpga_result fpgaSetUmsgAttributes (\n    fpga_handle handle,\n    uint64_t value\n) \n
                                             
                                            Writes usmg hint bit.
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource 
                                              •  
                                            • value Value to use for UMsg hint, Umsg hit is N wide bitvector where N = number of Umsgs. 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if input parameter combination is not valid. FPGA_EXCEPTION if input parameter fpga handle is not valid. 
                                            "},{"location":"opae-code/umsg_8h/#function-fpgatriggerumsg","title":"function fpgaTriggerUmsg","text":"
                                            Trigger Umsg. 
                                            fpga_result fpgaTriggerUmsg (\n    fpga_handle handle,\n    uint64_t value\n) \n
                                             
                                            Writes a 64-bit value to trigger low-latency accelerator notification mechanism (UMsgs).
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource 
                                              •  
                                            • value Value to use for UMsg 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if input parameter combination is not valid. FPGA_EXCEPTION if input parameter fpga handle is not valid. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/umsg.h
                                            "},{"location":"opae-code/umsg_8h_source/","title":"File umsg.h","text":"
                                            File List > docs > sw > include > opae > umsg.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_UMSG_H__\n#define __FPGA_UMSG_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaGetNumUmsg(fpga_handle handle, uint64_t *value);\n\nfpga_result fpgaSetUmsgAttributes(fpga_handle handle,\n                  uint64_t value);\n\nfpga_result fpgaTriggerUmsg(fpga_handle handle, uint64_t value);\n\nfpga_result fpgaGetUmsgPtr(fpga_handle handle, uint64_t **umsg_ptr);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_UMSG_H__\n
                                            "},{"location":"opae-code/userclk_8h/","title":"File userclk.h","text":"
                                            FileList > docs > sw > include > opae > userclk.h
                                             
                                            Go to the source code of this file.
                                             
                                            Functions for setting and get afu user clock. 
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/userclk_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaGetUserClock (fpga_handle handle, uint64_t * high_clk, uint64_t * low_clk, int flags) Get afu user clock high and low. fpga_result fpgaSetUserClock (fpga_handle handle, uint64_t high_clk, uint64_t low_clk, int flags) set afu user clock high and low"},{"location":"opae-code/userclk_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/userclk_8h/#function-fpgagetuserclock","title":"function fpgaGetUserClock","text":"
                                            Get afu user clock high and low. 
                                            fpga_result fpgaGetUserClock (\n    fpga_handle handle,\n    uint64_t * high_clk,\n    uint64_t * low_clk,\n    int flags\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource. 
                                              •  
                                            • high_clk AFU High user clock frequency in MHz. 
                                              •  
                                            • low_clk AFU Low user clock frequency in MHz. 
                                              •  
                                            • flags Flags Reserved.
                                              •  
                                             
                                            .*
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if invalid parameters were provided, or if the parameter combination is not valid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                            "},{"location":"opae-code/userclk_8h/#function-fpgasetuserclock","title":"function fpgaSetUserClock","text":"
                                            set afu user clock high and low 
                                            fpga_result fpgaSetUserClock (\n    fpga_handle handle,\n    uint64_t high_clk,\n    uint64_t low_clk,\n    int flags\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • handle Handle to previously opened accelerator resource. 
                                              •  
                                            • high_clk AFU High user clock frequency in MHz. 
                                              •  
                                            • low_clk AFU Low user clock frequency in MHz. 
                                              •  
                                            • flags Flags Reserved.
                                              •  
                                             
                                            .*
                                             
                                            Returns:
                                             
                                            FPGA_OK on success. FPGA_INVALID_PARAM if invalid parameters were provided, or if the parameter combination is not valid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/userclk.h
                                            "},{"location":"opae-code/userclk_8h_source/","title":"File userclk.h","text":"
                                            File List > docs > sw > include > opae > userclk.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_AFU_USER_CLOCK_H__\n#define __FPGA_AFU_USER_CLOCK_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaSetUserClock(fpga_handle handle,\n                uint64_t high_clk, uint64_t low_clk, int flags);\n\nfpga_result fpgaGetUserClock(fpga_handle handle,\n                uint64_t *high_clk, uint64_t *low_clk, int flags);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_AFU_USER_CLOCK_H__\n
                                            "},{"location":"opae-code/utils_8h/","title":"File utils.h","text":"
                                            FileList > docs > sw > include > opae > utils.h
                                             
                                            Go to the source code of this file.
                                             
                                            Utility functions and macros for the FPGA API. 
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            • #include <stdio.h>
                                              •  
                                            "},{"location":"opae-code/utils_8h/#public-functions","title":"Public Functions","text":"Type Name const char * fpgaErrStr (fpga_result e) Return human-readable error message."},{"location":"opae-code/utils_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/utils_8h/#function-fpgaerrstr","title":"function fpgaErrStr","text":"
                                            Return human-readable error message. 
                                            const char * fpgaErrStr (\n    fpga_result e\n) \n
                                             
                                            Returns a pointer to a human-readable error message corresponding to the provided fpga_error error code.
                                             
                                            Parameters:
                                             
                                               
                                            • e Error code (as returned by another FPGA API function 
                                              •  
                                             
                                            Returns:
                                             
                                            Pointer to a descriptive error message string 
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/utils.h
                                            "},{"location":"opae-code/utils_8h_source/","title":"File utils.h","text":"
                                            File List > docs > sw > include > opae > utils.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_UTILS_H__\n#define __FPGA_UTILS_H__\n\n#include <opae/types.h>\n#include <stdio.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nconst char *fpgaErrStr(fpga_result e);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_UTILS_H__\n
                                            "},{"location":"opae-code/version_8h/","title":"File version.h","text":"
                                            FileList > docs > sw > include > opae > version.h
                                             
                                            Go to the source code of this file.
                                             
                                               
                                            • #include <opae/types.h>
                                              •  
                                            "},{"location":"opae-code/version_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaGetOPAECBuildString (char * build_str, size_t len) Get build information about the OPAE library as a string. fpga_result fpgaGetOPAECVersion (fpga_version * version) Get version information about the OPAE library. fpga_result fpgaGetOPAECVersionString (char * version_str, size_t len) Get version information about the OPAE library as a string."},{"location":"opae-code/version_8h/#macros","title":"Macros","text":"Type Name define FPGA_BUILD_STR_MAX  41 define FPGA_VERSION_STR_MAX  10"},{"location":"opae-code/version_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/version_8h/#function-fpgagetopaecbuildstring","title":"function fpgaGetOPAECBuildString","text":"
                                            Get build information about the OPAE library as a string. 
                                            fpga_result fpgaGetOPAECBuildString (\n    char * build_str,\n    size_t len\n) \n
                                             
                                            Retrieve the build identifier of the OPAE library.
                                             
                                            Parameters:
                                             
                                               
                                            • build_str String to copy build information into 
                                              •  
                                            • len Length of build_str 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if build_str is NULL, FPGA_EXCEPTION if the version string cannot be copied into build_str, FPGA_OK otherwise. 
                                            "},{"location":"opae-code/version_8h/#function-fpgagetopaecversion","title":"function fpgaGetOPAECVersion","text":"
                                            Get version information about the OPAE library. 
                                            fpga_result fpgaGetOPAECVersion (\n    fpga_version * version\n) \n
                                             
                                            Retrieve major version, minor version, and revision information about the OPAE library.
                                             
                                            Parameters:
                                             
                                               
                                            • version FPGA version 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if any of the output parameters is NULL, FPGA_OK otherwise. 
                                            "},{"location":"opae-code/version_8h/#function-fpgagetopaecversionstring","title":"function fpgaGetOPAECVersionString","text":"
                                            Get version information about the OPAE library as a string. 
                                            fpga_result fpgaGetOPAECVersionString (\n    char * version_str,\n    size_t len\n) \n
                                             
                                            Retrieve major version, minor version, and revision information about the OPAE library, encoded in a human-readable string (e.g. \"1.0.0\").
                                             
                                            Parameters:
                                             
                                               
                                            • version_str String to copy version information into 
                                              •  
                                            • len Length of version_str 
                                              •  
                                             
                                            Returns:
                                             
                                            FPGA_INVALID_PARAM if version_str is NULL, FPGA_EXCEPTION if the version string cannot be copied into version_str, FPGA_OK otherwise. 
                                            "},{"location":"opae-code/version_8h/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/version_8h/#define-fpga_build_str_max","title":"define FPGA_BUILD_STR_MAX","text":"
                                            #define FPGA_BUILD_STR_MAX 41\n
                                            "},{"location":"opae-code/version_8h/#define-fpga_version_str_max","title":"define FPGA_VERSION_STR_MAX","text":"
                                            #define FPGA_VERSION_STR_MAX 10\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/version.h
                                            "},{"location":"opae-code/version_8h_source/","title":"File version.h","text":"
                                            File List > docs > sw > include > opae > version.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_VERSION_H__\n#define __FPGA_VERSION_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaGetOPAECVersion(fpga_version *version);\n\nfpga_result fpgaGetOPAECVersionString(char *version_str, size_t len);\n#define FPGA_VERSION_STR_MAX 10\n\nfpga_result fpgaGetOPAECBuildString(char *build_str, size_t len);\n#define FPGA_BUILD_STR_MAX 41\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_VERSION_H__\n
                                            "},{"location":"opae-code/vfio_8h/","title":"File vfio.h","text":"
                                            FileList > docs > sw > include > opae > vfio.h
                                             
                                            Go to the source code of this file.
                                             
                                            APIs to manage a PCIe device via vfio-pci. More...
                                             
                                               
                                            • #include <stdio.h>
                                              •  
                                            • #include <stdint.h>
                                              •  
                                            • #include <pthread.h>
                                              •  
                                            • #include <linux/vfio.h>
                                              •  
                                            • #include <opae/mem_alloc.h>
                                              •  
                                            • #include <opae/hash_map.h>
                                              •  
                                            "},{"location":"opae-code/vfio_8h/#classes","title":"Classes","text":"Type Name struct opae_vfio OPAE VFIO device abstraction. struct opae_vfio_buffer System DMA buffer. struct opae_vfio_device VFIO device. struct opae_vfio_device_irq Interrupt info. struct opae_vfio_device_region MMIO region info. struct opae_vfio_group VFIO group. struct opae_vfio_iova_range IO Virtual Address Range. struct opae_vfio_sparse_info MMIO sparse-mappable region info."},{"location":"opae-code/vfio_8h/#public-types","title":"Public Types","text":"Type Name enum opae_vfio_buffer_flags Flags for opae_vfio_buffer_allocate_ex() ."},{"location":"opae-code/vfio_8h/#public-functions","title":"Public Functions","text":"Type Name int opae_vfio_buffer_allocate (struct opae_vfio * v, size_t * size, uint8_t ** buf, uint64_t * iova) Allocate and map system buffer. int opae_vfio_buffer_allocate_ex (struct opae_vfio * v, size_t * size, uint8_t ** buf, uint64_t * iova, int flags) Allocate and map system buffer (extended w/ flags) int opae_vfio_buffer_free (struct opae_vfio * v, uint8_t * buf) Unmap and free a system buffer. struct opae_vfio_buffer * opae_vfio_buffer_info (struct opae_vfio * v, uint8_t * vaddr) Extract the internal data structure pointer for the given vaddr. void opae_vfio_close (struct opae_vfio * v) Release and close a VFIO device. int opae_vfio_irq_disable (struct opae_vfio * v, uint32_t index, uint32_t subindex) Disable an IRQ. int opae_vfio_irq_enable (struct opae_vfio * v, uint32_t index, uint32_t subindex, int event_fd) Enable an IRQ. int opae_vfio_irq_mask (struct opae_vfio * v, uint32_t index, uint32_t subindex) Mask an IRQ. int opae_vfio_irq_unmask (struct opae_vfio * v, uint32_t index, uint32_t subindex) Unmask an IRQ. int opae_vfio_open (struct opae_vfio * v, const char * pciaddr) Open and populate a VFIO device. int opae_vfio_region_get (struct opae_vfio * v, uint32_t index, uint8_t ** ptr, size_t * size) Query device MMIO region. int opae_vfio_secure_open (struct opae_vfio * v, const char * pciaddr, const char * token) Open and populate a VFIO device."},{"location":"opae-code/vfio_8h/#detailed-description","title":"Detailed Description","text":"
                                            Presents a simple interface for interacting with a PCIe device that is bound to the vfio-pci driver. See https://kernel.org/doc/Documentation/vfio.txt for a description of vfio-pci.
                                             
                                            Provides APIs for opening/closing the device, querying info about the MMIO regions of the device, and allocating/mapping and freeing/unmapping DMA buffers. 
                                            "},{"location":"opae-code/vfio_8h/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/vfio_8h/#enum-opae_vfio_buffer_flags","title":"enum opae_vfio_buffer_flags","text":"
                                            enum opae_vfio_buffer_flags {\n    OPAE_VFIO_BUF_PREALLOCATED = 1\n};\n
                                            "},{"location":"opae-code/vfio_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/vfio_8h/#function-opae_vfio_buffer_allocate","title":"function opae_vfio_buffer_allocate","text":"
                                            Allocate and map system buffer. 
                                            int opae_vfio_buffer_allocate (\n    struct opae_vfio * v,\n    size_t * size,\n    uint8_t ** buf,\n    uint64_t * iova\n) \n
                                             
                                            Allocate, map, and retrieve info for a system buffer capable of DMA. Saves an entry in the v->cont_buffers list. If the buffer is not explicitly freed by opae_vfio_buffer_free, it will be freed during opae_vfio_close.
                                             
                                            mmap is used for the allocation. If the size is greater than 2MB, then the allocation request is fulfilled by a 1GB huge page. Else, if the size is greater than 4096, then the request is fulfilled by a 2MB huge page. Else, the request is fulfilled by the non-huge page pool.
                                             
                                            Note:
                                             
                                            Allocations from the huge page pool require that huge pages be configured on the system. Huge pages may be configured on the kernel boot command prompt. Example default_hugepagesz=1G hugepagesz=1G hugepages=2 hugepagesz=2M hugepages=8
                                             
                                            Note:
                                             
                                            Huge pages may also be configured at runtime. Example sudo sh -c 'echo 8 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages' sudo sh -c 'echo 2 > /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages'
                                             
                                            Note:
                                             
                                            Be sure that the IOMMU is also enabled using the follow kernel boot command: intel_iommu=on
                                             
                                            Parameters:
                                             
                                               
                                            • v The open OPAE VFIO device. 
                                              •  
                                            • size A pointer to the requested size. The size may be rounded to the next page size prior to return from the function. 
                                              •  
                                            • buf Optional pointer to receive the virtual address for the buffer. Pass NULL to ignore. 
                                              •  
                                            • iova Optional pointer to receive the IOVA address for the buffer. Pass NULL to ignore. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error. Zero on success.
                                             
                                            Example  opae_vfio v;
                                             
                                            size_t sz; uint8_t *buf_2m_virt = NULL; uint8_t *buf_1g_virt = NULL; uint64_t buf_2m_iova = 0; uint64_t buf_1g_iova = 0;
                                             
                                            if (opae_vfio_open(&v, \"0000:00:00.0\")) {   // handle error } else {   sz = 2 * 1024 * 1024;   if (opae_vfio_buffer_allocate(&v,                                 &sz,                                 &buf_2m_virt,                                 &buf_2m_iova)) {     // handle allocation error   }
                                             
                                            sz = 1024 * 1024 * 1024;   if (opae_vfio_buffer_allocate(&v,                                 &sz,                                 &buf_1g_virt,                                 &buf_1g_iova)) {     // handle allocation error   } }
                                            "},{"location":"opae-code/vfio_8h/#function-opae_vfio_buffer_allocate_ex","title":"function opae_vfio_buffer_allocate_ex","text":"
                                            Allocate and map system buffer (extended w/ flags) 
                                            int opae_vfio_buffer_allocate_ex (\n    struct opae_vfio * v,\n    size_t * size,\n    uint8_t ** buf,\n    uint64_t * iova,\n    int flags\n) \n
                                             
                                            Allocate, map, and retrieve info for a system buffer capable of DMA. Saves an entry in the v->cont_buffers list. If the buffer is not explicitly freed by opae_vfio_buffer_free, it will be freed during opae_vfio_close, unless OPAE_VFIO_BUF_PREALLOCATED is used in which case the buffer is not freed by this library.
                                             
                                            When not using OPAE_VFIO_BUF_PREALLOCATED, mmap is used for the allocation. If the size is greater than 2MB, then the allocation request is fulfilled by a 1GB huge page. Else, if the size is greater than 4096, then the request is fulfilled by a 2MB huge page. Else, the request is fulfilled by the non-huge page pool.
                                             
                                            Parameters:
                                             
                                               
                                            • v The open OPAE VFIO device. 
                                              •  
                                            • size A pointer to the requested size. The size may be rounded to the next page size prior to return from the function. 
                                              •  
                                            • buf Optional pointer to receive the virtual address for the buffer/input buffer pointer when using OPAE_VFIO_BUF_PREALLOCATED. Pass NULL to ignore. 
                                              •  
                                            • iova Optional pointer to receive the IOVA address for the buffer. Pass NULL to ignore. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error. Zero on success.
                                             
                                            Example  opae_vfio v;
                                             
                                            size_t sz = MY_BUF_SIZE; uint8_t *prealloc_virt = NULL; uint64_t iova = 0;
                                             
                                            prealloc_virt = allocate_my_buffer(sz);
                                             
                                            if (opae_vfio_open(&v, \"0000:00:00.0\")) {   // handle error } else {   if (opae_vfio_buffer_allocate_ex(&v,                                    &sz,                                    &prealloc_virt,                                    &iova,                                    OPAE_VFIO_BUF_PREALLOCATED)) {     // handle allocation error   } }
                                            "},{"location":"opae-code/vfio_8h/#function-opae_vfio_buffer_free","title":"function opae_vfio_buffer_free","text":"
                                            Unmap and free a system buffer. 
                                            int opae_vfio_buffer_free (\n    struct opae_vfio * v,\n    uint8_t * buf\n) \n
                                             
                                            The buffer corresponding to buf must have been created by a previous call to opae_vfio_buffer_allocate.
                                             
                                            Parameters:
                                             
                                               
                                            • v The open OPAE VFIO device. 
                                              •  
                                            • buf The virtual address corresponding to the buffer to be freed. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error. Zero on success.
                                             
                                            Example  size_t sz; uint8_t *buf_2m_virt = NULL; uint64_t buf_2m_iova = 0;
                                             
                                            sz = 2 * 1024 * 1024; if (opae_vfio_buffer_allocate(&v,                               &sz,                               &buf_2m_virt,                               &buf_2m_iova)) {   // handle allocation error } else {   // use the buffer
                                             
                                            if (opae_vfio_buffer_free(&v, buf_2m_virt)) {     // handle free error   } }
                                            "},{"location":"opae-code/vfio_8h/#function-opae_vfio_buffer_info","title":"function opae_vfio_buffer_info","text":"
                                            Extract the internal data structure pointer for the given vaddr. 
                                            struct opae_vfio_buffer * opae_vfio_buffer_info (\n    struct opae_vfio * v,\n    uint8_t * vaddr\n) \n
                                             
                                            The virtual address vaddr must correspond to a buffer previously allocated by opae_vfio_buffer_allocate() or opae_vfio_buffer_allocate_ex().
                                             
                                            Parameters:
                                             
                                               
                                            • v The open OPAE VFIO device. 
                                              •  
                                            • vaddr The user virtual address of the desired buffer info struct. 
                                              •  
                                             
                                            Returns:
                                             
                                            NULL on lookup error. 
                                            "},{"location":"opae-code/vfio_8h/#function-opae_vfio_close","title":"function opae_vfio_close","text":"
                                            Release and close a VFIO device. 
                                            void opae_vfio_close (\n    struct opae_vfio * v\n) \n
                                             
                                            The given device pointer must have been previously initialized by opae_vfio_open. Releases all data structures, including any DMA buffer allocations that have not be explicitly freed by opae_vfio_buffer_free.
                                             
                                            Parameters:
                                             
                                               
                                            • v Storage for the device info. May be stack-resident.
                                              •  
                                             
                                            Example  opae_vfio v;
                                             
                                            if (opae_vfio_open(&v, \"0000:00:00.0\")) {   // handle error } else {   // interact with the device   ...   // free the device   opae_vfio_close(&v); }
                                             
                                            Example  $ sudo opaevfio -r 0000:00:00.0
                                            "},{"location":"opae-code/vfio_8h/#function-opae_vfio_irq_disable","title":"function opae_vfio_irq_disable","text":"
                                            Disable an IRQ. 
                                            int opae_vfio_irq_disable (\n    struct opae_vfio * v,\n    uint32_t index,\n    uint32_t subindex\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • v The open OPAE VFIO device. 
                                              •  
                                            • index The IRQ category. For MSI-X, use VFIO_PCI_MSIX_IRQ_INDEX. 
                                              •  
                                            • subindex The IRQ to disable. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error. Zero on success. 
                                            "},{"location":"opae-code/vfio_8h/#function-opae_vfio_irq_enable","title":"function opae_vfio_irq_enable","text":"
                                            Enable an IRQ. 
                                            int opae_vfio_irq_enable (\n    struct opae_vfio * v,\n    uint32_t index,\n    uint32_t subindex,\n    int event_fd\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • v The open OPAE VFIO device. 
                                              •  
                                            • index The IRQ category. For MSI-X, use VFIO_PCI_MSIX_IRQ_INDEX. 
                                              •  
                                            • subindex The IRQ to enable. 
                                              •  
                                            • event_fd The file descriptor, created by eventfd(). Interrupts will result in this event_fd being signaled. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error. Zero on success. 
                                            "},{"location":"opae-code/vfio_8h/#function-opae_vfio_irq_mask","title":"function opae_vfio_irq_mask","text":"
                                            Mask an IRQ. 
                                            int opae_vfio_irq_mask (\n    struct opae_vfio * v,\n    uint32_t index,\n    uint32_t subindex\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • v The open OPAE VFIO device. 
                                              •  
                                            • index The IRQ category. For MSI-X, use VFIO_PCI_MSIX_IRQ_INDEX. 
                                              •  
                                            • subindex The IRQ to mask. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error. Zero on success. 
                                            "},{"location":"opae-code/vfio_8h/#function-opae_vfio_irq_unmask","title":"function opae_vfio_irq_unmask","text":"
                                            Unmask an IRQ. 
                                            int opae_vfio_irq_unmask (\n    struct opae_vfio * v,\n    uint32_t index,\n    uint32_t subindex\n) \n
                                             
                                            Parameters:
                                             
                                               
                                            • v The open OPAE VFIO device. 
                                              •  
                                            • index The IRQ category. For MSI-X, use VFIO_PCI_MSIX_IRQ_INDEX. 
                                              •  
                                            • subindex The IRQ to unmask. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error. Zero on success. 
                                            "},{"location":"opae-code/vfio_8h/#function-opae_vfio_open","title":"function opae_vfio_open","text":"
                                            Open and populate a VFIO device. 
                                            int opae_vfio_open (\n    struct opae_vfio * v,\n    const char * pciaddr\n) \n
                                             
                                            Opens the PCIe device corresponding to the address given in pciaddr. The device must be bound to the vfio-pci driver prior to opening it. The data structures corresponding to IOVA space, MMIO regions, and DMA buffers are initialized.
                                             
                                            Parameters:
                                             
                                               
                                            • v Storage for the device info. May be stack-resident. 
                                              •  
                                            • pciaddr The PCIe address of the requested device. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error. Zero on success.
                                             
                                            Example  $ sudo opaevfio -i 0000:00:00.0 -u user -g group
                                             
                                            Example  opae_vfio v;
                                             
                                            if (opae_vfio_open(&v, \"0000:00:00.0\")) {   // handle error }
                                            "},{"location":"opae-code/vfio_8h/#function-opae_vfio_region_get","title":"function opae_vfio_region_get","text":"
                                            Query device MMIO region. 
                                            int opae_vfio_region_get (\n    struct opae_vfio * v,\n    uint32_t index,\n    uint8_t ** ptr,\n    size_t * size\n) \n
                                             
                                            Retrieves info describing the MMIO region with the given region index. The device structure v must have been previously opened by a call to opae_vfio_open.
                                             
                                            Parameters:
                                             
                                               
                                            • v The open OPAE VFIO device. 
                                              •  
                                            • index The zero-based index of the desired MMIO region. 
                                              •  
                                            • ptr Optional pointer to receive the virtual address for the region. Pass NULL to ignore. 
                                              •  
                                            • size Optional pointer to receive the size of the MMIO region. Pass NULL to ignore. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error (including index out-of-range). Zero on success.
                                             
                                            Example  opae_vfio v;
                                             
                                            uint8_t *fme_virt = NULL; uint8_t *port_virt = NULL; size_t fme_size = 0; size_t port_size = 0;
                                             
                                            if (opae_vfio_open(&v, \"0000:00:00.0\")) {   // handle error } else {   opae_vfio_region_get(&v, 0, &fme_virt, &fme_size);   opae_vfio_region_get(&v, 2, &port_virt, &port_size); }
                                            "},{"location":"opae-code/vfio_8h/#function-opae_vfio_secure_open","title":"function opae_vfio_secure_open","text":"
                                            Open and populate a VFIO device. 
                                            int opae_vfio_secure_open (\n    struct opae_vfio * v,\n    const char * pciaddr,\n    const char * token\n) \n
                                             
                                            Opens the PCIe device corresponding to the address given in pciaddr, using the VF token (GUID) given in token. The device must be bound to the vfio-pci driver prior to opening it. The data structures corresponding to IOVA space, MMIO regions, and DMA buffers are initialized.
                                             
                                            Parameters:
                                             
                                               
                                            • v Storage for the device info. May be stack-resident. 
                                              •  
                                            • pciaddr The PCIe address of the requested device. 
                                              •  
                                            • token The GUID representing the VF token. 
                                              •  
                                             
                                            Returns:
                                             
                                            Non-zero on error. Zero on success.
                                             
                                            Example  $ sudo opaevfio -i 0000:00:00.0 -u user -g group
                                             
                                            Example  opae_vfio v;
                                             
                                            if (opae_vfio_secure_open(&v, \"0000:00:00.0\",                           \"00f5ad6b-2edd-422e-9d1e-34124c686fec\")) {   // handle error }
                                             
                                            The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                            "},{"location":"opae-code/vfio_8h_source/","title":"File vfio.h","text":"
                                            File List > docs > sw > include > opae > vfio.h
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2020-2023, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __OPAE_VFIO_H__\n#define __OPAE_VFIO_H__\n\n#include <stdio.h>\n#include <stdint.h>\n#include <pthread.h>\n\n#include <linux/vfio.h>\n#include <opae/mem_alloc.h>\n#include <opae/hash_map.h>\n\nstruct opae_vfio_iova_range {\n    uint64_t start;             \n    uint64_t end;               \n    struct opae_vfio_iova_range *next;  \n};\n\nstruct opae_vfio_group {\n    char *group_device; \n    int group_fd;       \n};\n\nstruct opae_vfio_sparse_info {\n    uint32_t index;             \n    uint32_t offset;            \n    uint32_t size;              \n    uint8_t *ptr;               \n    struct opae_vfio_sparse_info *next; \n};\n\nstruct opae_vfio_device_region {\n    uint32_t region_index;              \n    uint8_t *region_ptr;                \n    size_t region_size;             \n    struct opae_vfio_sparse_info *region_sparse;    \n    struct opae_vfio_device_region *next;       \n};\n\nstruct opae_vfio_device_irq {\n    uint32_t flags;             \n    uint32_t index;             \n    uint32_t count;             \n    int32_t *event_fds;         \n    int32_t *masks;             \n    struct opae_vfio_device_irq *next;  \n};\n\nstruct opae_vfio_device {\n    int device_fd;                  \n    uint64_t device_config_offset;          \n    uint32_t device_num_regions;            \n    struct opae_vfio_device_region *regions;    \n    uint32_t device_num_irqs;           \n    struct opae_vfio_device_irq *irqs;      \n};\n\nstruct opae_vfio_buffer {\n    uint8_t *buffer_ptr;        \n    size_t buffer_size;     \n    uint64_t buffer_iova;       \n    int flags;          \n};\n\nstruct opae_vfio {\n    pthread_mutex_t lock;               \n    char *cont_device;              \n    char *cont_pciaddr;             \n    int cont_fd;                    \n    struct opae_vfio_iova_range *cont_ranges;   \n    struct mem_alloc iova_alloc;            \n    struct opae_vfio_group group;           \n    struct opae_vfio_device device;         \n    opae_hash_map cont_buffers;     \n};\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nint opae_vfio_open(struct opae_vfio *v,\n           const char *pciaddr);\n\nint opae_vfio_secure_open(struct opae_vfio *v,\n              const char *pciaddr,\n              const char *token);\n\nint opae_vfio_region_get(struct opae_vfio *v,\n             uint32_t index,\n             uint8_t **ptr,\n             size_t *size);\n\nint opae_vfio_buffer_allocate(struct opae_vfio *v,\n                  size_t *size,\n                  uint8_t **buf,\n                  uint64_t *iova);\n\nenum opae_vfio_buffer_flags {\n    OPAE_VFIO_BUF_PREALLOCATED = 1, \n};\n\nint opae_vfio_buffer_allocate_ex(struct opae_vfio *v,\n                 size_t *size,\n                 uint8_t **buf,\n                 uint64_t *iova,\n                 int flags);\n\nstruct opae_vfio_buffer *opae_vfio_buffer_info(struct opae_vfio *v,\n                           uint8_t *vaddr);\n\nint opae_vfio_buffer_free(struct opae_vfio *v,\n              uint8_t *buf);\n\nint opae_vfio_irq_enable(struct opae_vfio *v,\n             uint32_t index,\n             uint32_t subindex,\n             int event_fd);\n\nint opae_vfio_irq_unmask(struct opae_vfio *v,\n             uint32_t index,\n             uint32_t subindex);\n\nint opae_vfio_irq_mask(struct opae_vfio *v,\n               uint32_t index,\n               uint32_t subindex);\n\nint opae_vfio_irq_disable(struct opae_vfio *v,\n              uint32_t index,\n              uint32_t subindex);\n\nvoid opae_vfio_close(struct opae_vfio *v);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __OPAE_VFIO_H__\n
                                            "},{"location":"opae-code/dir_9a6968a8846ef48cff617fcd6355d7b4/","title":"Dir docs/sw/samples","text":"
                                            FileList > docs > sw > samples
                                            "},{"location":"opae-code/dir_9a6968a8846ef48cff617fcd6355d7b4/#directories","title":"Directories","text":"Type Name dir hello_events dir hello_fpga 
                                            The documentation for this class was generated from the following file docs/sw/samples/
                                            "},{"location":"opae-code/dir_d66a8e4b979fa79493bebe26e2602d2b/","title":"Dir docs/sw/samples/hello_events","text":"
                                            FileList > docs > sw > samples > hello_events
                                            "},{"location":"opae-code/dir_d66a8e4b979fa79493bebe26e2602d2b/#files","title":"Files","text":"Type Name file hello_events.c A code sample of using OPAE event API. 
                                            The documentation for this class was generated from the following file docs/sw/samples/hello_events/
                                            "},{"location":"opae-code/hello__events_8c/","title":"File hello_events.c","text":"
                                            FileList > docs > sw > samples > hello_events > hello_events.c
                                             
                                            Go to the source code of this file.
                                             
                                            A code sample of using OPAE event API. More...
                                             
                                               
                                            • #include <stdio.h>
                                              •  
                                            • #include <stdlib.h>
                                              •  
                                            • #include <string.h>
                                              •  
                                            • #include <unistd.h>
                                              •  
                                            • #include <getopt.h>
                                              •  
                                            • #include <poll.h>
                                              •  
                                            • #include <errno.h>
                                              •  
                                            • #include <sys/stat.h>
                                              •  
                                            • #include <pthread.h>
                                              •  
                                            • #include <opae/fpga.h>
                                              •  
                                            • #include <argsfilter.h>
                                              •  
                                            • #include \"mock/opae_std.h\"
                                              •  
                                            "},{"location":"opae-code/hello__events_8c/#classes","title":"Classes","text":"Type Name struct ras_inject_error"},{"location":"opae-code/hello__events_8c/#public-functions","title":"Public Functions","text":"Type Name void * error_thread (void * arg)  fpga_result find_fpga (fpga_properties device_filter, fpga_token * fpga, uint32_t * num_matches)  void help (void)  fpga_result inject_ras_fatal_error (fpga_token fme_token, uint8_t err)  int main (int argc, char * argv)  fpga_result parse_args (int argc, char * argv)  void print_err (const char * s, fpga_result res)  int usleep (unsigned)"},{"location":"opae-code/hello__events_8c/#macros","title":"Macros","text":"Type Name define FME_SYSFS_INJECT_ERROR  \"errors/inject_errors\" define GETOPT_STRING  \"hv\" define ON_ERR_GOTO (res, label, desc)"},{"location":"opae-code/hello__events_8c/#detailed-description","title":"Detailed Description","text":"
                                            This sample starts two processes. One process injects an artificial fatal error to sysfs; while the other tries to asynchronously capture and handle the event. This sample code exercises all major functions of the event API, including creating and destroying event handles, register and unregister events, polling on event file descriptor, and getting the OS object associated with an event. For a full discussion of OPAE event API, refer to event.h. 
                                            "},{"location":"opae-code/hello__events_8c/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/hello__events_8c/#function-error_thread","title":"function error_thread","text":"
                                            void * error_thread (\n    void * arg\n) \n
                                            "},{"location":"opae-code/hello__events_8c/#function-find_fpga","title":"function find_fpga","text":"
                                            fpga_result find_fpga (\n    fpga_properties device_filter,\n    fpga_token * fpga,\n    uint32_t * num_matches\n) \n
                                            "},{"location":"opae-code/hello__events_8c/#function-help","title":"function help","text":"
                                            void help (\n    void\n) \n
                                            "},{"location":"opae-code/hello__events_8c/#function-inject_ras_fatal_error","title":"function inject_ras_fatal_error","text":"
                                            fpga_result inject_ras_fatal_error (\n    fpga_token fme_token,\n    uint8_t err\n) \n
                                            "},{"location":"opae-code/hello__events_8c/#function-main","title":"function main","text":"
                                            int main (\n    int argc,\n    char * argv\n) \n
                                            "},{"location":"opae-code/hello__events_8c/#function-parse_args","title":"function parse_args","text":"
                                            fpga_result parse_args (\n    int argc,\n    char * argv\n) \n
                                            "},{"location":"opae-code/hello__events_8c/#function-print_err","title":"function print_err","text":"
                                            void print_err (\n    const char * s,\n    fpga_result res\n) \n
                                            "},{"location":"opae-code/hello__events_8c/#function-usleep","title":"function usleep","text":"
                                            int usleep (\n    unsigned\n) \n
                                            "},{"location":"opae-code/hello__events_8c/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/hello__events_8c/#define-fme_sysfs_inject_error","title":"define FME_SYSFS_INJECT_ERROR","text":"
                                            #define FME_SYSFS_INJECT_ERROR \"errors/inject_errors\"\n
                                            "},{"location":"opae-code/hello__events_8c/#define-getopt_string","title":"define GETOPT_STRING","text":"
                                            #define GETOPT_STRING \"hv\"\n
                                            "},{"location":"opae-code/hello__events_8c/#define-on_err_goto","title":"define ON_ERR_GOTO","text":"
                                            #define ON_ERR_GOTO (\n    res,\n    label,\n    desc\n) do {                                       \\\n        if ((res) != FPGA_OK ) {            \\ print_err ((desc), (res));  \\\n            goto label;                \\\n        }                                  \\\n    } while (0)\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/samples/hello_events/hello_events.c
                                            "},{"location":"opae-code/hello__events_8c_source/","title":"File hello_events.c","text":"
                                            File List > docs > sw > samples > hello_events > hello_events.c
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017-2021, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifdef HAVE_CONFIG_H\n#include <config.h>\n#endif // HAVE_CONFIG_H\n#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n#include <unistd.h>\n#include <getopt.h>\n#include <poll.h>\n#include <errno.h>\n#include <sys/stat.h>\n#include <pthread.h>\n\n#include <opae/fpga.h>\n#include <argsfilter.h>\n#include \"mock/opae_std.h\"\n\nint usleep(unsigned);\n\n#define FME_SYSFS_INJECT_ERROR \"errors/inject_errors\"\n\n#define ON_ERR_GOTO(res, label, desc)              \\\n    do {                                       \\\n        if ((res) != FPGA_OK) {            \\\n            print_err((desc), (res));  \\\n            goto label;                \\\n        }                                  \\\n    } while (0)\n\nvoid print_err(const char *s, fpga_result res)\n{\n    fprintf(stderr, \"Error %s: %s\\n\", s, fpgaErrStr(res));\n}\n\n// RAS Error Inject CSR\nstruct ras_inject_error {\n    union {\n        uint64_t csr;\n        struct {\n            /* Catastrophic  error */\n            uint64_t  catastrophicr_error : 1;\n            /* Fatal error */\n            uint64_t  fatal_error : 1;\n            /* Non-fatal error */\n            uint64_t  nonfatal_error : 1;\n            /* Reserved */\n            uint64_t  rsvd : 61;\n        };\n    };\n};\n\nfpga_result inject_ras_fatal_error(fpga_token fme_token, uint8_t err)\n{\n    fpga_result             res1       = FPGA_OK;\n    fpga_result             res2       = FPGA_OK;\n    fpga_handle             fme_handle = NULL;\n    struct ras_inject_error inj_error  = { {0} };\n    fpga_object             inj_err_object;\n\n    res1 = fpgaOpen(fme_token, &fme_handle, FPGA_OPEN_SHARED);\n    if (res1 != FPGA_OK) {\n        OPAE_ERR(\"Failed to open FPGA\");\n        return res1;\n    }\n\n    res1 = fpgaHandleGetObject(fme_handle, FME_SYSFS_INJECT_ERROR, &inj_err_object, 0);\n    ON_ERR_GOTO(res1, out_close, \"Failed to get Handle Object\");\n\n    // Inject fatal error\n    inj_error.fatal_error = err;\n\n    res1 = fpgaObjectWrite64(inj_err_object, inj_error.csr, 0);\n    ON_ERR_GOTO(res1, out_destroy_obj, \"Failed to Read Object\");\n\nout_destroy_obj:\n    res2 = fpgaDestroyObject(&inj_err_object);\n    ON_ERR_GOTO(res2, out_close, \"Failed to Destroy Object\");\nout_close:\n    res2 = fpgaClose(fme_handle);\n    if (res2 != FPGA_OK) {\n        OPAE_ERR(\"Failed to close FPGA\");\n    }\n\n    return res1 != FPGA_OK ? res1 : res2;\n}\n\nvoid *error_thread(void *arg)\n{\n    fpga_token token = (fpga_token) arg;\n    fpga_result res;\n\n    usleep(5000000);\n    printf(\"injecting error\\n\");\n    res = inject_ras_fatal_error(token, 1);\n    if (res != FPGA_OK)\n        print_err(\"setting inject error register\", res);\n\n    usleep(5000000);\n    printf(\"clearing error\\n\");\n    res = inject_ras_fatal_error(token, 0);\n    if (res != FPGA_OK)\n        print_err(\"clearing inject error register\", res);\n\n    return NULL;\n}\n\nvoid help(void)\n{\n    printf(\"\\n\"\n           \"hello_events\\n\"\n           \"OPAE Events API sample\\n\"\n           \"\\n\"\n           \"Usage:\\n\"\n           \"    hello_events [-hv] [-S <segment>] [-B <bus>] [-D <device>] [-F <function>] [PCI_ADDR]\\n\"\n           \"\\n\"\n           \"                -h,--help           Print this help\\n\"\n           \"                -v,--version        Print version info and exit\\n\"\n           \"\\n\");\n}\n\n/*\n * Parse command line arguments\n */\n#define GETOPT_STRING \"hv\"\nfpga_result parse_args(int argc, char *argv[])\n{\n    struct option longopts[] = {\n        { \"help\",    no_argument, NULL, 'h' },\n        { \"version\", no_argument, NULL, 'v' },\n        { NULL,      0,           NULL, 0   }\n    };\n\n    int getopt_ret;\n    int option_index;\n\n    while (-1 != (getopt_ret = getopt_long(argc, argv, GETOPT_STRING, longopts, &option_index))) {\n        const char *tmp_optarg = optarg;\n        /* checks to see if optarg is null and if not goes to value of optarg */\n        if ((optarg) && ('=' == *tmp_optarg)) {\n            ++tmp_optarg;\n        }\n\n    switch (getopt_ret) {\n    case 'h': /* help */\n        help();\n        return -1;\n\n    case 'v': /* version */\n        printf(\"hello_events %s %s%s\\n\",\n               OPAE_VERSION,\n               OPAE_GIT_COMMIT_HASH,\n               OPAE_GIT_SRC_TREE_DIRTY ? \"*\":\"\");\n        return -1;\n\n    default: /* invalid option */\n        fprintf(stderr, \"Invalid cmdline options\\n\");\n        return FPGA_EXCEPTION;\n    }\n    }\n\n    return FPGA_OK;\n}\n\nfpga_result find_fpga(fpga_properties device_filter,\n              fpga_token *fpga,\n              uint32_t *num_matches)\n{\n    fpga_properties filter = NULL;\n    fpga_result res        = FPGA_OK;\n    fpga_result dres       = FPGA_OK;\n\n    res = fpgaCloneProperties(device_filter, &filter);\n    ON_ERR_GOTO(res, out, \"cloning properties object\");\n\n    res = fpgaPropertiesSetObjectType(filter, FPGA_DEVICE);\n    ON_ERR_GOTO(res, out_destroy, \"setting interface ID\");\n\n    res = fpgaEnumerate(&filter, 1, fpga, 1, num_matches);\n    ON_ERR_GOTO(res, out_destroy, \"enumerating FPGAs\");\n\nout_destroy:\n    dres = fpgaDestroyProperties(&filter);\n    ON_ERR_GOTO(dres, out, \"destroying properties object\");\nout:\n    return (res == FPGA_OK) ? dres : res;\n}\n\nint main(int argc, char *argv[])\n{\n    fpga_token         fpga_device_token = NULL;\n    fpga_handle        fpga_device_handle = NULL;\n    uint32_t           num_matches = 1;\n    fpga_result        res1 = FPGA_OK;\n    fpga_result        res2 = FPGA_OK;\n    fpga_event_handle  eh;\n    uint64_t           count = 0;\n    int                res;\n    struct pollfd      pfd;\n    int                timeout = 10000;\n    int                poll_ret = 0;\n    ssize_t            bytes_read = 0;\n    pthread_t          errthr;\n    fpga_properties    device_filter = NULL;\n\n    res1 = fpgaGetProperties(NULL, &device_filter);\n    if (res1) {\n        print_err(\"failed to alloc properties\", res1);\n        return res1;\n    }\n\n    if (opae_set_properties_from_args(device_filter,\n                      &res1,\n                      &argc,\n                      argv)) {\n        print_err(\"failed arg parse\", res1);\n        res1 = FPGA_EXCEPTION;\n        goto out_exit;\n    } else if (res1) {\n        print_err(\"failed to set properties\", res1);\n        goto out_exit;\n    }\n\n    res1 = parse_args(argc, argv);\n    if ((int)res1 < 0)\n        goto out_exit;\n    ON_ERR_GOTO(res1, out_exit, \"parsing arguments\");\n\n    res1 = find_fpga(device_filter, &fpga_device_token, &num_matches);\n    ON_ERR_GOTO(res1, out_exit, \"finding FPGA accelerator\");\n\n    if (num_matches < 1) {\n        fprintf(stderr, \"No matches for address provided.\\n\");\n        res1 = FPGA_NOT_FOUND;\n        goto out_exit;\n    }\n\n    if (num_matches > 1) {\n        fprintf(stderr, \"Found more than one suitable slot.\\n\");\n    }\n\n    res1 = fpgaOpen(fpga_device_token, &fpga_device_handle, FPGA_OPEN_SHARED);\n    ON_ERR_GOTO(res1, out_destroy_tok, \"opening accelerator\");\n\n    res1 = fpgaCreateEventHandle(&eh);\n    ON_ERR_GOTO(res1, out_close, \"creating event handle\");\n\n    res1 = fpgaRegisterEvent(fpga_device_handle, FPGA_EVENT_ERROR, eh, 0);\n    ON_ERR_GOTO(res1, out_destroy_eh, \"registering an FME event\");\n\n    printf(\"Waiting for interrupts now...\\n\");\n\n    res = pthread_create(&errthr, NULL, error_thread, fpga_device_token);\n    if (res) {\n        printf(\"Failed to create error_thread.\\n\");\n        res1 = FPGA_EXCEPTION;\n        goto out_destroy_eh;\n    }\n\n\n    res1 = fpgaGetOSObjectFromEventHandle(eh, &pfd.fd);\n    ON_ERR_GOTO(res1, out_join, \"getting file descriptor\");\n\n    pfd.events = POLLIN;\n    poll_ret = poll(&pfd, 1, timeout);\n\n    if (poll_ret < 0) {\n        printf(\"Poll error errno = %s\\n\", strerror(errno));\n        res1 = FPGA_EXCEPTION;\n        goto out_join;\n    } else if (poll_ret == 0) {\n         printf(\"Poll timeout occurred\\n\");\n         res1 = FPGA_EXCEPTION;\n         goto out_join;\n    } else {\n         printf(\"FME Interrupt occurred\\n\");\n         bytes_read = opae_read(pfd.fd, &count, sizeof(count));\n         if (bytes_read <= 0)\n             printf(\"WARNING: error reading from poll fd: %s\\n\",\n                     bytes_read < 0 ? strerror(errno) : \"zero bytes read\");\n    }\n\n    res1 = fpgaUnregisterEvent(fpga_device_handle, FPGA_EVENT_ERROR, eh);\n    ON_ERR_GOTO(res1, out_join, \"unregistering an FME event\");\n\n    printf(\"Successfully tested Register/Unregister for FME events!\\n\");\n\nout_join:\n    pthread_join(errthr, NULL);\n\nout_destroy_eh:\n    res2 = fpgaDestroyEventHandle(&eh);\n    ON_ERR_GOTO(res2, out_close, \"deleting event handle\");\n\nout_close:\n    res2 = fpgaClose(fpga_device_handle);\n    ON_ERR_GOTO(res2, out_destroy_tok, \"closing accelerator\");\n\nout_destroy_tok:\n    res2 = fpgaDestroyToken(&fpga_device_token);\n    ON_ERR_GOTO(res2, out_exit, \"destroying token\");\n\nout_exit:\n    fpgaDestroyProperties(&device_filter);\n    return res1 != FPGA_OK ? res1 : res2;\n}\n
                                            "},{"location":"opae-code/dir_a3c160366dc832de1042e5d4d49ef034/","title":"Dir docs/sw/samples/hello_fpga","text":"
                                            FileList > docs > sw > samples > hello_fpga
                                            "},{"location":"opae-code/dir_a3c160366dc832de1042e5d4d49ef034/#files","title":"Files","text":"Type Name file hello_fpga.c A code sample illustrates the basic usage of the OPAE C API. 
                                            The documentation for this class was generated from the following file docs/sw/samples/hello_fpga/
                                            "},{"location":"opae-code/hello__fpga_8c/","title":"File hello_fpga.c","text":"
                                            FileList > docs > sw > samples > hello_fpga > hello_fpga.c
                                             
                                            Go to the source code of this file.
                                             
                                            A code sample illustrates the basic usage of the OPAE C API. More...
                                             
                                               
                                            • #include <stdio.h>
                                              •  
                                            • #include <stdlib.h>
                                              •  
                                            • #include <string.h>
                                              •  
                                            • #include <stdint.h>
                                              •  
                                            • #include <getopt.h>
                                              •  
                                            • #include <unistd.h>
                                              •  
                                            • #include <uuid/uuid.h>
                                              •  
                                            • #include <opae/fpga.h>
                                              •  
                                            • #include <argsfilter.h>
                                              •  
                                            • #include \"mock/opae_std.h\"
                                              •  
                                            "},{"location":"opae-code/hello__fpga_8c/#classes","title":"Classes","text":"Type Name struct cache_line struct config"},{"location":"opae-code/hello__fpga_8c/#public-attributes","title":"Public Attributes","text":"Type Name struct config config   = = { .open_flags = 0, .run_n3000 = 0 }"},{"location":"opae-code/hello__fpga_8c/#public-functions","title":"Public Functions","text":"Type Name fpga_result find_fpga (fpga_properties device_filter, fpga_guid afu_guid, fpga_token * accelerator_token, uint32_t * num_matches_accelerators)  fpga_result find_nlb_n3000 (fpga_handle accelerator_handle, uint64_t * afu_baddr)  void help (void)  int main (int argc, char * argv)  fpga_result parse_args (int argc, char * argv)  void print_err (const char * s, fpga_result res)  bool probe_for_ase (void)  int usleep (unsigned)"},{"location":"opae-code/hello__fpga_8c/#macros","title":"Macros","text":"Type Name define CACHELINE_ALIGNED_ADDR (p) ((p) >> LOG2_CL) define CL (x) ((x) * 64) define CSR_AFU_DSM_BASEL  0x0110 define CSR_CFG  0x0140 define CSR_CTL  0x0138 define CSR_DST_ADDR  0x0128 define CSR_NUM_LINES  0x0130 define CSR_SRC_ADDR  0x0120 define CSR_STATUS1  0x0168 define DSM_STATUS_TEST_COMPLETE  0x40 define FPGA_NLB0_UUID_H  0xd8424dc4a4a3c413 define FPGA_NLB0_UUID_L  0xf89e433683f9040b define GETOPT_STRING  \"hscv\" define LOG2_CL  6 define LPBK1_BUFFER_ALLOCATION_SIZE MB(2) define LPBK1_BUFFER_SIZE MB(1) define LPBK1_DSM_SIZE MB(2) define MB (x) ((x) * 1024 * 1024) define N3000_AFUID  \"9AEFFE5F-8457-0612-C000-C9660D824272\" define NLB0_AFUID  \"D8424DC4-A4A3-C413-F89E-433683F9040B\" define ON_ERR_GOTO (res, label, desc)  define TEST_TIMEOUT  30000"},{"location":"opae-code/hello__fpga_8c/#detailed-description","title":"Detailed Description","text":"
                                            The sample is a host application that demonstrates the basic steps of interacting with FPGA using the OPAE library. These steps include:
                                             
                                               
                                            • FPGA enumeration
                                              •  
                                            • Resource acquiring and releasing
                                              •  
                                            • Managing shared memory buffer
                                              •  
                                            • MMIO read and write
                                              •  
                                             
                                            The sample also demonstrates OPAE's object model, such as tokens, handles, and properties.
                                             
                                            The sample requires a native loopback mode (NLB) test image to be loaded on the FPGA. Refer to Quick Start Guide for full instructions on building, configuring, and running this code sample. 
                                            "},{"location":"opae-code/hello__fpga_8c/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/hello__fpga_8c/#variable-config","title":"variable config","text":"
                                            struct config config;\n
                                            "},{"location":"opae-code/hello__fpga_8c/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/hello__fpga_8c/#function-find_fpga","title":"function find_fpga","text":"
                                            fpga_result find_fpga (\n    fpga_properties device_filter,\n    fpga_guid afu_guid,\n    fpga_token * accelerator_token,\n    uint32_t * num_matches_accelerators\n) \n
                                            "},{"location":"opae-code/hello__fpga_8c/#function-find_nlb_n3000","title":"function find_nlb_n3000","text":"
                                            fpga_result find_nlb_n3000 (\n    fpga_handle accelerator_handle,\n    uint64_t * afu_baddr\n) \n
                                            "},{"location":"opae-code/hello__fpga_8c/#function-help","title":"function help","text":"
                                            void help (\n    void\n) \n
                                            "},{"location":"opae-code/hello__fpga_8c/#function-main","title":"function main","text":"
                                            int main (\n    int argc,\n    char * argv\n) \n
                                            "},{"location":"opae-code/hello__fpga_8c/#function-parse_args","title":"function parse_args","text":"
                                            fpga_result parse_args (\n    int argc,\n    char * argv\n) \n
                                            "},{"location":"opae-code/hello__fpga_8c/#function-print_err","title":"function print_err","text":"
                                            void print_err (\n    const char * s,\n    fpga_result res\n) \n
                                            "},{"location":"opae-code/hello__fpga_8c/#function-probe_for_ase","title":"function probe_for_ase","text":"
                                            bool probe_for_ase (\n    void\n) \n
                                            "},{"location":"opae-code/hello__fpga_8c/#function-usleep","title":"function usleep","text":"
                                            int usleep (\n    unsigned\n) \n
                                            "},{"location":"opae-code/hello__fpga_8c/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/hello__fpga_8c/#define-cacheline_aligned_addr","title":"define CACHELINE_ALIGNED_ADDR","text":"
                                            #define CACHELINE_ALIGNED_ADDR (\n    p\n) ((p) >> LOG2_CL )\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-cl","title":"define CL","text":"
                                            #define CL (\n    x\n) ((x) * 64)\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-csr_afu_dsm_basel","title":"define CSR_AFU_DSM_BASEL","text":"
                                            #define CSR_AFU_DSM_BASEL 0x0110\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-csr_cfg","title":"define CSR_CFG","text":"
                                            #define CSR_CFG 0x0140\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-csr_ctl","title":"define CSR_CTL","text":"
                                            #define CSR_CTL 0x0138\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-csr_dst_addr","title":"define CSR_DST_ADDR","text":"
                                            #define CSR_DST_ADDR 0x0128\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-csr_num_lines","title":"define CSR_NUM_LINES","text":"
                                            #define CSR_NUM_LINES 0x0130\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-csr_src_addr","title":"define CSR_SRC_ADDR","text":"
                                            #define CSR_SRC_ADDR 0x0120\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-csr_status1","title":"define CSR_STATUS1","text":"
                                            #define CSR_STATUS1 0x0168\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-dsm_status_test_complete","title":"define DSM_STATUS_TEST_COMPLETE","text":"
                                            #define DSM_STATUS_TEST_COMPLETE 0x40\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-fpga_nlb0_uuid_h","title":"define FPGA_NLB0_UUID_H","text":"
                                            #define FPGA_NLB0_UUID_H 0xd8424dc4a4a3c413\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-fpga_nlb0_uuid_l","title":"define FPGA_NLB0_UUID_L","text":"
                                            #define FPGA_NLB0_UUID_L 0xf89e433683f9040b\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-getopt_string","title":"define GETOPT_STRING","text":"
                                            #define GETOPT_STRING \"hscv\"\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-log2_cl","title":"define LOG2_CL","text":"
                                            #define LOG2_CL 6\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-lpbk1_buffer_allocation_size","title":"define LPBK1_BUFFER_ALLOCATION_SIZE","text":"
                                            #define LPBK1_BUFFER_ALLOCATION_SIZE MB (2)\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-lpbk1_buffer_size","title":"define LPBK1_BUFFER_SIZE","text":"
                                            #define LPBK1_BUFFER_SIZE MB (1)\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-lpbk1_dsm_size","title":"define LPBK1_DSM_SIZE","text":"
                                            #define LPBK1_DSM_SIZE MB (2)\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-mb","title":"define MB","text":"
                                            #define MB (\n    x\n) ((x) * 1024 * 1024)\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-n3000_afuid","title":"define N3000_AFUID","text":"
                                            #define N3000_AFUID \"9AEFFE5F-8457-0612-C000-C9660D824272\"\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-nlb0_afuid","title":"define NLB0_AFUID","text":"
                                            #define NLB0_AFUID \"D8424DC4-A4A3-C413-F89E-433683F9040B\"\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-on_err_goto","title":"define ON_ERR_GOTO","text":"
                                            #define ON_ERR_GOTO (\n    res,\n    label,\n    desc\n) do {                                       \\\n        if ((res) != FPGA_OK ) {            \\ print_err ((desc), (res));  \\\n            goto label;                \\\n        }                                  \\\n    } while (0)\n
                                            "},{"location":"opae-code/hello__fpga_8c/#define-test_timeout","title":"define TEST_TIMEOUT","text":"
                                            #define TEST_TIMEOUT 30000\n
                                             
                                            The documentation for this class was generated from the following file docs/sw/samples/hello_fpga/hello_fpga.c
                                            "},{"location":"opae-code/hello__fpga_8c_source/","title":"File hello_fpga.c","text":"
                                            File List > docs > sw > samples > hello_fpga > hello_fpga.c
                                             
                                            Go to the documentation of this file. 
                                             
                                            // Copyright(c) 2017-2022, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifdef HAVE_CONFIG_H\n#include <config.h>\n#endif // HAVE_CONFIG_H\n#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n#include <stdint.h>\n#include <getopt.h>\n#include <unistd.h>\n\n#include <uuid/uuid.h>\n#include <opae/fpga.h>\n#include <argsfilter.h>\n#include \"mock/opae_std.h\"\n\nint usleep(unsigned);\n\n#ifndef TEST_TIMEOUT\n#define TEST_TIMEOUT 30000\n#endif // TEST_TIMEOUT\n\n#ifndef CL\n# define CL(x)                       ((x) * 64)\n#endif // CL\n#ifndef LOG2_CL\n# define LOG2_CL                     6\n#endif // LOG2_CL\n#ifndef MB\n# define MB(x)                       ((x) * 1024 * 1024)\n#endif // MB\n\n#define CACHELINE_ALIGNED_ADDR(p) ((p) >> LOG2_CL)\n\n#define LPBK1_BUFFER_SIZE            MB(1)\n#define LPBK1_BUFFER_ALLOCATION_SIZE MB(2)\n#define LPBK1_DSM_SIZE               MB(2)\n#define CSR_SRC_ADDR                 0x0120\n#define CSR_DST_ADDR                 0x0128\n#define CSR_CTL                      0x0138\n#define CSR_STATUS1                  0x0168\n#define CSR_CFG                      0x0140\n#define CSR_NUM_LINES                0x0130\n#define DSM_STATUS_TEST_COMPLETE     0x40\n#define CSR_AFU_DSM_BASEL            0x0110\n\n/* NLB0 AFU_ID */\n#define NLB0_AFUID \"D8424DC4-A4A3-C413-F89E-433683F9040B\"\n\n/* NLB0 AFU_ID for N3000 */\n#define N3000_AFUID \"9AEFFE5F-8457-0612-C000-C9660D824272\"\n#define FPGA_NLB0_UUID_H 0xd8424dc4a4a3c413\n#define FPGA_NLB0_UUID_L 0xf89e433683f9040b\n\n\n/*\n * macro to check return codes, print error message, and goto cleanup label\n * NOTE: this changes the program flow (uses goto)!\n */\n#define ON_ERR_GOTO(res, label, desc)              \\\n    do {                                       \\\n        if ((res) != FPGA_OK) {            \\\n            print_err((desc), (res));  \\\n            goto label;                \\\n        }                                  \\\n    } while (0)\n\n/* Type definitions */\ntypedef struct {\n    uint32_t uint[16];\n} cache_line;\n\nvoid print_err(const char *s, fpga_result res)\n{\n    fprintf(stderr, \"Error %s: %s\\n\", s, fpgaErrStr(res));\n}\n\n/*\n * Global configuration of bus, set during parse_args()\n * */\nstruct config {\n    int open_flags;\n    int run_n3000;\n}\n\nconfig = {\n    .open_flags = 0,\n    .run_n3000 = 0\n};\n\nvoid help(void)\n{\n    printf(\"\\n\"\n           \"hello_fpga\\n\"\n           \"OPAE Native Loopback 0 (NLB0) sample\\n\"\n           \"\\n\"\n           \"Usage:\\n\"\n           \"        hello_fpga [-schv] [-S <segment>] [-B <bus>] [-D <device>] [-F <function>] [PCI_ADDR]\\n\"\n           \"\\n\"\n           \"                -s,--shared         Open accelerator in shared mode\\n\"\n           \"                -c,--n3000          Assume N3000 MMIO layout\\n\"\n           \"                -h,--help           Print this help\\n\"\n           \"                -v,--version        Print version info and exit\\n\"\n           \"\\n\");\n}\n\n#define GETOPT_STRING \"hscv\"\nfpga_result parse_args(int argc, char *argv[])\n{\n    struct option longopts[] = {\n        { \"help\",    no_argument, NULL, 'h' },\n        { \"shared\",  no_argument, NULL, 's' },\n        { \"n3000\",   no_argument, NULL, 'c' },\n        { \"version\", no_argument, NULL, 'v' },\n        { NULL,      0,           NULL,  0  }\n    };\n\n    int getopt_ret;\n    int option_index;\n\n    char version[32];\n    char build[32];\n\n    while (-1 != (getopt_ret = getopt_long(argc, argv, GETOPT_STRING,\n                        longopts, &option_index))) {\n        const char *tmp_optarg = optarg;\n        /* Checks to see if optarg is null and if not it goes to value of optarg */\n        if ((optarg) && ('=' == *tmp_optarg)) {\n            ++tmp_optarg;\n        }\n\n        switch (getopt_ret) {\n        case 'h':\n            help();\n            return -1;\n        case 's':\n            config.open_flags |= FPGA_OPEN_SHARED;\n            break;\n        case 'c':\n            config.run_n3000 = 1;\n            break;\n        case 'v':\n            fpgaGetOPAECVersionString(version, sizeof(version));\n            fpgaGetOPAECBuildString(build, sizeof(build));\n            printf(\"hello_fpga %s %s\\n\",\n                   version, build);\n            return -1;\n\n        default: /* invalid option */\n            fprintf(stderr, \"Invalid cmdline option \\n\");\n            return FPGA_EXCEPTION;\n        }\n    }\n\n    return FPGA_OK;\n}\n\nfpga_result find_fpga(fpga_properties device_filter,\n              fpga_guid afu_guid,\n              fpga_token *accelerator_token,\n              uint32_t *num_matches_accelerators)\n{\n    fpga_properties filter = NULL;\n    fpga_result res1;\n    fpga_result res2 = FPGA_OK;\n\n    res1 = fpgaCloneProperties(device_filter, &filter);\n    ON_ERR_GOTO(res1, out, \"cloning properties object\");\n\n    res1 = fpgaPropertiesSetObjectType(filter, FPGA_ACCELERATOR);\n    ON_ERR_GOTO(res1, out_destroy, \"setting object type\");\n\n    res1 = fpgaPropertiesSetGUID(filter, afu_guid);\n    ON_ERR_GOTO(res1, out_destroy, \"setting GUID\");\n\n    res1 = fpgaEnumerate(&filter, 1, accelerator_token, 1, num_matches_accelerators);\n    ON_ERR_GOTO(res1, out_destroy, \"enumerating accelerators\");\n\nout_destroy:\n    res2 = fpgaDestroyProperties(&filter);\n    ON_ERR_GOTO(res2, out, \"destroying properties object\");\nout:\n    return res1 != FPGA_OK ? res1 : res2;\n}\n\n/* Is the FPGA simulated with ASE? */\nbool probe_for_ase(void)\n{\n    fpga_result r = FPGA_OK;\n    uint16_t device_id = 0;\n    fpga_properties filter = NULL;\n    uint32_t num_matches = 1;\n    fpga_token fme_token;\n\n    /* Connect to the FPGA management engine */\n    fpgaGetProperties(NULL, &filter);\n    fpgaPropertiesSetObjectType(filter, FPGA_DEVICE);\n\n    /* Connecting to one is sufficient to find ASE */\n    fpgaEnumerate(&filter, 1, &fme_token, 1, &num_matches);\n    if (0 != num_matches) {\n        /* Retrieve the device ID of the FME */\n        fpgaDestroyProperties(&filter);\n        fpgaGetProperties(fme_token, &filter);\n        r = fpgaPropertiesGetDeviceID(filter, &device_id);\n        fpgaDestroyToken(&fme_token);\n    }\n    fpgaDestroyProperties(&filter);\n\n    /* ASE's device ID is 0xa5e */\n    return ((FPGA_OK == r) && (0xa5e == device_id));\n}\n\nfpga_result find_nlb_n3000(fpga_handle accelerator_handle,\n               uint64_t *afu_baddr)\n{\n\n    fpga_result res1 = FPGA_OK;\n    int end_of_list = 0;\n    int nlb0_found = 0;\n    uint64_t header = 0;\n    uint64_t uuid_hi = 0;\n    uint64_t uuid_lo = 0;\n    uint64_t next_offset = 0;\n    uint64_t nlb0_offset = 0;\n\n    /* find NLB0 in AFU */\n    do {\n        // Read the next feature header\n        res1 = fpgaReadMMIO64(accelerator_handle, 0, nlb0_offset, &header);\n        ON_ERR_GOTO(res1, out_exit, \"fpgaReadMMIO64\");\n        res1 = fpgaReadMMIO64(accelerator_handle, 0, nlb0_offset+8, &uuid_lo);\n        ON_ERR_GOTO(res1, out_exit, \"fpgaReadMMIO64\");\n        res1 = fpgaReadMMIO64(accelerator_handle, 0, nlb0_offset+16, &uuid_hi);\n        ON_ERR_GOTO(res1, out_exit, \"fpgaReadMMIO64\");\n        // printf(\"%zx: %zx %zx %zx\\n\", nlb0_offset, header, uuid_lo, uuid_hi);\n        if ((((header >> 60) & 0xf) == 0x1) &&\n            (uuid_lo == FPGA_NLB0_UUID_L) && (uuid_hi == FPGA_NLB0_UUID_H)) {\n            nlb0_found = 1;\n            break;\n        }\n        // End of the list flag\n        end_of_list = (header >> 40) & 1;\n        // Move to the next feature header\n        next_offset = (header >> 16) & 0xffffff;\n        if ((next_offset == 0xffff) || (next_offset == 0)) {\n            nlb0_found = 0;\n            break;\n        }\n        nlb0_offset = nlb0_offset + next_offset;\n    } while (!end_of_list);\n\n    if (!nlb0_found) {\n        printf(\"AFU NLB0 not found\\n\");\n        return FPGA_EXCEPTION;\n    }\n\n    printf(\"AFU NLB0 found @ %zx\\n\", nlb0_offset);\n    *afu_baddr = nlb0_offset;\n    return FPGA_OK;\n\nout_exit:\n    return FPGA_EXCEPTION;\n}\n\nint main(int argc, char *argv[])\n{\n    fpga_token         accelerator_token;\n    fpga_handle        accelerator_handle;\n    fpga_guid          guid;\n    uint32_t           num_matches_accelerators = 0;\n    uint32_t           use_ase;\n\n    volatile uint64_t *dsm_ptr    = NULL;\n    volatile uint64_t *status_ptr = NULL;\n    volatile uint64_t *input_ptr  = NULL;\n    volatile uint64_t *output_ptr = NULL;\n\n    uint64_t           dsm_wsid;\n    uint64_t           input_wsid;\n    uint64_t           output_wsid;\n    uint32_t           i;\n    uint32_t           timeout;\n    fpga_result        res1 = FPGA_OK;\n    fpga_result        res2 = FPGA_OK;\n    fpga_properties    device_filter = NULL;\n\n    res1 = fpgaGetProperties(NULL, &device_filter);\n    if (res1 != FPGA_OK) {\n        print_err(\"failed to allocate properties.\\n\", res1);\n        return 1;\n    }\n\n    if (opae_set_properties_from_args(device_filter,\n                      &res1,\n                      &argc,\n                      argv)) {\n        print_err(\"failed arg parse.\\n\", res1);\n        res1 = FPGA_EXCEPTION;\n        goto out_exit;\n    } else if (res1) {\n        print_err(\"failed to set properties.\\n\", res1);\n        goto out_exit;\n    }\n\n    res1 = parse_args(argc, argv);\n    if ((int)res1 < 0)\n        goto out_exit;\n    ON_ERR_GOTO(res1, out_exit, \"parsing arguments\");\n\n    if (config.run_n3000) {\n        if (uuid_parse(N3000_AFUID, guid) < 0)\n            res1 = FPGA_EXCEPTION;\n    } else {\n        if (uuid_parse(NLB0_AFUID, guid) < 0)\n            res1 = FPGA_EXCEPTION;\n    }\n\n    ON_ERR_GOTO(res1, out_exit, \"parsing guid\");\n\n    use_ase = probe_for_ase();\n    if (use_ase) {\n        printf(\"Running in ASE mode\\n\");\n    }\n\n    /* Look for accelerator with NLB0_AFUID */\n    res1 = find_fpga(device_filter,\n             guid,\n             &accelerator_token,\n             &num_matches_accelerators);\n    ON_ERR_GOTO(res1, out_exit, \"finding FPGA accelerator\");\n\n    if (num_matches_accelerators == 0) {\n        res1 = FPGA_NOT_FOUND;\n    }\n    ON_ERR_GOTO(res1, out_exit, \"no matching accelerator\");\n\n    if (num_matches_accelerators > 1) {\n        printf(\"Found more than one suitable accelerator. \");\n    }\n\n\n    /* Open accelerator and map MMIO */\n    res1 = fpgaOpen(accelerator_token, &accelerator_handle, config.open_flags);\n    ON_ERR_GOTO(res1, out_destroy_tok, \"opening accelerator\");\n\n    res1 = fpgaMapMMIO(accelerator_handle, 0, NULL);\n    ON_ERR_GOTO(res1, out_close, \"mapping MMIO space\");\n\n\n    /* Allocate buffers */\n    res1 = fpgaPrepareBuffer(accelerator_handle, LPBK1_DSM_SIZE,\n                (void **)&dsm_ptr, &dsm_wsid, 0);\n    ON_ERR_GOTO(res1, out_close, \"allocating DSM buffer\");\n\n    res1 = fpgaPrepareBuffer(accelerator_handle, LPBK1_BUFFER_ALLOCATION_SIZE,\n               (void **)&input_ptr, &input_wsid, 0);\n    ON_ERR_GOTO(res1, out_free_dsm, \"allocating input buffer\");\n\n    res1 = fpgaPrepareBuffer(accelerator_handle, LPBK1_BUFFER_ALLOCATION_SIZE,\n               (void **)&output_ptr, &output_wsid, 0);\n    ON_ERR_GOTO(res1, out_free_input, \"allocating output buffer\");\n\n    printf(\"Running Test\\n\");\n\n    /* Initialize buffers */\n    memset((void *)dsm_ptr,    0,    LPBK1_DSM_SIZE);\n    memset((void *)input_ptr,  0xAF, LPBK1_BUFFER_SIZE);\n    memset((void *)output_ptr, 0xBE, LPBK1_BUFFER_SIZE);\n\n    cache_line *cl_ptr = (cache_line *)input_ptr;\n    for (i = 0; i < LPBK1_BUFFER_SIZE / CL(1); ++i) {\n        cl_ptr[i].uint[15] = i+1; /* set the last uint in every cacheline */\n    }\n\n\n    /* Reset accelerator */\n    res1 = fpgaReset(accelerator_handle);\n    ON_ERR_GOTO(res1, out_free_output, \"resetting accelerator\");\n\n    uint64_t nlb_base_addr = 0;\n\n    if (config.run_n3000) {\n        res1 = find_nlb_n3000(accelerator_handle, &nlb_base_addr);\n        ON_ERR_GOTO(res1, out_free_output, \"finding nlb in AFU\");\n    }\n\n    /* Program DMA addresses */\n    uint64_t iova = 0;\n    res1 = fpgaGetIOAddress(accelerator_handle, dsm_wsid, &iova);\n    ON_ERR_GOTO(res1, out_free_output, \"getting DSM IOVA\");\n\n    res1 = fpgaWriteMMIO64(accelerator_handle, 0, nlb_base_addr + CSR_AFU_DSM_BASEL, iova);\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_AFU_DSM_BASEL\");\n\n    res1 = fpgaWriteMMIO32(accelerator_handle, 0, nlb_base_addr + CSR_CTL, 0);\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_CFG\");\n    res1 = fpgaWriteMMIO32(accelerator_handle, 0, nlb_base_addr + CSR_CTL, 1);\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_CFG\");\n\n    res1 = fpgaGetIOAddress(accelerator_handle, input_wsid, &iova);\n    ON_ERR_GOTO(res1, out_free_output, \"getting input IOVA\");\n    res1 = fpgaWriteMMIO64(accelerator_handle, 0, nlb_base_addr + CSR_SRC_ADDR, CACHELINE_ALIGNED_ADDR(iova));\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_SRC_ADDR\");\n\n    res1 = fpgaGetIOAddress(accelerator_handle, output_wsid, &iova);\n    ON_ERR_GOTO(res1, out_free_output, \"getting output IOVA\");\n    res1 = fpgaWriteMMIO64(accelerator_handle, 0, nlb_base_addr + CSR_DST_ADDR, CACHELINE_ALIGNED_ADDR(iova));\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_DST_ADDR\");\n\n    res1 = fpgaWriteMMIO32(accelerator_handle, 0, nlb_base_addr + CSR_NUM_LINES, LPBK1_BUFFER_SIZE / CL(1));\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_NUM_LINES\");\n    res1 = fpgaWriteMMIO32(accelerator_handle, 0, nlb_base_addr + CSR_CFG, 0x42000);\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_CFG\");\n\n    status_ptr = dsm_ptr + DSM_STATUS_TEST_COMPLETE/sizeof(uint64_t);\n\n\n    /* Start the test */\n    res1 = fpgaWriteMMIO32(accelerator_handle, 0, nlb_base_addr + CSR_CTL, 3);\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_CFG\");\n\n    /* Wait for test completion */\n    timeout = TEST_TIMEOUT;\n    while (0 == ((*status_ptr) & 0x1)) {\n        usleep(100);\n        if (!use_ase && (--timeout == 0)) {\n            res1 = FPGA_EXCEPTION;\n            ON_ERR_GOTO(res1, out_free_output, \"test timed out\");\n        }\n    }\n\n    /* Stop the device */\n    res1 = fpgaWriteMMIO32(accelerator_handle, 0, nlb_base_addr + CSR_CTL, 7);\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_CFG\");\n\n    /* Wait for the AFU's read/write traffic to complete */\n    uint32_t afu_traffic_trips = 0;\n    while (afu_traffic_trips < 100) {\n        /*\n         * CSR_STATUS1 holds two 32 bit values: num pending reads and writes.\n         * Wait for it to be 0.\n         */\n        uint64_t s1;\n        res1 = fpgaReadMMIO64(accelerator_handle, 0, nlb_base_addr + CSR_STATUS1, &s1);\n        ON_ERR_GOTO(res1, out_free_output, \"reading CSR_STATUS1\");\n        if (s1 == 0) {\n            break;\n        }\n\n        afu_traffic_trips += 1;\n        usleep(1000);\n    }\n\n    /* Check output buffer contents */\n    for (i = 0; i < LPBK1_BUFFER_SIZE; i++) {\n        if (((uint8_t *)output_ptr)[i] != ((uint8_t *)input_ptr)[i]) {\n            fprintf(stderr, \"Output does NOT match input \"\n                \"at offset %i!\\n\", i);\n            break;\n        }\n    }\n\n    printf(\"Done Running Test\\n\");\n\n\n    /* Release buffers */\nout_free_output:\n    res2 = fpgaReleaseBuffer(accelerator_handle, output_wsid);\n    ON_ERR_GOTO(res2, out_free_input, \"releasing output buffer\");\nout_free_input:\n    res2 = fpgaReleaseBuffer(accelerator_handle, input_wsid);\n    ON_ERR_GOTO(res2, out_free_dsm, \"releasing input buffer\");\nout_free_dsm:\n    res2 = fpgaReleaseBuffer(accelerator_handle, dsm_wsid);\n    ON_ERR_GOTO(res2, out_unmap, \"releasing DSM buffer\");\n\n    /* Unmap MMIO space */\nout_unmap:\n    res2 = fpgaUnmapMMIO(accelerator_handle, 0);\n    ON_ERR_GOTO(res2, out_close, \"unmapping MMIO space\");\n\n    /* Release accelerator */\nout_close:\n    res2 = fpgaClose(accelerator_handle);\n    ON_ERR_GOTO(res2, out_destroy_tok, \"closing accelerator\");\n\n    /* Destroy token */\nout_destroy_tok:\n    res2 = fpgaDestroyToken(&accelerator_token);\n    ON_ERR_GOTO(res2, out_exit, \"destroying token\");\n\nout_exit:\n    fpgaDestroyProperties(&device_filter);\n    return res1 != FPGA_OK ? res1 : res2;\n}\n
                                            "},{"location":"opae-code/namespaces/","title":"Namespace List","text":"
                                            Here is a list of all namespaces with brief descriptions:
                                             
                                               
                                            • namespace opae 
                                                 
                                              • namespace fpga 
                                                   
                                                • namespace types 
                                                     
                                                  • namespace detail 
                                                    •  
                                                   
                                                  •  
                                                 
                                                •  
                                               
                                              •  
                                            • namespace std 
                                              •  
                                            "},{"location":"opae-code/classes/","title":"Class Index","text":""},{"location":"opae-code/classes/#b","title":"b","text":"
                                               
                                            • busy (opae::fpga::types)
                                              •  
                                            "},{"location":"opae-code/classes/#c","title":"c","text":"
                                               
                                            • cache_line
                                              •  
                                            • config
                                              •  
                                            "},{"location":"opae-code/classes/#e","title":"e","text":"
                                               
                                            • error (opae::fpga::types)
                                              •  
                                            • event (opae::fpga::types)
                                              •  
                                            • except (opae::fpga::types)
                                              •  
                                            • exception (opae::fpga::types)
                                              •  
                                            "},{"location":"opae-code/classes/#f","title":"f","text":"
                                               
                                            • fpga_error_info
                                              •  
                                            • fpga_metric
                                              •  
                                            • fpga_metric_info
                                              •  
                                            • fpga_version
                                              •  
                                            "},{"location":"opae-code/classes/#g","title":"g","text":"
                                               
                                            • guid_t (opae::fpga::types)
                                              •  
                                            "},{"location":"opae-code/classes/#h","title":"h","text":"
                                               
                                            • handle (opae::fpga::types)
                                              •  
                                            "},{"location":"opae-code/classes/#i","title":"i","text":"
                                               
                                            • invalid_param (opae::fpga::types)
                                              •  
                                            "},{"location":"opae-code/classes/#m","title":"m","text":"
                                               
                                            • mem_alloc
                                              •  
                                            • mem_link
                                              •  
                                            • metric_threshold
                                              •  
                                            "},{"location":"opae-code/classes/#n","title":"n","text":"
                                               
                                            • no_access (opae::fpga::types)
                                              •  
                                            • no_daemon (opae::fpga::types)
                                              •  
                                            • no_driver (opae::fpga::types)
                                              •  
                                            • no_memory (opae::fpga::types)
                                              •  
                                            • not_found (opae::fpga::types)
                                              •  
                                            • not_supported (opae::fpga::types)
                                              •  
                                            "},{"location":"opae-code/classes/#o","title":"o","text":"
                                               
                                            • opae_uio
                                              •  
                                            • opae_uio_device_region
                                              •  
                                            • opae_vfio
                                              •  
                                            • opae_vfio_buffer
                                              •  
                                            • opae_vfio_device
                                              •  
                                            • opae_vfio_device_irq
                                              •  
                                            • opae_vfio_device_region
                                              •  
                                            • opae_vfio_group
                                              •  
                                            • opae_vfio_iova_range
                                              •  
                                            • opae_vfio_sparse_info
                                              •  
                                            "},{"location":"opae-code/classes/#p","title":"p","text":"
                                               
                                            • properties (opae::fpga::types)
                                              •  
                                            • pvalue (opae::fpga::types)
                                              •  
                                            "},{"location":"opae-code/classes/#r","title":"r","text":"
                                               
                                            • reconf_error (opae::fpga::types)
                                              •  
                                            • ras_inject_error
                                              •  
                                            "},{"location":"opae-code/classes/#s","title":"s","text":"
                                               
                                            • shared_buffer (opae::fpga::types)
                                              •  
                                            • src_location (opae::fpga::types)
                                              •  
                                            • sysobject (opae::fpga::types)
                                              •  
                                            "},{"location":"opae-code/classes/#t","title":"t","text":"
                                               
                                            • token (opae::fpga::types)
                                              •  
                                            • type_t (opae::fpga::types::event)
                                              •  
                                            • threshold
                                              •  
                                            "},{"location":"opae-code/classes/#v","title":"v","text":"
                                               
                                            • version (opae::fpga::types)
                                              •  
                                             
                                            ## \\
                                             
                                               
                                            • _fpga_token_header
                                              •  
                                            • _opae_hash_map
                                              •  
                                            • _opae_hash_map_item
                                              •  
                                            "},{"location":"opae-code/hierarchy/","title":"Class Hierarchy","text":"
                                            This inheritance list is sorted roughly, but not completely, alphabetically:
                                             
                                               
                                            • class opae::fpga::types::error An error object represents an error register for a resource. 
                                              •  
                                            • class opae::fpga::types::event Wraps fpga event routines in OPAE C. 
                                              •  
                                            • class opae::fpga::types::handle An allocated accelerator resource. 
                                              •  
                                            • class opae::fpga::types::properties Wraps an OPAE fpga_properties object. 
                                              •  
                                            • class opae::fpga::types::shared_buffer Host/AFU shared memory blocks. 
                                              •  
                                            • class opae::fpga::types::src_location Identify a particular line in a source file. 
                                              •  
                                            • class opae::fpga::types::sysobject Wraps the OPAE fpga_object primitive. 
                                              •  
                                            • class opae::fpga::types::token Wraps the OPAE fpga_token primitive. 
                                              •  
                                            • class opae::fpga::types::version 
                                              •  
                                            • struct _fpga_token_header Internal token type header. 
                                              •  
                                            • struct _opae_hash_map Hash map object. 
                                              •  
                                            • struct _opae_hash_map_item List link item. 
                                              •  
                                            • struct cache_line 
                                              •  
                                            • struct config 
                                              •  
                                            • struct fpga_error_info 
                                              •  
                                            • struct fpga_metric Metric struct. 
                                              •  
                                            • struct fpga_metric_info Metric info struct. 
                                              •  
                                            • struct fpga_version Semantic version. 
                                              •  
                                            • struct mem_alloc 
                                              •  
                                            • struct mem_link Provides an API for allocating/freeing a logical address space. 
                                              •  
                                            • struct metric_threshold 
                                              •  
                                            • struct opae::fpga::types::event::type_t C++ struct that is interchangeable with fpga_event_type enum. 
                                              •  
                                            • struct opae::fpga::types::guid_t Representation of the guid member of a properties object. 
                                              •  
                                            • struct opae::fpga::types::pvalue Wraps OPAE properties defined in the OPAE C API by associating an fpga_properties reference with the getters and setters defined for a property.
                                              •  
                                            • struct opae_uio OPAE UIO device abstraction. 
                                              •  
                                            • struct opae_uio_device_region MMIO region info. 
                                              •  
                                            • struct opae_vfio OPAE VFIO device abstraction. 
                                              •  
                                            • struct opae_vfio_buffer System DMA buffer. 
                                              •  
                                            • struct opae_vfio_device VFIO device. 
                                              •  
                                            • struct opae_vfio_device_irq Interrupt info. 
                                              •  
                                            • struct opae_vfio_device_region MMIO region info. 
                                              •  
                                            • struct opae_vfio_group VFIO group. 
                                              •  
                                            • struct opae_vfio_iova_range IO Virtual Address Range. 
                                              •  
                                            • struct opae_vfio_sparse_info MMIO sparse-mappable region info. 
                                              •  
                                            • struct ras_inject_error 
                                              •  
                                            • struct threshold Threshold struct. 
                                              •  
                                            • class std::exception 
                                                 
                                              • class opae::fpga::types::except Generic OPAE exception. 
                                                   
                                                • class opae::fpga::types::busy busy exception 
                                                  •  
                                                • class opae::fpga::types::exception exception exception 
                                                  •  
                                                • class opae::fpga::types::invalid_param invalid_param exception
                                                  •  
                                                • class opae::fpga::types::no_access no_access exception
                                                  •  
                                                • class opae::fpga::types::no_daemon no_daemon exception
                                                  •  
                                                • class opae::fpga::types::no_driver no_driver exception
                                                  •  
                                                • class opae::fpga::types::no_memory no_memory exception
                                                  •  
                                                • class opae::fpga::types::not_found not_found exception
                                                  •  
                                                • class opae::fpga::types::not_supported not_supported exception
                                                  •  
                                                • class opae::fpga::types::reconf_error reconf_error exception
                                                  •  
                                                 
                                                •  
                                              • class opae::fpga::types::except Generic OPAE exception. 
                                                   
                                                • class opae::fpga::types::busy busy exception 
                                                  •  
                                                • class opae::fpga::types::exception exception exception 
                                                  •  
                                                • class opae::fpga::types::invalid_param invalid_param exception
                                                  •  
                                                • class opae::fpga::types::no_access no_access exception
                                                  •  
                                                • class opae::fpga::types::no_daemon no_daemon exception
                                                  •  
                                                • class opae::fpga::types::no_driver no_driver exception
                                                  •  
                                                • class opae::fpga::types::no_memory no_memory exception
                                                  •  
                                                • class opae::fpga::types::not_found not_found exception
                                                  •  
                                                • class opae::fpga::types::not_supported not_supported exception
                                                  •  
                                                • class opae::fpga::types::reconf_error reconf_error exception
                                                  •  
                                                 
                                                •  
                                              • class opae::fpga::types::except Generic OPAE exception. 
                                                   
                                                • class opae::fpga::types::busy busy exception 
                                                  •  
                                                • class opae::fpga::types::exception exception exception 
                                                  •  
                                                • class opae::fpga::types::invalid_param invalid_param exception
                                                  •  
                                                • class opae::fpga::types::no_access no_access exception
                                                  •  
                                                • class opae::fpga::types::no_daemon no_daemon exception
                                                  •  
                                                • class opae::fpga::types::no_driver no_driver exception
                                                  •  
                                                • class opae::fpga::types::no_memory no_memory exception
                                                  •  
                                                • class opae::fpga::types::not_found not_found exception
                                                  •  
                                                • class opae::fpga::types::not_supported not_supported exception
                                                  •  
                                                • class opae::fpga::types::reconf_error reconf_error exception
                                                  •  
                                                 
                                                •  
                                              • class opae::fpga::types::except Generic OPAE exception. 
                                                   
                                                • class opae::fpga::types::busy busy exception 
                                                  •  
                                                • class opae::fpga::types::exception exception exception 
                                                  •  
                                                • class opae::fpga::types::invalid_param invalid_param exception
                                                  •  
                                                • class opae::fpga::types::no_access no_access exception
                                                  •  
                                                • class opae::fpga::types::no_daemon no_daemon exception
                                                  •  
                                                • class opae::fpga::types::no_driver no_driver exception
                                                  •  
                                                • class opae::fpga::types::no_memory no_memory exception
                                                  •  
                                                • class opae::fpga::types::not_found not_found exception
                                                  •  
                                                • class opae::fpga::types::not_supported not_supported exception
                                                  •  
                                                • class opae::fpga::types::reconf_error reconf_error exception
                                                  •  
                                                 
                                                •  
                                              • class opae::fpga::types::except Generic OPAE exception. 
                                                   
                                                • class opae::fpga::types::busy busy exception 
                                                  •  
                                                • class opae::fpga::types::exception exception exception 
                                                  •  
                                                • class opae::fpga::types::invalid_param invalid_param exception
                                                  •  
                                                • class opae::fpga::types::no_access no_access exception
                                                  •  
                                                • class opae::fpga::types::no_daemon no_daemon exception
                                                  •  
                                                • class opae::fpga::types::no_driver no_driver exception
                                                  •  
                                                • class opae::fpga::types::no_memory no_memory exception
                                                  •  
                                                • class opae::fpga::types::not_found not_found exception
                                                  •  
                                                • class opae::fpga::types::not_supported not_supported exception
                                                  •  
                                                • class opae::fpga::types::reconf_error reconf_error exception
                                                  •  
                                                 
                                                •  
                                              • class opae::fpga::types::except Generic OPAE exception. 
                                                   
                                                • class opae::fpga::types::busy busy exception 
                                                  •  
                                                • class opae::fpga::types::exception exception exception 
                                                  •  
                                                • class opae::fpga::types::invalid_param invalid_param exception
                                                  •  
                                                • class opae::fpga::types::no_access no_access exception
                                                  •  
                                                • class opae::fpga::types::no_daemon no_daemon exception
                                                  •  
                                                • class opae::fpga::types::no_driver no_driver exception
                                                  •  
                                                • class opae::fpga::types::no_memory no_memory exception
                                                  •  
                                                • class opae::fpga::types::not_found not_found exception
                                                  •  
                                                • class opae::fpga::types::not_supported not_supported exception
                                                  •  
                                                • class opae::fpga::types::reconf_error reconf_error exception
                                                  •  
                                                 
                                                •  
                                              • class opae::fpga::types::except Generic OPAE exception. 
                                                   
                                                • class opae::fpga::types::busy busy exception 
                                                  •  
                                                • class opae::fpga::types::exception exception exception 
                                                  •  
                                                • class opae::fpga::types::invalid_param invalid_param exception
                                                  •  
                                                • class opae::fpga::types::no_access no_access exception
                                                  •  
                                                • class opae::fpga::types::no_daemon no_daemon exception
                                                  •  
                                                • class opae::fpga::types::no_driver no_driver exception
                                                  •  
                                                • class opae::fpga::types::no_memory no_memory exception
                                                  •  
                                                • class opae::fpga::types::not_found not_found exception
                                                  •  
                                                • class opae::fpga::types::not_supported not_supported exception
                                                  •  
                                                • class opae::fpga::types::reconf_error reconf_error exception
                                                  •  
                                                 
                                                •  
                                              • class opae::fpga::types::except Generic OPAE exception. 
                                                   
                                                • class opae::fpga::types::busy busy exception 
                                                  •  
                                                • class opae::fpga::types::exception exception exception 
                                                  •  
                                                • class opae::fpga::types::invalid_param invalid_param exception
                                                  •  
                                                • class opae::fpga::types::no_access no_access exception
                                                  •  
                                                • class opae::fpga::types::no_daemon no_daemon exception
                                                  •  
                                                • class opae::fpga::types::no_driver no_driver exception
                                                  •  
                                                • class opae::fpga::types::no_memory no_memory exception
                                                  •  
                                                • class opae::fpga::types::not_found not_found exception
                                                  •  
                                                • class opae::fpga::types::not_supported not_supported exception
                                                  •  
                                                • class opae::fpga::types::reconf_error reconf_error exception
                                                  •  
                                                 
                                                •  
                                              • class opae::fpga::types::except Generic OPAE exception. 
                                                   
                                                • class opae::fpga::types::busy busy exception 
                                                  •  
                                                • class opae::fpga::types::exception exception exception 
                                                  •  
                                                • class opae::fpga::types::invalid_param invalid_param exception
                                                  •  
                                                • class opae::fpga::types::no_access no_access exception
                                                  •  
                                                • class opae::fpga::types::no_daemon no_daemon exception
                                                  •  
                                                • class opae::fpga::types::no_driver no_driver exception
                                                  •  
                                                • class opae::fpga::types::no_memory no_memory exception
                                                  •  
                                                • class opae::fpga::types::not_found not_found exception
                                                  •  
                                                • class opae::fpga::types::not_supported not_supported exception
                                                  •  
                                                • class opae::fpga::types::reconf_error reconf_error exception
                                                  •  
                                                 
                                                •  
                                              • class opae::fpga::types::except Generic OPAE exception. 
                                                   
                                                • class opae::fpga::types::busy busy exception 
                                                  •  
                                                • class opae::fpga::types::exception exception exception 
                                                  •  
                                                • class opae::fpga::types::invalid_param invalid_param exception
                                                  •  
                                                • class opae::fpga::types::no_access no_access exception
                                                  •  
                                                • class opae::fpga::types::no_daemon no_daemon exception
                                                  •  
                                                • class opae::fpga::types::no_driver no_driver exception
                                                  •  
                                                • class opae::fpga::types::no_memory no_memory exception
                                                  •  
                                                • class opae::fpga::types::not_found not_found exception
                                                  •  
                                                • class opae::fpga::types::not_supported not_supported exception
                                                  •  
                                                • class opae::fpga::types::reconf_error reconf_error exception
                                                  •  
                                                 
                                                •  
                                              • class opae::fpga::types::except Generic OPAE exception. 
                                                   
                                                • class opae::fpga::types::busy busy exception 
                                                  •  
                                                • class opae::fpga::types::exception exception exception 
                                                  •  
                                                • class opae::fpga::types::invalid_param invalid_param exception
                                                  •  
                                                • class opae::fpga::types::no_access no_access exception
                                                  •  
                                                • class opae::fpga::types::no_daemon no_daemon exception
                                                  •  
                                                • class opae::fpga::types::no_driver no_driver exception
                                                  •  
                                                • class opae::fpga::types::no_memory no_memory exception
                                                  •  
                                                • class opae::fpga::types::not_found not_found exception
                                                  •  
                                                • class opae::fpga::types::not_supported not_supported exception
                                                  •  
                                                • class opae::fpga::types::reconf_error reconf_error exception
                                                  •  
                                                 
                                                •  
                                               
                                              •  
                                            "},{"location":"opae-code/modules/","title":"Modules","text":"
                                            Here is a list of all modules:
                                            "},{"location":"opae-code/todo/","title":"Todo List","text":""},{"location":"opae-code/todo/#global-fpgaregisterevent-fpga_handle-handle-fpga_event_type-event_type-fpga_event_handle-event_handle-uint32_t-flags","title":"Global fpgaRegisterEvent  (fpga_handle handle, fpga_event_type event_type, fpga_event_handle event_handle, uint32_t flags)","text":"
                                            define if calling fpgaRegisterEvent multiple times with the same event_handle is an error condition or if it is silently ignored.  define if calling fpgaUnregisterEvent multiple times with the same event_handle is an error condition or if it is silently ignored.
                                            "},{"location":"opae-code/pages/","title":"Class List","text":"
                                            Here are the classes, structs, unions and interfaces with brief descriptions:
                                            "},{"location":"opae-code/class_members/","title":"Class Members","text":""},{"location":"opae-code/class_members/#a","title":"a","text":"
                                               
                                            • allocated (mem_alloc)
                                              •  
                                            • address (mem_link)
                                              •  
                                            • accelerator_state (opae::fpga::types::properties)
                                              •  
                                            • allocate (opae::fpga::types::shared_buffer)
                                              •  
                                            • attach (opae::fpga::types::shared_buffer)
                                              •  
                                            • as_string (opae::fpga::types::version)
                                              •  
                                            • as_struct (opae::fpga::types::version)
                                              •  
                                            "},{"location":"opae-code/class_members/#b","title":"b","text":"
                                               
                                            • bus (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • buckets (_opae_hash_map)
                                              •  
                                            • busy (opae::fpga::types::busy)
                                              •  
                                            • buf_ (opae::fpga::types::except)
                                              •  
                                            • bbs_id (opae::fpga::types::properties)
                                              •  
                                            • bbs_version (opae::fpga::types::properties)
                                              •  
                                            • bytes (opae::fpga::types::sysobject)
                                              •  
                                            • build (opae::fpga::types::version)
                                              •  
                                            • buffer_iova (opae_vfio_buffer)
                                              •  
                                            • buffer_ptr (opae_vfio_buffer)
                                              •  
                                            • buffer_size (opae_vfio_buffer)
                                              •  
                                            "},{"location":"opae-code/class_members/#c","title":"c","text":"
                                               
                                            • cleanup_context (_opae_hash_map)
                                              •  
                                            • can_clear (fpga_error_info, opae::fpga::types::error)
                                              •  
                                            • c_type (opae::fpga::types::error, opae::fpga::types::guid_t, opae::fpga::types::handle, opae::fpga::types::properties, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject, opae::fpga::types::token)
                                              •  
                                            • close (opae::fpga::types::handle)
                                              •  
                                            • capabilities (opae::fpga::types::properties)
                                              •  
                                            • copy_ (opae::fpga::types::pvalue)
                                              •  
                                            • copy_t (opae::fpga::types::pvalue)
                                              •  
                                            • compare (opae::fpga::types::shared_buffer)
                                              •  
                                            • cont_buffers (opae_vfio)
                                              •  
                                            • cont_device (opae_vfio)
                                              •  
                                            • cont_fd (opae_vfio)
                                              •  
                                            • cont_pciaddr (opae_vfio)
                                              •  
                                            • cont_ranges (opae_vfio)
                                              •  
                                            • count (opae_vfio_device_irq)
                                              •  
                                            • catastrophicr_error (ras_inject_error)
                                              •  
                                            • csr (ras_inject_error)
                                              •  
                                            "},{"location":"opae-code/class_members/#d","title":"d","text":"
                                               
                                            • device (_fpga_token_header, opae::fpga::types::properties, opae_vfio)
                                              •  
                                            • device_id (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • data_ (opae::fpga::types::guid_t)
                                              •  
                                            • device_fd (opae_uio, opae_vfio_device)
                                              •  
                                            • device_path (opae_uio)
                                              •  
                                            • device_config_offset (opae_vfio_device)
                                              •  
                                            • device_num_irqs (opae_vfio_device)
                                              •  
                                            • device_num_regions (opae_vfio_device)
                                              •  
                                            "},{"location":"opae-code/class_members/#e","title":"e","text":"
                                               
                                            • error (opae::fpga::types::error, opae::fpga::types::event::type_t)
                                              •  
                                            • error_info_ (opae::fpga::types::error)
                                              •  
                                            • error_num_ (opae::fpga::types::error)
                                              •  
                                            • event (opae::fpga::types::event)
                                              •  
                                            • event_handle_ (opae::fpga::types::event)
                                              •  
                                            • except (opae::fpga::types::except)
                                              •  
                                            • exception (opae::fpga::types::exception)
                                              •  
                                            • enumerate (opae::fpga::types::token)
                                              •  
                                            • event_fds (opae_vfio_device_irq)
                                              •  
                                            • end (opae_vfio_iova_range)
                                              •  
                                            "},{"location":"opae-code/class_members/#f","title":"f","text":"
                                               
                                            • function (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • flags (_opae_hash_map, opae_vfio_buffer, opae_vfio_device_irq)
                                              •  
                                            • free (mem_alloc)
                                              •  
                                            • fill (opae::fpga::types::shared_buffer)
                                              •  
                                            • file (opae::fpga::types::src_location)
                                              •  
                                            • file_ (opae::fpga::types::src_location)
                                              •  
                                            • fn (opae::fpga::types::src_location)
                                              •  
                                            • fn_ (opae::fpga::types::src_location)
                                              •  
                                            • fatal_error (ras_inject_error)
                                              •  
                                            "},{"location":"opae-code/class_members/#g","title":"g","text":"
                                               
                                            • guid (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • group_name (fpga_metric_info)
                                              •  
                                            • get (opae::fpga::types::error, opae::fpga::types::event, opae::fpga::types::properties, opae::fpga::types::sysobject)
                                              •  
                                            • guid_t (opae::fpga::types::guid_t)
                                              •  
                                            • get_token (opae::fpga::types::handle)
                                              •  
                                            • get_ (opae::fpga::types::pvalue)
                                              •  
                                            • get_value (opae::fpga::types::pvalue)
                                              •  
                                            • getter_t (opae::fpga::types::pvalue)
                                              •  
                                            • get_parent (opae::fpga::types::token)
                                              •  
                                            • group (opae_vfio)
                                              •  
                                            • group_device (opae_vfio_group)
                                              •  
                                            • group_fd (opae_vfio_group)
                                              •  
                                            "},{"location":"opae-code/class_members/#h","title":"h","text":"
                                               
                                            • hash_seed (_opae_hash_map)
                                              •  
                                            • hysteresis (metric_threshold)
                                              •  
                                            • handle_ (opae::fpga::types::event, opae::fpga::types::handle, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject)
                                              •  
                                            • handle (opae::fpga::types::handle)
                                              •  
                                            "},{"location":"opae-code/class_members/#i","title":"i","text":"
                                               
                                            • interface (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • isvalid (fpga_metric)
                                              •  
                                            • interrupt (opae::fpga::types::event::type_t)
                                              •  
                                            • invalidate (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                              •  
                                            • is_set (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                              •  
                                            • is_set_ (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                              •  
                                            • invalid_param (opae::fpga::types::invalid_param)
                                              •  
                                            • io_address (opae::fpga::types::shared_buffer)
                                              •  
                                            • io_address_ (opae::fpga::types::shared_buffer)
                                              •  
                                            • iova_alloc (opae_vfio)
                                              •  
                                            • irqs (opae_vfio_device)
                                              •  
                                            • index (opae_vfio_device_irq, opae_vfio_sparse_info)
                                              •  
                                            • is_valid (threshold)
                                              •  
                                            "},{"location":"opae-code/class_members/#k","title":"k","text":"
                                               
                                            • key_cleanup (_opae_hash_map)
                                              •  
                                            • key_compare (_opae_hash_map)
                                              •  
                                            • key_hash (_opae_hash_map)
                                              •  
                                            • key (_opae_hash_map_item)
                                              •  
                                            "},{"location":"opae-code/class_members/#l","title":"l","text":"
                                               
                                            • lower_c_threshold (metric_threshold)
                                              •  
                                            • lower_nc_threshold (metric_threshold)
                                              •  
                                            • lower_nr_threshold (metric_threshold)
                                              •  
                                            • loc_ (opae::fpga::types::except)
                                              •  
                                            • local_memory_size (opae::fpga::types::properties)
                                              •  
                                            • len_ (opae::fpga::types::shared_buffer)
                                              •  
                                            • line (opae::fpga::types::src_location)
                                              •  
                                            • line_ (opae::fpga::types::src_location)
                                              •  
                                            • lock (opae_vfio)
                                              •  
                                            "},{"location":"opae-code/class_members/#m","title":"m","text":"
                                               
                                            • magic (_fpga_token_header)
                                              •  
                                            • metric_num (fpga_metric, fpga_metric_info)
                                              •  
                                            • metric_datatype (fpga_metric_info)
                                              •  
                                            • metric_guid (fpga_metric_info)
                                              •  
                                            • metric_name (fpga_metric_info, metric_threshold)
                                              •  
                                            • metric_type (fpga_metric_info)
                                              •  
                                            • metric_units (fpga_metric_info)
                                              •  
                                            • major (fpga_version)
                                              •  
                                            • minor (fpga_version)
                                              •  
                                            • MAX_EXCEPT (opae::fpga::types::except)
                                              •  
                                            • msg_ (opae::fpga::types::except)
                                              •  
                                            • mmio_ptr (opae::fpga::types::handle)
                                              •  
                                            • model (opae::fpga::types::properties)
                                              •  
                                            • masks (opae_vfio_device_irq)
                                              •  
                                            "},{"location":"opae-code/class_members/#n","title":"n","text":"
                                               
                                            • num_buckets (_opae_hash_map)
                                              •  
                                            • next (_opae_hash_map_item, mem_link, opae_uio_device_region, opae_vfio_device_irq, opae_vfio_device_region, opae_vfio_iova_range, opae_vfio_sparse_info)
                                              •  
                                            • name (fpga_error_info, opae::fpga::types::error)
                                              •  
                                            • no_access (opae::fpga::types::no_access)
                                              •  
                                            • no_daemon (opae::fpga::types::no_daemon)
                                              •  
                                            • no_driver (opae::fpga::types::no_driver)
                                              •  
                                            • no_memory (opae::fpga::types::no_memory)
                                              •  
                                            • not_found (opae::fpga::types::not_found)
                                              •  
                                            • not_supported (opae::fpga::types::not_supported)
                                              •  
                                            • none (opae::fpga::types::properties)
                                              •  
                                            • num_errors (opae::fpga::types::properties)
                                              •  
                                            • num_interrupts (opae::fpga::types::properties)
                                              •  
                                            • num_mmio (opae::fpga::types::properties)
                                              •  
                                            • num_slots (opae::fpga::types::properties)
                                              •  
                                            • nonfatal_error (ras_inject_error)
                                              •  
                                            "},{"location":"opae-code/class_members/#o","title":"o","text":"
                                               
                                            • object_id (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • objtype (_fpga_token_header)
                                              •  
                                            • open_flags (config)
                                              •  
                                            • operator= (opae::fpga::types::error, opae::fpga::types::guid_t, opae::fpga::types::handle, opae::fpga::types::properties, opae::fpga::types::pvalue, opae::fpga::types::shared_buffer, opae::fpga::types::src_location, opae::fpga::types::sysobject)
                                              •  
                                            • operator fpga_event_type (opae::fpga::types::event::type_t)
                                              •  
                                            • operator fpga_event_handle (opae::fpga::types::event)
                                              •  
                                            • os_object (opae::fpga::types::event)
                                              •  
                                            • os_object_ (opae::fpga::types::event)
                                              •  
                                            • operator fpga_result (opae::fpga::types::except)
                                              •  
                                            • operator uint8_t * (opae::fpga::types::guid_t)
                                              •  
                                            • operator== (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                              •  
                                            • open (opae::fpga::types::handle)
                                              •  
                                            • operator fpga_handle (opae::fpga::types::handle)
                                              •  
                                            • operator fpga_properties (opae::fpga::types::properties)
                                              •  
                                            • operator copy_t (opae::fpga::types::pvalue)
                                              •  
                                            • owner (opae::fpga::types::shared_buffer)
                                              •  
                                            • operator fpga_object (opae::fpga::types::sysobject)
                                              •  
                                            • operator fpga_token (opae::fpga::types::token)
                                              •  
                                            • offset (opae_vfio_sparse_info)
                                              •  
                                            "},{"location":"opae-code/class_members/#p","title":"p","text":"
                                               
                                            • patch (fpga_version)
                                              •  
                                            • prev (mem_link)
                                              •  
                                            • ptr_t (opae::fpga::types::error, opae::fpga::types::event, opae::fpga::types::handle, opae::fpga::types::properties, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject, opae::fpga::types::token)
                                              •  
                                            • power_thermal (opae::fpga::types::event::type_t)
                                              •  
                                            • parse (opae::fpga::types::guid_t)
                                              •  
                                            • props_ (opae::fpga::types::guid_t, opae::fpga::types::properties, opae::fpga::types::pvalue)
                                              •  
                                            • parent (opae::fpga::types::properties)
                                              •  
                                            • properties (opae::fpga::types::properties)
                                              •  
                                            • pvalue (opae::fpga::types::pvalue)
                                              •  
                                            • ptr (opae_vfio_sparse_info)
                                              •  
                                            "},{"location":"opae-code/class_members/#q","title":"q","text":"
                                               
                                            • qualifier_name (fpga_metric_info)
                                              •  
                                            "},{"location":"opae-code/class_members/#r","title":"r","text":"
                                               
                                            • run_n3000 (config)
                                              •  
                                            • read_value (opae::fpga::types::error)
                                              •  
                                            • register_event (opae::fpga::types::event)
                                              •  
                                            • res_ (opae::fpga::types::except)
                                              •  
                                            • read_csr32 (opae::fpga::types::handle)
                                              •  
                                            • read_csr64 (opae::fpga::types::handle)
                                              •  
                                            • reconfigure (opae::fpga::types::handle)
                                              •  
                                            • reset (opae::fpga::types::handle)
                                              •  
                                            • reconf_error (opae::fpga::types::reconf_error)
                                              •  
                                            • read (opae::fpga::types::shared_buffer)
                                              •  
                                            • release (opae::fpga::types::shared_buffer)
                                              •  
                                            • read64 (opae::fpga::types::sysobject)
                                              •  
                                            • regions (opae_uio, opae_vfio_device)
                                              •  
                                            • region_index (opae_uio_device_region, opae_vfio_device_region)
                                              •  
                                            • region_page_offset (opae_uio_device_region)
                                              •  
                                            • region_ptr (opae_uio_device_region, opae_vfio_device_region)
                                              •  
                                            • region_size (opae_uio_device_region, opae_vfio_device_region)
                                              •  
                                            • region_sparse (opae_vfio_device_region)
                                              •  
                                            • rsvd (ras_inject_error)
                                              •  
                                            "},{"location":"opae-code/class_members/#s","title":"s","text":"
                                               
                                            • segment (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • subsystem_device_id (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • subsystem_vendor_id (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • size (mem_link, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject, opae_vfio_sparse_info)
                                              •  
                                            • socket_id (opae::fpga::types::properties)
                                              •  
                                            • set_ (opae::fpga::types::pvalue)
                                              •  
                                            • setter_t (opae::fpga::types::pvalue)
                                              •  
                                            • shared_buffer (opae::fpga::types::shared_buffer)
                                              •  
                                            • size_t (opae::fpga::types::shared_buffer)
                                              •  
                                            • src_location (opae::fpga::types::src_location)
                                              •  
                                            • sysobject (opae::fpga::types::sysobject)
                                              •  
                                            • sysobject_ (opae::fpga::types::sysobject)
                                              •  
                                            • start (opae_vfio_iova_range)
                                              •  
                                            "},{"location":"opae-code/class_members/#t","title":"t","text":"
                                               
                                            • token_ (opae::fpga::types::error, opae::fpga::types::handle, opae::fpga::types::sysobject, opae::fpga::types::token)
                                              •  
                                            • type_ (opae::fpga::types::event::type_t, opae::fpga::types::event)
                                              •  
                                            • type_t (opae::fpga::types::event::type_t)
                                              •  
                                            • type (opae::fpga::types::properties, opae::fpga::types::sysobject)
                                              •  
                                            • token (opae::fpga::types::token)
                                              •  
                                            • threshold_name (threshold)
                                              •  
                                            "},{"location":"opae-code/class_members/#u","title":"u","text":"
                                               
                                            • uint (cache_line)
                                              •  
                                            • upper_c_threshold (metric_threshold)
                                              •  
                                            • upper_nc_threshold (metric_threshold)
                                              •  
                                            • upper_nr_threshold (metric_threshold)
                                              •  
                                            • update (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                              •  
                                            "},{"location":"opae-code/class_members/#v","title":"v","text":"
                                               
                                            • vendor_id (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • value_cleanup (_opae_hash_map)
                                              •  
                                            • value (_opae_hash_map_item, fpga_metric, threshold)
                                              •  
                                            • virt_ (opae::fpga::types::shared_buffer)
                                              •  
                                            "},{"location":"opae-code/class_members/#w","title":"w","text":"
                                               
                                            • what (opae::fpga::types::except)
                                              •  
                                            • write_csr32 (opae::fpga::types::handle)
                                              •  
                                            • write_csr512 (opae::fpga::types::handle)
                                              •  
                                            • write_csr64 (opae::fpga::types::handle)
                                              •  
                                            • write (opae::fpga::types::shared_buffer)
                                              •  
                                            • wsid (opae::fpga::types::shared_buffer)
                                              •  
                                            • wsid_ (opae::fpga::types::shared_buffer)
                                              •  
                                            • write64 (opae::fpga::types::sysobject)
                                              •  
                                            "},{"location":"opae-code/class_members/#_1","title":"~","text":"
                                               
                                            • ~error (opae::fpga::types::error)
                                              •  
                                            • ~event (opae::fpga::types::event)
                                              •  
                                            • ~handle (opae::fpga::types::handle)
                                              •  
                                            • ~properties (opae::fpga::types::properties)
                                              •  
                                            • ~shared_buffer (opae::fpga::types::shared_buffer)
                                              •  
                                            • ~sysobject (opae::fpga::types::sysobject)
                                              •  
                                            • ~token (opae::fpga::types::token)
                                              •  
                                            "},{"location":"opae-code/class_members/#_2","title":"@","text":"
                                               
                                            • @1 (ras_inject_error)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/","title":"Class Member Functions","text":""},{"location":"opae-code/class_member_functions/#a","title":"a","text":"
                                               
                                            • allocate (opae::fpga::types::shared_buffer)
                                              •  
                                            • attach (opae::fpga::types::shared_buffer)
                                              •  
                                            • as_string (opae::fpga::types::version)
                                              •  
                                            • as_struct (opae::fpga::types::version)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#b","title":"b","text":"
                                               
                                            • busy (opae::fpga::types::busy)
                                              •  
                                            • bytes (opae::fpga::types::sysobject)
                                              •  
                                            • build (opae::fpga::types::version)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#c","title":"c","text":"
                                               
                                            • c_type (opae::fpga::types::error, opae::fpga::types::guid_t, opae::fpga::types::handle, opae::fpga::types::properties, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject, opae::fpga::types::token)
                                              •  
                                            • can_clear (opae::fpga::types::error)
                                              •  
                                            • close (opae::fpga::types::handle)
                                              •  
                                            • compare (opae::fpga::types::shared_buffer)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#e","title":"e","text":"
                                               
                                            • error (opae::fpga::types::error)
                                              •  
                                            • event (opae::fpga::types::event)
                                              •  
                                            • except (opae::fpga::types::except)
                                              •  
                                            • exception (opae::fpga::types::exception)
                                              •  
                                            • enumerate (opae::fpga::types::token)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#f","title":"f","text":"
                                               
                                            • fill (opae::fpga::types::shared_buffer)
                                              •  
                                            • file (opae::fpga::types::src_location)
                                              •  
                                            • fn (opae::fpga::types::src_location)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#g","title":"g","text":"
                                               
                                            • get (opae::fpga::types::error, opae::fpga::types::event, opae::fpga::types::properties, opae::fpga::types::sysobject)
                                              •  
                                            • guid_t (opae::fpga::types::guid_t)
                                              •  
                                            • get_token (opae::fpga::types::handle)
                                              •  
                                            • get_value (opae::fpga::types::pvalue)
                                              •  
                                            • get_parent (opae::fpga::types::token)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#h","title":"h","text":"
                                               
                                            • handle (opae::fpga::types::handle)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#i","title":"i","text":"
                                               
                                            • invalidate (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                              •  
                                            • is_set (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                              •  
                                            • invalid_param (opae::fpga::types::invalid_param)
                                              •  
                                            • io_address (opae::fpga::types::shared_buffer)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#l","title":"l","text":"
                                               
                                            • line (opae::fpga::types::src_location)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#m","title":"m","text":"
                                               
                                            • mmio_ptr (opae::fpga::types::handle)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#n","title":"n","text":"
                                               
                                            • name (opae::fpga::types::error)
                                              •  
                                            • no_access (opae::fpga::types::no_access)
                                              •  
                                            • no_daemon (opae::fpga::types::no_daemon)
                                              •  
                                            • no_driver (opae::fpga::types::no_driver)
                                              •  
                                            • no_memory (opae::fpga::types::no_memory)
                                              •  
                                            • not_found (opae::fpga::types::not_found)
                                              •  
                                            • not_supported (opae::fpga::types::not_supported)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#o","title":"o","text":"
                                               
                                            • operator= (opae::fpga::types::error, opae::fpga::types::guid_t, opae::fpga::types::handle, opae::fpga::types::properties, opae::fpga::types::pvalue, opae::fpga::types::shared_buffer, opae::fpga::types::src_location, opae::fpga::types::sysobject)
                                              •  
                                            • operator fpga_event_type (opae::fpga::types::event::type_t)
                                              •  
                                            • operator fpga_event_handle (opae::fpga::types::event)
                                              •  
                                            • os_object (opae::fpga::types::event)
                                              •  
                                            • operator fpga_result (opae::fpga::types::except)
                                              •  
                                            • operator uint8_t * (opae::fpga::types::guid_t)
                                              •  
                                            • operator== (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                              •  
                                            • open (opae::fpga::types::handle)
                                              •  
                                            • operator fpga_handle (opae::fpga::types::handle)
                                              •  
                                            • operator fpga_properties (opae::fpga::types::properties)
                                              •  
                                            • operator copy_t (opae::fpga::types::pvalue)
                                              •  
                                            • owner (opae::fpga::types::shared_buffer)
                                              •  
                                            • operator fpga_object (opae::fpga::types::sysobject)
                                              •  
                                            • operator fpga_token (opae::fpga::types::token)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#p","title":"p","text":"
                                               
                                            • parse (opae::fpga::types::guid_t)
                                              •  
                                            • properties (opae::fpga::types::properties)
                                              •  
                                            • pvalue (opae::fpga::types::pvalue)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#r","title":"r","text":"
                                               
                                            • read_value (opae::fpga::types::error)
                                              •  
                                            • register_event (opae::fpga::types::event)
                                              •  
                                            • read_csr32 (opae::fpga::types::handle)
                                              •  
                                            • read_csr64 (opae::fpga::types::handle)
                                              •  
                                            • reconfigure (opae::fpga::types::handle)
                                              •  
                                            • reset (opae::fpga::types::handle)
                                              •  
                                            • reconf_error (opae::fpga::types::reconf_error)
                                              •  
                                            • read (opae::fpga::types::shared_buffer)
                                              •  
                                            • release (opae::fpga::types::shared_buffer)
                                              •  
                                            • read64 (opae::fpga::types::sysobject)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#s","title":"s","text":"
                                               
                                            • shared_buffer (opae::fpga::types::shared_buffer)
                                              •  
                                            • size (opae::fpga::types::shared_buffer, opae::fpga::types::sysobject)
                                              •  
                                            • src_location (opae::fpga::types::src_location)
                                              •  
                                            • sysobject (opae::fpga::types::sysobject)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#t","title":"t","text":"
                                               
                                            • type_t (opae::fpga::types::event::type_t)
                                              •  
                                            • type (opae::fpga::types::sysobject)
                                              •  
                                            • token (opae::fpga::types::token)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#u","title":"u","text":"
                                               
                                            • update (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#w","title":"w","text":"
                                               
                                            • what (opae::fpga::types::except)
                                              •  
                                            • write_csr32 (opae::fpga::types::handle)
                                              •  
                                            • write_csr512 (opae::fpga::types::handle)
                                              •  
                                            • write_csr64 (opae::fpga::types::handle)
                                              •  
                                            • write (opae::fpga::types::shared_buffer)
                                              •  
                                            • wsid (opae::fpga::types::shared_buffer)
                                              •  
                                            • write64 (opae::fpga::types::sysobject)
                                              •  
                                            "},{"location":"opae-code/class_member_functions/#_1","title":"~","text":"
                                               
                                            • ~error (opae::fpga::types::error)
                                              •  
                                            • ~event (opae::fpga::types::event)
                                              •  
                                            • ~handle (opae::fpga::types::handle)
                                              •  
                                            • ~properties (opae::fpga::types::properties)
                                              •  
                                            • ~shared_buffer (opae::fpga::types::shared_buffer)
                                              •  
                                            • ~sysobject (opae::fpga::types::sysobject)
                                              •  
                                            • ~token (opae::fpga::types::token)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/","title":"Class Member Variables","text":""},{"location":"opae-code/class_member_variables/#a","title":"a","text":"
                                               
                                            • allocated (mem_alloc)
                                              •  
                                            • address (mem_link)
                                              •  
                                            • accelerator_state (opae::fpga::types::properties)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#b","title":"b","text":"
                                               
                                            • bus (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • buckets (_opae_hash_map)
                                              •  
                                            • buf_ (opae::fpga::types::except)
                                              •  
                                            • bbs_id (opae::fpga::types::properties)
                                              •  
                                            • bbs_version (opae::fpga::types::properties)
                                              •  
                                            • buffer_iova (opae_vfio_buffer)
                                              •  
                                            • buffer_ptr (opae_vfio_buffer)
                                              •  
                                            • buffer_size (opae_vfio_buffer)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#c","title":"c","text":"
                                               
                                            • cleanup_context (_opae_hash_map)
                                              •  
                                            • can_clear (fpga_error_info)
                                              •  
                                            • capabilities (opae::fpga::types::properties)
                                              •  
                                            • copy_ (opae::fpga::types::pvalue)
                                              •  
                                            • cont_buffers (opae_vfio)
                                              •  
                                            • cont_device (opae_vfio)
                                              •  
                                            • cont_fd (opae_vfio)
                                              •  
                                            • cont_pciaddr (opae_vfio)
                                              •  
                                            • cont_ranges (opae_vfio)
                                              •  
                                            • count (opae_vfio_device_irq)
                                              •  
                                            • catastrophicr_error (ras_inject_error)
                                              •  
                                            • csr (ras_inject_error)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#d","title":"d","text":"
                                               
                                            • device (_fpga_token_header, opae::fpga::types::properties, opae_vfio)
                                              •  
                                            • device_id (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • data_ (opae::fpga::types::guid_t)
                                              •  
                                            • device_fd (opae_uio, opae_vfio_device)
                                              •  
                                            • device_path (opae_uio)
                                              •  
                                            • device_config_offset (opae_vfio_device)
                                              •  
                                            • device_num_irqs (opae_vfio_device)
                                              •  
                                            • device_num_regions (opae_vfio_device)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#e","title":"e","text":"
                                               
                                            • error_info_ (opae::fpga::types::error)
                                              •  
                                            • error_num_ (opae::fpga::types::error)
                                              •  
                                            • event_handle_ (opae::fpga::types::event)
                                              •  
                                            • error (opae::fpga::types::event::type_t)
                                              •  
                                            • event_fds (opae_vfio_device_irq)
                                              •  
                                            • end (opae_vfio_iova_range)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#f","title":"f","text":"
                                               
                                            • function (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • flags (_opae_hash_map, opae_vfio_buffer, opae_vfio_device_irq)
                                              •  
                                            • free (mem_alloc)
                                              •  
                                            • file_ (opae::fpga::types::src_location)
                                              •  
                                            • fn_ (opae::fpga::types::src_location)
                                              •  
                                            • fatal_error (ras_inject_error)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#g","title":"g","text":"
                                               
                                            • guid (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • group_name (fpga_metric_info)
                                              •  
                                            • get_ (opae::fpga::types::pvalue)
                                              •  
                                            • group (opae_vfio)
                                              •  
                                            • group_device (opae_vfio_group)
                                              •  
                                            • group_fd (opae_vfio_group)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#h","title":"h","text":"
                                               
                                            • hash_seed (_opae_hash_map)
                                              •  
                                            • hysteresis (metric_threshold)
                                              •  
                                            • handle_ (opae::fpga::types::event, opae::fpga::types::handle, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#i","title":"i","text":"
                                               
                                            • interface (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • isvalid (fpga_metric)
                                              •  
                                            • interrupt (opae::fpga::types::event::type_t)
                                              •  
                                            • is_set_ (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                              •  
                                            • io_address_ (opae::fpga::types::shared_buffer)
                                              •  
                                            • iova_alloc (opae_vfio)
                                              •  
                                            • irqs (opae_vfio_device)
                                              •  
                                            • index (opae_vfio_device_irq, opae_vfio_sparse_info)
                                              •  
                                            • is_valid (threshold)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#k","title":"k","text":"
                                               
                                            • key_cleanup (_opae_hash_map)
                                              •  
                                            • key_compare (_opae_hash_map)
                                              •  
                                            • key_hash (_opae_hash_map)
                                              •  
                                            • key (_opae_hash_map_item)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#l","title":"l","text":"
                                               
                                            • lower_c_threshold (metric_threshold)
                                              •  
                                            • lower_nc_threshold (metric_threshold)
                                              •  
                                            • lower_nr_threshold (metric_threshold)
                                              •  
                                            • loc_ (opae::fpga::types::except)
                                              •  
                                            • local_memory_size (opae::fpga::types::properties)
                                              •  
                                            • len_ (opae::fpga::types::shared_buffer)
                                              •  
                                            • line_ (opae::fpga::types::src_location)
                                              •  
                                            • lock (opae_vfio)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#m","title":"m","text":"
                                               
                                            • magic (_fpga_token_header)
                                              •  
                                            • metric_num (fpga_metric, fpga_metric_info)
                                              •  
                                            • metric_datatype (fpga_metric_info)
                                              •  
                                            • metric_guid (fpga_metric_info)
                                              •  
                                            • metric_name (fpga_metric_info, metric_threshold)
                                              •  
                                            • metric_type (fpga_metric_info)
                                              •  
                                            • metric_units (fpga_metric_info)
                                              •  
                                            • major (fpga_version)
                                              •  
                                            • minor (fpga_version)
                                              •  
                                            • MAX_EXCEPT (opae::fpga::types::except)
                                              •  
                                            • msg_ (opae::fpga::types::except)
                                              •  
                                            • model (opae::fpga::types::properties)
                                              •  
                                            • masks (opae_vfio_device_irq)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#n","title":"n","text":"
                                               
                                            • num_buckets (_opae_hash_map)
                                              •  
                                            • next (_opae_hash_map_item, mem_link, opae_uio_device_region, opae_vfio_device_irq, opae_vfio_device_region, opae_vfio_iova_range, opae_vfio_sparse_info)
                                              •  
                                            • name (fpga_error_info)
                                              •  
                                            • none (opae::fpga::types::properties)
                                              •  
                                            • num_errors (opae::fpga::types::properties)
                                              •  
                                            • num_interrupts (opae::fpga::types::properties)
                                              •  
                                            • num_mmio (opae::fpga::types::properties)
                                              •  
                                            • num_slots (opae::fpga::types::properties)
                                              •  
                                            • nonfatal_error (ras_inject_error)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#o","title":"o","text":"
                                               
                                            • object_id (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • objtype (_fpga_token_header)
                                              •  
                                            • open_flags (config)
                                              •  
                                            • os_object_ (opae::fpga::types::event)
                                              •  
                                            • offset (opae_vfio_sparse_info)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#p","title":"p","text":"
                                               
                                            • patch (fpga_version)
                                              •  
                                            • prev (mem_link)
                                              •  
                                            • power_thermal (opae::fpga::types::event::type_t)
                                              •  
                                            • props_ (opae::fpga::types::guid_t, opae::fpga::types::properties, opae::fpga::types::pvalue)
                                              •  
                                            • parent (opae::fpga::types::properties)
                                              •  
                                            • ptr (opae_vfio_sparse_info)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#q","title":"q","text":"
                                               
                                            • qualifier_name (fpga_metric_info)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#r","title":"r","text":"
                                               
                                            • run_n3000 (config)
                                              •  
                                            • res_ (opae::fpga::types::except)
                                              •  
                                            • regions (opae_uio, opae_vfio_device)
                                              •  
                                            • region_index (opae_uio_device_region, opae_vfio_device_region)
                                              •  
                                            • region_page_offset (opae_uio_device_region)
                                              •  
                                            • region_ptr (opae_uio_device_region, opae_vfio_device_region)
                                              •  
                                            • region_size (opae_uio_device_region, opae_vfio_device_region)
                                              •  
                                            • region_sparse (opae_vfio_device_region)
                                              •  
                                            • rsvd (ras_inject_error)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#s","title":"s","text":"
                                               
                                            • segment (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • subsystem_device_id (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • subsystem_vendor_id (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • size (mem_link, opae_vfio_sparse_info)
                                              •  
                                            • socket_id (opae::fpga::types::properties)
                                              •  
                                            • set_ (opae::fpga::types::pvalue)
                                              •  
                                            • sysobject_ (opae::fpga::types::sysobject)
                                              •  
                                            • start (opae_vfio_iova_range)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#t","title":"t","text":"
                                               
                                            • token_ (opae::fpga::types::error, opae::fpga::types::handle, opae::fpga::types::sysobject, opae::fpga::types::token)
                                              •  
                                            • type_ (opae::fpga::types::event::type_t, opae::fpga::types::event)
                                              •  
                                            • type (opae::fpga::types::properties)
                                              •  
                                            • threshold_name (threshold)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#u","title":"u","text":"
                                               
                                            • uint (cache_line)
                                              •  
                                            • upper_c_threshold (metric_threshold)
                                              •  
                                            • upper_nc_threshold (metric_threshold)
                                              •  
                                            • upper_nr_threshold (metric_threshold)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#v","title":"v","text":"
                                               
                                            • vendor_id (_fpga_token_header, opae::fpga::types::properties)
                                              •  
                                            • value_cleanup (_opae_hash_map)
                                              •  
                                            • value (_opae_hash_map_item, fpga_metric, threshold)
                                              •  
                                            • virt_ (opae::fpga::types::shared_buffer)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#w","title":"w","text":"
                                               
                                            • wsid_ (opae::fpga::types::shared_buffer)
                                              •  
                                            "},{"location":"opae-code/class_member_variables/#_1","title":"@","text":"
                                               
                                            • @1 (ras_inject_error)
                                              •  
                                            "},{"location":"opae-code/class_member_typedefs/","title":"Class Member Typedefs","text":""},{"location":"opae-code/class_member_typedefs/#c","title":"c","text":"
                                               
                                            • copy_t (opae::fpga::types::pvalue)
                                              •  
                                            "},{"location":"opae-code/class_member_typedefs/#g","title":"g","text":"
                                               
                                            • getter_t (opae::fpga::types::pvalue)
                                              •  
                                            "},{"location":"opae-code/class_member_typedefs/#p","title":"p","text":"
                                               
                                            • ptr_t (opae::fpga::types::error, opae::fpga::types::event, opae::fpga::types::handle, opae::fpga::types::properties, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject, opae::fpga::types::token)
                                              •  
                                            "},{"location":"opae-code/class_member_typedefs/#s","title":"s","text":"
                                               
                                            • setter_t (opae::fpga::types::pvalue)
                                              •  
                                            • size_t (opae::fpga::types::shared_buffer)
                                              •  
                                            "},{"location":"opae-code/class_member_enums/","title":"Class Member Enums","text":""},{"location":"opae-code/namespace_members/","title":"Namespace Members","text":""},{"location":"opae-code/namespace_members/#a","title":"a","text":"
                                               
                                            • assert_fpga_ok (opae::fpga::types::detail)
                                              •  
                                            "},{"location":"opae-code/namespace_members/#e","title":"e","text":"
                                               
                                            • exception_fn (opae::fpga::types::detail)
                                              •  
                                            "},{"location":"opae-code/namespace_members/#i","title":"i","text":"
                                               
                                            • is_ok (opae::fpga::types::detail)
                                              •  
                                            "},{"location":"opae-code/namespace_members/#o","title":"o","text":"
                                               
                                            • opae_exceptions (opae::fpga::types::detail)
                                              •  
                                            "},{"location":"opae-code/namespace_member_functions/","title":"Namespace Member Functions","text":""},{"location":"opae-code/namespace_member_functions/#a","title":"a","text":"
                                               
                                            • assert_fpga_ok (opae::fpga::types::detail)
                                              •  
                                            "},{"location":"opae-code/namespace_member_functions/#i","title":"i","text":"
                                               
                                            • is_ok (opae::fpga::types::detail)
                                              •  
                                            "},{"location":"opae-code/namespace_member_variables/","title":"Namespace Member Variables","text":""},{"location":"opae-code/namespace_member_variables/#o","title":"o","text":"
                                               
                                            • opae_exceptions (opae::fpga::types::detail)
                                              •  
                                            "},{"location":"opae-code/namespace_member_typedefs/","title":"Namespace Member Typedefs","text":""},{"location":"opae-code/namespace_member_typedefs/#e","title":"e","text":"
                                               
                                            • exception_fn (opae::fpga::types::detail)
                                              •  
                                            "},{"location":"opae-code/namespace_member_enums/","title":"Namespace Member Enums","text":""},{"location":"opae-code/functions/","title":"Functions","text":""},{"location":"opae-code/functions/#e","title":"e","text":"
                                               
                                            • error_thread (hello_events.c)
                                              •  
                                            "},{"location":"opae-code/functions/#f","title":"f","text":"
                                               
                                            • fpgaClose (access.h)
                                              •  
                                            • fpgaOpen (access.h)
                                              •  
                                            • fpgaReset (access.h)
                                              •  
                                            • fpgaGetIOAddress (buffer.h)
                                              •  
                                            • fpgaPrepareBuffer (buffer.h)
                                              •  
                                            • fpgaReleaseBuffer (buffer.h)
                                              •  
                                            • fpgaCloneToken (enum.h)
                                              •  
                                            • fpgaDestroyToken (enum.h)
                                              •  
                                            • fpgaEnumerate (enum.h)
                                              •  
                                            • fpgaClearAllErrors (error.h)
                                              •  
                                            • fpgaClearError (error.h)
                                              •  
                                            • fpgaGetErrorInfo (error.h)
                                              •  
                                            • fpgaReadError (error.h)
                                              •  
                                            • fpgaCreateEventHandle (event.h)
                                              •  
                                            • fpgaDestroyEventHandle (event.h)
                                              •  
                                            • fpgaGetOSObjectFromEventHandle (event.h)
                                              •  
                                            • fpgaRegisterEvent (event.h)
                                              •  
                                            • fpgaUnregisterEvent (event.h)
                                              •  
                                            • fpgaFinalize (init.h)
                                              •  
                                            • fpgaInitialize (init.h)
                                              •  
                                            • fpgaAssignPortToInterface (manage.h)
                                              •  
                                            • fpgaAssignToInterface (manage.h)
                                              •  
                                            • fpgaReconfigureSlot (manage.h)
                                              •  
                                            • fpgaReleaseFromInterface (manage.h)
                                              •  
                                            • fpgaGetMetricsByIndex (metrics.h)
                                              •  
                                            • fpgaGetMetricsByName (metrics.h)
                                              •  
                                            • fpgaGetMetricsInfo (metrics.h)
                                              •  
                                            • fpgaGetMetricsThresholdInfo (metrics.h)
                                              •  
                                            • fpgaGetNumMetrics (metrics.h)
                                              •  
                                            • fpgaMapMMIO (mmio.h)
                                              •  
                                            • fpgaReadMMIO32 (mmio.h)
                                              •  
                                            • fpgaReadMMIO64 (mmio.h)
                                              •  
                                            • fpgaUnmapMMIO (mmio.h)
                                              •  
                                            • fpgaWriteMMIO32 (mmio.h)
                                              •  
                                            • fpgaWriteMMIO512 (mmio.h)
                                              •  
                                            • fpgaWriteMMIO64 (mmio.h)
                                              •  
                                            • fpgaClearProperties (properties.h)
                                              •  
                                            • fpgaCloneProperties (properties.h)
                                              •  
                                            • fpgaDestroyProperties (properties.h)
                                              •  
                                            • fpgaGetProperties (properties.h)
                                              •  
                                            • fpgaGetPropertiesFromHandle (properties.h)
                                              •  
                                            • fpgaPropertiesGetAcceleratorState (properties.h)
                                              •  
                                            • fpgaPropertiesGetBBSID (properties.h)
                                              •  
                                            • fpgaPropertiesGetBBSVersion (properties.h)
                                              •  
                                            • fpgaPropertiesGetBus (properties.h)
                                              •  
                                            • fpgaPropertiesGetCapabilities (properties.h)
                                              •  
                                            • fpgaPropertiesGetDevice (properties.h)
                                              •  
                                            • fpgaPropertiesGetDeviceID (properties.h)
                                              •  
                                            • fpgaPropertiesGetFunction (properties.h)
                                              •  
                                            • fpgaPropertiesGetGUID (properties.h)
                                              •  
                                            • fpgaPropertiesGetInterface (properties.h)
                                              •  
                                            • fpgaPropertiesGetLocalMemorySize (properties.h)
                                              •  
                                            • fpgaPropertiesGetModel (properties.h)
                                              •  
                                            • fpgaPropertiesGetNumErrors (properties.h)
                                              •  
                                            • fpgaPropertiesGetNumInterrupts (properties.h)
                                              •  
                                            • fpgaPropertiesGetNumMMIO (properties.h)
                                              •  
                                            • fpgaPropertiesGetNumSlots (properties.h)
                                              •  
                                            • fpgaPropertiesGetObjectID (properties.h)
                                              •  
                                            • fpgaPropertiesGetObjectType (properties.h)
                                              •  
                                            • fpgaPropertiesGetParent (properties.h)
                                              •  
                                            • fpgaPropertiesGetSegment (properties.h)
                                              •  
                                            • fpgaPropertiesGetSocketID (properties.h)
                                              •  
                                            • fpgaPropertiesGetSubsystemDeviceID (properties.h)
                                              •  
                                            • fpgaPropertiesGetSubsystemVendorID (properties.h)
                                              •  
                                            • fpgaPropertiesGetVendorID (properties.h)
                                              •  
                                            • fpgaPropertiesSetAcceleratorState (properties.h)
                                              •  
                                            • fpgaPropertiesSetBBSID (properties.h)
                                              •  
                                            • fpgaPropertiesSetBBSVersion (properties.h)
                                              •  
                                            • fpgaPropertiesSetBus (properties.h)
                                              •  
                                            • fpgaPropertiesSetCapabilities (properties.h)
                                              •  
                                            • fpgaPropertiesSetDevice (properties.h)
                                              •  
                                            • fpgaPropertiesSetDeviceID (properties.h)
                                              •  
                                            • fpgaPropertiesSetFunction (properties.h)
                                              •  
                                            • fpgaPropertiesSetGUID (properties.h)
                                              •  
                                            • fpgaPropertiesSetInterface (properties.h)
                                              •  
                                            • fpgaPropertiesSetLocalMemorySize (properties.h)
                                              •  
                                            • fpgaPropertiesSetModel (properties.h)
                                              •  
                                            • fpgaPropertiesSetNumErrors (properties.h)
                                              •  
                                            • fpgaPropertiesSetNumInterrupts (properties.h)
                                              •  
                                            • fpgaPropertiesSetNumMMIO (properties.h)
                                              •  
                                            • fpgaPropertiesSetNumSlots (properties.h)
                                              •  
                                            • fpgaPropertiesSetObjectID (properties.h)
                                              •  
                                            • fpgaPropertiesSetObjectType (properties.h)
                                              •  
                                            • fpgaPropertiesSetParent (properties.h)
                                              •  
                                            • fpgaPropertiesSetSegment (properties.h)
                                              •  
                                            • fpgaPropertiesSetSocketID (properties.h)
                                              •  
                                            • fpgaPropertiesSetSubsystemDeviceID (properties.h)
                                              •  
                                            • fpgaPropertiesSetSubsystemVendorID (properties.h)
                                              •  
                                            • fpgaPropertiesSetVendorID (properties.h)
                                              •  
                                            • fpgaUpdateProperties (properties.h)
                                              •  
                                            • fpgaDestroyObject (sysobject.h)
                                              •  
                                            • fpgaHandleGetObject (sysobject.h)
                                              •  
                                            • fpgaObjectGetObject (sysobject.h)
                                              •  
                                            • fpgaObjectGetObjectAt (sysobject.h)
                                              •  
                                            • fpgaObjectGetSize (sysobject.h)
                                              •  
                                            • fpgaObjectGetType (sysobject.h)
                                              •  
                                            • fpgaObjectRead (sysobject.h)
                                              •  
                                            • fpgaObjectRead64 (sysobject.h)
                                              •  
                                            • fpgaObjectWrite64 (sysobject.h)
                                              •  
                                            • fpgaTokenGetObject (sysobject.h)
                                              •  
                                            • fpgaGetNumUmsg (umsg.h)
                                              •  
                                            • fpgaGetUmsgPtr (umsg.h)
                                              •  
                                            • fpgaSetUmsgAttributes (umsg.h)
                                              •  
                                            • fpgaTriggerUmsg (umsg.h)
                                              •  
                                            • fpgaGetUserClock (userclk.h)
                                              •  
                                            • fpgaSetUserClock (userclk.h)
                                              •  
                                            • fpgaErrStr (utils.h)
                                              •  
                                            • fpgaGetOPAECBuildString (version.h)
                                              •  
                                            • fpgaGetOPAECVersion (version.h)
                                              •  
                                            • fpgaGetOPAECVersionString (version.h)
                                              •  
                                            • find_fpga (hello_events.c, hello_fpga.c)
                                              •  
                                            • find_nlb_n3000 (hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/functions/#h","title":"h","text":"
                                               
                                            • help (hello_events.c, hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/functions/#i","title":"i","text":"
                                               
                                            • inject_ras_fatal_error (hello_events.c)
                                              •  
                                            "},{"location":"opae-code/functions/#m","title":"m","text":"
                                               
                                            • mem_alloc_add_free (mem_alloc.h)
                                              •  
                                            • mem_alloc_destroy (mem_alloc.h)
                                              •  
                                            • mem_alloc_get (mem_alloc.h)
                                              •  
                                            • mem_alloc_init (mem_alloc.h)
                                              •  
                                            • mem_alloc_put (mem_alloc.h)
                                              •  
                                            • main (hello_events.c, hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/functions/#o","title":"o","text":"
                                               
                                            • opae_hash_map_add (hash_map.h)
                                              •  
                                            • opae_hash_map_destroy (hash_map.h)
                                              •  
                                            • opae_hash_map_find (hash_map.h)
                                              •  
                                            • opae_hash_map_init (hash_map.h)
                                              •  
                                            • opae_hash_map_is_empty (hash_map.h)
                                              •  
                                            • opae_hash_map_remove (hash_map.h)
                                              •  
                                            • opae_u64_key_compare (hash_map.h)
                                              •  
                                            • opae_u64_key_hash (hash_map.h)
                                              •  
                                            • opae_print (log.h)
                                              •  
                                            • opae_uio_close (uio.h)
                                              •  
                                            • opae_uio_open (uio.h)
                                              •  
                                            • opae_uio_region_get (uio.h)
                                              •  
                                            • opae_vfio_buffer_allocate (vfio.h)
                                              •  
                                            • opae_vfio_buffer_allocate_ex (vfio.h)
                                              •  
                                            • opae_vfio_buffer_free (vfio.h)
                                              •  
                                            • opae_vfio_buffer_info (vfio.h)
                                              •  
                                            • opae_vfio_close (vfio.h)
                                              •  
                                            • opae_vfio_irq_disable (vfio.h)
                                              •  
                                            • opae_vfio_irq_enable (vfio.h)
                                              •  
                                            • opae_vfio_irq_mask (vfio.h)
                                              •  
                                            • opae_vfio_irq_unmask (vfio.h)
                                              •  
                                            • opae_vfio_open (vfio.h)
                                              •  
                                            • opae_vfio_region_get (vfio.h)
                                              •  
                                            • opae_vfio_secure_open (vfio.h)
                                              •  
                                            "},{"location":"opae-code/functions/#p","title":"p","text":"
                                               
                                            • parse_args (hello_events.c, hello_fpga.c)
                                              •  
                                            • print_err (hello_events.c, hello_fpga.c)
                                              •  
                                            • probe_for_ase (hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/functions/#u","title":"u","text":"
                                               
                                            • usleep (hello_events.c, hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/macros/","title":"Macros","text":""},{"location":"opae-code/macros/#a","title":"a","text":"
                                               
                                            • ASSERT_FPGA_OK (except.h)
                                              •  
                                            "},{"location":"opae-code/macros/#c","title":"c","text":"
                                               
                                            • CACHELINE_ALIGNED_ADDR (hello_fpga.c)
                                              •  
                                            • CL (hello_fpga.c)
                                              •  
                                            • CSR_AFU_DSM_BASEL (hello_fpga.c)
                                              •  
                                            • CSR_CFG (hello_fpga.c)
                                              •  
                                            • CSR_CTL (hello_fpga.c)
                                              •  
                                            • CSR_DST_ADDR (hello_fpga.c)
                                              •  
                                            • CSR_NUM_LINES (hello_fpga.c)
                                              •  
                                            • CSR_SRC_ADDR (hello_fpga.c)
                                              •  
                                            • CSR_STATUS1 (hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/macros/#d","title":"d","text":"
                                               
                                            • DSM_STATUS_TEST_COMPLETE (hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/macros/#f","title":"f","text":"
                                               
                                            • FPGA_ERROR_NAME_MAX (types.h)
                                              •  
                                            • FPGA_METRIC_STR_SIZE (types.h)
                                              •  
                                            • fpga_is_parent_child (types.h)
                                              •  
                                            • FPGA_BUILD_STR_MAX (version.h)
                                              •  
                                            • FPGA_VERSION_STR_MAX (version.h)
                                              •  
                                            • FME_SYSFS_INJECT_ERROR (hello_events.c)
                                              •  
                                            • FPGA_NLB0_UUID_H (hello_fpga.c)
                                              •  
                                            • FPGA_NLB0_UUID_L (hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/macros/#g","title":"g","text":"
                                               
                                            • GETOPT_STRING (hello_events.c, hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/macros/#l","title":"l","text":"
                                               
                                            • LOG2_CL (hello_fpga.c)
                                              •  
                                            • LPBK1_BUFFER_ALLOCATION_SIZE (hello_fpga.c)
                                              •  
                                            • LPBK1_BUFFER_SIZE (hello_fpga.c)
                                              •  
                                            • LPBK1_DSM_SIZE (hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/macros/#m","title":"m","text":"
                                               
                                            • MB (hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/macros/#n","title":"n","text":"
                                               
                                            • N3000_AFUID (hello_fpga.c)
                                              •  
                                            • NLB0_AFUID (hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/macros/#o","title":"o","text":"
                                               
                                            • OPAECXX_HERE (except.h)
                                              •  
                                            • OPAE_DBG (log.h)
                                              •  
                                            • OPAE_DEFAULT_LOGLEVEL (log.h)
                                              •  
                                            • OPAE_ERR (log.h)
                                              •  
                                            • OPAE_MSG (log.h)
                                              •  
                                            • OPAE_UIO_PATH_MAX (uio.h)
                                              •  
                                            • ON_ERR_GOTO (hello_events.c, hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/macros/#t","title":"t","text":"
                                               
                                            • TEST_TIMEOUT (hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/macros/#_","title":"_","text":"
                                               
                                            • __SHORT_FILE__ (log.h)
                                              •  
                                            "},{"location":"opae-code/variables/","title":"Variables","text":""},{"location":"opae-code/variables/#c","title":"c","text":"
                                               
                                            • config (hello_fpga.c)
                                              •  
                                            "},{"location":"opae-code/variables/#f","title":"f","text":"
                                               
                                            • fpga_event_handle (types.h)
                                              •  
                                            • fpga_guid (types.h)
                                              •  
                                            • fpga_handle (types.h)
                                              •  
                                            • fpga_metric (types.h)
                                              •  
                                            • fpga_metric_info (types.h)
                                              •  
                                            • fpga_object (types.h)
                                              •  
                                            • fpga_properties (types.h)
                                              •  
                                            • fpga_token (types.h)
                                              •  
                                            • fpga_token_header (types.h)
                                              •  
                                            • fpga_accelerator_state (types_enum.h)
                                              •  
                                            • fpga_buffer_flags (types_enum.h)
                                              •  
                                            • fpga_event_type (types_enum.h)
                                              •  
                                            • fpga_interface (types_enum.h)
                                              •  
                                            • fpga_metric_datatype (types_enum.h)
                                              •  
                                            • fpga_metric_type (types_enum.h)
                                              •  
                                            • fpga_objtype (types_enum.h)
                                              •  
                                            • fpga_open_flags (types_enum.h)
                                              •  
                                            • fpga_reconf_flags (types_enum.h)
                                              •  
                                            • fpga_result (types_enum.h)
                                              •  
                                            • fpga_sysobject_flags (types_enum.h)
                                              •  
                                            • fpga_sysobject_type (types_enum.h)
                                              •  
                                            "},{"location":"opae-code/variables/#m","title":"m","text":"
                                               
                                            • metric_threshold (types.h)
                                              •  
                                            "},{"location":"opae-code/variables/#o","title":"o","text":"
                                               
                                            • opae_hash_map (hash_map.h)
                                              •  
                                            • opae_hash_map_flags (hash_map.h)
                                              •  
                                            • opae_hash_map_item (hash_map.h)
                                              •  
                                            • opae_loglevel (log.h)
                                              •  
                                            • opae_vfio_buffer_flags (vfio.h)
                                              •  
                                            "},{"location":"opae-code/variables/#t","title":"t","text":"
                                               
                                            • threshold (types.h)
                                              •  
                                            "},{"location":"opae-code/variables/#_","title":"_","text":"
                                               
                                            • _opae_hash_map_flags (hash_map.h)
                                              •  
                                            "},{"location":"opae-code/links/","title":"Links","text":"
                                            * Related Pages     * Todo List   * Modules   * Class List     * struct _fpga_token_header     * struct _opae_hash_map     * struct _opae_hash_map_item     * struct cache_line     * struct config     * struct fpga_error_info     * struct fpga_metric     * struct fpga_metric_info     * struct fpga_version     * struct mem_alloc     * struct mem_link     * struct metric_threshold     * union metric_value     * namespace opae       * namespace opae::fpga         * namespace opae::fpga::types           * class opae::fpga::types::busy           * namespace opae::fpga::types::detail           * class opae::fpga::types::error           * class opae::fpga::types::event             * struct opae::fpga::types::event::type_t           * class opae::fpga::types::except           * class opae::fpga::types::exception           * struct opae::fpga::types::guid_t           * class opae::fpga::types::handle           * class opae::fpga::types::invalid_param           * class opae::fpga::types::no_access           * class opae::fpga::types::no_daemon           * class opae::fpga::types::no_driver           * class opae::fpga::types::no_memory           * class opae::fpga::types::not_found           * class opae::fpga::types::not_supported           * class opae::fpga::types::properties           * struct opae::fpga::types::pvalue           * class opae::fpga::types::reconf_error           * class opae::fpga::types::shared_buffer           * class opae::fpga::types::src_location           * class opae::fpga::types::sysobject           * class opae::fpga::types::token           * class opae::fpga::types::version     * struct opae_uio     * struct opae_uio_device_region     * struct opae_vfio     * struct opae_vfio_buffer     * struct opae_vfio_device     * struct opae_vfio_device_irq     * struct opae_vfio_device_region     * struct opae_vfio_group     * struct opae_vfio_iova_range     * struct opae_vfio_sparse_info     * struct ras_inject_error     * namespace std     * struct threshold   * Namespace ListNamespace List   * Namespace Members   * Namespace Member Functions   * Namespace Member Variables   * Namespace Member Typedefs   * Namespace Member Enumerations   * Class Index   * Class Hierarchy   * Class Members   * Class Member Functions   * Class Member Variables   * Class Member Typedefs   * Class Member Enumerations   * Files     * docs       * docs/sw         * docs/sw/include           * docs/sw/include/opae             * access.h             * access.h source             * buffer.h             * buffer.h source             * docs/sw/include/opae/cxx               * core.h               * core.h source               * docs/sw/include/opae/cxx/core                 * errors.h                 * errors.h source                 * events.h                 * events.h source                 * except.h                 * except.h source                 * handle.h                 * handle.h source                 * properties.h                 * properties.h source                 * pvalue.h                 * pvalue.h source                 * shared_buffer.h                 * shared_buffer.h source                 * sysobject.h                 * sysobject.h source                 * token.h                 * token.h source                 * version.h                 * version.h source             * enum.h             * enum.h source             * error.h             * error.h source             * event.h             * event.h source             * fpga.h             * fpga.h source             * hash_map.h             * hash_map.h source             * init.h             * init.h source             * log.h             * log.h source             * manage.h             * manage.h source             * mem_alloc.h             * mem_alloc.h source             * metrics.h             * metrics.h source             * mmio.h             * mmio.h source             * properties.h             * properties.h source             * sysobject.h             * sysobject.h source             * types.h             * types.h source             * types_enum.h             * types_enum.h source             * uio.h             * uio.h source             * umsg.h             * umsg.h source             * userclk.h             * userclk.h source             * utils.h             * utils.h source             * version.h             * version.h source             * vfio.h             * vfio.h source         * docs/sw/samples           * docs/sw/samples/hello_events             * hello_events.c             * hello_events.c source           * docs/sw/samples/hello_fpga             * hello_fpga.c             * hello_fpga.c source   * File Variables   * File Functions   * File Macros
                                            "}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Open FPGA Stack Overview","text":"
                                             
                                            Open FPGA Stack (OFS): OFS is an open-source hardware and software framework that reduces the development time for creating your custom platform.   Provided within this framework are reference designs targeting different Intel\u00ae FPGA devices with upstreamed drivers and management software tools.  
                                             
                                             
                                            The reference shells, called FPGA Interface Manager (FIMs), provide an integrated, timing closed design with the most common interfaces for host attach applications. After selecting your starting shell, you can add or subtract interfaces depending on your application requirements. Then leverage the build scripts, RTL, unit tests, Universal Verification Methodology (UVM) environment, software and drivers for this reference shell as a starting point for your own FPGA platform solution.
                                             
                                            OFS currently targets Intel Stratix\u00ae 10 and Intel Agilex\u00ae 7 FPGA Device Families. To find out more about Intel FPGAs, visit the Intel Stratix 10 and Intel Agilex 7 pages at Intel.com.
                                             
                                            "},{"location":"#how-can-i-start-using-ofs","title":"How Can I Start Using OFS?","text":"
                                               
                                            1.  
                                              If you are interested in a production card that uses OFS for your workload application development or for full deployment, please refer to the OFS Board Catalog.  
                                               
                                              2.  
                                            2.  
                                              If you are an FPGA developer, refer to our [FPGA Developer Journey Guide] to understand the OFS development flow as well as the reference shells, tools and development boards you can use to gest started.  FPGA Developers interested in oneAPI should reference this guide as well.
                                               
                                              3.  
                                            3.  
                                              If you are a software developer, refer to our Software Tab for driver and user space tool development resources.
                                               
                                              4.  
                                            4.  
                                              If you are an application developer, preview our overview video on how OFS can help you develop FPGA-based workloads and review one of the AFU Developer Guides to find the OFS resources available for creating your own application workload.
                                               
                                              5.  
                                             
                                               
                                            • Accelerator Functional Unit (AFU) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach
                                              •  
                                            • Accelerator Functional Unit (AFU) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach
                                              •  
                                            • Accelerator Functional Unit (AFU) Developer Guide: OFS for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach
                                              •  
                                             
                                            Beyond the resources we have on this site, you can navigate to the OFS GitHub repos by clicking the GitHub repository icon at the top left corner of this site page.
                                             
                                            "},{"location":"#what-fim-reference-designs-are-available","title":"What FIM Reference Designs Are Available?","text":"
                                            Below summarizes the five current reference designs (aka FIMs) you can choose from:
                                             
                                            OFS FIM Targeting Intel\u00ae Agilex\u00ae7 PCIe Attach (2xR-tile, F-tile)
                                             
                                             Key Feature Description Target OPN AGIB027R29A1E2VR3 PCIe R-tile PCIe* Gen5x8 Virtualization 5 physical functions/3 virtual functions with ability to expand Memory * Two Fabric DDR4 channels, x64 (no ECC), 2666 MHz, 8GB Ethernet 2x4x25GbE, 2x200GbE, 2x400GbE Hard Processor System Not enabled Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration)* Platform Management Controller Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported OneAPI OneAPI Acceleration Support Package (ASP) provided with compiled FIM to support OneAPI Runtime Software Support * Linux DFL drivers targeting OFS FIMs* OPAE Software Development Kit* OPAE Tools Target Board Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) 
                                            Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae PCIe Attach Reference FIM documentation collection.
                                             
                                             
                                            OFS FIM Targeting Intel\u00ae Agilex\u00ae7 PCIe Attach (2xF-tile)
                                             
                                             Key Feature Description Target OPN AGFB027R24C2E2VR2 PCIe F-tile PCIe* Gen4x16 Virtualization 5 physical functions/3 virtual functions with ability to expand Memory 3 DDR Channels:* One HPS DDR4 bank, x40 (x32 Data and x8 ECC), 2400 MHz, 1GB each* Two Fabric DDR4 banks, x64 (no ECC), 2400 MHz, 8GB Ethernet 2x4x25GbE Hard Processor System 64-bit quad core Arm\u00ae Cortex\u00ae-A53 MPCore with integrated peripherals. Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration)* Platform Management Controller Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported OneAPI OneAPI Acceleration Support Package (ASP) provided with compiled FIM to support OneAPI Runtime Software Support * Linux DFL drivers targeting OFS FIMs* OPAE Software Development Kit* OPAE Tools Target Board Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) 
                                            Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae PCIe Attach Reference FIM documentation collection.
                                             
                                             
                                            OFS FIM Targeting Intel\u00ae Agilex\u00ae7 PCIe Attach (P-tile, E-tile)
                                             
                                             Key Feature Description Target OPN AGFB014R24A2E2V PCIe P-tile PCIe* Gen4x16 Virtualization 5 physical functions/3 virtual functions with ability to expand Memory 5 DDR Channels:* One HPS DDR4 bank, x40 (x32 Data and x8 ECC), 1200 MHz, 1GB each* Four Fabric DDR4 banks, x32 (no ECC), 1200 MHz, 4GB Ethernet 2x4x25GbE, 2x4x10GbE or 2x100GbE Hard Processor System 64-bit quad core Arm\u00ae Cortex\u00ae-A53 MPCore with integrated peripherals. Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration)* Platform Controller Management Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported OneAPI OneAPI Acceleration Support Package (ASP) provided with compiled FIM to support OneAPI Runtime Software Support * Linux DFL drivers targeting OFS FIMs* OPAE Software Development Kit* OPAE Tools Target Board Intel\u00ae FPGA SmartNIC N6001-PL 
                                            Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae PCIe Attach Reference FIM documentation collection.
                                             
                                             
                                            OFS FIM Features Targeting Intel\u00ae Agilex\u00ae 7 SoC Attach
                                             
                                             Key Feature Description Device OPN AGFC023R25A2E2VR0 PCIe P-tile PCIe* Gen4x16 to the HostP-tile PCIe* Gen4x16 to the SoC (IceLake-D) Virtualization Host: 2 physical functions SoC:   1 physical function and 3 Virtual functions Memory Four Fabric DDR4 banks, x40 (optional ECC, be configured as x32 and ECC x8 ), 1200 MHz, 4GB Ethernet 2x4x25GbE Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration) * Platform Controller Management Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported Software Support * Linux DFL drivers targeting OFS FIMs * OPAE Software Development Kit * OPAE Tools Target Board Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL 
                                            Note: Source code for BMC RTL/Firmware that works with this reference FIM can be obtained by contacting your Intel Sales Representative.
                                             
                                            Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae SoC Attach Reference FIM documentation collection.
                                             
                                             
                                            OFS FIM Targeting Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach
                                             
                                             Key Feature Description Device OPN 1SX280HN2F43E2VG Ethernet Configuration 1x10GbE example with 2x100GbE capability PCIe Gen3x16 EMIF Up to four DDR channels PF/VF 1 PF/3 VFs Management FPGA Management Engine (FME) with FIM management registers Interface Arm\u00ae AMBA\u00ae4 AXI Interface HLD support oneAPI Software Kernel code upstreamed to Linux.org Target Board Intel\u00ae FPGA Programmable Acceleration Card D5005 
                                            Click here for the OFS Collateral for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach Reference FIM documentation.
                                             
                                            To find information on the latest releases, go to the Discussions Tab in the OFS GitHub repository.
                                             
                                            "},{"location":"#open-fpga-stack-repositories","title":"Open FPGA Stack Repositories","text":"
                                            Accessing OFS ingredients to use within the development framework is easy.  The github.com/OFS site provides all the hardware and software repositories in one location.
                                             Development Focus Repository Folder Description Hardware ofs-agx7-pcie-attach Provides RTL, unit tests, and build scripts to create an Intel\u00ae Agilex\u00ae 7 FIM and is leveraged as a starting point for a custom PCIe Attach design.  The reference FIM targets an Intel\u00ae FPGA SmartNIC N6001-PL Platform or the Intel Agilex 7 FPGA F-Series Development Kit (2x F-Tile) User Guide. Hardware ofs-f2000x-pl Provides RTL, unit tests, and build scripts to create Intel\u00aeAgilex\u00ae FIM and is leveraged as a starting point for a custom SoC Attach design.  The reference FIM targets an Intel\u00ae FPGA IPU F2000X-PL Platform. Hardware ofs-d5005 Provides RTL, unit tests, and build scripts to create Intel\u00ae Stratix 10\u00ae FIM and is leveraged as a starting point for a custom PCIe Attach design.  The reference FIM targets an Intel\u00ae FPGA PAC D5005 development board. Hardware oneapi-asp Contains the files to generate the support package that works with the reference shells and allows you to use OneAPI. This is an optional repository for developers interested in OneAPI Hardware ofs-fim-common Provides RTL components that are shared among all new platforms that are introduced in OFS.  This folder is a subumodule in each platform repository folder. Hardware examples-afu Provides simple Accelerator Functional Unit (AFU) examples you can use as a template for starting your own workload design. Hardware ofs-platform-afu-bbb Contains the hardware code to build a standard interface between the FIM and your workload. Software linux-dfl This repository is a mirror of the linux.org Git site and contains the most up-to-date drivers that are being developed and upstreamed for OFS platforms. Software meta-ofs This repository provides the Linux\u00ae DFL kernel and the OPAE SDK for the Yocto\u00ae Project. Software opae-sdk Contains the ingredients to build the OFS Open Programmable Acceleration Engine (OPAE) Software Development Kit which provides APIs and userspace tools for OFS FPGA management. Software opae-sim This repository is used to build the AFU Hardware/Software Co-Simulation Environment workload developers can use to ensure their AFU can work with the OFS software stack. Software linux-dfl-backport A place for finding and leveraging out-of-tree backported drivers for older OS versions . Software opae-legacy Supports OFS platforms built on the legacy version of OPAE software.  Not used in current OFS designs Documentation ofs.github.io Contains the hardware and software collateral that surfaces on the OFS website: https://ofs.github.io 
                                            "},{"location":"hw/common/doc_modules/links/","title":"AFU Dev","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/","title":"Software Reference Manual: Open FPGA Stack","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#10-introduction","title":"1.0 Introduction","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#11-audience","title":"1.1 Audience","text":"
                                            The information presented in this document is intended to be used by software developers looking to increase their knowledge of the OPAE SDK user-space software stack and the kernel-space linux-dfl drivers. This information is intended as a starting point, with links to where users can deep dive on specific topics.
                                             
                                            Former OPAE and DFL software documents were combined with the Software Reference Manual to reduce clutter and more cleanly document OFS software capabilities. The following documents are no longer available as standalone as of ofs-2023.3:
                                             
                                               
                                            • Quick Start Guide
                                              •  
                                            • OPAE Installation Guide
                                              •  
                                            • OPAE C API Programming Guide
                                              •  
                                            • OPAE Python Bindings
                                              •  
                                            • OPAE Plugin Developers Guide
                                              •  
                                            • Open Programmable Accelerator Engine (OPAE) Linux Device Driver Architecture
                                              •  
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#12-terminology","title":"1.2 Terminology","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#13-opae-software-development-kit-sdk","title":"1.3 OPAE Software Development Kit (SDK)","text":"
                                            The OPAE C library is a lightweight user-space library that provides abstraction for FPGA resources in a compute environment. Built on top of the OPAE Intel\u00ae FPGA driver stack that supports Intel\u00ae FPGA platforms, the library abstracts away hardware specific and OS specific details and exposes the underlying FPGA resources as a set of features accessible from within software programs running on the host. The OPAE source code is available on the OPAE SDK repository.
                                             
                                            By providing a unified C API, the library supports different FPGA integration and deployment models, ranging from single-node systems with one or a few FPGA devices to large-scale FPGA deployments in a data center. At one end of the spectrum, the API supports a simple application using a PCIe link to reconfigure the FPGA with different accelerator functions. At the other end of the spectrum, resource management and orchestration services in a data center can use this API to discover and select FPGA resources and then allocate them for use by acceleration workloads.
                                             
                                            The OPAE provides consistent interfaces to crucial components of the platform. OPAE does not constrain frameworks and applications by making optimizations with limited applicability. When the OPAE does provide convenience functions or optimizations, they are optional. For example, the OPAE provides an interface to allocate physically contiguous buffers in system memory that user-space software and an accelerator can share. This interface enables the most basic feature set of allocating and sharing a large page of memory in one API call. However, it does not provide a malloc()-like interface backed by a memory pool or slab allocator. Higher layers of the software stack can make such domain-specific optimizations.
                                             
                                            Most of the information related to OPAE can be found on the official OFS Site and in the OPAE SDK repository. The following is a summary of the information present on this web page:
                                             
                                               
                                            • Configuration options present in the OPAE SDK build and installation flow
                                              •  
                                            • The steps required to build a sample OPAE application
                                              •  
                                            • An explanation of the basic application flow
                                              •  
                                            • A reference for the C, C++, and Python APIs
                                              •  
                                            • An explanation of the OPAE Linux Device Driver Architecture
                                              •  
                                            • Definitions for the various user-facing OPAE SDK tools
                                              •  
                                             
                                            The remaining sections on OPAE in this document are unique and build on basic principles explained in opae.github.io.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#table-1-additional-websites-and-links","title":"Table 1: Additional Websites and Links","text":"Document Link OPAE SDK on github OPAE SDK repository OPAE Documents OFS Site pybind11 https://pybind11.readthedocs.io/en/stable/ CLI11 https://github.com/CLIUtils/CLI11 spdlog https://github.com/gabime/spdlog"},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#table-2-ofs-hardware-terminology","title":"Table 2: OFS Hardware Terminology","text":"Term Description FPGA: Field Programmable Gate Array a discrete or integrated device connecting to a host CPU via PCIe or other type of interconnects. Accelerator Function Unit (AFU) The AFU is the supplied implementation of an accelerator, typically in HDL. AFUs implement a function such as compression, encryption, or mathematical operations.The Quartus Prime Pro software synthesizes the RTL logic into a bitstream. Accelerator Function (AF) The AF is the compiled binary for an AFU. An AF is a raw binary file (.rbf) bitstream. A tool (fpgaconf) reconfigures the FPGA using an AF bitstream. Reconfiguration The process of reprogramming the FPGA with a different AF."},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#20-opae-c-api","title":"2.0 OPAE C API","text":"
                                            The following OPAE data structures and functions integrate AFUs into the OPAE environment.  The OPAE C API models these data structures and functions. For more information on the object  models refer to the Object model section.
                                             
                                               
                                            • Accelerator: An accelerator is an allocable accelerator function implemented in an FPGA.  An accelerator tracks the  ownership of an AFU (or part of it) for a process that uses it. Multiple processes can share an accelerator.
                                              •  
                                            • Device: The OPAE enumerates and models two device types: the FPGA and the AFU.
                                              •  
                                            • Events: Events are asynchronous notifications. The FPGA driver triggers particular events to indicate error conditions. Accelerator logic can also define its own events. User applications can choose to be notified when particular events occur and respond appropriately.
                                              •  
                                            • Shared memory buffers: Software allocates shared memory buffers in user process memory on the host. Shared memory buffers facilitate data transfers between the user process and the  accelerator that it owns.
                                              •  
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#21-libopae-c","title":"2.1 libopae-c","text":"
                                            Linking with this library is straightforward. Code using the  OPAE library should include the header file fpga.h. Taking the GCC compiler on Linux as an example, here is the simplest compile and link command:
                                             
                                            gcc myprog.c -I</path/to/fpga.h> -L</path/to/libopae-c.so> -lopae-c -luuid -ljson-c -lpthread
                                             
                                            .. note::
                                             
                                            The OPAE library uses the third-party `libuuid` and `libjson-c` libraries that are not distributed with \nthe OPAE library. Make sure to install these libraries.\n
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#sample-code","title":"Sample Code","text":"
                                            The library source includes two code samples. Use these samples to learn how to call functions in the library. Build and run these samples to determine if your installation and environment are set up properly. 
                                             
                                            Refer to the Running the Hello FPGA Example chapter in the Intel\u00ae Acceleration Stack Quick Start Guide for for Intel Programmable Acceleration Card with Intel Arria\u00ae 10 GX FPGA for more information about using the sample code.  
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#high-level-directory-structure","title":"High-Level Directory Structure","text":"
                                            Building and installing the OPAE library results in the following directory structure on the Linux OS. Windows and MacOS have similar directories and files.
                                             Directory & Files Contents include/opae Directory containing all header files include/opae/fpga.h Top-level header for user code to include include/opae/access.h Header file for accelerator acquire/release, MMIO, memory management, event handling, and so on include/opae/bitstream.h Header file for bitstream manipulation functions include/opae/common.h Header file for error reporting functions include/opae/enum.h Header file for AFU enumeration functions include/opae/manage.h Header file for FPGA management functions include/opae/types.h Various type definitions lib Directory containing shared library files lib/libopae-c.so The shared dynamic library for linking with the user application doc Directory containing API documentation doc/html Directory for documentation of HTML format doc/latex Directory for documentation of LaTex format doc/man Directory for documentation of Unix man page format"},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#object-models","title":"Object Models","text":"
                                               
                                            • fpga_objtype: An enum type that represents the type of an FPGA resource, either FPGA_DEVICE or FPGA_ACCELERATOR.  An FPGA_DEVICE object corresponds to a physical FPGA device. Only FPGA_DEVICE objects can invoke management functions. The FPGA_ACCELERATOR represents an instance of an AFU. 
                                              •  
                                            • fpga_token: An opaque type that represents a resource known to, but not necessarily owned by, the calling process. The calling process must own a resource before it can invoke functions of the resource.
                                              •  
                                            • fpga_handle: An opaque type that represents a resource owned by the calling process. The API functions fpgaOpen() and fpgaClose() acquire and release ownership of a resource that an fpga_handle represents. (Refer to the Functions section for more information.)
                                              •  
                                            • fpga_properties: An opaque type for a properties object. Your applications use these properties to query and search for appropriate resources. The  FPGA Resource Properties section documents properties visible to your applications.
                                              •  
                                            • fpga_event_handle: An opaque handle the FPGA driver uses to notify your application about an event. 
                                              •  
                                            • fpga_event_type: An enum type that represents the types of events. The following are valid values:  FPGA_EVENT_INTERRUPT, FPGA_EVENT_ERROR, and FPGA_EVENT_POWER_THERMAL. (The Intel Programmable Acceleration Card (PAC) with Intel Arria 10 GX FPGA does not handle thermal and power events.)
                                              •  
                                            • fpga_result: An enum type to represent the result of an API function. If the function returns successfully the result is FPGA_OK. Otherwise, the result is the appropriate error codes. Function fpgaErrStr() translates an error code into human-readable strings.
                                              •  
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#functions","title":"Functions","text":"
                                            The table below groups important API calls by their functionality. For more information about each of the functions, refer to the  OPAE C API reference manual.
                                             Functionality API Call FPGA Accelerator Description Enumeration fpgaEnumerate() Yes Yes Query FPGA resources that match certain properties Enumeration: Properties fpga[Get, Update, Clear, Clone, Destroy Properties]() Yes Yes Manage fpga_properties life cycle fpgaPropertiesGet[Prop]() Yes Yes Get the specified property Prop, from the FPGA Resource Properties table fpgaPropertiesSet[Prop]() Yes Yes Set the specified property Prop, from the FPGA Resource Properties table Access: Ownership fpga[Open, Close]() Yes Yes Acquire/release ownership Access: Reset fpgaReset() Yes Yes Reset an accelerator Access: Event handling fpga[Register, Unregister]Event() Yes Yes Register/unregister an event to be notified about fpga[Create, Destroy]EventHandle() Yes Yes Manage fpga_event_handle life cycle Access: MMIO fpgaMapMMIO(), fpgaUnMapMMIO() Yes Yes Map/unmap MMIO space fpgaGetMMIOInfo() Yes Yes Get information about the specified MMIO space fpgaReadMMIO[32, 64]() Yes Yes Read a 32-bit or 64-bit value from MMIO space fpgaWriteMMIO[32, 64]() Yes Yes Write a 32-bit or 64-bit value to MMIO space Memory management: Shared memory fpga[Prepare, Release]Buffer() Yes Yes Manage memory buffer shared between the calling process and an accelerator fpgaGetIOAddress() Yes Yes Return the device I/O address of a shared memory buffer fpgaBindSVA() Yes Yes Bind IOMMU shared virtual addressing Management: Reconfiguration fpgaReconfigureSlot() Yes No Replace an existing AFU with a new one Error report fpgaErrStr() Yes Yes Map an error code to a human readable string 
                                            .. note::
                                             
                                            The UMsg APIs are not supported for the Intel(R) PAC cards. They will be deprecated in a future release.\n
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#fpga-resource-propertie","title":"FPGA Resource Propertie","text":"
                                            Applications query resource properties by specifying the property name for Prop in the  fpgaPropertiesGet[Prop]() and fpgaPropertiesSet[Prop]() functions. The FPGA and Accelerator columns state whether or not the Property is available for the FPGA or Accelerator objects.
                                             Property FPGA Accelerator Description Parent No Yes fpga_token of the parent object ObjectType Yes Yes The type of the resource: either FPGA_DEVICE or FPGA_ACCELERATOR Bus Yes Yes The bus number Device Yes Yes The PCI device number Function Yes Yes The PCI function number SocketId Yes Yes The socket ID DeviceId Yes Yes The device ID NumSlots Yes No Number of AFU slots available on an FPGA_DEVICE resource BBSID Yes No The FPGA Interface Manager (FIM) ID of an FPGA_DEVICE resource BBSVersion Yes No The FIM version of an FPGA_DEVICE resource VendorId Yes No The vendor ID of an FPGA_DEVICE resource GUID Yes Yes The GUID of an FPGA_DEVICE or FPGA_ACCELERATOR resource NumMMIO No Yes The number of MMIO space of an FPGA_ACCELERATOR resource NumInterrupts No Yes The number of interrupts of an FPGA_ACCELERATOR resource AcceleratorState No Yes The state of an FPGA_ACCELERATOR resource: either FPGA_ACCELERATOR_ASSIGNED or FPGA_ACCELERATOR_UNASSIGNED"},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#opae-c-api-return-codes","title":"OPAE C API Return Codes","text":"
                                            The OPAE C library returns a code for every exported public API function.  FPGA_OK indicates successful completion of the requested operation. Any return code other than FPGA_OK indicates an error or unexpected behavior. When using the OPAE C API, always check the API return codes. 
                                             Error Code Description FPGA_OK Operation completed successfully FPGA_INVALID_PARAM Invalid parameter supplied FPGA_BUSY Resource is busy FPGA_EXCEPTION An exception occurred FPGA_NOT_FOUND A required resource was not found FPGA_NO_MEMORY Not enough memory to complete operation FPGA_NOT_SUPPORTED Requested operation is not supported FPGA_NO_DRIVER Driver is not loaded FPGA_NO_DAEMON FPGA Daemon (fpgad) is not running FPGA_NO_ACCESS Insufficient privileges or permissions FPGA_RECONF_ERROR Error while reconfiguring FPGA"},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#211-device-abstraction","title":"2.1.1 Device Abstraction","text":"
                                            The OPAE C API relies on two base abstractions concerning how the FIM and accelerator are presented to and manipulated by the user. The FIM is concerned with management functionality. Access to the FIM and its interfaces is typically restricted to privileged (root) users. The accelerator contains the user-defined logic in its reconfigurable region. Most OPAE end-user applications are concerned with querying and opening the accelerator device, then interacting with the AFU via MMIO and shared memory.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2111-device-types","title":"2.1.1.1 Device types","text":"
                                            The C enum fpga_objtype defines two variants. The FPGA_DEVICE variant corresponds to the FIM portion of the device, and the FPGA_ACCELERATOR refers to the accelerator, also known as the AFU.
                                             
                                            An FPGA_DEVICE refers loosely to the sysfs tree rooted at the dfl-fme.X directory, for example /sys/class/fpga_region/region0/dfl-fme.0, and its associated device file /dev/dfl-fme.0.
                                             
                                            An FPGA_ACCELERATOR refers loosely to the sysfs tree rooted at the dfl-port.X directory, for example /sys/class/fpga_region/region0/dfl-port.0, and its associated device file /dev/dfl-port.0.
                                             
                                            The number X in dfl-fme.X and dfl-port.X refers to a numeric ID that is assigned by the DFL device driver to uniquely identify an instance of the FIM/Accelerator. Systems with multiple FPGA acceleration devices will have multiple dfl-fme.X\u2019s and matching dfl-port.X\u2019s.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2112-tokens-and-handles","title":"2.1.1.2 Tokens and Handles","text":"
                                            An fpga_token is an opaque data structure that uniquely represents an FPGA_DEVICE or an FPGA_ACCELERATOR. Tokens convey existence, but not ownership. Tokens are retrieved via the OPAE enumeration process described below using the fpgaEnumerate() call.
                                             
                                            An fpga_handle is an opaque data structure that corresponds to an opened device instance, whether FPGA_DEVICE or FPGA_ACCELERATOR. A Handle is obtained from a token via the fpgaOpen() call. A handle conveys that the /dev/dfl-fme.X or /dev/dfl-port.X device file has been opened and is ready for interaction via its IOCTL interface.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#212-enumeration","title":"2.1.2 Enumeration","text":"
                                            Enumeration is the process by which an OPAE application becomes aware of the existence of FPGA_DEVICE\u2019s and FPGA_ACCELERATOR\u2019s. Refer to the signature of the fpgaEnumerate() call:
                                             
                                            fpga_result fpgaEnumerate(const fpga_properties *filters,\nuint32_t num_filters,\nfpaa_token *tokens,\nuint32_t max_tokens,\nuint32_t *num_matches);\n
                                             
                                            Figure 1 fpgaEnumerate()
                                             
                                            The typical enumeration flow involves an initial call to fpgaEnumerate() to discover the number of available tokens.
                                             
                                            uint32_t num_matches = 0;\nfpgaEnumerate(NULL, 0, NULL, 0, &num_matches);\n
                                             
                                            Figure 2 Discovering Number of Tokens
                                             
                                            Once the number of available tokens is known, the application can allocate the correct amount of space to hold the tokens:
                                             
                                            fpga_token *tokens;\nuint32_t num_tokens = num_matches;\ntokens = (fpga_token *)calloc(num_tokens, sizeof(fpga_token));\nfpgaEnumerate(NULL, 0, tokens, num_tokens, &num_matches);\n
                                             
                                            Figure 3 Enumerating All Tokens
                                             
                                            Note that parameters filters and num_filters were not used in the preceding example, as they were NULL and 0. When no filtering criteria are provided, fpgaEnumerate() returns all tokens that can be enumerated.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2121-fpga_properties-and-filtering","title":"2.1.2.1 fpga_properties and Filtering","text":"
                                            An fpga_properties is an opaque data structure used to retrieve all of the properties concerning an FPGA_DEVICE or FPGA_ACCELERATOR. These properties can be included in the filters parameter to fpgaEnumerate() to select tokens by specific criteria.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#21211-common-properties","title":"2.1.2.1.1 Common Properties","text":"
                                            Table 3 lists the set of properties that are common to FPGA_DEVICE and FPGA_ACCELERATOR:
                                              Property  Description fpga_guid guid; FPGA_DEVICE: PR Interface ID FPGA_ACCELERATOR: AFU ID fpga_token parent; FPGA_DEVICE: always NULL FPGA_ACCELERATOR: the token of the corresponding FPGA_DEVICE, if any. Otherwise, NULL. fpga_objtype objtype; FPGA_DEVICE or FPGA_ACCELERATOR uint16_t segment; The segment portion of the PCIe address: ssss:bb:dd.f uint8_t bus; 
                                            The bus portion of the PCIe address:
                                             
                                            ssss:bb:dd.f
                                             uint8_t device; 
                                            The device portion of the PCIe address:
                                             
                                            ssss:bb:dd.f
                                             uint8_t function; The function portion of the PCIe address: ssss:bb:dd.f uint64_t object_id; A unique 64-bit value that identifies this token on the system. uint16_t vendor_id; The PCIe Vendor ID uint16_t device_id; The PCIe Device ID uint32_t num_errors; The number of error sysfs nodes available for this token. fpga_interface interface; An identifier for the underlying plugin-based access method. 
                                            Table 3 Common Properties
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#21212-fpga_device-properties","title":"2.1.2.1.2 FPGA_DEVICE Properties","text":"
                                            Table 4 lists the set of properties that are specific to FPGA_DEVICE token types.
                                             Property Description uint64_t bbs_id; FIM-specific Blue Bitstream ID fpga_version bbs_version; BBS version 
                                            Table 4 FPGA_DEVICE Properties
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#21213-fpga_accelerator-properties","title":"2.1.2.1.3 FPGA_ACCELERATOR Properties","text":"
                                            Table 5 lists the set of properties that are specific to FPGA_ACCELERATOR token types.
                                             Property Description fpga_accelerator_state state; Whether the Accelerator is currently open uint32_t num_mmio; The number of MMIO regions available uint32_t num_interrupts; The number of interrupts available 
                                            Table 5 FPGA_ACCELERATOR Properties
                                             
                                            Following is an example of using fpga_properties to enumerate a specific AFU:
                                             
                                            #define NLB0_AFU \"D8424DC4-A4A3-C413-F89E-433683F9040B\"\nfpga_properties filter = NULL;\nfpga_guid afu_id;\nfpgaGetProperties(NULL, &filter); // NULL: a new empty properties\nfpgaPropertiesSetObjectType(filter, FPGA_ACCELERATOR);\nuuid_parse(NLB0_AFU, afu_id);\nfpgaPropertiesSetGuid(filter, afu_id);\nfpgaEnumerate(&filter, 1, tokens, num_tokens, &num_matches);\n
                                             Relevant Links:  - fpga_guid - fpgaGetProperties - fpgaPropertiesSetObjectType - fpgaPropertiesSetGUID
                                             
                                            Figure 4 Filtering During Enumeration
                                             
                                            Note that fpga_properties and fpga_token\u2019s are allocated resources that must be freed by their respective API calls, ie fpgaDestroyProperties() and fpgaDestroyToken().
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#213-access","title":"2.1.3 Access","text":"
                                            Once a token is discovered and returned to the caller by fpgaEnumerate(), the token can be converted into a handle by fpgaOpen(). Upon a successful call to fpgaOpen(), the associated /dev/dfl-fme.X (FPGA_DEVICE) or /dev/dfl-port.X (FPGA_ACCELERATOR) is opened and ready for use. Having acquired an fpga_handle, the application can then use the handle with any of the OPAE APIs that require an fpga_handle as an input parameter.
                                             
                                            Like tokens and properties, handles are allocated resources. When a handle is no longer needed, it should be closed and released by calling fpgaClose().
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#214-events","title":"2.1.4 Events","text":"
                                            Event registration in OPAE is a two-step process. First, the type of event must be identified. The following fpga_event_type variants are defined:
                                             Event Description FPGA_EVENT_INTERRUPT AFU interrupt FPGA_EVENT_ERROR Infrastructure error event (FME/Port Error) 
                                            Table 6 FPGA Event Types
                                             
                                            Once the desired event type is known, an fpga_event_handle is created via fpgaCreateEventHandle(). Once the event handle is available, the event notification is registered using fpgaRegisterEvent(). In the example below, note the use of the flags field for passing the desired IRQ vector when the event type is FPGA_EVENT_INTERRUPT. With the event registered, the application can then use fpgaGetOSObjectFromEventHandle() to obtain a file descriptor for use with the poll() system call. When the interrupt occurs, the file descriptor will be set to the signaled state by the DFL driver.
                                             
                                            fpga_event_handle event_handle = NULL;\nint fd = -1;\nfpgaCreateEventHandle(&event_handle);\nfpgaRegisterEvent(fpga_handle, FPGA_EVENT_INTERRUPT,\nevent_handle, irq_vector);\nfpgaGetOSObjectFromEventHandle(event_handle, &fd);\n
                                             
                                            Figure 5 Creating and Registering Events
                                             
                                            When an event notification is no longer needed, it should be released by calling fpgaUnregisterEvent(). Like device handles, event handles are allocated resources that must be freed when no longer used. To free an event handle, use the fpgaDestroyEventHandle() call.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#215-mmio-and-shared-memory","title":"2.1.5 MMIO and Shared Memory","text":"
                                            Communication with the AFU is achieved via reading and writing CSRs and by reading and writing to AFU/host shared memory buffers. An AFU\u2019s CSRs are memory-mapped into the application process address space by way of the fpgaMapMMIO() call.
                                             
                                            uint32_t mmio_num = 0;\nfpgaMapMMIO(fpga_handle, mmio_num, NULL);\nfpgaWriteMMIO64(fpga_handle, mmio_num, MY_CSR, 0xa);\n
                                             
                                            Figure 6 Mapping and Accessing CSRs
                                             
                                            The second parameter, mmio_num, is the zero-based index identifying the desired MMIO region. The maximum number of MMIO regions for a particular handle is found by accessing the num_mmio property. Refer to the fpgaPropertiesGetNumMMIO() call.
                                             
                                            Once the AFU CSRs are mapped into the process address space, the application can use the fpgaReadMMIO**XX**() and fpgaWriteMMIO**XX**() family of functions, eg fpgaReadMMIO64() and fpgaWriteMMIO64(). When an MMIO region is no longer needed, it should be unmapped from the process address space using the fpgaUnmapMMIO() call.
                                             
                                            Shared memory buffers are allocated by way of the fpgaPrepareBuffer() call.
                                             
                                            fpga_result fpgaPrepareBuffer(fpga_handle handle,\nuint64_t len,\nvoid **buf_addr,\nuint64_t *wsid,\nint flags);\n
                                             
                                            Figure 7 fpgaPrepareBuffer()
                                             
                                            Three buffer lengths are supported by this allocation method:
                                             Length Description 4096 (4KiB) No memory configuration needed. 2097152 (2MiB) Requires 2MiB huge pages to be allocated. 1073741824 (1GiB) Requires 1GiB huge pages to be allocated. 
                                            Table 7 fpgaPrepareBuffer() Lengths
                                             
                                            echo 8 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages\necho 2 > /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages\n
                                             
                                            Figure 8 Configuring Huge Pages
                                             
                                            The buf_addr parameter to fpgaPrepareBuffer() is a pointer to a void * that accepts the user virtual base address of the newly-created buffer. The wsid parameter is a pointer to a uint64_t that receives a unique workspace ID for the buffer allocation. This workspace ID is used in subsequent calls to fpgaReleaseBuffer(), which should be called when the buffer is no longer needed and in calls to fpgaGetIOAddress() which is used to query the IO base address of the buffer. The IO base address can be programmed into the AFU by means of the AFU CSR space. For example, here is a code snippet from the hello_fpga sample that demonstrates programming a shared buffer\u2019s IO base address into an AFU CSR in MMIO region 0:
                                             
                                            #define LOG2_CL 6\n#define CACHELINE_ALIGNED_ADDR(p) ((p) >> LOG2_CL)\nfpgaGetIOAddress(accelerator_handle, input_wsid, &iova);\nfpgaWriteMMIO64(accelerator_handle, 0, nlb_base_addr + CSR_SRC_ADDR,\nCACHELINE_ALIGNED_ADDR(iova));\n
                                             
                                            Figure 9 Programming Shared Memory
                                             
                                            If applications need to map a shared buffer that has been allocated by some other means than fpgaPrepareBuffer(), then the flags parameter can be set to FPGA_BUF_PREALLOCATED. This causes fpgaPrepareBuffer() to skip the allocation portion of the call and to only memory map the given buf_addr into the application process address space.
                                             
                                            Buffers can also be allocated and mapped as read-only by specifying FPGA_BUF_READ_ONLY.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#216-management","title":"2.1.6 Management","text":"
                                            The management feature in OPAE concerns re-programming the programmable region of the Port. To program the Port bitstream, pass a handle to the FPGA_DEVICE associated with the desired Port. The slot parameter identifies which Port to program in the case of multi-port implementations. Most designs will only pass zero as the slot parameter. The bitstream parameter is a buffer that contains the entire bitstream contents, including the JSON bitstream header information. The bitstream_len field gives the length of bitstream in bytes.
                                             
                                            fpgaReconfigureSlot() first checks whether the FPGA_ACCELERATOR corresponding to the FPGA_DEVICE in fme_handle is open. If it is open, then the programming request is aborted with an error code. The application may pass FPGA_RECONF_FORCE in the flags parameter in order to avoid this open check and forcefully program the bitstream.
                                             
                                            fpga_result fpgaReconfigureSlot(fpga_handle fme_handle,\nuint32_t slot,\nconst uint8_t *bitstream,\nsize_t bitstream_len,\nint flags);\n
                                             
                                            Figure 10 fpgaReconfigureSlot()
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#217-errors","title":"2.1.7 Errors","text":"
                                            The OPAE errors API provides a means to query and clear both FPGA_DEVICE and FPGA_ACCELERATOR errors. Each FPGA device exports a collection of error registers via the DFL drivers\u2019 sysfs tree, for both the FME and the Port. Each register is typically an unsigned 64-bit mask of the current errors, where each bit or some collection of bits specifies an error type. An error is signaled if its bit or collection of bits is non-zero. Note that the 32-bit error index may vary from one process execution to the next. Applications should use fpgaGetErrorInfo() and examine the error name returned in the struct fpga_error_info to identify the desired 64-bit error mask.
                                             
                                            struct fpga_error_info {\nchar name[FPGA_ERROR_NAME_MAX];\nbool can_clear;\n};\n
                                             
                                            Figure 11 struct fpga_error_info
                                             
                                            Each 64-bit mask of errors is assigned a unique 32-bit integer index and a unique name. Given an fpga_token and an error index, fpgaGetErrorInfo() retrieves the struct fpga_error_info corresponding to the error.
                                             
                                            fpga_result fpgaGetErrorInfo(fpga_token token,\nuint32_t error_num,\nstruct fpga_error_info *error_info);\n
                                             
                                            Figure 12 fpgaGetErrorInfo()
                                             
                                            fpgaReadError() provides access to the raw 64-bit error mask, given the unique error index. fpgaClearError() clears the errors for a particular index. fpgaClearAllErrors() clears all the errors for the given fpga_token.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#218-metrics","title":"2.1.8 Metrics","text":"
                                            The OPAE metrics API refers to a group of functions and data structures that allow querying the various device metrics from the Board Management Controller component of the FPGA device. A metric is described by an instance of struct fpga_metric_info.
                                             
                                            typedef struct fpga_metric_info {\nuint64_t metric_num;\nfpga_guid metric_guid;\nchar qualifier_name[FPGA_METRIC_STR_SIZE];\nchar group_name[FPGA_METRIC_STR_SIZE];\nchar metric_name[FPGA_METRIC_STR_SIZE];\nchar metric_units[FPGA_METRIC_STR_SIZE];\nenum fpga_metric_datatype metric_datatype;\nenum fpga_metric_type metric_type;\n} fpga_metric_info;\n
                                             Relevant Links:  - fpga_metric_datatype - fpga_metric_type
                                             
                                            Figure 13 fpga_metric_info
                                             
                                            The group_name field holds a string describing the broad categorization of the metric. Some sample values for group_name are \u201cthermal_mgmt\u201d and \u201cpower_mgmt\u201d. The metric_name field contains the metric\u2019s name. The number and names of metrics may vary from one FPGA platform to the next. The qualifier_name field is a concatenation of group_name and metric_name, with a colon character in between. The metric_units field contains the string name of the unit of measurement for the specific metric. Some examples for metric_units are \u201cVolts\u201d, \u201cAmps\u201d, and \u201cCelsius\u201d.
                                             
                                            The metric_datatype field uniquely identifies the underlying C data type for the metric\u2019s value:
                                             
                                            enum fpga_metric_datatype {\nFPGA_METRIC_DATATYPE_INT,\nFPGA_METRIC_DATATYPE_FLOAT,\nFPGA_METRIC_DATATYPE_DOUBLE,\nFPGA_METRIC_DATATYPE_BOOL,\nFPGA_METRIC_DATATYPE_UNKNOWN\n};\n
                                             
                                            Figure 14 enum fpga_metric_datatype
                                             
                                            The metric_type field classifies the metric into a broad category. This information is redundant with the group_name field.
                                             
                                            enum fpga_metric_type {\nFPGA_METRIC_TYPE_POWER,\nFPGA_METRIC_TYPE_THERMAL,\nFPGA_METRIC_TYPE_PERFORMANCE_CTR,\nFPGA_METRIC_TYPE_AFU,\nFPGA_METRIC_TYPE_UNKNOWN\n};\n
                                             
                                            Figure 15 enum fpga_metric_type
                                             
                                            In order to enumerate the information for each of the metrics available from the FPGA device, determine the number of metrics using fpgaGetNumMetrics().
                                             
                                            uint64_t num_metrics = 0;\nfpgaGetNumMetrics(handle, &num_metrics);\n
                                             
                                            Figure 16 Determining Number of Metrics
                                             
                                            This call retrieves the number of available metrics for the FPGA_DEVICE that is opened behind the handle parameter to the call. Refer to 2.1.3 Access for information about the fpgaOpen() call. When the number of available metrics is known, allocate a buffer large enough to hold that many fpga_metric_info data structures, and call fpgaGetMetricsInfo() to populate the entries:
                                             
                                            fpga_metric_info *metric_info;\nuint64_t metric_infos = num_metrics;\nmetric_info = calloc(num_metrics, sizeof(fpga_metric_info));\nfpgaGetMetricsInfo(handle, metric_info, &metric_infos);\n
                                             
                                            Figure 17 Querying Metrics Info
                                             
                                            The fpga_metric structure is the representation of a metric\u2019s value:
                                             
                                            typedef struct fpga_metric {\nuint64_t metric_num;\nmetric_value value;\nbool isvalid;\n} fpga_metric;\n
                                             Relevant Links: - metric_value
                                             
                                            Figure 18 struct fpga_metric
                                             
                                            The metric_num field matches the metric_num field of the fpga_metric_info structure. value contains the metric value, which is encoded in the C data type identified by the metric_datatype field of fpga_metric_info. Finally, the isvalid field denotes whether the metric value is valid.
                                             
                                            There are two methods of obtaining a metric\u2019s value, given the information in the fpga_metric_info structure:
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2181-querying-metric-values-by-index","title":"2.1.8.1 Querying Metric Values by Index","text":"
                                            fpgaGetMetricsByIndex() retrieves a metric value using the metric_num field of the metric info:
                                             
                                            uint64_t metric_num = metric_info[0]->metric_num;\nfpga_metric metric0;\nfpgaGetMetricsByIndex(handle, &metric_num, 1, &metric0);\n
                                             
                                            Figure 19 Retrieve Metric by Index
                                             
                                            This call allows retrieving one or more metric values, each identified by their unique metric_num. The second and fourth parameters allow passing arrays so that multiple values can be fetched in a single call.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2182-querying-metric-values-by-name","title":"2.1.8.2 Querying Metric Values by Name","text":"
                                            fpgaGetMetricsByName() retrieves a metric value using the metric_name field of the metric info:
                                             
                                            char *metric_name = metric_info[1]->metric_name;\nfpga_metric metric1;\nfpgaGetMetricsByName(handle, &metric_name, 1, &metric1);\n
                                             
                                            This call also allows retrieving one or more metric values, each identified by their unique metric_name. The second and fourth parameters allow passing arrays so that multiple values can be fetched in a single call.
                                             
                                            The fpgaGetMetricsThresholdInfo() call is provided for legacy implementations only. It should be considered deprecated for current and future FPGA designs.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#219-sysobject","title":"2.1.9 SysObject","text":"
                                            When the hardware access method in use is the DFL drivers (see 2.3.2 libxfpga Plugin), the sysfs tree rooted at the struct _fpga_token\u2019s sysfspath member is accessible via the OPAE SDK SysObject API. The SysObject API provides an abstraction to search, traverse, read, and write sysfs entities. These sysfs entities may take the form of directories, which are referred to as containers, or files, which are referred to as attributes. Figure 20 enum fpga_sysobject_type shows the API\u2019s means of distinguishing between the two types.
                                             
                                            enum fpga_sysobject_type {\nFPGA_OBJECT_CONTAINER = (1u << 0),\nFPGA_OBJECT_ATTRIBUTE = (1u << 1)\n};\n
                                             
                                            Figure 20 enum fpga_sysobject_type
                                             
                                            The SysObject API introduces another opaque structure type, fpga_object. An fpga_object can be queried from an fpga_token or an fpga_handle by way of the fpgaTokenGetObject() and fpgaHandleGetObject() API\u2019s.
                                             
                                            fpga_result fpgaTokenGetObject(fpga_token token, const char *name,\nfpga_object *object, int flags);\nfpga_result fpgaHandleGetObject(fpga_handle handle, const char *name,\nfpga_object *object, int flags);\n
                                             
                                            Figure 21 fpgaTokenGetObject() / fpgaHandleGetObject()
                                             
                                            The remainder of the SysObject API is broken into two categories of calls, depending on the fpga_object\u2019s type. The type of an fpga_object is learned via fpgaObjectGetType().
                                             
                                            fpga_result fpgaObjectGetType(fpga_object obj,\nenum fpga_sysobject_type *type);\n
                                             
                                            Figure 22 fpgaObjectGetType()
                                             
                                            When an fpga_object is no longer needed, it should be freed via fpgaDestroyObject().
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2191-fpga_object_container-apis","title":"2.1.9.1 FPGA_OBJECT_CONTAINER API\u2019s","text":"
                                            For directory sysfs entities, passing a value of FPGA_OBJECT_RECURSE_ONE or FPGA_OBJECT_RECURSE_ALL in the flags parameter to fpgaTokenGetObject() or fpgaHandleGetObject() causes these two API\u2019s to treat the target object as either a single-layer or multi-layer directory structure, making its child entities available for query via fpgaObjectGetObject() and fpgaObjectGetObjectAt().
                                             
                                            fpga_result fpgaObjectGetObject(fpga_object parent, const char *name,\nfpga_object *object, int flags);\nfpga_result fpgaObjectGetObjectAt(fpga_object parent, size_t idx,\nfpga_object *object);\n
                                             
                                            Figure 23 fpgaObjectGetObject() / fpgaObjectGetObjectAt()
                                             
                                            Any child object resulting from fpgaObjectGetObject() or fpgaObjectGetObjectAt() must be freed via fpgaDestroyObject() when it is no longer needed.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2192-fpga_object_attribute-apis","title":"2.1.9.2 FPGA_OBJECT_ATTRIBUTE API\u2019s","text":"
                                            Attribute sysfs entities may be queried for their size and read from or written to. In order to determine the size of an attribute\u2019s data, use fpgaObjectGetSize().
                                             
                                            fpga_result fpgaObjectGetSize(fpga_object obj,\nuint32_t *value, int flags);\n
                                             
                                            Figure 24 fpgaObjectGetSize()
                                             
                                            Attributes containing arbitrary string data can be read with fpgaObjectRead().
                                             
                                            fpga_result fpgaObjectRead(fpga_object obj, uint8_t *buffer,\nsize_t offset, size_t len, int flags);\n
                                             
                                            Figure 25 fpgaObjectRead()
                                             
                                            If an attribute contains an unsigned integer value, its value can be read with fpgaObjectRead64() and written with fpgaObjectWrite64().
                                             
                                            fpga_result fpgaObjectRead64(fpga_object obj,\nuint64_t *value, int flags);\nfpga_result fpgaObjectWrite64(fpga_object obj,\nuint64_t value, int flags);\n
                                             
                                            Figure 26 fpgaObjectRead64() / fpgaObjectWrite64()
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2110-utilities","title":"2.1.10 Utilities","text":"
                                            The fpga_result enumeration defines a set of error codes used throughout OPAE. In order to convert an fpga_result error code into a printable string, the application can use the fpgaErrStr() call.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#22-dfl-driver-ioctl-interfaces","title":"2.2 DFL Driver IOCTL Interfaces","text":"
                                            The DFL drivers export an IOCTL interface which the libxfpga.so plugin consumes in order to query and configure aspects of the FME and Port. These interfaces are used only internally by the SDK; they are not customer-facing. The description here is provided for completeness only.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#221-port-reset","title":"2.2.1 Port Reset","text":"
                                            The DFL_FPGA_PORT_RESET ioctl is used by the fpgaReset() call in order to perform a Port reset. The fpga_handle passed to fpgaReset() must be a valid open handle to an FPGA_ACCELERATOR. The ioctl requires no input/output parameters.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#222-port-information","title":"2.2.2 Port Information","text":"
                                            The DFL_FPGA_PORT_GET_INFO ioctl is used to query properties of the Port, notably the number of associated MMIO regions. The ioctl requires a pointer to a struct dfl_fpga_port_info.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#223-mmio-region-information","title":"2.2.3 MMIO Region Information","text":"
                                            The DFL_FPGA_PORT_GET_REGION_INFO ioctl is used to query the details of an MMIO region. The ioctl requires a pointer to a struct dfl_fpga_port_region_info. The index field of the struct is populated by the caller, and the padding, size, and offset values are populated by the DFL driver.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#224-shared-memory-mapping-and-unmapping","title":"2.2.4 Shared Memory Mapping and Unmapping","text":"
                                            The DFL_FPGA_PORT_DMA_MAP ioctl is used to map a memory buffer into the application\u2019s process address space. The ioctl requires a pointer to a struct dfl_fpga_port_dma_map.
                                             
                                            The DFL_FPGA_PORT_DMA_UNMAP ioctl is used to unmap a memory buffer from the application\u2019s process address space. The ioctl requires a pointer to a struct dfl_fpga_port_dma_unmap.
                                             
                                            These ioctls provide the underpinnings of the fpgaPrepareBuffer() and fpgaReleaseBuffer() calls.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#225-number-of-port-error-irqs","title":"2.2.5 Number of Port Error IRQs","text":"
                                            The DFL_FPGA_PORT_ERR_GET_IRQ_NUM ioctl is used to query the number of Port error interrupt vectors available. The ioctl requires a pointer to a uint32_t that receives the Port error interrupt count.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#226-port-error-interrupt-configuration","title":"2.2.6 Port Error Interrupt Configuration","text":"
                                            The DFL_FPGA_PORT_ERR_SET_IRQ ioctl is used to configure one or more file descriptors for the Port Error interrupt. The ioctl requires a pointer to a struct dfl_fpga_irq_set. The values stored in the evtfds field of this struct should be populated with the event file descriptors for the interrupt, as returned by the eventfd() C standard library API.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#227-number-of-afu-interrupts","title":"2.2.7 Number of AFU Interrupts","text":"
                                            The DFL_FPGA_PORT_UINT_GET_IRQ_NUM ioctl is used to query the number of AFU interrupt vectors available. The ioctl requires a pointer to a uint32_t that receives the AFU interrupt count.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#228-user-afu-interrupt-configuration","title":"2.2.8 User AFU Interrupt Configuration","text":"
                                            The DFL_FPGA_PORT_UINT_SET_IRQ ioctl is used to configure one or more file descriptors for the AFU interrupt. The ioctl requires a pointer to a struct dfl_fpga_irq_set. The values stored in the evtfds field of this struct should be populated with the event file descriptors for the interrupt, as returned by the eventfd() C standard library API.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#229-partial-reconfiguration","title":"2.2.9 Partial Reconfiguration","text":"
                                            The DFL_FPGA_FME_PORT_PR ioctl is used to update the logic stored in the Port\u2019s programmable region. This ioctl must be issued on the device file descriptor corresponding to the FPGA_DEVICE (/dev/dfl-fme.X). The ioctl requires a pointer to a struct dfl_fpga_fme_port_pr with each of the fields populated.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2210-number-of-fme-error-irqs","title":"2.2.10 Number of FME Error IRQs","text":"
                                            The DFL_FPGA_FME_ERR_GET_IRQ_NUM ioctl is used to query the number of FME error interrupt vectors available. The ioctl requires a pointer to a uint32_t that receives the FME error interrupt count.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2211-fme-error-interrupt-configuration","title":"2.2.11 FME Error Interrupt Configuration","text":"
                                            The DFL_FPGA_FME_ERR_SET_IRQ ioctl is used to configure one or more file descriptors for the FME Error interrupt. The ioctl requires a pointer to a struct dfl_fpga_irq_set. The values stored in the evtfds field of this struct should be populated with the event file descriptors for the interrupt, as returned by the eventfd() C standard library API. as returned by the eventfd() C standard library API.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#23-plugin-manager","title":"2.3 Plugin Manager","text":"
                                            The OPAE Plugin Manager refers to initialization code in libopae-c that examines an FPGA device\u2019s PCIe Vendor and Device ID and makes an association between a particular FPGA device and its access method. OPAE currently supports three device access methods:
                                             
                                            Access Method
                                             
                                            Plugin Module
                                             Device Feature List drivers libxfpga.so Virtual Function I/O libopae-v.so AFU Simulation Environment libase.so 
                                            Table 9 Plugin Device Access Methods
                                             
                                            The Plugin Manager allows code that is written to a specific API signature to access FPGA hardware via different mechanisms. In other words, the end user codes to the OPAE API; and the OPAE API, based on configuration data, routes the hardware access to the device via different means.
                                             
                                            As an example, consider an API configuration that accesses FPGA device_A via the Device Feature List drivers and that accesses FPGA device_B via VFIO. The application is coded against the OPAE API.
                                             
                                            As part of its initialization process, the application enumerates and discovers an fpga_token corresponding to device_A. That fpga_token is opened and its MMIO region 0 is mapped via a call to fpgaMapMMIO().
                                             
                                            The API configuration for device_A is such that the fpga_handle corresponding to device_A routes its hardware access calls through libxfpga.so. The call to fpgaMapMMIO() is redirected to libxfpga.so\u2019s implementation of the MMIO mapping function, xfpga_fpgaMapMMIO(). As a result, the call to xfpga_fpgaMapMMIO() uses its AFU file descriptor to communicate with the DFL driver to map the MMIO region.
                                             
                                            Subsequently, the application enumerates and discovers an fpga_token corresponding to device_B. That fpga_token is opened and its MMIO region 0 is mapped via a call to fpgaMapMMIO().
                                             
                                            The API configuration for device_B is such that the fpga_handle corresponding to device_B routes its hardware access calls through libopae-v.so. The call to fpgaMapMMIO() is redirected to libopae-v.so\u2019s implementation of the MMIO mapping function, vfio_fpgaMapMMIO(). As a result, the call to vfio_fpgaMapMMIO() uses the MMIO mapping performed by libopaevfio.so during initialization of the VFIO session.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#231-plugin-model","title":"2.3.1 Plugin Model","text":"
                                            The OPAE SDK plugin model is facilitated by its use of opaque C structures for fpga_token and fpga_handle. These types are both declared as void *; and this allows the parameters to the OPAE SDK functions to take different forms, depending on the layer of the SDK being used.
                                             
                                            At the topmost layer, for example when calling fpgaEnumerate(), the output fpga_token parameter array is actually an array of pointers to opae_wrapped_token struct\u2019s.
                                             
                                            typedef struct _opae_wrapped_token {\nuint32_t magic;\nfpga_token opae_token;\nuint32_t ref_count;\nstruct _opae_wrapped_token *prev;\nstruct _opae_wrapped_token *next;\nopae_api_adapter_table *adapter_table;\n} opae_wrapped_token;\n
                                             
                                            Figure 27 opae_wrapped_token
                                             
                                            An opae_wrapped_token, as the name suggests, is a thin wrapper around the lower-layer token which is stored in struct member opae_token. The adapter_table struct member is a pointer to a plugin-specific adapter table. The adapter table provides a mapping between the top-layer opae_wrapped_token and its underlying plugin-specific API entry points, which are called using the opae_token struct member (the lower-level token).
                                             
                                            typedef struct _opae_api_adapter_table {\nstruct _opae_api_adapter_table *next;\nopae_plugin plugin;\n...\nfpga_result (*fpgaEnumerate)(const fpga_properties *filters,\nuint32_t num_filters,\nfpga_token *tokens,\nuint32_t max_tokens,\nuint32_t *num_matches);\n...\nint (*initialize)(void);\nint (*finalize)(void);\n} opae_api_adapter_table;\n
                                             
                                            Figure 28 opae_api_adapter_table
                                             
                                            When libopae-c loads, the plugin manager uses the plugin configuration data to open and configure a session to each of the required plugin libraries. During this configuration process, each plugin is passed an empty adapter table struct. The purpose of the plugin configuration is to populate this adapter table struct with each of the plugin-specific API entry points.
                                             
                                            When the top-level fpgaEnumerate() is called, each adapter table\u2019s plugin-specific fpgaEnumerate() struct member is called; and the output fpga_token\u2019s are collected. At this point, these fpga_token\u2019s are the lower-level token structure types. Before the top-level fpgaEnumerate() returns, these plugin-specific tokens are wrapped inside opae_wrapped_token structures, along with a pointer to the respective adapter table.
                                             
                                            After enumeration is complete, the application goes on to call other top-level OPAE SDK functions with the wrapped tokens. Each top-level entry point which accepts an fpga_token knows that it is actually being passed an opae_wrapped_token. With this knowledge, the entry point peeks inside the wrapped token and calls through to the plugin-specific API entry point using the adapter table, passing the lower-level opae_token struct member.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#232-libxfpga-plugin","title":"2.3.2 libxfpga Plugin","text":"
                                            2.3.1 Plugin Model introduced the concept of an opae_wrapped_token and a corresponding plugin-specific token structure. libxfpga.so is the plugin library that implements the DFL driver hardware access method. Its plugin-specific token data structure is struct _fpga_token.
                                             
                                            struct _fpga_token {\nfpga_token_header hdr;\nuint32_t device_instance;\nuint32_t subdev_instance;\nchar sysfspath[SYSFS_PATH_MAX];\nchar devpath[DEV_PATH_MAX];\nstruct error_list *errors;\n};\n
                                             Relevant Links: - fpga_token_header - error_list
                                             
                                            Figure 29 struct _fpga_token
                                             
                                            A struct _fpga_token corresponding to the Port will have sysfspath and devpath members that contain strings like the following example paths:
                                             
                                            sysfspath: \u201c/sys/class/fpga_region/region0/dfl-port.0\u201d\ndevpath: \u201c/dev/dfl-port.0\u201d\n
                                             
                                            Figure 30 libxfpga Port Token
                                             
                                            Likewise, a struct _fpga_token corresponding to the FME will have sysfspath and devpath members that contain strings like the following example paths:
                                             
                                            sysfspath: \u201c/sys/class/fpga_region/region0/dfl-fme.0\u201d\ndevpath: \u201c/dev/dfl-fme.0\u201d\n
                                             
                                            Figure 31 libxfpga FME Token
                                             
                                            When a call to the top-level fpgaOpen() is made, the lower-level token is unwrapped and passed to xfpga_fpgaOpen(). In return, xfpga_fpgaOpen() opens the character device file identified by the devpath member of the struct _fpga_token. It then allocates and initializes an instance of libxfpga.so\u2019s plugin-specific handle data structure, struct _fpga_handle.
                                             
                                            struct _fpga_handle {\npthread_mutex_t lock;\nuint64_t magic;\nfpga_token token;\nint fddev;\nint fdfpgad;\nuint32_t num_irqs;\nuint32_t irq_set;\nstruct wsid_tracker *wsid_root;\nstruct wsid_tracker *mmio_root;\nvoid *umsg_virt;\nuint64_t umsg_size;\nuint64_t *umsg_iova;\nbool metric_enum_status;\nfpga_metric_vector fpga_enum_metric_vector;\nvoid *bmc_handle;\nstruct _fpga_bmc_metric *_bmc_metric_cache_value;\nuint64_t num_bmc_metric;\nuint32_t flags;\n};\n
                                             Relevant Links: - wsid_tracker - fpga_metric_vector - _fpga_bmc_metric
                                             
                                            Figure 32 struct _fpga_handle
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#233-libopae-v-plugin","title":"2.3.3 libopae-v Plugin","text":"
                                            libopae-v.so is the plugin library that implements the VFIO hardware access method. Its plugin-specific token data structure is vfio_token.
                                             
                                            #define USER_MMIO_MAX 8\ntypedef struct _vfio_token {\nfpga_token_header hdr;\nfpga_guid compat_id;\npci_device_t *device;\nuint32_t region;\nuint32_t offset;\nuint32_t mmio_size;\nuint32_t pr_control;\nuint32_t user_mmio_count;\nuint32_t user_mmio[USER_MMIO_MAX];\nuint64_t bitstream_id;\nuint64_t bitstream_mdata;\nuint8_t num_ports;\nstruct _vfio_token *parent;\nstruct _vfio_token *next;\nvfio_ops ops;\n} vfio_token;\n
                                             Relevant Links: - fpga_token_header - pci_device_t - vfio_ops
                                             
                                            Figure 33 vfio_token
                                             
                                            When a call to the top-level fpgaOpen() is made, the lower-level token is unwrapped and passed to vfio_fpgaOpen(). In return, vfio_fpgaOpen() opens the VFIO device matching the device address found in the input vfio_token. It then allocates and initializes an instance of libopae-v.so\u2019s plugin-specific handle data structure, vfio_handle.
                                             
                                            typedef struct _vfio_handle {\nuint32_t magic;\nstruct _vfio_token *token;\nvfio_pair_t *vfio_pair;\nvolatile uint8_t *mmio_base;\nsize_t mmio_size;\npthread_mutex_t lock;\n#define OPAE_FLAG_HAS_AVX512 (1u << 0)\nuint32_t flags;\n} vfio_handle;\n
                                             Relevant Links: - vfio_token - vfio_pair_t
                                             
                                            Figure 34 vfio_handle
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2331-supporting-libraries","title":"2.3.3.1 Supporting Libraries","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#23311-libopaevfio","title":"2.3.3.1.1 libopaevfio","text":"
                                            libopaevfio.so is OPAE\u2019s implementation of the Linux kernel\u2019s Virtual Function I/O interface. This VFIO interface presents a generic means of configuring and accessing PCIe endpoints from a user-space process via a supporting Linux kernel device driver, vfio-pci.
                                             
                                            libopaevfio.so provides APIs for opening/closing a VFIO device instance, for mapping/unmapping MMIO spaces, for allocating/freeing DMA buffers, and for configuring interrupts for the device.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#23312-libopaemem","title":"2.3.3.1.2 libopaemem","text":"
                                            Each DMA buffer allocation made by libopaevfio.so\u2019s opae_vfio_buffer_allocate() and opae_vfio_buffer_allocate_ex() APIs requires a backing I/O Virtual Address range. These address ranges are discovered at VFIO device open time by way of the VFIO_IOMMU_GET_INFO ioctl.
                                             
                                            Each range specifies a large contiguous block of I/O Virtual Address space. The typical DMA buffer allocation size is significantly less than one of these IOVA blocks, so the division of each block into allocatable segments must be managed so that multiple DMA buffer allocations can be made from a single block. In other words, the IOVA blocks must be memory-managed in order to make efficient use of them.
                                             
                                            libopaemem.so provides a generic means of managing a large memory space, consisting of individual large memory blocks of contiguous address space. When a DMA buffer allocation is requested, libopaevfio.so uses this generic memory manager to carve out a small chunk of contiguous IOVA address space in order for the DMA mapping to be made. The IOVA space corresponding to the allocation is marked as allocated, and the rest of the large block remains as allocatable space within the memory manager. Subsequent de-allocation returns a chunk of IOVA space to the free state, coalescing contiguous chunks as they are freed. The allocations and de-allocations of the IOVA space can occur in any order with respect to each other. libopaemem.so tracks both the allocated and free block space, carving out small chunks from the large IOVA blocks on allocations, and coalescing small chunks back into larger ones on frees.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2332-configuring-pcie-virtual-functions","title":"2.3.3.2 Configuring PCIe Virtual Functions","text":"
                                            Before an AFU can be accessed with VFIO, the FPGA Physical Function must be configured to enable its Virtual Functions. Then, each VF must be bound to the vfio-pci Linux kernel driver.
                                             
                                            As of the Arrow Creek program, the FPGA hardware allows multiple AFU\u2019s to co-exist by placing each AFU in its own PCIe Virtual Function. Upon system startup, no PCIe VF\u2019s exist. The pci_device command can be used to enable the VF\u2019s and their AFU\u2019s. First, use the lspci command to examine the current device topology:
                                             
                                            # lspci | grep cel\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nb1:00.1 Processing accelerators: Intel Corporation Device bcce\nb1:00.2 Processing accelerators: Intel Corporation Device bcce\nb1:00.3 Processing accelerators: Intel Corporation Device bcce\nb1:00.4 Processing accelerators: Intel Corporation Device bcce\n
                                             
                                            Figure 35 lspci Device Topology
                                             
                                            In this example, VF\u2019s are controlled by PF 0, as highlighted in Figure 35 lspci Device Topology. In the figure, each PF is shown as having the Arrow Creek PF PCIe device ID of bcce.
                                             
                                            Now, use the pci_device command to enable three VF\u2019s for PF0:
                                             
                                            # pci_device 0000:b1:00.0 vf 3\n# lspci | grep cel\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nb1:00.1 Processing accelerators: Intel Corporation Device bcce\nb1:00.2 Processing accelerators: Intel Corporation Device bcce\nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device\nb1:00.4 Processing accelerators: Intel Corporation Device bcce\nb1:00.5 Processing accelerators: Intel Corporation Device bccf\nb1:00.6 Processing accelerators: Intel Corporation Device bccf\nb1:00.7 Processing accelerators: Intel Corporation Device bccf\n
                                             
                                            Figure 36 Enable Virtual Functions
                                             
                                            Figure 20 Enable Virtual Functions shows that three VF\u2019s were created. Each VF is shown as having the Arrow Creek VF PCIe device ID of bccf.
                                             
                                            Now, each Virtual Function must be bound to the vfio-pci Linux kernel driver so that it can be accessed via VFIO:
                                             
                                            # opaevfio -i -u myuser -g mygroup 0000:b1:00.5\nBinding (0x8086,0xbccf) at 0000:b1:00.5 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.5 is 318\n
                                             
                                            Figure 37 Bind VF's to vfio-pci
                                             
                                            Here, myuser and mygroup identify the unprivileged user/group that requires access to the device. The opaevfio command will change the ownership of the device per the values given.
                                             
                                            Once the VF\u2019s are bound to vfio-pci, the OPAE SDK will find and enumerate them with libopae-v.so:
                                             
                                            # fpgainfo port\n//****** PORT ******//\nObject Id : 0xEF00000\nPCIe s:b:d.f : 0000:B1:00.0\nDevice Id : 0xBCCE\nSocket Id : 0x00\n//****** PORT ******//\nObject Id : 0xE0B1000000000000\nPCIe s:b:d.f : 0000:B1:00.7\nDevice Id : 0xBCCF\nSocket Id : 0x01\nAccelerator GUID : 4dadea34-2c78-48cb-a3dc-5b831f5cecbb\n//****** PORT ******//\nObject Id : 0xC0B1000000000000\nPCIe s:b:d.f : 0000:B1:00.6\nDevice Id : 0xBCCF\nSocket Id : 0x01\nAccelerator GUID : 823c334c-98bf-11ea-bb37-0242ac130002\n//****** PORT ******//\nObject Id : 0xA0B1000000000000\nPCIe s:b:d.f : 0000:B1:00.5\nDevice Id : 0xBCCF\nSocket Id : 0x01\nAccelerator GUID : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n
                                             
                                            Figure 38 List VF's with fpgainfo
                                             
                                            When the VF\u2019s are no longer needed, they can be unbound from the vfio-pci driver:
                                             
                                            # opaevfio -r 0000:b1:00.5\nReleasing (0x8086,0xbccf) at 0000:b1:00.5 from vfio-pci\n# opaevfio -r 0000:b1:00.6\nReleasing (0x8086,0xbccf) at 0000:b1:00.6 from vfio-pci\n# opaevfio -r 0000:b1:00.7\nReleasing (0x8086,0xbccf) at 0000:b1:00.7 from vfio-pci\n
                                             
                                            Figure 39 Unbind VF's from vfio-pci
                                             
                                            Finally, the VF\u2019s can be disabled:
                                             
                                            # pci_device 0000:b1:00.0 vf 0\n# lspci | grep cel\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nb1:00.1 Processing accelerators: Intel Corporation Device bcce\nb1:00.2 Processing accelerators: Intel Corporation Device bcce\nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device\nb1:00.4 Processing accelerators: Intel Corporation Device bcce\n
                                             
                                            Figure 40 Disable Virtual Functions
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#24-application-flow","title":"2.4 Application Flow","text":"
                                            A typical OPAE application that interacts with an AFU via MMIO and shared memory will have a flow similar to the one described in this section.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#241-create-filter-criteria","title":"2.4.1 Create Filter Criteria","text":"
                                            Refer to 2.1.2 Enumeration. When enumerating AFU\u2019s, if no filtering criteria is specified, then fpgaEnumerate() returns fpga_token\u2019s for each AFU that is present in the system. In order to limit the enumeration search to a specific AFU, create an fpga_properties object and set its guid to that of the desired AFU:
                                             
                                            #define MY_AFU_GUID \u201c57fa0b03-ab4f-4b02-b4eb-d3fe1ec18518\u201d\nfpga_properties filter = NULL;\nfpga_guid guid;\nfpgaGetProperties(NULL, &filter);\nuuid_parse(MY_AFU_GUID, guid);\nfpgaPropertiesSetGUID(filter, guid);\n
                                             
                                            Figure 41 Flow: Create Filter Criteria
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#242-enumerate-the-afu","title":"2.4.2 Enumerate the AFU","text":"
                                            With the filtering criteria in place, enumerate to obtain an fpga_token for the AFU:
                                             
                                            fpga_token afu_token = NULL;\nuint32_t num_matches = 0;\nfpgaEnumerate(&filter, 1, &afu_token, 1, &num_matches);\n
                                             
                                            Figure 42 Flow: Enumerate the AFU
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#243-open-the-afu","title":"2.4.3 Open the AFU","text":"
                                            After finding an fpga_token for the AFU using fpgaEnumerate(), the token must be opened with fpgaOpen() to establish a session with the AFU. The process of opening an fpga_token creates an fpga_handle:
                                             
                                            fpga_handle afu_handle = NULL;\nfpgaOpen(afu_token, &afu_handle, 0);\n
                                             
                                            Figure 43 Flow: Open the AFU
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#244-map-mmio-region","title":"2.4.4 Map MMIO Region","text":"
                                            In order to access the MMIO region of the AFU to program its CSR\u2019s, the region must first be mapped into the application\u2019s process address space. This is accomplished using fpgaMapMMIO():
                                             
                                            uint32_t region = 0;\nfpgaMapMMIO(afu_handle, region, NULL);\n
                                             
                                            Figure 44 Flow: Map MMIO Region
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#245-allocate-dma-buffers","title":"2.4.5 Allocate DMA Buffers","text":"
                                            If the AFU is DMA-capable, shared memory buffers can be allocated and mapped into the process address space and the IOMMU with fpgaPrepareBuffer(). Refer to Figure 8 Configuring Huge Pages for instructions on configuring 2MiB and 1GiB huge pages.
                                             
                                            #define BUF_SIZE (2 * 1024 * 1024)\nvolatile uint64_t *src_ptr = NULL;\nuint64_t src_wsid = 0;\nvolatile uint64_t *dest_ptr = NULL;\nuint64_t dest_wsid = 0;\nfpgaPrepareBuffer(afu_handle, BUF_SIZE, (void **)&src_ptr,\n&src_wsid, 0);\nfpgaPrepareBuffer(afu_handle, BUF_SIZE, (void **)&dest_ptr,\n&dest_wsid, 0);\nmemset(src_ptr, 0xaf, BUF_SIZE);\nmemset(dest_ptr, 0xbe, BUF_SIZE);\n
                                             
                                            Figure 45 Flow: Allocate DMA Buffers
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#246-make-afu-aware-of-dma-buffers","title":"2.4.6 Make AFU Aware of DMA Buffers","text":"
                                            The process by which locations of shared memory buffers and their sizes are made known to the AFU is entirely AFU-specific. This example shows the method used by the Native Loopback AFU. Each buffer I/O virtual address is cacheline-aligned and programmed into a unique AFU CSR; then the buffer size in lines is programmed into a length CSR:
                                             
                                            #define CSR_SRC_ADDR 0x000A // AFU-specific\n#define CSR_DEST_ADDR 0x000B // AFU-specific\n#define CSR_NUM_LINES 0x000C // AFU-specific\nuint64_t src_iova = 0;\nuint64_t dest_iova = 0;\nfpgaGetIOAddress(afu_handle, src_wsid, &src_iova);\nfpgaGetIOAddress(afu_handle, dest_wsid, &dest_iova);\nfpgaWriteMMIO64(afu_handle, 0, CSR_SRC_ADDR, src_iova >> 6);\nfpgaWriteMMIO64(afu_handle, 0, CSR_DEST_ADDR, dest_iova >> 6);\nfpgaWriteMMIO32(afu_handle, 0, CSR_NUM_LINES, BUF_SIZE / 64);\n
                                             
                                            Figure 46 Flow: Make AFU Aware of DMA Buffers
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#247-initiate-an-acceleration-task","title":"2.4.7 Initiate an Acceleration Task","text":"
                                            With the shared buffer configuration complete, the AFU can be told to initiate the acceleration task. This process is AFU-specific. The Native Loopback AFU starts the acceleration task by writing a value to its control CSR:
                                             
                                            #define CSR_CTRL 0x000D // AFU-specific\nfpgaWriteMMIO32(afu_handle, 0, CSR_CTRL, 3);\n
                                             
                                            Figure 47 Initiate an Acceleration Task
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#248-wait-for-task-completion","title":"2.4.8 Wait for Task Completion","text":"
                                            Once the acceleration task is initiated, the application may poll the AFU for a completion status. This process is AFU-specific. The AFU may provide a status CSR for the application to poll; or the AFU may communicate status to the application by means of a result code written to a shared buffer.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#249-free-dma-buffers","title":"2.4.9 Free DMA Buffers","text":"
                                            When the acceleration task completes and the AFU is quiesced such that there are no outstanding memory transactions targeted for the shared memory, the DMA buffers can be unmapped and freed using fpgaReleaseBuffer():
                                             
                                            fpgaReleaseBuffer(afu_handle, src_wsid);\nfpgaReleaseBuffer(afu_handle, dest_wsid);\n
                                             
                                            Figure 48 Flow: Free DMA Buffers
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2410-unmap-mmio-region","title":"2.4.10 Unmap MMIO Region","text":"
                                            The MMIO regions should also be unmapped using fpgaUnmapMMIO():
                                             
                                            fpgaUnmapMMIO(afu_handle, region);\n
                                             
                                             
                                            Figure 49 Flow: Unmap MMIO Region
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2411-close-the-afu","title":"2.4.11 Close the AFU","text":"
                                            The AFU handle should be closed via fpgaClose() to release its resources:
                                             
                                             
                                            fpgaClose(afu_handle);\n
                                             
                                             
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#2412-release-the-tokens-and-properties","title":"2.4.12 Release the Tokens and Properties","text":"
                                            The fpga_token\u2019s returned by fpgaEnumerate() should be destroyed using the fpgaDestroyToken() API. The fpga_properties objects should be destroyed using the fpgaDestroyProperties() API:
                                             
                                            fpgaDestroyToken(&afu_token);\nfpgaDestroyProperties(&filter);\n
                                             
                                            Figure 51 Flow: Release the Tokens and Properties
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#30-opae-c-api","title":"3.0 OPAE C++ API","text":"
                                            The OPAE C++ API refers to a C++ layer that sits on top of the OPAE C API, providing object-oriented implementations of the main OPAE C API abstractions: properties, tokens, handles, dma buffers, etc. Like the OPAE C API, the C++ API headers contain Doxygen markup for each of the provided classes.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#31-libopae-cxx-core","title":"3.1 libopae-cxx-core","text":"
                                            The implementation files for the C++ API are compiled into libopae-cxx-core.so. A convenience header, core.h, provides a quick means of including each of the C++ API headers. Each of the types comprising the C++ API is located within the opae::fpga::types C++ namespace.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#311-properties","title":"3.1.1 Properties","text":"
                                            Class properties provides the C++ implementation of the fpga_properties type and its associated APIs.
                                             
                                            properties::ptr_t filter = properties::get();\n
                                             
                                            Figure 52 C++ Create New Empty Properties
                                             
                                            Class properties provides member variables for each fpga_properties item that can be manipulated with fpgaPropertiesGet\u2026() and fpgaPropertiesSet\u2026(). For example, to set the AFU ID in a properties instance and to set that instance\u2019s type to FPGA_ACCELERATOR:
                                             
                                            #define MY_AFU_ID \u201c8ad74241-d13b-48eb-b428-7986dcbcab14\u201d\nfilter->guid.parse(MY_AFU_ID);\nfilter->type = FPGA_ACCELERATOR;\n
                                             
                                            Figure 53 C++ Properties Set GUID and Type
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#312-tokens","title":"3.1.2 Tokens","text":"
                                            Class token provides the C++ implementation of the fpga_token type and its associated APIs. Class token also provides the enumerate() static member function:
                                             
                                            std::vector<token::ptr_t> tokens = token::enumerate({filter});\nif (tokens.size() < 1) {\n// flag error and return\n}\ntoken::ptr_t tok = tokens[0];\n
                                             
                                            Figure 54 C++ Enumeration
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#313-handles","title":"3.1.3 Handles","text":"
                                            Class handle provides the C++ implementation of the fpga_handle type and its associated APIs. The handle class provides member functions for opening and closing a token, for reading and writing to MMIO space, and for reconfiguring the FPGA\u2019s Programmable Region.
                                             
                                            handle::ptr_t accel = handle::open(tok, 0);\n
                                             
                                            Figure 55 C++ Opening a Handle
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#314-shared-memory","title":"3.1.4 Shared Memory","text":"
                                            The shared_buffer class provides member functions for allocating and releasing DMA buffers, for querying buffer attributes, and for reading and writing buffers.
                                             
                                            #define BUF_SIZE (2 * 1024 * 1024)\nshared_buffer::ptr_t input = shared_buffer::allocate(accel, BUF_SIZE);\nshared_buffer::ptr_t output = shared_buffer::allocate(accel, BUF_SIZE);\nstd::fill_n(input->c_type(), BUF_SIZE, 0xaf);\nstd::fill_n(output->c_type(), BUF_SIZE, 0xbe);\n
                                             
                                            Figure 56 C++ Allocate and Init Buffers
                                             
                                            Once DMA buffers have been allocated, their IO addresses are programmed into AFU-specific CSRs to enable the DMA. Here, the IO address of each buffer is aligned to the nearest cache line before programming it into the AFU CSR space. The number of cache lines is then programmed into the appropriate AFU CSR.
                                             
                                            #define CSR_SRC_ADDR 0x000A // AFU-specific\n#define CSR_DEST_ADDR 0x000B // AFU-specific\n#define CSR_NUM_LINES 0x000C // AFU-specific\n#define LOG2_CL 6\naccel->write_csr64(CSR_SRC_ADDR, input->io_address() >> LOG2_CL);\naccel->write_csr64(CSR_DEST_ADDR, output->io_address() >> LOG2_CL);\naccel->write_csr32(CSR_NUM_LINES, BUF_SIZE / 64);\n
                                             
                                            Figure 57 C++ Make the AFU Aware of DMA Buffers
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#315-events","title":"3.1.5 Events","text":"
                                            The event class provides member functions for event registration. In order to register an event, provide the handle::ptr_t for the desired device, along with the event type and optional flags.
                                             
                                            int vect = 2;\nevent::ptr_t evt = event::register_event(accel,\nFPGA_EVENT_INTERRUPT,\nvect);\nint evt_fd = evt.os_object();\n
                                             
                                            Figure 58 C++ Event Registration
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#316-errors","title":"3.1.6 Errors","text":"
                                            Class error provides a means of querying the device errors given a token::ptr_t. The token and integer ID provided to the error::get() static member function uniquely identify one of the 64-bit error masks associated with the token.
                                             
                                            error::ptr_t err = error::get(tok, 0); \n
                                             
                                            Figure 59 C++ Query Device Errors
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#317-sysobject","title":"3.1.7 SysObject","text":"
                                            Class sysobject is the C++ implementation of the OPAE SysObject API. sysobject provides a means of creating class instances via its two sysobject::get() static member functions. A third non-static sysobject::get() enables creating a sysobject instance given a parent sysobject instance. The read64() and write64() member functions allow reading and writing the sysobject\u2019s value as a 64-bit unsigned integer. The bytes() member functions allow reading a sysobject\u2019s value as a raw byte stream.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#40-opae-python-api","title":"4.0 OPAE Python API","text":"
                                            The OPAE Python API is built on top of the OPAE C++ Core API and its object model. Because of this, developing OPAE applications in Python is very similar to developing OPAE applications in C++ which significantly reduces the learning curve required to adapt to the Python API. While the object model remains the same, some static factory functions in the OPAE C++ Core API have been moved to module level methods in the OPAE Python API with the exception of the properties class. The goal of the OPAE Python API is to enable fast prototyping, test automation, infrastructure managment, and an easy to use framework for FPGA resource interactions that don\u2019t rely on software algorithms with a high runtime complexity.
                                             
                                            The major benefits of using pybind11 for developing the OPAE Python API include, but are not limited to, the following features of pybind11:
                                             
                                               
                                            • Uses C++ 11 standard library although it can use C++ 14 or C++17.
                                              •  
                                            • Automatic conversions of shared_ptr types
                                              •  
                                            • Built-in support for numpy and Eigen numerical libraries
                                              •  
                                            • Interoperable with the Python C API
                                              •  
                                             
                                            Currently, the only Python package that is part of OPAE is opae.fpga. Because opae.fpga is built on top of the opae-cxx-core API, it does require that the runtime libraries for both opae-cxx-core and opae-c be installed on the system (as well as any other libraries they depend on). Those libraries can be installed using the opae-libs package (from either RPM or DEB format - depending on your Linux distribution). Installation for the Python OPAE bindings are included in the Getting Started Guide for your platform. 
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#41-opae","title":"4.1 opae","text":"
                                            The Python API is coded as a pybind11 project, which allows C++ code to directly interface with Python internals. Each C++ API concept is encoded into a Python equivalent. The functionality exists as a Python extension module, compiled into _opae.so.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#411-enumeration","title":"4.1.1 Enumeration","text":"
                                            Enumeration is somewhat simplified as compared to the OPAE C/C++ APIs. The fpga.enumerate() function accepts keyword arguments for each of the property names that are defined in the C++ API. As an example, to enumerate for an FPGA_ACCELERATOR by its GUID:
                                             
                                            from opae import fpga\nMY_ACCEL = \u201cd573b29e-176f-4cb7-b810-efbf7be34cc9\u201d\ntokens = fpga.enumerate(type=fpga.ACCELERATOR, guid=MY_ACCEL)\nassert tokens, \u201cNo accelerator matches {}\u201d.format(MY_ACCEL)\n
                                             
                                            Figure 60 Python Enumeration
                                             
                                            The return value from the fpga.enumerate() function is a list of all the token objects matching the search criteria.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#412-properties","title":"4.1.2 Properties","text":"
                                            Querying properties for a token or handle is also a bit different in the Python API. In order to query properties for one of these objects, pass the object to the fpga.properties() constructor. The return value is a properties object with each of the property names defined as instance attributes.
                                             
                                            prop = fpga.properties(tokens[0])\nprint(f'0x{prop.vendor_id:04x} 0x{prop.device_id:04x}')\n
                                             
                                            Figure 61 Python Get Token Properties
                                             
                                            Properties objects may also be created by invoking the fpga.properties() constructor, passing the same keyword arguments as those to fpga.enumerate(). Properties objects created in this way are also useful for enumeration purposes:
                                             
                                            props = fpga.properties(type=fpga.ACCELERATOR)\ntokens = fpga.enumerate([props])\n
                                             
                                            Figure 62 Python Properties Constructor
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#413-tokens","title":"4.1.3 Tokens","text":"
                                            Tokens overload both the __getitem__ and __getattr__ methods to enable the SysObject API. Both of the following are valid forms of accessing the \u2018errors/first_error\u2019 sysfs attribute, given a token object:
                                             
                                            tok = tokens[0]\nferr = tok['errors/first_error']\nprint(f'first error: 0x{ferr.read64():0x}')\nprint('0x{:0x}'.format(tok.errors.first_error.read64()))\n
                                             
                                            Figure 63 Python Tokens and SysObject API
                                             
                                            Tokens also implement a find() method, which accepts a glob expression in order to search sysfs. The following example finds the \u201cid\u201d sysfs entry in the given token\u2019s sysfs tree.
                                             
                                            my_id = tok.find(\u2018i?\u2019)\nprint(f'{my_id.read64()}')\n
                                             
                                            Figure 64 Python Token Find
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#414-handles","title":"4.1.4 Handles","text":"
                                            Tokens are converted to handles by way of the fpga.open() function. The flags (second) parameter to fpga.open() may be zero or fpga.OPEN_SHARED.
                                             
                                            with fpga.open(tok, fpga.OPEN_SHARED) as handle:\n    use_the_handle(handle)\n
                                             
                                            Figure 65 Python Open Handle
                                             
                                            Like token objects, handle objects overload __getitem__ and __getattr__ methods to enable the SysObject API. handle also provides a find() method similar to token\u2019s find().
                                             
                                            err = handle['errors/errors']\nprint(f'errors: 0x{err.read64():0x}')\nprint('first error: 0x{:0x}'.format(\nhandle.errors.first_error.read64()))\nmy_id = handle.find('i?')\nprint(f'{my_id.read64()}')\n
                                             
                                            Figure 66 Python Handles and SysObject API
                                             
                                            Partial reconfiguration is provided by class handle\u2019s reconfigure() method. The first parameter, slot, will be zero in most designs. The second parameter is an opened file descriptor to the file containing the GBS image. The third parameter, flags, defaults to zero.
                                             
                                            with open(\u2018my.gbs\u2019, \u2018rb\u2019) as fd:\n    handle.reconfigure(0, fd)\n
                                             
                                            Figure 67 Python Partial Reconfiguration
                                             
                                            Device reset is accomplished by means of handle\u2019s reset() method, which takes no parameters.
                                             
                                            Finally for handles, CSR space reads are accomplished via read_csr32() and read_csr64(). Both methods accept the register offset as the first parameter and an optional csr_space index, which defaults to zero, as the second parameter. CSR space writes are accomplished via write_csr32() and write_csr64(). Both methods accept the register offset as the first parameter, the value to write as the second, and an optional csr_space index, which defaults to zero, as the third.
                                             
                                            print(\u20190x{:0x}\u2019.format(handle.read_csr32(0x000a)))\nprint(\u20180x{:0x}\u2019.format(handle.read_csr64(0x000c)))\nhandle.write_csr32(0x000b, 0xdecafbad, 2)\nhandle.write_csr64(0x000e, 0xc0cac01adecafbad, 2)\n
                                             
                                            Figure 68 Python Read/Write CSR
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#415-shared-memory","title":"4.1.5 Shared Memory","text":"
                                            The fpga.allocate_shared_buffer() function provides access to the OPAE memory allocator. The allocation sizes and required huge page configurations are the same as those noted in 2.1.5 MMIO and Shared Memory.
                                             
                                            The fpga.allocate_shared_buffer() function returns an object instance of type shared_buffer. The shared_buffer class implements methods size(), wsid(), and io_address(), which return the buffer size in bytes, the unique workspace ID, and the IO address respectively:
                                             
                                            buf = fpga.allocate_shared_buffer(handle, 4096)\nprint(f\u2019size: {buf.size()}\u2019)\nprint(f\u2019wsid: 0x{buf.wsid():0x}\u2019)\nprint(f\u2019io_address: 0x{buf.io_address():0x}\u2019)\n
                                             
                                            Figure 69 Python Allocate Shared Memory
                                             
                                            The shared_buffer class implements a fill() method which takes an integer parameter which is applied to each byte of the buffer (similar to C standard library\u2019s memset()). The compare() method compares the contents of the first size bytes of one buffer to another. The value returned from compare() is the same as the C standard library\u2019s memcmp(). The copy() method copies the first size bytes of the calling buffer into the argument buffer.
                                             
                                            b0 = fpga.allocate_shared_buffer(handle, 4096)\nb1 = fpga.allocate_shared_buffer(handle, 4096)\nb0.fill(0xa5)\nb1.fill(0xa5)\nprint(f'compare: {b0.compare(b1, 4096)}')\nb1.fill(0xa0)\nb0.copy(b1, 4096)\nprint(f'compare: {b0.compare(b1, 4096)}')\n
                                             
                                            Figure 70 Python Buffer Fill, Copy, Compare
                                             
                                            shared_buffer\u2019s read32() and read64() methods read a 32- or 64-bit value from the given offset. The write32() and write64() methods write a 32- or 64-bit value to the given offset.
                                             
                                            print(f'value at 0: 0x{b0.read32(0):0x}')\nprint(f'value at 4: 0x{b0.read64(4):0x}')\nb0.write32(0xabadbeef, 0)\nb0.write64(0xdecafbadabadbeef, 4)\nprint(f'value at 0: 0x{b0.read32(0):0x}')\nprint(f'value at 4: 0x{b0.read64(4):0x}')\n
                                             
                                            Figure 71 Python Buffer Read and Write
                                             
                                            The shared_buffer class provides three polling methods: poll(), poll32(), and poll64(). Each method takes an offset as its first parameter. The second parameter is a value and the third is a mask. The value and mask parameters are 8 bits wide for poll(), 32 bits wide for poll32(), and 64 bits wide for poll64(). The fourth and last parameter is a timeout value which defaults to 1000 microseconds.
                                             
                                            Each polling method reads the n-bit wide item at offset and applies (logical AND) the mask to that value. The masked value created in the previous step is then compared to the second parameter, value. If the two values are equal, then the method returns true immediately. Otherwise, the method continues to loop, attempting the same comparison over and over without sleeping. Finally, if the elapsed time from the beginning of the call to the current time is greater than or equal to the timeout value, then the method times out and returns false.
                                             
                                            if b0.poll32(0, 0xbebebebe, 0xffffffff, 250):\n    print(\u2018Got it!\u2019)\n
                                             
                                            Figure 72 Python Buffer Poll
                                             
                                            The shared_buffer split() method allows creating two or more buffer objects from one larger buffer object. The return value is a list of shared_buffer instances whose sizes match the arguments given to split().
                                             
                                            b1, b2 = b1.split(2048, 2048)\nprint(f'b1 io_address: 0x{b1.io_address():0x}')\nprint(f'b2 io_address: 0x{b2.io_address():0x}')\n
                                             
                                            Figure 73 Python Splitting Buffer
                                             
                                            Finally, the shared_buffer class implements the Python buffer protocol to support memoryview objects. The Python buffer protocol allows access to an object\u2019s underlying memory without copying that memory. As a brief example:
                                             
                                            mv = memoryview(b1)\nassert mv\nassert mv[0] == 0xbe\nb1[15] = int(65536)\nassert struct.unpack('<L', bytearray(b1[15:19]))[0] == 65536\n
                                             
                                            Figure 74 Python memoryview
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#416-events","title":"4.1.6 Events","text":"
                                            Given a handle and an event type, the fpga.register_event() function returns an object of type event. The event class implements one method, os_object(), which returns the underlying file descriptor that can be used to poll for the event:
                                             
                                            import select\nevt = fpga.register_event(handle, fpga.EVENT_ERROR)\nos_object = evt.os_object()\nreceived_event = False\nepoll = select.epoll()\nepoll.register(os_object, select.EPOLLIN)\nfor fileno, ev in epoll.poll(1):\n    if fileno == os_object:\n        received_event = True\n        print(f'received: {received_event}')\n
                                             
                                            Figure 75 Python Events
                                             
                                            In addition to fpga.EVENT_ERROR, fpga.EVENT_INTERRUPT, and fpga.EVENT_POWER_THERMAL are also supported.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#417-errors","title":"4.1.7 Errors","text":"
                                            Given a token, the fpga.errors() function returns a list of objects of type error. Each error instance represents a 64-bit mask of error values. The error bit masks are platform-dependent. Each error instance has two attributes: name and can_clear and one method: read_value() which returns the 64-bit error mask.
                                             
                                            for e in fpga.errors(tok):\n    print(f\u2019name: \"{e.name}\"\u2019)\n    print(f\u2019can_clear: {e.can_clear}\u2019)\n    print(f\u2019value: {e.read_value()}\u2019)\n
                                             
                                            Figure 76 Python Get Errors
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#418-sysobject","title":"4.1.8 SysObject","text":"
                                            The Python API\u2019s SysObject implementation is introduced in 4.1.3 Tokens and 4.1.4 Handles. When the index operator (__getitem__) or attribute reference (__getattr__) is used and the referenced string or attribute name corresponds to a sysfs entry in the sysfs path of either a token or a handle, then an object of type sysobject is returned.
                                             
                                            The size() method returns the length of the sysfs entry in bytes. Note that a typical sysfs entry is terminated with a \u2018\\n\u2019 followed by the \u2018\\0\u2019 NULL terminator. The bytes() method returns the sysfs entry\u2019s value as a string.
                                             
                                            afu_id = tok['afu_id']\nassert afu_id\nprint(f'size: {afu_id.size()} bytes: {afu_id.bytes().rstrip()}')\n
                                             
                                            Figure 77 Python sysobject as Bytes
                                             
                                            The sysobject read64() and write64() methods provide a means to read and write a sysfs entry\u2019s value as an unsigned 64-bit integer. The sysobject class itself also implements the __getitem__ and __getattr__ methods so that a sysobject of type FPGA_OBJECT_CONTAINER can retrieve sysobject instances for child sysfs entries.
                                             
                                            errs = tok['errors']\nfirst = errs['first_error']\nassert first\nprint(f'first 0x{first.read64():0x}')\n
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#418-python-example-application","title":"4.1.8 Python Example Application","text":"
                                            The following example is an implementation of the sample, hello_fpga.c, which is designed to configure the NLB0 diagnostic accelerator for a simple loopback.
                                             
                                            import time\nfrom opae import fpga\n\nNLB0 = \"d8424dc4-a4a3-c413-f89e-433683f9040b\"\nCTL = 0x138\nCFG = 0x140\nNUM_LINES = 0x130\nSRC_ADDR = 0x0120\nDST_ADDR = 0x0128\nDSM_ADDR = 0x0110\nDSM_STATUS = 0x40\n\ndef cl_align(addr):\n    return addr >> 6\n\ntokens = fpga.enumerate(type=fpga.ACCELERATOR, guid=NLB0)\nassert tokens, \"Could not enumerate accelerator: {}\".format(NlB0)\n\nwith fpga.open(tokens[0], fpga.OPEN_SHARED) as handle:\n    src = fpga.allocate_shared_buffer(handle, 4096)\n    dst = fpga.allocate_shared_buffer(handle, 4096)\n    dsm = fpga.allocate_shared_buffer(handle, 4096)\n    handle.write_csr32(CTL, 0)\n    handle.write_csr32(CTL, 1)\n    handle.write_csr64(DSM_ADDR, dsm.io_address())\n    handle.write_csr64(SRC_ADDR, cl_align(src.io_address())) # cacheline-aligned\n    handle.write_csr64(DST_ADDR, cl_align(dst.io_address())) # cacheline-aligned\n    handle.write_csr32(CFG, 0x42000)\n    handle.write_csr32(NUM_LINES, 4096/64)\n    handle.write_csr32(CTL, 3)\n    while dsm[DSM_STATUS] & 0x1 == 0:\n        time.sleep(0.001)\n    handle.write_csr32(CTL, 7)\n
                                             
                                            This example shows how one might reprogram (Partial Reconfiguration) an accelerator on a given bus, 0x5e, using a bitstream file, m0.gbs.
                                             
                                            from opae import fpga\n\nBUS = 0x5e\nGBS = 'm0.gbs'\ntokens = fpga.enumerate(type=fpga.DEVICE, bus=BUS)\nassert tokens, \"Could not enumerate device on bus: {}\".format(BUS)\nwith open(GBS, 'rb') as fd, fpga.open(tokens[0]) as device:\n    device.reconfigure(0, fd)\n
                                             
                                            Figure 78 Python sysobject Container
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#50-management-interfaces-opaeadmin","title":"5.0 Management Interfaces - opae.admin","text":"
                                            While the OPAE SDK C, C++, and Python APIs focus on presenting the AFU and all its related functionality to the end user, there is also a need for a maintenance functionality to aid in configuring the platform and performing secure firmware updates for the FPGA device and its components.  opae.admin is a Python framework which provides abstractions for performing these types of maintenance tasks on FPGA devices.  opae.admin provides Python classes which model the FPGA and the sysfs interfaces provided by the DFL drivers.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#51-sysfs","title":"5.1 sysfs","text":"
                                            opae.admin\u2019s sysfs module provides abstractions for interacting with sysfs nodes, which comprise the base entity abstraction of opae.admin.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#511-sysfs_node","title":"5.1.1 sysfs_node","text":"
                                            A sysfs_node is an object that tracks a unique path within a sysfs directory tree.  sysfs_node provides methods for finding and constructing other sysfs_node objects, based on the root path of the parent sysfs_node object.  sysfs_node also provides a mechanism to read and write sysfs file contents.  sysfs_node serves as the base class for many of the sysfs module\u2019s other classes.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#512-pci_node","title":"5.1.2 pci_node","text":"
                                            A pci_node is a sysfs_node that is rooted at /sys/bus/pci/devices.  Each pci_node has a unique PCIe address corresponding to the PCIe device it represents.  Methods for finding the pci_node\u2019s children, for determining the PCIe device tree rooted at the node, for manipulating the node\u2019s PCIe address, for determining the vendor and device ID\u2019s, and for removing, unbinding, and rescanning the device are provided.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#513-sysfs_driver","title":"5.1.3 sysfs_driver","text":"
                                            A sysfs_driver is a sysfs_node that provides a method for unbinding a sysfs_device object.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#514-sysfs_device","title":"5.1.4 sysfs_device","text":"
                                            A sysfs_device is a sysfs_node that is located under /sys/class or /sys/bus.  sysfs_device provides the basis for opae.admin\u2019s FPGA enumeration capability.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#515-pcie_device","title":"5.1.5 pcie_device","text":"
                                            A pcie_device is a sysfs_device that is rooted at /sys/bus/pci/devices.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#52-fpga","title":"5.2 fpga","text":"
                                            opae.admin\u2019s fpga module provides classes which abstract an FPGA and its components.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#521-region","title":"5.2.1 region","text":"
                                            A region is a sysfs_node that has an associated Linux character device, rooted at /dev.  Methods for opening the region\u2019s character device file and for interacting with the character device via its IOCTL interface are provided.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#522-fme","title":"5.2.2 fme","text":"
                                            An fme is a region that represents an FPGA device\u2019s FME component.  An fme provides accessors for the PR interface ID, the various bus paths that may exist under an FME, and the BMC firmware revision information.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#523-port","title":"5.2.3 port","text":"
                                            A port is a region that represents an FPGA device\u2019s Port component.  A port provides an accessor for the Port AFU ID.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#524-fpga_base","title":"5.2.4 fpga_base","text":"
                                            An fpga_base is a sysfs_device that provides accessors for the FPGA device\u2019s FME, for the FPGA device\u2019s Port, and for the secure update sysfs controls.  fpga_base provides routines for enabling and disabling AER and for performing device RSU.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#525-fpga","title":"5.2.5 fpga","text":"
                                            An fpga (derived from fpga_base) is the basis for representing the FPGA device in opae.admin.  Utilities such as fpgasupdate rely on fpga\u2019s enum classmethod to enumerate all of the FPGA devices in the system.  In order for a device to enumerate via this mechanism, it must be bound to the dfl-pci driver at the time of enumeration.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#53-opaeadmin-utilities","title":"5.3 opae.admin Utilities","text":"
                                            Several utilities are written on top of opae.admin\u2019s class abstractions.  The following sections highlight some of the most commonly-used utilities.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#531-fpgasupdate","title":"5.3.1 fpgasupdate","text":"
                                            fpgasupdate, or FPGA Secure Update, is used to apply firmware updates to the components of the FPGA.  As the name implies, these updates target a secure FPGA device, one that has the ability to implement a secure root of trust. The command-line interface to fpgasupdate was designed to be as simple as possible for the end user.  The command simply takes a path to the firmware update file to be applied and the PCIe address of the targeted FPGA device.
                                             
                                            # fpgasupdate update-file.bin 0000:b2:00.0\n
                                             
                                            Figure 79 fpgasupdate Interface
                                             
                                            fpgasupdate can apply a variety of firmware image updates. | Image| Description| | -----| -----| |Programmable Region Image| .gbs or Green BitStream| |SR Root Key Hash|  Static Region RKH| |PR Root Key Hash|  Programmable Region RKH| |FPGA Firmware Image|   Static Region Device Firmware| |PR Authentication Certificate| Programmable Region Auth Cert| |BMC Firmware Image|    Board Management Controller Firmware| |SR Thermal Image|  Static Region Thermal Sensor Thresholds| |PR Thermal Image|  Programmable Region Thermal Sensor Thresholds| |CSK Cancelation|   Code Signing Key Cancelation Request| |SDM Image| Secure Device Manager Firmware|
                                             
                                            Table 10 fpgasupdate Image Types
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#532-pci_device","title":"5.3.2 pci_device","text":"
                                            pci_device is a utility that provides a convenient interface to some of the Linux Kernel\u2019s standard PCIe device capabilities.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5321-pci_device-aer-subcommand","title":"5.3.2.1 pci_device aer subcommand","text":"
                                            The aer dump subcommand displays the Correctable, Fatal, and NonFatal device errors.
                                             
                                            # pci_device 0000:b2:00.0 aer dump\n
                                             
                                            Figure 80 pci_device aer dump
                                             
                                            The aer mask subcommand displays, masks, or unmasks errors using the syntax of the setpci command.
                                             
                                            # pci_device 0000:b2:00.0 aer mask show\n0x00010000 0x000031c1\n# pci_device 0000:b2:00.0 aer mask all\n# pci_device 0000:b2:00.0 aer mask off\n# pci_device 0000:b2:00.0 aer mask 0x01010101 0x10101010\n
                                             
                                            Figure 81 pci_device aer mask
                                             
                                            The aer clear subcommand clears the current errors.
                                             
                                            # pci_device 0000:b2:00.0 aer clear\naer clear errors: 00000000\n
                                             
                                            Figure 82 pci_device aer clear
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5322-pci_device-unbind-subcommand","title":"5.3.2.2 pci_device unbind subcommand","text":"
                                            The unbind subcommand unbinds the target device from the currently-bound device driver.
                                             
                                            # pci_device 0000:b2:00.0 unbind\n
                                             
                                            Figure 83 pci_device unbind
                                             
                                            In order to re-bind the device to a driver, eg dfl-pci, use the following commands:
                                             
                                            # cd /sys/bus/pci/drivers/dfl-pci\n# echo 0000:b2:00.0 > bind\n
                                             
                                            Figure 84 Re-binding a Driver
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5323-pci_device-rescan-subcommand","title":"5.3.2.3 pci_device rescan subcommand","text":"
                                            The rescan subcommand triggers a PCIe bus rescan of all PCIe devices.
                                             
                                            # pci_device 0000:b2:00.0 rescan\n
                                             
                                            Figure 85 pci_device rescan
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5324-pci_device-remove-subcommand","title":"5.3.2.4 pci_device remove subcommand","text":"
                                            The remove subcommand removes the target device from Linux kernel management.
                                             
                                            # pci_device 0000:b2:00.0 remove\n
                                             
                                            Figure 86 pci_device remove
                                             
                                            Note: a reboot may be required in order to re-establish the Linux kernel management for the device.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5325-pci_device-topology-subcommand","title":"5.3.2.5 pci_device topology subcommand","text":"
                                            The topology subcommand shows a tab-delimited depiction of the target device as it exists in the PCIe device tree in the Linux kernel.
                                             
                                            # pci_device 0000:b2:00.0 topology\n[pci_address(0000:3a:00.0), pci_id(0x8086, 0x2030)] (pcieport)\n    [pci_address(0000:3b:00.0), pci_id(0x10b5, 0x8747)] (pcieport)\n        [pci_address(0000:3c:09.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:b2:00.0), pci_id(0x8086, 0x0b30)] (dfl-pci)\n        [pci_address(0000:3c:11.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:43:00.0), pci_id(0x8086, 0x0b32)] (no driver)\n        [pci_address(0000:3c:08.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:3d:00.1), pci_id(0x8086, 0x0d58)] (i40e)\n            [pci_address(0000:3d:00.0), pci_id(0x8086, 0x0d58)] (i40e)\n        [pci_address(0000:3c:10.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:41:00.0), pci_id(0x8086, 0x0d58)] (i40e)\n            [pci_address(0000:41:00.1), pci_id(0x8086, 0x0d58)] (i40e)\n
                                             
                                            Figure 87 pci_device topology
                                             
                                            The green output indicates the target device.  The other endpoint devices are shown in blue.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5326-pci_device-vf-subcommand","title":"5.3.2.6 pci_device vf subcommand","text":"
                                            The vf subcommand allows setting the value of the sriov_numvfs sysfs node of the target device.  This is useful in scenarios where device functionality is presented in the form of one or more PCIe Virtual Functions.
                                             
                                            # pci_device 0000:b2:00.0 vf 3\n# pci_device 0000:b2:00.0 vf 0\n
                                             
                                            Figure 88 pci_device vf
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#533-rsu","title":"5.3.3 rsu","text":"
                                            rsu is a utility that performs Remote System Update.  rsu is used subsequent to programming a firmware update or other supported file type with fpgasupdate, in order to reset the targeted FPGA entity so that a newly-loaded firmware image becomes active.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5331-rsu-bmc-subcommand","title":"5.3.3.1 rsu bmc subcommand","text":"
                                            The bmc subcommand causes a Board Management Controller reset.  This command is used to apply a previous fpgasupdate of a BMC firmware image.  The --page argument selects the desired boot image.  Valid values for --page are \u2018user\u2019 and \u2018factory\u2019.
                                             
                                            # rsu bmc --page user 0000:b2:00.0\n# rsu bmc --page factory 0000:b2:00.0\n
                                             
                                            Figure 89 rsu bmc
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5332-rsu-retimer-subcommand","title":"5.3.3.2 rsu retimer subcommand","text":"
                                            The retimer subcommand causes a Parkvale reset (specific to Vista Creek).  This command is used to apply a previous fpgasupdate of a BMC firmware image (the Parkvale firmware is contained within the BMC firmware image).  The retimer subcommand causes only the Parkvale to reset.
                                             
                                            # rsu retimer 0000:b2:00.0\n
                                             
                                            Figure 90 rsu retimer
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5333-rsu-fpga-subcommand","title":"5.3.3.3 rsu fpga subcommand","text":"
                                            The fpga subcommand causes a reconfiguration of the FPGA Static Region.  This command is used to apply a previous fpgasupdate of the Static Region image.  The --page argument selects the desired boot image.  Valid values for --page are \u2018user1\u2019, \u2018user2\u2019, and \u2018factory\u2019.
                                             
                                            # rsu fpga --page user1 0000:b2:00.0\n# rsu fpga --page user2 0000:b2:00.0\n# rsu fpga --page factory 0000:b2:00.0\n
                                             
                                            Figure 91 rsu fpga
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5334-rsu-sdm-subcommand","title":"5.3.3.4 rsu sdm subcommand","text":"
                                            The sdm subcommand causes a reset of the Secure Device Manager.  This command is used to apply a previous fpgasupdate of the SDM image.
                                             
                                            # rsu sdm 0000:b2:00.0\n
                                             
                                            Figure 92 rsu sdm
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#5335-rsu-fpgadefault-subcommand","title":"5.3.3.5 rsu fpgadefault subcommand","text":"
                                            The fpgadefault subcommand can be used to display the default FPGA boot sequence; and it can be used to select the image to boot on the next reset of the FPGA. When given without additional parameters, the fpgadefault subcommand displays the default FPGA boot sequence:
                                             
                                            # rsu fpgadefault 0000:b2:00.0\n
                                             
                                            Figure 93 rsu Displaying FPGA Boot Sequence
                                             
                                            The parameters to the fpgadefault subcommand are --page and --fallback.  The --page parameter accepts \u2018user1\u2019, \u2018user2\u2019, or \u2018factory\u2019, specifying the desired page to boot the FPGA from on the next reset.  Note that this subcommand does not actually cause the reset to occur.  Please refer to rsu fpga subcommand for an example of resetting the FPGA using the rsu command.
                                             
                                            # rsu fpgadefault --page user1 0000:b2:00.0\n# rsu fpgadefault --page user2 0000:b2:00.0\n# rsu fpgadefault --page factory 0000:b2:00.0\n
                                             
                                            Figure 94 rsu Select FPGA Boot Image
                                             
                                            The --fallback parameter accepts a comma-separated list of the keywords \u2018user1\u2019, \u2018user2\u2019, and \u2018factory\u2019.  These keywords, in conjunction with the --page value are used to determine a fallback boot sequence for the FPGA. The fallback boot sequence is used to determine which FPGA image to load in the case of a boot failure.  For example, given the following command, the FPGA would attempt to boot in the order \u2018factory\u2019, \u2018user1\u2019, \u2018user2\u2019.  That is to say, if the \u2018factory\u2019 image failed to boot, then the \u2018user1\u2019 image would be tried.  Failing to boot \u2018user1\u2019, the \u2018user2\u2019 image would be tried.
                                             
                                            # rsu fpgadefault --page factory --fallback user1,user2 0000:b2:00.0\n
                                             
                                            Figure 95 rsu Select FPGA Boot Image and Fallback
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#60-sample-applications","title":"6.0 Sample Applications","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#61-afu-test-framework","title":"6.1 afu-test Framework","text":"
                                            afu-test refers to a test-writing framework that exists as a set of C++ classes written on top of the OPAE C++ bindings. The first class, afu, serves as the base class for the test application abstraction. Class afu provides integration with CLI11, a C++ \u201911 command-line parsing framework, and with spdlog, a C++ logging library. The second class, command represents a unique test sequence that is called by the afu object. Instances of the command class implement the test-specific workload.
                                             
                                            class afu {\npublic:\nafu(const char *name,\nconst char *afu_id = nullptr,\nconst char *log_level = nullptr);\nint open_handle(const char *afu_id);\nint main(int argc, char *argv[]);\nvirtual int run(CLI::App *app, command::ptr_t test);\ntemplate<class T>\nCLI::App *register_command();\n};\n
                                             
                                            Figure 96 C++ class afu
                                             
                                            The afu class constructor initializes the CLI11 command parser with some general, application-wide parameters.
                                             Subcommand Description -g,--guid Accelerator AFU ID. -p,--pci-address Address of the accelerator device. -l,--log-level Requested spdlog output level. -s,--shared Open the AFU in shared mode? -t,--timeout Application timeout in milliseconds. 
                                            Figure 97 class afu Application Parameters
                                             
                                            The register_command() member function adds a test command instance to the afu object. Each test command that an afu object is capable of executing is registered during the test\u2019s startup code. For instance, here is the hssi application\u2019s use of register_command():
                                             
                                            hssi_afu app;\nint main(int argc, char *argv[])\n{\napp.register_command<hssi_10g_cmd>();\napp.register_command<hssi_100g_cmd>();\napp.register_command<hssi_pkt_filt_10g_cmd>();\napp.register_command<hssi_pkt_filt_100g_cmd>();\n\u2026\napp.main(argc, argv);\n}\n
                                             
                                            Figure 98 hssi's app.register_command()
                                             
                                            Next, the afu instance\u2019s main() member function is called. main() initializes the spdlog instance, searches its database of registered commands to find the command matching the test requested from the command prompt, uses the open_handle() member function to enumerate for the requested AFU ID, and calls its run() member function, passing the CLI::App and the test command variables. The run() member function initializes a test timeout mechanism, then calls the command parameter\u2019s run() to invoke the test-specific logic.
                                             
                                            With all the boiler-plate of application startup, configuration, and running handled by the afu class, the test-specific command class is left to implement only a minimum number of member functions:
                                             
                                            class command {\npublic:\nvirtual const char *name() const = 0;\nvirtual const char *description() const = 0;\nvirtual int run(afu *afu, CLI::App *app) = 0;\nvirtual void add_options(CLI::App *app) { }\nvirtual const char *afu_id() const { return nullptr; }\n};\n
                                             
                                            Figure 99 class command
                                             
                                            The name() member function gives the unique command name. Some examples of names from the hssi app are hssi_10g, hssi_100g, pkt_filt_10g, and pkt_filt_100g. The description() member function gives a brief description that is included in the command-specific help output. add_options() adds command-specific command-line options. afu_id() gives the AFU ID for the command, in string form. Finally, run() implements the command-specific test functionality.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#62-afu-test-based-samples","title":"6.2 afu-test Based Samples","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#621-dummy_afu","title":"6.2.1 dummy_afu","text":"
                                            The dummy_afu application is a afu-test based application that implements three commands: mmio, ddr, and lpbk.
                                             Target Description # dummy_afu mmio Targets special scratchpad area implemented by the AFU. # dummy_afu ddr Execute dummy_afu-specific DDR test. # dummy_afu lpbk Execute a simple loopback test."},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#622-host_exerciser","title":"6.2.2 host_exerciser","text":"
                                            host_exerciser markdown document.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#623-hssi","title":"6.2.3 hssi","text":"
                                            hssi markdown document.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#70-other-utilities","title":"7.0 Other Utilities","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#71-opaeio","title":"7.1 opae.io","text":"
                                            opae.io markdown document.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#72-bitstreaminfo","title":"7.2 bitstreaminfo","text":"
                                            The bitstreaminfo command prints diagnostic information about firmware image files that have been passed through the PACSign utility. PACSign prepends secure block 0 and secure block 1 data headers to the images that it processes. These headers contain signature hashes and other metadata that are consumed by the BMC firmware during a secure update.
                                             
                                            To run bitstreaminfo, pass the path to the desired firmware image file:
                                             
                                            # bitstreaminfo my_file.bin \n
                                             
                                            Figure 100 Running bitstreaminfo
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#73-fpgareg","title":"7.3 fpgareg","text":"
                                            The fpgareg command prints the register spaces for the following fpga device components:
                                             Command Description # fpgareg 0000:b1:00.0 pcie Walks and prints the DFL for the device. # fpgareg 0000:b1:00.0 bmc Prints the BMC registers for the device. # fpgareg 0000:b1:00.0 hssi Prints the HSSI registers for the device. # fpgareg 0000:b1:00.0 acc Prints the AFU register spaces. 
                                            Figure 101 fpgareg Commands
                                             
                                            Note that fpgareg is only available as of Arrow Creek ADP and forward. It will not work with prior platforms, eg N3000.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#74-opaevfio","title":"7.4 opaevfio","text":"
                                            opaevfio markdown document.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#80-building-opae","title":"8.0 Building OPAE","text":"
                                            The OPAE SDK uses the cmake build and configuration system, version >= 3.10. The basic steps required to build the SDK from source are:
                                             
                                            Install prerequisite packages.
                                             
                                            $ git clone <https://github.com/OFS/opae-sdk.git>\n$ cd opae-sdk\n$ mkdir build\n$ cd build\n$ cmake .. <cmake options>\n$ make\n
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#81-installing-prerequisite-packages","title":"8.1 Installing Prerequisite Packages","text":"
                                            The OPAE SDK is intended to build and run on modern Linux distributions. The SDK contains a set of system configuration scripts to aid the system configuration process.
                                             Script Target Operating System centos.sh CentOS 8.x fedora.sh Fedora 33/34 ubuntu.sh Ubuntu 20.04 LTS 
                                            Table 11 System Configuration Scripts
                                             
                                            For more information on the environment setup instructions for your exact platform, please visit https://ofs.github.io/ and refer to your platform's User Guide.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#82-cloning-the-sdk-repository","title":"8.2 Cloning the SDK repository","text":"
                                            $ git clone https://github.com/OFS/opae-sdk.git\n
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#83-cmake-options","title":"8.3 CMake Options","text":"Option Description Values Default -DCMAKE_BUILD_TYPE Configure debugging info 
                                            Debug
                                             
                                            Release
                                             
                                            Coverage
                                             
                                            RelWithDebInfo
                                             RelWithDebInfo -DCMAKE_INSTALL_PREFIX Root install path /usr/local -DOPAE_BUILD_SPHINX_DOC Enable/Disable docs ON/OFF OFF -DOPAE_BUILD_TESTS Enable/Disable unit tests ON/OFF OFF -DOPAE_ENABLE_MOCK Enable/Disable mock driver for unit tests ON/OFF OFF -DOPAE_INSTALL_RPATH Enable/Disable rpath for install ON/OFF OFF -DOPAE_VERSION_LOCAL Local version string -DOPAE_PRESERVE_REPOS Preserve local changes to external repos? ON/OFF OFF -D OPAE_BUILD_LIBOPAE_CXX Enable C++ bindings ON/OFF ON -DOPAE_WITH_PYBIND11 Enable pybind11 ON/OFF ON -D OPAE_BUILD_PYTHON_DIST Enable Python bindings ON/OFF OFF -DOPAE_BUILD_LIBOPAEVFIO Build libopaevfio.so ON/OFF ON -D OPAE_BUILD_PLUGIN_VFIO Build libopae-v.so ON/OFF ON -DOPAE_BUILD_LIBOPAEUIO Build libopaeuio.so ON/OFF ON -DOPAE_BUILD_LIBOFS Build OFS Copy Engine ON/OFF ON -DOPAE_BUILD_SAMPLES Build Samples ON/OFF ON -DOPAE_BUILD_LEGACY Build legacy repo ON/OFF OFF -DOPAE_LEGACY_TAG Specify legacy build tag master -DOPAE_WITH_CLI11 Enable apps which use CLI11 ON/OFF ON -DOPAE_WITH_SPDLOG Enable apps which use spdlog ON/OFF ON -DOPAE_WITH_LIBEDIT Enable apps which use libedit ON/OFF ON -DOPAE_WITH_HWLOC Enable apps which use hwloc ON/OFF ON -DOPAE_WITH_TBB Enable apps which use Thread Building Blocks ON/OFF ON -DOPAE_MINIMAL_BUILD Enable/Disable minimal build. When set to ON, disable CLI11, spdlog, libedit, hwloc, tbb ON/OFF OFF 
                                            Table 12 CMake Options
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#84-building-opae-for-debug","title":"8.4 Building OPAE for Debug","text":"
                                            $ cmake .. -DCMAKE_BUILD_TYPE=Debug\n
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#85-creating-rpms","title":"8.5 Creating RPMs","text":"
                                            To ease the RPM creation process, the OPAE SDK provides a simple RPM creation script. The parameters to the RPM create script are fedora or rhel, depending on which distribution is targeted. For rhel, the build flag -DOPAE_MINIMAL_BUILD is set to ON, omitting the binaries which have dependencies on external components that RHEL does not include in its base repositories.
                                             
                                            In order to create RPMs for Fedora, run the create script on a system loaded with all the Fedora build prerequisites. If prerequisites are missing, the create script will complain until they are resolved.
                                             
                                            In order to create RPMs for RHEL, run the create script on a system loaded with all the RHEL build prerequisites. If prerequisites are missing, the create script will complain until they are resolved.
                                             
                                            $ cd opae-sdk\n$ ./packaging/opae/rpm/create fedora\n-OR-\n$ ./packaging/opae/rpm/create rhel\n
                                             
                                             Figure 102 RPM Creation
                                             
                                            After running the create script, the RPM files will be located in the packaging/opae/rpm directory.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#851-updating-the-rpm-version-information","title":"8.5.1 Updating the RPM Version Information","text":"
                                            The RPMs will be versioned according to the information found in the file packaging/opae/version. Edit this file to update the version information, then re-run the create script to create the RPMs.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#90-debugging-opae","title":"9.0 Debugging OPAE","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#91-enabling-debug-logging","title":"9.1   Enabling Debug Logging","text":"
                                            The OPAE SDK has a built-in debug logging facility.  To enable it, set the cmake flag -DCMAKE_BUILD_TYPE=Debug and then use the following environment variables: | Variable| Description| | ----- | ----- | |LIBOPAE_LOG=1| Enable debug logging output.  When not set, only critical error messages are displayed.| |LIBOPAE_LOGFILE=file.log|  Capture debug log output to file.log.  When not set, the logging appears on stdout and stderr.  The file must appear in a relative path or it can be rooted at /tmp.|
                                             
                                             Table 13 Logging Environment Variables
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#92-gdb","title":"9.2 GDB","text":"
                                            To enable gdb-based debugging, the cmake configuration step must specify a value for -DCMAKE_BUILD_TYPE of either Debug or RelWithDebInfo so that debug symbols are included in the output binaries. The OPAE SDK makes use of dynamically-loaded library modules.  When debugging with gdb, the best practice is to remove all OPAE SDK libraries from the system installation paths to ensure that library modules are only loaded from the local build tree:
                                             
                                            $ cd opae-sdk/build\n$ LD_LIBRARY_PATH=$PWD/lib gdb --args <some_opae_executable> <args>\n
                                             
                                            Figure 103 Debugging with GDB
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#100-adding-new-device-support","title":"10.0 Adding New Device Support","text":"
                                            As of OPAE 2.2.0 the SDK has transitioned to a single configuration file model.  The libraries, plugins, and applications obtain their runtime configuration during startup by examining a single JSON configuration file.  In doing so, the original configuration file formats for libopae-c and fpgad have been deprecated in favor of the respective sections in the new configuration file. 
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#101-configuration-file-search-order","title":"10.1 Configuration File Search Order","text":"
                                            By default the OPAE SDK will install its configuration file to /etc/opae/opae.cfg. 
                                             
                                            /etc/opae/opae.cfg \n
                                             
                                            Figure 104 Default Configuration File
                                             
                                            The SDK searches for the configuration file during startup by employing the following search algorithm: 
                                             
                                            First, the environment variable LIBOPAE_CFGFILE is examined.  If it is set to a path that represents a valid path to a configuration file, then that configuration file path is used, and the search is complete. 
                                             
                                            Next, the HOME environment variable is examined.  If its value is valid, then it is prepended to the following set of relative paths.  If HOME is not set, then the search continues with the value of the current user\u2019s home path as determined by getpwuid().  The home path, if any, determined by getpwuid() is prepended to the following set of relative paths.  Searching completes successfully if any of these home-relative search paths is valid. 
                                             
                                            /.local/opae.cfg \n\n/.local/opae/opae.cfg \n\n/.config/opae/opae.cfg \n
                                             
                                            Figure 105 HOME Relative Search Paths
                                             
                                            Finally, the configuration file search continues with the following system-wide paths.  If any of these paths is found to contain a configuration file, then searching completes successfully. 
                                             
                                            usr/local/etc/opae/opae.cfg \n\n/etc/opae/opae.cfg \n
                                             
                                            Figure 106 System Search Paths
                                             
                                            If the search exhausts all of the possible configuration file locations without finding a configuration file, then an internal default configuration is used.  This internal default configuration matches that of the opae.cfg file shipped with the OPAE SDK. 
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#102-configuration-file-format","title":"10.2 Configuration File Format","text":"
                                            The OPAE SDK configuration file is stored in JSON formatted text.  The file has two main sections: \u201cconfigs\u201d and \u201cconfigurations\u201d.  The \u201cconfigs\u201d section is an array of strings.  Each value in the \u201cconfigs\u201d array is a key into the data stored in the \u201cconfigurations\u201d section.  If a key is present in \u201cconfigs\u201d, then that key is searched for and processed in \u201cconfigurations\u201d.  If the key is not found in \u201cconfigs\u201d, then that section of \u201cconfigurations\u201d will not be processed, irrespective of whether it exists in \u201cconfigurations\u201d. 
                                             
                                            { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      ... \n\n    } \n\n  }, \n\n  \u201cconfigs\u201d: [ \n\n    \u201cc6100\u201d \n\n  ] \n\n} \n
                                             
                                            Figure 107 Keyed Configurations
                                             
                                            Each keyed section in \u201cconfigurations\u201d has four top-level entries: \u201cenabled\u201d, \u201cplatform\u201d, \u201cdevices\u201d, \u201copae\u201d. 
                                             
                                            { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      \u201cenabled\u201d: true, \n\n      \u201cplatform\u201d: \u201cIntel Acceleration Development Platform C6100\u201d, \n\n      \u201cdevices\u201d: [ \n\n        { \u201cname\u201d: \u201cc6100_pf\u201d, \u201cid\u201d: [ ... ] }, \n\n        { \u201cname\u201d: \u201cc6100_vf\u201d, \u201cid\u201d: [ ... ] } \n\n      ], \n\n      \u201copae\u201d: { \n\n        ... \n\n      } \n\n    } \n\n  }, \n\n} \n
                                             
                                            Figure 108 Configurations Format
                                             
                                            The \u201cenabled\u201d key holds a Boolean value.  If the value is false or if the \u201cenabled\u201d key is omitted, then that configuration is skipped when parsing the file.  The \u201cplatform\u201d key holds a string that identifies the current configuration item as a product family.  The \u201cdevices\u201d key contains the device descriptions. 
                                             
                                            \u201cdevices\u201d is an array of objects that contain a \u201cname\u201d and an \u201cid\u201d key.  The \u201cname\u201d is a shorthand descriptor for a device PF or VF.  The value of \u201cname\u201d appears elsewhere in the current \u201cconfigurations\u201d section in order to uniquely identify the device.  \u201cid\u201d is an array of four strings, corresponding to the PCIe Vendor ID, Device ID, Subsystem Vendor ID, and Subsystem Device ID of the device.  The entries corresponding to Vendor ID and Device ID must contain valid 16-bit hex integers.  The entries corresponding to Subsystem Vendor ID and Subsystem Device ID may be 16-bit hex integers or the special wildcard string \u201c*\u201d, which indicates a don\u2019t care condition. 
                                             
                                            The remaining sections in this chapter outline the format of the \u201copae\u201d configurations key. 
                                             
                                            \u201cplugin\u201d: libopae-c and libopae-v 
                                             
                                            The \u201cplugin\u201d key in the \u201copae\u201d section of a configuration is an array of OPAE SDK plugin configuration data.  Each item in the array matches one or more PF or VF devices to a plugin library module. 
                                             
                                            { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      ... \n\n      \u201copae\u201d: { \n\n        \u201cplugin\u201d: [ \n\n          { \n\n            \u201cenabled\u201d: true, \n\n            \u201cmodule\u201d: \u201clibxfpga.so\u201d, \n\n            \u201cdevices\u201d: [ \u201cc6100_pf\u201d ], \n\n            \u201cconfiguration\u201d: {} \n\n          }, \n\n          { \n\n            \u201cenabled\u201d: true, \n\n            \u201cmodule\u201d: \u201clibopae-v.so\u201d, \n\n            \u201cdevices\u201d: [ \u201cc6100_pf\u201d, \u201cc6100_vf\u201d ], \n\n            \u201cconfiguration\u201d: {} \n\n          } \n\n        ], \n\n      } \n\n    } \n\n  }, \n\n} \n
                                             
                                            Figure 109 \"opae\" / \"plugin\" key/
                                             
                                            If the \u201cenabled\u201d key is false or if it is omitted, then that \u201cplugin\u201d array entry is skipped, and parsing continues.  The \u201cmodule\u201d key is a string that identifies the desired plugin module library for the entry.  The \u201cdevices\u201d array lists one or more PF/VF identifiers.  Each array value must be a string, and it must match a device that is described in the \u201cconfigurations\u201d \u201cdevices\u201d section.  The \u201cconfiguration\u201d key of the \u201cplugin\u201d section specifies a unique plugin-specific configuration.  Currently, libopae-c and libopae-v use no plugin-specific config, so these keys are left empty. 
                                             
                                            \u201cfpgainfo\u201d: fpgainfo application 
                                             
                                            The \u201cfpgainfo\u201d key in the \u201copae\u201d section of a configuration is an array of fpgainfo plugin configuration data.  Each item in the array matches one or more PF or VF devices to an fpgainfo plugin library module. 
                                             
                                            { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      ... \n\n      \u201copae\u201d: { \n\n        \u201cfpgainfo\u201d: [ \n\n          { \n\n            \u201cenabled\u201d: true, \n\n            \u201cmodule\u201d: \u201clibboard_c6100.so\u201d, \n\n            \u201cdevices\u201d: [ \n\n              { \u201cdevice\u201d: \u201cc6100_pf\u201d, \u201cfeature_id\u201d: \u201c0x12\u201d }, \n\n              { \u201cdevice\u201d: \u201cc6100_vf\u201d, \u201cfeature_id\u201d: \u201c0x12\u201d } \n\n            ] \n\n          } \n\n        ], \n\n      } \n\n    } \n\n  }, \n\n} \n
                                             
                                            Figure 110 \"opae\" / \"fpgainfo\" key
                                             
                                            If the \u201cenabled\u201d key is false or if it is omitted, then that \u201cfpgainfo\u201d array entry is skipped, and parsing continues.  The \u201cmodule\u201d key is a string that identifies the desired fpgainfo module library for the entry.  Each \u201cdevices\u201d array entry gives a PF/VF identifier in its \u201cdevice\u201d key and a DFL feature ID in its \u201cfeature_id\u201d key. 
                                             
                                            \u201cfpgad\u201d: fpgad daemon process 
                                             
                                            The \u201cfpgad\u201d key in the \u201copae\u201d section of a configuration is an array of fpgad plugin configuration data.  Each item in the array matches one or more PF or VF devices to an fpgad plugin library module. 
                                             
                                            { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      ... \n\n      \u201copae\u201d: { \n\n        \u201cfpgad\u201d: [ \n\n          { \n\n            \u201cenabled\u201d: true, \n\n            \u201cmodule\u201d: \u201clibfpgad-vc.so\u201d, \n\n            \u201cdevices\u201d: [ \u201cc6100_pf\u201d ], \n\n            \u201cconfiguration\u201d: { \n\n              ... \n\n            } \n\n          } \n\n        ], \n\n      } \n\n    } \n\n  }, \n\n} \n
                                             
                                            Figure 111 \"opae\" / \"fpgad\" key 
                                             
                                            If the \u201cenabled\u201d key is false or if it is omitted, then that \u201cfpgad\u201d array entry is skipped, and parsing continues.  The \u201cmodule\u201d key is a string that identifies the desired fpgad plugin module library for the entry.  The \u201cdevices\u201d array lists one or more PF/VF identifiers.  Each array value must be a string, and it must match a device that is described in the \u201cconfigurations\u201d \u201cdevices\u201d section.  The \u201cconfiguration\u201d key of the \u201cfpgad\u201d section specifies a unique plugin-specific configuration. 
                                             
                                            \u201crsu\u201d: rsu script 
                                             
                                            The \u201crsu\u201d key in the \u201copae\u201d section of a configuration is an array of rsu script configuration data.  Each item in the array matches one or more PF devices to an rsu configuration. 
                                             
                                            { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      ... \n\n      \u201copae\u201d: { \n\n        \u201crsu\u201d: [ \n\n          { \n\n            \u201cenabled\u201d: true, \n\n            \u201cdevices\u201d: [ \u201cc6100_pf\u201d ], \n\n            \u201cfpga_default_sequences\u201d: \u201ccommon_rsu_sequences\u201d \n\n          } \n\n        ], \n\n      } \n\n    } \n\n  }, \n\n  \u201ccommon_rsu_sequences\u201d: [ \n\n    ... \n\n  ] \n\n} \n
                                             
                                            Figure 112 \"opae\" / \"rsu\" key  
                                             
                                            If the \u201cenabled\u201d key is false or if it is omitted, then that \u201crsu\u201d array entry is skipped, and parsing continues.  When disabled, the device(s) mentioned in that array entry will not be available for the rsu command. The \u201cdevices\u201d array lists one or more PF identifiers.  Each array value must be a string, and it must match a device that is described in the \u201cconfigurations\u201d \u201cdevices\u201d section.  The \u201cfpga_default_sequences\u201d key of the \u201crsu\u201d section specifies a JSON key.  The configuration searches for that JSON key at the global level of the configuration file, and when found applies its value as the valid set of fpga boot sequences that can be used with the rsu fpgadefault subcommand. 
                                             
                                            C \u201cfpgareg\u201d: fpgareg script
                                             
                                            The \u201cfpgareg\u201d key in the \u201copae\u201d section of a configuration is an array of fpgareg script configuration data.  Each item in the array matches one or more PF/VF devices to an fpgareg configuration. 
                                             
                                            ```C { 
                                             
                                            \u201cconfigurations\u201d: { 
                                             
                                            \u201cc6100\u201d: {\n\n  ...\n\n  \u201copae\u201d: {\n\n    \u201cfpgareg\u201d: [\n\n      {\n\n        \u201cenabled\u201d: true,\n\n        \u201cdevices\u201d: [ \u201cc6100_pf\u201d, \u201cc6100_vf\u201d ]\n\n      }\n\n    ],\n\n  }\n\n}\n
                                             
                                            }, 
                                             
                                            }   ```
                                             
                                            Figure 113 \"opae\" / \"fpgareg\" key   
                                             
                                            If the \u201cenabled\u201d key is false or if it is omitted, then that \u201cfpgareg\u201d array entry is skipped, and parsing continues.  When disabled, the device(s) mentioned in that array entry will not be available for the fpgareg command. The \u201cdevices\u201d array lists one or more PF/VF identifiers.  Each array value must be a string, and it must match a device that is described in the \u201cconfigurations\u201d \u201cdevices\u201d section.   
                                             
                                            \u201copae.io\u201d: opae.io application \n
                                             
                                            The \u201copae.io\u201d key in the \u201copae\u201d section of a configuration is an array of opae.io configuration data.  Each item in the array matches one or more PF/VF devices to an opae.io platform string. 
                                             
                                            { \n\n  \u201cconfigurations\u201d: { \n\n    \u201cc6100\u201d: { \n\n      ... \n\n      \u201copae\u201d: { \n\n        \u201copae.io\u201d: [ \n\n          { \n\n            \u201cenabled\u201d: true, \n\n            \u201cdevices\u201d: [ \u201cc6100_pf\u201d, \u201cc6100_vf\u201d ] \n\n          } \n\n        ], \n\n      } \n\n    } \n\n  }, \n\n} \n
                                             
                                            Figure 114 \"opae\" / \"opae.io\" key    
                                             
                                            If the \u201cenabled\u201d key is false or if it is omitted, then that \u201copae.io\u201d array entry is skipped, and parsing continues.  When disabled, the device(s) mentioned in that array entry will continue to be available for the opae.io command.  The device(s) platform string will not be shown in the opae.io ls command.  The \u201cdevices\u201d array lists one or more PF/VF identifiers.  Each array value must be a string, and it must match a device that is described in the \u201cconfigurations\u201d \u201cdevices\u201d section.   
                                             
                                            Libxfpga \u2013 Updating the Metrics API 
                                             
                                            Edit libraries/plugins/xfpga/sysfs.c.  Find the definition of the opae_id_to_hw_type() function.  Update the function to add the new vendor/device ID to hw_type mapping. 
                                             
                                            This mapping is used by the SDK\u2019s metrics API to determine the method of accessing the board sensor information and is very specific to the underlying BMC implementation.  It may be necessary to add a new hw_type value and to update the logic in libraries/plugins/xfpga/metrics. 
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#110-dfl-linux-kernel-drivers","title":"11.0 DFL Linux Kernel Drivers","text":"
                                            OFS DFL driver software provides the bottom-most API to FPGA platforms. Libraries such as OPAE and frameworks such as DPDK are consumers of the APIs provided by OFS. Applications may be built on top of these frameworks and libraries. The OFS software does not cover any out-of-band management interfaces. OFS driver software is designed to be extendable, flexible, and provide for bare-metal and virtualized functionality.
                                             
                                            The OFS driver software can be found in the OFS repository - linux-dfl, under the linux-dfl specific category. This repository has an associated OFS repository - linux-dfl that includes the following information:
                                             
                                               
                                            • An description of the three available branch archetypes
                                              •  
                                            • Configuration tweaks required while building the kernel
                                              •  
                                            • A functional description of the available DFL framework
                                              •  
                                            • Descriptions for all currently available driver modules that support FPGA DFL board solutions
                                              •  
                                            • Steps to create a new DFL driver
                                              •  
                                            • Steps to port a DFL driver patch
                                              •  
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#111-hardware-architecture","title":"11.1 Hardware Architecture","text":"
                                            The Linux Operating System treats the FPGA hardware as a PCIe* device. A predefined data structure, Device Feature List (DFL), allows for dynamic feature discovery in an Intel FPGA solution.
                                             
                                             
                                            The Linux Device Driver implements PCIe Single Root I/O Virtualization (SR-IOV) for the creation of Virtual Functions (VFs). The device driver can release individual accelerators for assignment to virtual machines (VMs).
                                             
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#112-fpga-management-engine-fme","title":"11.2 FPGA Management Engine (FME)","text":"
                                            The FPGA Management Engine provides error reporting, reconfiguration, performance reporting, and other infrastructure functions. Each FPGA has one FME which is always accessed through the Physical Function (PF). The Intel Xeon\u00ae Processor with Integrated FPGA also performs power and thermal management. These functions are not available on the Intel Programmable Acceleration Card (PAC).
                                             
                                            User-space applications can acquire exclusive access to the FME using open(), and release it using close(). Device access may be managed by standard Linux interfaces and tools.
                                             
                                            If an application terminates without freeing the FME or Port resources, Linux closes all file descriptors owned by the terminating process, freeing those resources.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#113-port","title":"11.3 Port","text":"
                                            A Port represents the interface between two components: * The FPGA Interface Manager (FIM) which is part of the static FPGA fabric * The Accelerator Function Unit (AFU) which is the partially reconfigurable region
                                             
                                            The Port controls the communication from software to the AFU and makes features such as reset and debug available.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#114-accelerator-function-unit-afu","title":"11.4 Accelerator Function Unit (AFU)","text":"
                                            An AFU attaches to a Port. The AFU provides a 256 KB memory mapped I/O (MMIO) region for accelerator-specific control registers.
                                             
                                               
                                            • Use open() on the Port device to acquire access to an AFU associated with the Port device.
                                              •  
                                            • Use close()on the Port device to release the AFU associated with the Port device.
                                              •  
                                            • Use mmap() on the Port device to map accelerator MMIO regions.
                                              •  
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#115-partial-reconfiguration-pr","title":"11.5 Partial Reconfiguration (PR)","text":"
                                            Use PR to reconfigure an AFU from a bitstream file. Successful reconfiguration has the following requirement:
                                             
                                               
                                            • You must generate the reconfiguration AFU for the exact FIM. The AFU and FIM are compatible if their interface IDs match. You can verify this match by comparing the interface ID in the bitstream header against the interface ID that is exported by the driver in sysfs.
                                              •  
                                             
                                            In all other cases PR fails and may cause system instability.
                                             
                                            Platforms that support 512-bit Partial Reconfiguration require binutils >= version 2.25.
                                             
                                            Close any software programs accessing the FPGA, including those running in a virtualized host before initiating PR. For virtualized environments, the recommended sequence is as follows:
                                             
                                               
                                            1. Unload the driver from the guest
                                              2.  
                                            2. Release the VF from the guest
                                              3.  
                                             
                                            Releasing the VF from the guest while an application on the guest is still accessing its resources may lead to VM instabilities. We recommend closing all applications accessing the VF in the guest before releasing the VF.
                                             
                                               
                                            1. Disable SR-IOV
                                              2.  
                                            2. Perform PR
                                              3.  
                                            3. Enable SR-IOV
                                              4.  
                                            4. Assign the VF to the guest
                                              5.  
                                            5. Load the driver in the guest
                                              6.  
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#116-fpga-virtualization","title":"11.6 FPGA Virtualization","text":"
                                            To enable accelerator access from applications running on a VM, create a VF for the port using the following process:
                                             
                                               
                                            1.  
                                              Release the Port from the PF using the associated ioctl on the FME device.
                                               
                                              2.  
                                            2.  
                                              Use the following command to enable SR-IOV and VFs. Each VF can own a single Port with an AFU. In the following command, N is the number of Port released from the PF.
                                               
                                              3.  
                                             
                                                echo N > $PCI_DEVICE_PATH/sriov_numvfs\n
                                             
                                            The number, 'N', cannot be greater than the number of supported VFs. This can be read from $PCI_DEVICE_PATH/sriov_totalvfs.
                                             
                                               
                                            1.  
                                              Pass the VFs through to VMs using hypervisor interfaces.
                                               
                                              2.  
                                            2.  
                                              Access the AFU on a VF from applications running on the VM using the same driver inside the VM.
                                               
                                              3.  
                                             
                                            Creating VFs is only supported for port devices. Consequently, PR and other management functions are only available through the PF.    
                                             
                                             
                                            If assigning multiple devices to the same VM on a guest IOMMU, you may need to increase the hard_limit option in order to avoid hitting a limit of pinned memory. The hard limit should be more than (VM memory size x Number of PCIe devices).
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#117-driver-organization","title":"11.7 Driver Organization","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1171-pcie-module-device-driver","title":"11.7.1 PCIe Module Device Driver","text":"
                                            FPGA devices appear as a PCIe devices. Once enumeration detects a PCIe PF or VF, the Linux OS loads the FPGA PCIe device driver. The device driver performs the following functions:
                                             
                                               
                                            1. Walks through the Device Feature List in PCIe device base address register (BAR) memory to discover features and their sub-features and creates necessary platform devices.
                                              2.  
                                            2. Enables SR-IOV.
                                              3.  
                                            3. Introduces the feature device infrastructure, which abstracts operations for sub-features and provides common functions to feature device drivers.
                                              4.  
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1172-pcie-module-device-driver-functions","title":"11.7.2 PCIe Module Device Driver Functions","text":"
                                            The PCIe Module Device Driver performs the following functions:
                                             
                                               
                                            1. PCIe discovery, device enumeration, and feature discovery.
                                              2.  
                                            2. Creates sysfs directories for the device, FME, and Port.
                                              3.  
                                            3. Creates the platform driver instances, causing the Linux kernel to load their respective drivers.
                                              4.  
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1173-fme-platform-module-device-driver","title":"11.7.3 FME Platform Module Device Driver","text":"
                                            The FME Platform Module Device Driver loads automatically after the PCIe driver creates the FME Platform Module. It provides the following features for FPGA management:
                                             
                                               
                                            1.  
                                              Power and thermal management, error reporting, performance reporting, and other infrastructure functions. You can access these functions via sysfs interfaces the FME driver provides.
                                               
                                              2.  
                                            2.  
                                              Partial Reconfiguration. During PR sub-feature initialization, the FME driver registers the FPGA Manager framework to support PR. When the FME receives the relevant ioctl request from user-space, it invokes the common interface function from the FPGA Manager to reconfigure the AFU using PR.
                                               
                                              3.  
                                            3.  
                                              Port management for virtualization (releasing/assigning port device).
                                               
                                              4.  
                                             
                                            After a port device is released, you can use the PCIe driver SR-IOV interfaces to create/destroy VFs.
                                             
                                            For more information, refer to \"FPGA Virtualization\".
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1174-fme-platform-module-device-driver-functions","title":"11.7.4 FME Platform Module Device Driver Functions","text":"
                                            The FME Platform Module Device Driver performs the the following functions:
                                             
                                               
                                            • Creates the FME character device node.
                                              •  
                                            • Creates the FME sysfs files and implements the FME sysfs file accessors.
                                              •  
                                            • Implements the FME private feature sub-drivers.
                                              •  
                                            • FME private feature sub-drivers:     * FME Header     * Partial Reconfiguration     * Global Error     * Global Performance
                                              •  
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1175-port-platform-module-device-driver","title":"11.7.5 Port Platform Module Device Driver","text":"
                                            After the PCIe Module Device Driver creates the Port Platform Module device, the FPGA Port and AFU driver are loaded.  This module provides an interface for user-space applications to access the individual accelerators, including basic reset control on the Port, AFU MMIO region export, DMA buffer mapping service, and remote debug functions.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1176-port-platform-module-device-driver-functions","title":"11.7.6 Port Platform Module Device Driver Functions","text":"
                                            The Port Platform Module Device Driver performs the the following functions:
                                             
                                               
                                            • Creates the Port character device node.
                                              •  
                                            • Creates the Port sysfs files and implements the Port sysfs file accessors.
                                              •  
                                            • Implements the following Port private feature sub-drivers.     * Port Header     * AFU     * Port Error     * Signal Tap
                                              •  
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#1177-opae-fpga-driver-interface","title":"11.7.7 OPAE FPGA Driver Interface","text":"
                                            The user-space interface consists of a sysfs hierarchy and ioctl requests. Most kernel attributes can be accessed/modified via sysfs nodes in this hierarchy. More complex I/O operations are controlled via ioctl requests. The OPAE API implementation, libopae-c, has been designed to use this interface to interact with the OPAE FPGA kernel drivers.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#120-plugin-development","title":"12.0 Plugin Development","text":""},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#121-overview","title":"12.1 Overview","text":"
                                            Beginning with OPAE C library version 1.2.0, OPAE implements a plugin-centric model. This guide serves as a reference to define the makeup of an OPAE C API plugin and to describe a sequence of steps that one may follow when constructing an OPAE C API plugin.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#122-plugin-required-functions","title":"12.2 Plugin Required Functions","text":"
                                            An OPAE C API plugin is a runtime-loadable shared object library, also known as a module. On Linux systems, the dl family of APIs from libdl are used to interact with shared objects. Refer to \"man dlopen\" and \"man dlsym\" for examples of using the libdl API.
                                             
                                            An OPAE C API plugin implements one required function. This function is required to have C linkage, so that its name is not mangled.
                                             
                                                int opae_plugin_configure(opae_api_adapter_table *table, const char *config);\n
                                             
                                            During initialization, the OPAE plugin manager component loads each plugin, searching for its opae_plugin_configure function. If none is found, then the plugin manager rejects that plugin. When it is found, opae_plugin_configure is called passing a pointer to a freshly-created opae_api_adapter_table and a buffer consisting of configuration data for the plugin.
                                             
                                            The job of the opae_plugin_configure function is to populate the given adapter table with each of the plugin's API entry points and to consume and comprehend the given configuration data in preparation for initialization.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#123-opae-api-adapter-table","title":"12.3 OPAE API Adapter Table","text":"
                                            The adapter table is a data structure that contains function pointer entry points for each of the OPAE APIs implemented by a plugin. In this way, it adapts the plugin-specific behavior to the more general case of a flat C API. Note that OPAE applications are only required to link with opae-c. In other words, the name of the plugin library should not appear on the linker command line. In this way, plugins are truly decoupled from the OPAE C API, and they are required to adapt to the strict API specification by populating the adapter table only. No other linkage is required nor recommended.
                                             
                                            adapter.h contains the definition of the opae_api_adapter_table. An abbreviated version is depicted below, along with supporting type opae_plugin:
                                             
                                                typedef struct _opae_plugin {\n        char *path;\n        void *dl_handle;\n    } opae_plugin;\n\n    typedef struct _opae_api_adapter_table {\n\n        struct _opae_api_adapater_table *next;\n        opae_plugin plugin;\n\n        fpga_result (*fpgaOpen)(fpga_token token, fpga_handle *handle,\n                                int flags);\n\n        fpga_result (*fpgaClose)(fpga_handle handle);\n\n        ...\n\n        fpga_result (*fpgaEnumerate)(const fpga_properties *filters,\n                                     uint32_t num_filters, fpga_token *tokens,\n                                     uint32_t max_tokens,\n                                     uint32_t *num_matches);\n\n        ...\n\n        // configuration functions\n        int (*initialize)(void);\n        int (*finalize)(void);\n\n        // first-level query\n        bool (*supports_device)(const char *device_type);\n        bool (*supports_host)(const char *hostname);\n\n    } opae_api_adapter_table;\n
                                             
                                            Some points worth noting are that the adapter tables are organized in memory by adding them to a linked list data structure. This is the use of the next structure member. (The list management is handled by the plugin manager.) The plugin structure member contains the handle to the shared object instance, as created by dlopen. This handle is used in the plugin's opae_plugin_configure to load plugin entry points. A plugin need only implement the portion of the OPAE C API that a target application needs. Any API entry points that are not supported should be left as NULL pointers (the default) in the adapter table. When an OPAE API that has no associated entry point in the adapter table is called, the result for objects associated with that plugin will be FPGA_NOT_SUPPORTED.
                                             
                                            The following code illustrates a portion of the opae_plugin_configure for a theoretical OPAE C API plugin libfoo.so:
                                             
                                                /* foo_plugin.c */\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        adapter->fpgaOpen = dlsym(adapter->plugin.dl_handle, \"foo_fpgaOpen\");\n        adapter->fpgaClose =\n                dlsym(adapter->plugin.dl_handle, \"foo_fpgaClose\");\n\n        ...\n\n        adapter->fpgaEnumerate =\n                dlsym(adapter->plugin.dl_handle, \"foo_fpgaEnumerate\");\n\n        ...\n\n        return 0;\n    }\n
                                             
                                            Notice that the implementations of the API entry points for plugin libfoo.so are prefixed with foo_. This is the recommended practice to avoid name collisions and to enhance the debugability of the application. Upon successful configuration, opae_plugin_configure returns 0 to indicate success. A non-zero return value indicates failure and causes the plugin manager to reject the plugin from futher consideration.
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#124-plugin-optional-functions","title":"12.4 Plugin Optional Functions","text":"
                                            Once the plugin manager loads and configures each plugin, it uses the adapter table to call back into the plugin so that it can be made ready for runtime. This is the job of the opae_plugin_initialize entry point, whose signature is defined as:
                                             
                                                int opae_plugin_initialize(void);\n
                                             
                                            The function takes no parameters, as the configuration data was already given to the plugin by opae_plugin_configure. opae_plugin_initialize returns 0 if no errors were encountered during initialization. A non-zero return code indicates that plugin initialization failed. A plugin makes its opae_plugin_initialize available to the plugin manager by populating the adapter table's initialize entry point as shown:
                                             
                                                /* foo_plugin.c */\n\n    int foo_plugin_initialize(void)\n    {\n        ...\n\n        return 0; /* success */\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->initialize =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_initialize\");\n\n        ...\n\n        return 0;\n    }\n
                                             
                                            If a plugin does not implement an opae_plugin_initialize entry point, then the initialize member of the adapter table should be left uninitialized. During plugin initialization, if a plugin has no opae_plugin_initialize entry in its adapter table, the plugin initialization step will be skipped, and the plugin will be considered to have initialized successfully.
                                             
                                            Once plugin initialization is complete for all loaded plugins, the system is considered to be running and fully functional.
                                             
                                            During teardown, the plugin manager uses the adapter table to call into each plugin's opae_plugin_finalize entry point, whose signature is defined as:
                                             
                                                int opae_plugin_finalize(void);\n
                                             
                                            opae_plugin_finalize returns 0 if no errors were encountered during teardown. A non-zero return code indicates that plugin teardown failed. A plugin makes its opae_plugin_finalize available to the plugin manager by populating the adapter table's finalize entry point as shown:
                                             
                                                /* foo_plugin.c */\n\n    int foo_plugin_finalize(void)\n    {\n        ...\n\n        return 0; /* success */\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->finalize =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_finalize\");\n\n        ...\n\n        return 0;\n    }\n
                                             
                                            If a plugin does not implement an opae_plugin_finalize entry point, then the finalize member of the adapter table should be left uninitialized. During plugin cleanup, if a plugin has no opae_plugin_finalize entry point in its adapter table, the plugin finalize step will be skipped, and the plugin will be considered to have finalized successfully.
                                             
                                            In addition to initialize and finalize, an OPAE C API plugin has two further optional entry points that relate to device enumeration. During enumeration, when a plugin is being considered for a type of device, the plugin may provide input on that decision by exporting an opae_plugin_supports_device entry point in the adapter table:
                                             
                                                bool opae_plugin_supports_device(const char *device_type);\n
                                             
                                            opae_plugin_supports_device returns true if the given device type is supported and false if it is not. A false return value from opae_plugin_supports_device causes device enumeration to skip the plugin.
                                             
                                            Populating the opae_plugin_supports_device is done as:
                                             
                                                /* foo_plugin.c */\n\n    bool foo_plugin_supports_device(const char *device_type)\n    {\n        if (/* device_type is supported */)\n            return true;\n\n        ...\n\n        return false;\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->supports_device =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_supports_device\");\n\n        ...\n\n        return 0;\n    }\n
                                             
                                            .. note::\n    The `opae_plugin_supports_device` mechanism serves as a placeholder only.\n    It is not implemented in the current version of the OPAE C API.\n
                                             
                                            Similarly to determining whether a plugin supports a type of device, a plugin may also answer questions about network host support by populating an opae_plugin_supports_host entry point in the adapter table:
                                             
                                                bool opae_plugin_supports_host(const char *hostname);\n
                                             
                                            opae_plugin_supports_host returns true if the given hostname is supported and false if it is not. A false return value from opae_plugin_supports_host causes device enumeration to skip the plugin.
                                             
                                            Populating the opae_plugin_supports_host is done as:
                                             
                                                /* foo_plugin.c */\n\n    bool foo_plugin_supports_host(const char *hostname)\n    {\n        if (/* hostname is supported */)\n            return true;\n\n        ...\n\n        return false;\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->supports_host =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_supports_host\");\n\n        ...\n\n        return 0;\n    }\n
                                             
                                            .. note::\n    The `opae_plugin_supports_host` mechanism serves as a placeholder only.\n    It is not implemented in the current version of the OPAE C API.\n
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#125-plugin-construction","title":"12.5 Plugin Construction","text":"
                                            The steps required to implement an OPAE C API plugin, libfoo.so, are:
                                             
                                               
                                            • Create foo_plugin.c: implements opae_plugin_configure, opae_plugin_initialize, opae_plugin_finalize, opae_plugin_supports_device, and opae_plugin_supports_host as described in the previous sections.
                                              •  
                                            • Create foo_plugin.h: implements function prototypes for each of the plugin-specific OPAE C APIs.
                                              •  
                                             
                                                /* foo_plugin.h */\n\n    fpga_result foo_fpgaOpen(fpga_token token, fpga_handle *handle,\n                             int flags);\n\n    fpga_result foo_fpgaClose(fpga_handle handle);\n\n    ...\n\n    fpga_result foo_fpgaEnumerate(const fpga_properties *filters,\n                                  uint32_t num_filters, fpga_token *tokens,\n                                  uint32_t max_tokens,\n                                  uint32_t *num_matches);\n    ...\n
                                             
                                               
                                            • Create foo_types.h: implements plugin-specific types for opaque data structures.
                                              •  
                                             
                                                /* foo_types.h */\n\n    struct _foo_token {\n        ...\n    };\n\n    struct _foo_handle {\n        ...\n    };\n\n    struct _foo_event_handle {\n        ...\n    };\n\n    struct _foo_object {\n        ...\n    };\n
                                             
                                               
                                            • Create foo_enum.c: implements foo_fpgaEnumerate, foo_fpgaCloneToken, and foo_fpgaDestroyToken.
                                              •  
                                            • Create foo_open.c: implements foo_fpgaOpen.
                                              •  
                                            • Create foo_close.c: implements foo_fpgaClose.
                                              •  
                                            • Create foo_props.c: implements foo_fpgaGetProperties, foo_fpgaGetPropertiesFromHandle, foo_fpgaUpdateProperties
                                              •  
                                            • Create foo_mmio.c: implements foo_fpgaMapMMIO, foo_fpgaUnmapMMIO foo_fpgaWriteMMIO64, foo_fpgaReadMMIO64, foo_fpgaWriteMMIO32, foo_fpgaReadMMIO32.
                                              •  
                                            • Create foo_buff.c: implements foo_fpgaPrepareBuffer, foo_fpgaReleaseBuffer, foo_fpgaGetIOAddress.
                                              •  
                                            • Create foo_error.c: implements foo_fpgaReadError, foo_fpgaClearError, foo_fpgaClearAllErrors, foo_fpgaGetErrorInfo.
                                              •  
                                            • Create foo_event.c: implements foo_fpgaCreateEventHandle, foo_fpgaDestroyEventHandle, foo_fpgaGetOSObjectFromEventHandle, foo_fpgaRegisterEvent, foo_fpgaUnregisterEvent.
                                              •  
                                            • Create foo_reconf.c: implements foo_fpgaReconfigureSlot.
                                              •  
                                            • Create foo_obj.c: implements foo_fpgaTokenGetObject, foo_fpgaHandleGetObject, foo_fpgaObjectGetObject, foo_fpgaDestroyObject, foo_fpgaObjectGetSize, foo_fpgaObjectRead, foo_fpgaObjectRead64, foo_fpgaObjectWrite64.
                                              •  
                                            • Create foo_clk.c: implements foo_fpgaSetUserClock, foo_fpgaGetUserClock.
                                              •  
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#130-building-a-sample-application","title":"13.0 Building a Sample Application","text":"
                                            Building a sample application The library source includes code samples. Use these samples to learn how to call functions in the library. Build and run these samples as quick sanity checks to determine if your installation and environment are set up properly.
                                             
                                            In this guide, we will build hello_fpga.c. This is the \"Hello World!\" example of using the library. This code searches for a predefined and known AFU called \"Native Loopback Adapter\" on the FPGA. If found, it acquires ownership and then interacts with the AFU by sending it a 2MB message and waiting for the message to be echoed back. This code exercises all major components of the API except for AFU reconfiguration: AFU search, enumeration, access, MMIO, and memory management.
                                             
                                            You can also find the source for hello_fpga in the samples directory of the OPAE SDK repository on GitHub.
                                             
                                                int main(int argc, char *argv[])\n    {\n        fpga_properties    filter = NULL;\n        fpga_token         afu_token;\n        fpga_handle        afu_handle;\n        fpga_guid          guid;\n        uint32_t           num_matches;\n\n        volatile uint64_t *dsm_ptr    = NULL;\n        volatile uint64_t *status_ptr = NULL;\n        volatile uint64_t *input_ptr  = NULL;\n        volatile uint64_t *output_ptr = NULL;\n\n        uint64_t        dsm_wsid;\n        uint64_t        input_wsid;\n        uint64_t        output_wsid;\n        fpga_result     res = FPGA_OK;\n\n        if (uuid_parse(NLB0_AFUID, guid) < 0) {\n            fprintf(stderr, \"Error parsing guid '%s'\\n\", NLB0_AFUID);\n            goto out_exit;\n        }\n\n        /* Look for accelerator by its \"afu_id\" */\n        res = fpgaGetProperties(NULL, &filter);\n        ON_ERR_GOTO(res, out_exit, \"creating properties object\");\n\n        res = fpgaPropertiesSetObjectType(filter, FPGA_ACCELERATOR);\n        ON_ERR_GOTO(res, out_destroy_prop, \"setting object type\");\n\n        res = fpgaPropertiesSetGuid(filter, guid);\n        ON_ERR_GOTO(res, out_destroy_prop, \"setting GUID\");\n\n        /* TODO: Add selection via BDF / device ID */\n\n        res = fpgaEnumerate(&filter, 1, &afu_token, 1, &num_matches);\n        ON_ERR_GOTO(res, out_destroy_prop, \"enumerating accelerators\");\n\n        if (num_matches < 1) {\n            fprintf(stderr, \"accelerator not found.\\n\");\n            res = fpgaDestroyProperties(&filter);\n            return FPGA_INVALID_PARAM;\n        }\n\n        /* Open accelerator and map MMIO */\n        res = fpgaOpen(afu_token, &afu_handle, 0);\n        ON_ERR_GOTO(res, out_destroy_tok, \"opening accelerator\");\n\n        res = fpgaMapMMIO(afu_handle, 0, NULL);\n        ON_ERR_GOTO(res, out_close, \"mapping MMIO space\");\n\n        /* Allocate buffers */\n        res = fpgaPrepareBuffer(afu_handle, LPBK1_DSM_SIZE,\n                    (void **)&dsm_ptr, &dsm_wsid, 0);\n        ON_ERR_GOTO(res, out_close, \"allocating DSM buffer\");\n\n        res = fpgaPrepareBuffer(afu_handle, LPBK1_BUFFER_ALLOCATION_SIZE,\n                   (void **)&input_ptr, &input_wsid, 0);\n        ON_ERR_GOTO(res, out_free_dsm, \"allocating input buffer\");\n\n        res = fpgaPrepareBuffer(afu_handle, LPBK1_BUFFER_ALLOCATION_SIZE,\n                   (void **)&output_ptr, &output_wsid, 0);\n        ON_ERR_GOTO(res, out_free_input, \"allocating output buffer\");\n\n        printf(\"Running Test\\n\");\n\n        /* Initialize buffers */\n        memset((void *)dsm_ptr,    0,    LPBK1_DSM_SIZE);\n        memset((void *)input_ptr,  0xAF, LPBK1_BUFFER_SIZE);\n        memset((void *)output_ptr, 0xBE, LPBK1_BUFFER_SIZE);\n\n        cache_line *cl_ptr = (cache_line *)input_ptr;\n        for (uint32_t i = 0; i < LPBK1_BUFFER_SIZE / CL(1); ++i) {\n            cl_ptr[i].uint[15] = i+1; /* set the last uint in every cacheline */\n        }\n\n        /* Reset accelerator */\n        res = fpgaReset(afu_handle);\n        ON_ERR_GOTO(res, out_free_output, \"resetting accelerator\");\n\n        /* Program DMA addresses */\n        uint64_t iova;\n        res = fpgaGetIOAddress(afu_handle, dsm_wsid, &iova);\n        ON_ERR_GOTO(res, out_free_output, \"getting DSM IOVA\");\n\n        res = fpgaWriteMMIO64(afu_handle, 0, CSR_AFU_DSM_BASEL, iova);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_AFU_DSM_BASEL\");\n\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 0);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 1);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        res = fpgaGetIOAddress(afu_handle, input_wsid, &iova);\n        ON_ERR_GOTO(res, out_free_output, \"getting input IOVA\");\n        res = fpgaWriteMMIO64(afu_handle, 0, CSR_SRC_ADDR, CACHELINE_ALIGNED_ADDR(iova));\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_SRC_ADDR\");\n\n        res = fpgaGetIOAddress(afu_handle, output_wsid, &iova);\n        ON_ERR_GOTO(res, out_free_output, \"getting output IOVA\");\n        res = fpgaWriteMMIO64(afu_handle, 0, CSR_DST_ADDR, CACHELINE_ALIGNED_ADDR(iova));\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_DST_ADDR\");\n\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_NUM_LINES, LPBK1_BUFFER_SIZE / CL(1));\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_NUM_LINES\");\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CFG, 0x42000);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        status_ptr = dsm_ptr + DSM_STATUS_TEST_COMPLETE/8;\n\n        /* Start the test */\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 3);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        /* Wait for test completion */\n        while (0 == ((*status_ptr) & 0x1)) {\n            usleep(100);\n        }\n\n        /* Stop the device */\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 7);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        /* Check output buffer contents */\n        for (uint32_t i = 0; i < LPBK1_BUFFER_SIZE; i++) {\n            if (((uint8_t*)output_ptr)[i] != ((uint8_t*)input_ptr)[i]) {\n                fprintf(stderr, \"Output does NOT match input \"\n                    \"at offset %i!\\n\", i);\n                break;\n            }\n        }\n\n        printf(\"Done Running Test\\n\");\n\n        /* Release buffers */\n    out_free_output:\n        res = fpgaReleaseBuffer(afu_handle, output_wsid);\n        ON_ERR_GOTO(res, out_free_input, \"releasing output buffer\");\n    out_free_input:\n        res = fpgaReleaseBuffer(afu_handle, input_wsid);\n        ON_ERR_GOTO(res, out_free_dsm, \"releasing input buffer\");\n    out_free_dsm:\n        res = fpgaReleaseBuffer(afu_handle, dsm_wsid);\n        ON_ERR_GOTO(res, out_unmap, \"releasing DSM buffer\");\n\n        /* Unmap MMIO space */\n    out_unmap:\n        res = fpgaUnmapMMIO(afu_handle, 0);\n        ON_ERR_GOTO(res, out_close, \"unmapping MMIO space\");\n\n        /* Release accelerator */\n    out_close:\n        res = fpgaClose(afu_handle);\n        ON_ERR_GOTO(res, out_destroy_tok, \"closing accelerator\");\n\n        /* Destroy token */\n    out_destroy_tok:\n        res = fpgaDestroyToken(&afu_token);\n        ON_ERR_GOTO(res, out_destroy_prop, \"destroying token\");\n\n        /* Destroy properties object */\n    out_destroy_prop:\n        res = fpgaDestroyProperties(&filter);\n        ON_ERR_GOTO(res, out_exit, \"destroying properties object\");\n\n    out_exit:\n        return res;\n\n    }\n
                                             
                                            Linking with the OPAE library is straightforward. Code using this library should include the header file fpga.h. Taking the GCC compiler on Linux as an example, the minimalist compile and link line should look like:
                                             
                                            gcc -std=c99 hello_fpga.c -I/usr/local/include -L/usr/local/lib -lopae-c -luuid -ljson-c -lpthread -o hello_fpga\n
                                             
                                            The API uses some features from the C99 language standard. The -std=c99 switch is required if the compiler does not support C99 by default.
                                             
                                            Third-party library dependency: The library internally uses libuuid and libjson-c. But they are not distributed as part of the library. Make sure you have these libraries properly installed. The -c flag may not be necessary depending on  the platform being used. 
                                             
                                            sudo ./hello_fpga -c\nRunning Test\nRunning on bus 0x08.\nAFU NLB0 found @ 28000\nDone Running Test\n
                                             To run the hello_fpga application:
                                             
                                            sudo ./hello_fpga\nRunning Test\nDone\n
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#appendix-a-integrating-an-n6001-based-custom-platform-with-dfl-and-opae","title":"Appendix A - Integrating an N6001 Based Custom Platform with DFL and OPAE","text":"
                                            The process of adding a custom device to the DFL framework and OPAE SDK requires changes to files  in several locations. In this section we will walk through the additions necessary to instruct the kernel's probe and match function to associate your new device with OPAE, choose the correct OPAE plugin to associate with your board, and change basic descriptors to properly display the name of your new custom platform when using OPAE's management interfaces. 
                                             
                                            This section does not require useage of an entirely new platform - we will use the Intel N6001's FIM design as our base, and alter only those settings in the PCIe IP necessary to change the following PCIe IDs for both the PF and VF:
                                             
                                               
                                            • Vendor ID
                                              •  
                                            • Device ID
                                              •  
                                            • Subsystem Vendor ID
                                              •  
                                            • Subsystem Device ID
                                              •  
                                             
                                            This document will not cover the process by which a FIM can be modified to support these new IDs. Refer to section 5.7 How to  change the PCIe device ID and Vendor ID in  F2000x FIM Developer guide or section 10.0 How to  change the PCIe device ID and Vendor ID in the N6001 FIM Developer guide for an overview of this process. The following sections assume you have a FIM that has been configured with new IDs. This FIM can be loaded onto your N6001 board using either the SOF via a USB-BlasterII cable and the Quartus Programmer, or by using the OPAE command fpgasupdate consuming a compatible binary programming file. The following values will be used as our new device IDs.
                                             ID Type PF0 PF0-VF Vendor ID 0xff00 0xff00 Device ID 0x1234 0x5555 Subsystem Vendor ID 0xff00 0xff00 Subsystem Device ID 0x0x5678 0x5678 
                                            The only value that differs between PF0 and PF0-VF in this example is the Device ID, and all other values do not currently exist in either the OPAE SDK or linux-dfl drivers. You will need to download the OPAE SDK and linux-dfl sources from GitHub and modify their contents. We will be using a validated Agilex OFS release to pair our OPAE SDK and linux-dfl versions together. Refer to the below table for a list of the exact version we will use for each software component and where you can  learn how to build and install. Your versions may not match those in the document.
                                             Software Version Build Instructions OPAE SDK 2.3.0-1 4.0 OPAE Software Development Kit linux-dfl ofs-2022.3-2 3.0 Intel OFS DFL Kernel Drivers 
                                            The following steps will enable your device to use the OPAE SDK. We will call our new device the \"Intel FPGA Programmable Acceleration Card N6002\". This device is identical to the Intel FPGA Programmable Acceleration Card N6001, and will use the pre-existing plugins and feature ID associated with that device. We will also use the enum value FPGA_HW_DCP_N6002 to describe our new board in the code. These plugins can be customized as you become more familiar with the OPAE SDK software. 
                                             
                                               
                                            1. Download the OPAE SDK from GitHub. File paths assume the user is in the directory opae-sdk.
                                              2.  
                                            2. Open the file binaries/opae.io/opae/io/config.py. Add a new configuration entry under DEFAULT_OPAE_IO_CONFIG. Save and exit.
                                              3.  
                                             
                                            Example:
                                             
                                            (0xff00, 0x1234, 0xff00, 0x5678) : {\n        'platform': 'Intel FPGA Programmable Acceleration Card N6002'\n    },\n
                                             
                                               
                                            1. Open the file libraries/libopae-c/cfg-file.c. Add two new entries (one for PF0 and PF0-VF) under STATIC libopae_config_data default_libopae_config_table[]. Add  two new entries under STATIC fpgainfo_config_data default_fpgainfo_config_table[]. Save and exit.
                                              2.  
                                             
                                            Example:
                                             
                                            STATIC fpgainfo_config_data default_fpgainfo_config_table[] = {\n...\n{ 0xff00, 0x1234, 0xff00, 0x5678, 0x12, \"libboard_n6000.so\", NULL,\n\"Intel FPGA Programmable Acceleration Card N6002\" },                  // N6002 PF0\n{ 0xff00, 0x5555, 0xff00, 0x5678, 0x12, \"libboard_n6000.so\", NULL,\n\"Intel FPGA Programmable Acceleration Card N6002\" },                  // N6002 PF0-VF\n
                                             
                                            Example:
                                             
                                            STATIC libopae_config_data default_libopae_config_table[] = {\n...\n{ 0xff00, 0x1234, 0xff00,          0x5678,          \"libxfpga.so\",  \"{}\", 0 }    , // N6002 PF0\n{ 0xff00, 0x5555, 0xff00,          0x5678,          \"libxfpga.so\",  \"{}\", 0 }    , // N6002 PF0-VF\n
                                             4. Open the file libraries/plugins/xfpga/metrics/metric_utils.c. Add one entry to the switch case under enum_fpga_metrics(fpga_handle handle). The enum value used should match the enum set in step 6.  Add a new condition to the if statement beginning if (((_fpga_enum_metric->hw_type == FPGA_HW_DCP_N3000). Save and exit.
                                             
                                            Example:
                                             
                                                             // DCP VC DC\n                 case FPGA_HW_DCP_N3000:\n                 case FPGA_HW_DCP_D5005:\n                 case FPGA_HW_DCP_N6002:\n                 case FPGA_HW_DCP_N5010: {\n                 ...\n
                                             
                                            Example:
                                             
                                                                        if (((_fpga_enum_metric->hw_type == FPGA_HW_DCP_N3000) ||\n                                 (_fpga_enum_metric->hw_type == FPGA_HW_DCP_D5005) ||\n                                 (_fpga_enum_metric->hw_type == FPGA_HW_DCP_N6002) ||\n                                 (_fpga_enum_metric->hw_type == FPGA_HW_DCP_N5010)) &&\n
                                             5. Open the file libraries/plugins/xfpga/sysfs.c. Add a new set of switch cases under enum fpga_hw_type opae_id_to_hw_type(uint16_t vendor_id, uint16_t device_id). The enum value used should match the enum value set in step 6. Save and exit.
                                             
                                            Example:
                                             
                                                        if (vendor_id == 0xff00) {        \n                 switch (device_id) {\n                         case 0x1234:\n                         case 0x5555:\n                                 hw_type = FPGA_HW_DCP_N6002;\n                         break;\n                 default:\n                         OPAE_ERR(\"unknown device id: 0x%04x\", device_id);\n
                                             6. Open the file libraries/plugins/xfpga/types_int.h. Add your new enum value under enum fpga_hw_type. Save and exit.
                                             
                                            Example:
                                             
                                            enum fpga_hw_type {\n         FPGA_HW_MCP,\n         FPGA_HW_DCP_RC,\n         FPGA_HW_DCP_D5005,\n         FPGA_HW_DCP_N3000,\n         FPGA_HW_DCP_N5010,\n         FPGA_HW_DCP_N6002,\n         FPGA_HW_UNKNOWN\n};\n
                                             7. Open the file opae.cfg. Create a new entry for device \"n6002\" by copying the entry for \"n6001,\"\" substituting our new values. Add one new entry under \"configs\" for the name \"n6002.\" Save and exit.
                                             
                                            Example:
                                             
                                                \"n6002\": {\n      \"enabled\": true,\n      \"platform\": \"Intel Acceleration Development Platform N6002\",\n      \"devices\": [\n        { \"name\": \"n6002_pf\", \"id\": [ \"0xff00\", \"0x1234\", \"0xff00\", \"0x5678\" ] },\n        { \"name\": \"n6002_vf\", \"id\": [ \"0xff00\", \"0x5555\", \"0xff00\", \"0x5678\" ] }\n      ],\n      \"opae\": {\n        \"plugin\": [\n          {\n            \"enabled\": true,\n            \"module\": \"libxfpga.so\",\n            \"devices\": [ \"n6002_pf\" ],\n            \"configuration\": {}\n          },\n          {\n            \"enabled\": true,\n            \"module\": \"libopae-v.so\",\n            \"devices\": [ \"n6002_pf\", \"n6002_vf\" ],\n            \"configuration\": {}\n          }\n        ],\n        \"fpgainfo\": [\n          {\n            \"enabled\": true,\n            \"module\": \"libboard_n6000.so\",\n            \"devices\": [\n              { \"device\": \"n6002_pf\", \"feature_id\": \"0x12\" },\n              { \"device\": \"n6002_vf\", \"feature_id\": \"0x12\" }\n            ]\n          }\n        ],\n        \"fpgad\": [\n         {\n            \"enabled\": true,\n            \"module\": \"libfpgad-vc.so\",\n            \"devices\": [ \"n6002_pf\" ],\n            \"configuration\": {\n              \"cool-down\": 30,\n              \"get-aer\":     [ \"setpci -s %s ECAP_AER+0x08.L\",\n                               \"setpci -s %s ECAP_AER+0x14.L\" ],\n              \"disable-aer\": [ \"setpci -s %s ECAP_AER+0x08.L=0xffffffff\",\n                               \"setpci -s %s ECAP_AER+0x14.L=0xffffffff\" ],\n              \"set-aer\":     [ \"setpci -s %s ECAP_AER+0x08.L=0x%08x\",\n                               \"setpci -s %s ECAP_AER+0x14.L=0x%08x\" ],\n              \"sensor-overrides\": [],\n              \"monitor-seu\": false\n            }\n          }\n        ],\n        \"rsu\": [\n          {\n            \"enabled\": true,\n            \"devices\": [ \"n6002_pf\" ],\n            \"fpga_default_sequences\": \"common_rsu_sequences\"\n          }\n        ],\n        \"fpgareg\": [\n          {\n            \"enabled\": true,\n            \"devices\": [ \"n6002_pf\", \"n6002_vf\" ]\n          }\n        ],\n        \"opae.io\": [\n          {\n            \"enabled\": true,\n            \"devices\": [ \"n6002_pf\", \"n6002_vf\" ]\n          }\n        ]\n      }\n    },\n
                                             
                                            Example:
                                             
                                            \"configs\": [\n     \"mcp\",\n     \"a10gx\",\n     \"d5005\",\n     \"n3000\",\n     \"n5010\",\n     \"n5013\",\n     \"n5014\",\n     \"n6000\",\n     \"n6001\",\n     \"n6002\",\n     ...\n
                                             These conclude our integration of our new platform with the OPAE SDK. The next step is to download the source for linux-dfl (as shown above)  and configure our new kernel's match and probe function to associate the DFL drivers with our new custom platform. The following file path assumes the user is in the directory linux-dfl 1. Open the file drivers/fpga/dfl-pci.c. Define a list of necessary ID values at the top of the file. Use these values to add two new entries under pci_device_id cci_pcie_id_tbl[], one for PF0 and the other for PF0-VF. Save and exit.
                                             
                                            Example:
                                             
                                            /* N6002 IDS */\n#define PCIE_DEVICE_ID_PF_N6002                 0x1234\n#define PCIE_VENDOR_ID_PF_N6002                 0xff00\n#define PCIE_SUBDEVICE_ID_PF_N6002              0x5678\n#define PCIE_DEVICE_ID_VF_N6002                 0x5555\n
                                             
                                            Example:
                                             
                                            static struct pci_device_id cci_pcie_id_tbl[] = {\n        ...\n        {PCI_DEVICE_SUB(PCIE_VENDOR_ID_PF_N6002, PCIE_DEVICE_ID_PF_N6002,\n                    PCIE_VENDOR_ID_PF_N6002, PCIE_SUBDEVICE_ID_PF_N6002),}, //N6002 PF0\n        {PCI_DEVICE_SUB(PCIE_VENDOR_ID_PF_N6002, PCIE_DEVICE_ID_VF_N6002,\n                    PCIE_VENDOR_ID_PF_N6002, PCIE_SUBDEVICE_ID_PF_N6002),}, //N6002 PF0-VF\n                    ...\n
                                             
                                            This concludes our integration our new platform with the linux-dfl driver set. Build and install the linux-dfl enabled kernel and the OPAE SDK userspace libraries as discussed in their relevant sections in the user guide linked above. If the above conditions have been met, and your N6001 board has been configured with this new \"N6002\" FIM, you should see the following output when running the command \"fpgainfo fme\" (your Bitstream ID, PR Interface ID, and Image Info entries may differ). Check that the board's display name at the top and values for Vendor/Device/SubVendor/Subdevice IDs are correct.
                                             
                                            Intel Acceleration Development Platform N6002\nBoard Management Controller NIOS FW version: 3.11.0\nBoard Management Controller Build version: 3.11.0\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0xFF00\nDevice Id                        : 0x1234\nSubVendor Id                     : 0xFF00\nSubDevice Id                     : 0x5678\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010202C8AD72D7\nBitstream Version                : 5.0.1\nPr Interface Id                  : 8df219e3-cf25-5e77-8689-f57102d54435\nBoot Page                        : user1\nFactory Image Info               : a2b5fd0e7afca4ee6d7048f926e75ac2\nUser1 Image Info                 : 9804075d2e8a71a192ec89602f2f5544\nUser2 Image Info                 : 9804075d2e8a71a192ec89602f2f5544\n
                                             
                                            "},{"location":"hw/common/reference_manual/ofs_sw/mnl_sw_ofs/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                            Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                             
                                            OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/","title":"oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack","text":""},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#10-overview","title":"1.0 Overview","text":""},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#11-about-this-document","title":"1.1 About this Document","text":"
                                            This document serves as a reference manual for platform designers wanting to enable oneAPI support on their Open FPGA Stack(OFS) platforms. The document describes essential hardware and software components required for enabling this design flow using OFS. Implementation details for oneapi-asp for Open FPGA Stack(OFS) reference platforms is covered towards the end of the document.
                                             
                                            Note: Table 1-1 in oneAPI Accelerator Support Package (ASP): Getting Started User Guide lists OFS reference platforms.
                                             
                                            For more information about developing application kernels for FPGA using Intel\u00ae oneAPI Base Toolkit (Base Kit) refer to Intel\u00ae FPGA Add-on for oneAPI Base Toolkit webpage.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#12-terminology","title":"1.2 Terminology","text":"
                                            This table defines some of the common terms used when discussing OFS. 
                                             
                                            Table 1-1: Terminology
                                             Term Abbreviation Description Open FPGA Stack OFS A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. Accelerator Functional Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside.  This AFU may or may not be a partial reconfiguration region. FPGA Interface Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs.  The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. High Level Design HLD For the purpose of this guide, this term refers to designing with Intel High Level Design tools like Intel\u00ae oneAPI Base Toolkit (Base Kit). oneAPI Accelerator Support Package oneAPI ASP A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and other OFS hardware, software components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Intel\u00ae FPGA Basic Building Blocks BBB Basic Building Blocks (BBB) for Intel\u00ae FPGAs is a suite of application building blocks and shims like Memory Properties Factory (MPF). BBB Memory Properties Factory BBB MPF Intel\u00ae FPGA BBB MPF block provides features like virtual to physical address (VTP), ordering read responses, read/write hazard detection, and masked (partial) writes. oneapi-asp uses MPF VTP feature. Open Programmable Acceleration Engine Software Development Kit OPAE SDK A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. Platform Interface Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Device Feature List DFL A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. Best Known Configuration BKC The exact hardware configuration Intel has optimized and validated the solution against. SYCL - SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL\u2122) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Installable Client Driver ICD Intel\u00ae FPGA Runtime for OpenCL\u2122 Software Technology supports the OpenCL ICD extension from the Khronos Group\u2122. The OpenCL ICD extension allows you to have multiple OpenCL implementations on your system. With the OpenCL ICD Loader Library, you may choose from a list of installed platforms and execute OpenCL API calls that are specific to your OpenCL implementation of choice. FPGA Client Driver FCD Intel\u00ae FPGA Runtime for OpenCL\u2122 Software Technology supports FPGA Client Driver(FCD) extension. FCD allows the runtime to automatically find and load the oneAPI ASP libraries at host run time 
                                            Note: oneapi-asp was referred to as ofs-hld-shim in OFS (Agilex & Stratix 10 OFS) and OpenCL AFU Shim (ofs-opencl-afu-shim) in OFS early access(EA) release (for Intel\u00ae Stratix 10\u00ae FPGA with Intel\u00ae FPGA PAC D5005 as reference platform).
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#13-prerequisites","title":"1.3 Prerequisites","text":"
                                            The content in this manual requires readers to be familiar with:
                                             
                                               
                                            • Hardware and software components of Open FPGA Stack, especially the following:
                                                 
                                              • FPGA Interface Manager(FIM)
                                                   
                                                • Intel\u00ae Stratix 10\u00ae FPGA:
                                                     
                                                  • FIM Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs
                                                    •  
                                                  • FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs
                                                    •  
                                                   
                                                  •  
                                                • Intel\u00ae Agilex\u00ae 7 FPGA:
                                                     
                                                  • Technical Reference Manual: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs
                                                    •  
                                                  • FPGA Interface Manager Developer Guide for Intel\u00ae Agilex\u00ae 7 PCIe Attach
                                                    •  
                                                   
                                                  •  
                                                 
                                                •  
                                              • Accelerator Functional Unit(AFU)
                                                   
                                                • Intel\u00ae Stratix 10\u00ae FPGA: OFS AFU Development Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs
                                                  •  
                                                • Intel\u00ae Agilex\u00ae 7 FPGA: AFU Development Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs
                                                  •  
                                                 
                                                •  
                                              • Software Reference Manual: Open FPGA Stack (OPAE SDK and Linux-DFL section)
                                                •  
                                              • ofs-platform-afu-bbb
                                                •  
                                               
                                              •  
                                             
                                            In addition to above, developers must be familiar with the following tools & concepts:
                                             
                                               
                                            • Intel\u00ae Quartus\u00ae Prime Design Software (Intel\u00ae Quartus\u00ae software revisions, Platform Designer, compilation Flows, timing analysis, compilation reports, understanding FPGA resource utilization, programming Intel\u00ae FPGAs) 
                                              •  
                                            • Partial Reconfiguration (PR)
                                              •  
                                            • FPGA Peripheral IPs (PCIe, External Memory IP, Ethernet IP) 
                                              •  
                                            • Avalon\u00ae Interface
                                              •  
                                            • Scripting (TCL, Python, Shell scripts) 
                                              •  
                                            • Verilog, SystemVerilog 
                                              •  
                                            • C++ 
                                              •  
                                            • Familiarity with SYCL 
                                              •  
                                            • Familiarity with oneAPI compilation process for FPGAs & oneAPI code samples 
                                              •  
                                            • Familiarity with oneAPI Accelerator Support Package (ASP): Getting Started User Guide
                                              •  
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#14-introduction-to-oneapi-on-open-fpga-stackofs","title":"1.4 Introduction to oneAPI on Open FPGA Stack(OFS)","text":"
                                            The Intel\u00ae oneAPI Base Toolkit (Base Kit) is a core set of tools and libraries for developing high-performance, data-centric applications across diverse architectures (CPUs, GPUs and FPGAs). It features an industry-leading C++ compiler that implements SYCL, an evolution of C++ for heterogeneous computing.
                                             
                                            Figure 1-1 shows the high-level representation of oneAPI application developers using FPGAs for acceleration. The runtime flow consists of a host code running on a processor and an application kernel code running on an FPGA. Open FPGA Stack enables vendors to enable support for this flow on their platforms.
                                             
                                            Figure 1-1: oneAPI Application on OFS Platforms
                                             
                                             
                                            Intel\u00ae oneAPI Base Toolkit (Base Kit) consists of a compiler and runtime environment. The compiler converts a SYCL kernel (FPGA application code) into a hardware circuit. This hardware circuit requires additional logic to communicate with the runtime and FPGA board peripherals. This additional logic is provided by oneAPI Accelerator Support Package(oneAPI ASP). oneAPI ASP consists of hardware components that enable this generated hardware circuit to communicate with the host processor as well as software components that enable the runtime to identify and communicate with the kernel.
                                             
                                            Figure 1-2 shows the workload design steps and steps in which the Intel\u00ae oneAPI Base Toolkit (Base Kit) requires oneAPI ASP as input. For more information about workload development and how workload developers target a specific platform during compilation, refer to Intel oneAPI Programming Guide. The next section introduces oneAPI ASP.
                                             
                                            Figure 1-2: High Level Design Flow for FPGAs with Intel\u00ae oneAPI Base Toolkit (Base Kit)
                                             
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#15-introduction-to-oneapi-accelerator-support-packageasp","title":"1.5 Introduction to oneAPI Accelerator Support Package(ASP)","text":"
                                            As mentioned in previous section, oneAPI ASP is a collection of hardware and software components that interface with the hardware circuit generated by the oneAPI compiler. The hardware circuit generated by the oneAPI compiler from a oneAPI kernel is referred to as the kernel system. While the kernel system consists of logic controlled by the workload developer's specifications, the kernel system interfaces are generated by the oneAPI compiler based on specifications provided by the oneAPI ASP designer. These specifications are input to the compiler using XML files (discussed in section 2.0).
                                             
                                            Note: All the interfaces generated by the oneAPI compiler are Avalon\u00ae Interfaces.
                                             
                                            Figure 1-3: Kernel System Interfaces
                                             
                                             
                                            Figure 1-3 shows a high-level representation of an OFS hardware design and interfaces to/from kernel_system. The numbered arrows depict the following:
                                             
                                               
                                            • Path 1 represents host-to-External Memory Interface (EMIF)
                                              •  
                                            • Path 2 represents the host to kernel interface
                                              •  
                                            • Path 3 represents kernel to EMIF
                                              •  
                                            • Path 4 represents kernel to Unified Shared Memory (USM) Interface
                                              •  
                                            • Path 5 represents kernel to HSSI interface
                                              •  
                                             
                                            oneAPI ASP hardware components can be divided into 3 categories:
                                             
                                               
                                            1. RTL components: constituting various interface logic, for example, host to External Memory Interface (EMIF), kernel to EMIF interface, host to kernel interface, kernel to host memory interface as well as additional components to handle kernel control signals and perform Direct Memory Access (DMA)
                                              2.  
                                            2. XML files: for describing hardware interfaces and compilation environment to Intel\u00ae oneAPI Base Toolkit (Base Kit)
                                              3.  
                                            3. Scripts: to control compile flow
                                              4.  
                                             
                                            In addition to the hardware components, a software layer is required for handling I/O operations between oneAPI runtime and the board. The oneAPI ASP software layer can be divided into 2 categories:
                                             
                                               
                                            1. Memory Mapped Device (MMD) Layer: required by the host & runtime to communicate with the oneAPI kernel & other oneAPI ASP hardware registers
                                              2.  
                                            2. oneAPI ASP utilities: required to setup and diagnose the board
                                              3.  
                                             
                                            The MMD uses API provided by OPAE SDK to communicate with the device. The FPGA driver is provided by the linux-DFL kernel driver.
                                             
                                            Figure 1-4 shows how the above oneAPI ASP components tie into Open FPGA Stack.
                                             
                                            Figure 1-4: Open FPGA Stack (OFS) components
                                             
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#20-xml-files-in-oneapi-asp","title":"2.0 XML Files in oneAPI ASP","text":"
                                            The kernel system interfaces generated by the oneAPI compiler are based on specifications provided by oneAPI ASP developer. An XML file, called board_spec.xml is used to pass the specifications to the oneAPI compiler. oneAPI ASP developers must create this XML file for their boards.
                                             
                                            In addition to board_spec.xml, the Intel\u00ae oneAPI Base Toolkit (Base Kit) relies on another XML file called board_env.xml to get information about the board environment. The board_env.xml file helps the runtime setup board installation.
                                             
                                            The next section explains the contents of board_spec.xml. Section 2.2 covers contents of board_env.xml file.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#21-board_specxml-file","title":"2.1 board_spec.xml File","text":"
                                            A typical board_spec.xml structure is shown in Fig 2-1. In addition to kernel system interfaces, the board_spec.xml file is also used to specify other compilation details like Intel\u00ae Quartus\u00ae Prime Pro Edition Software version used in platform design, compilation scripts to help control Intel\u00ae Quartus\u00ae software compile flows, FPGA resource utilization details.
                                             
                                            Elements of board_spec.xml file are summarized in table 2-1. Each element has additional attributes and parameters. Details are covered in respective sections for each element.
                                             
                                            Figure 2-1: board_spec.xml File Structure
                                             
                                             
                                            Table 2-1: Elements of board_spec.xml
                                             Element Use of Element Attributes board Used to specify a name for the board and the version of Intel\u00ae Quartus\u00ae Prime Pro Edition Software used to develop the platform design. This board name is used to identify the board design to be compiled and must match the name of the directory in which board_spec.xml resides. version, name compile Used to describe different compile flows project, revision, qsys_file, generic_kernel, generate_cmd, synthesize_cmd, auto_migrate device Used to specify the FPGA device model file for the FPGA part on the board. device_model, used_resources global_mem Different attributes in this element are used to provide details about the external memory used as the global memory for the FPGA oneAPI kernel/application. name, max_bandwidth, interleaved_bytes, config_addr, default, interface host Used to specify the offset at which the kernel resides. kernel_config interfaces Used to specify control signals to oneAPI kernels interface, kernel_clk_reset channels Used to describe interface to stream data directly between kernels and I/O interface 
                                            The compiler expects a separate board_spec.xml file for every board variant a platform supports. Board variants are different hardware design implementations for the same platform, a oneAPI ASP can have multiple board variants. A oneAPI kernel developer can select the board variant suitable for their application at compile time.
                                             
                                            A board_spec.xml file must be located at the top most level of each board variant's hardware directory (the hardware directory is specified by board_env.xml, please refer to section 2.2 for details on hardware element). For example, a separate board_spec.xml file for each board variant for OFS reference platforms is located in oneapi-asp/Platform-Name/hardware/Board-Variant/ directory, where Platform-Name is n6001 for Agilex OFS and d5005 for Stratix 10 OFS.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#211-board-element","title":"2.1.1 board Element","text":"
                                            The board element of the board_spec.xml file provides the Intel\u00ae Quartus\u00ae Prime Pro Edition Software version and the name of the board.
                                             
                                            Table 2-2: Attributes for the board Element
                                             Attribute Description version The version of the board. The board version should match the version of the Intel\u00ae Quartus\u00ae Prime Pro Edition Software you use to develop the platform design. The oneAPI compiler uses this value to perform environment checks for supported version during application compile name The name of the accelerator board, which must match the name of the directory in which the board_spec.xml file resides. The name must contain a combination of only letters, numbers, underscores (_), hyphens (-), or periods (.) (for example, ofs_n6000). 
                                            Example below shows the board element populated for a board designed with Intel\u00ae Quartus\u00ae Prime Pro Edition Software version 22.3 and board variant named \"Agilex_brd1\".
                                             
                                            Note: A board variant name is different from a platform directory name. Please see Note in section 2.2 for more information on board variants.
                                             
                                            Figure 2-2: board Element
                                             
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#212-compile-element","title":"2.1.2 compile Element","text":"
                                            Depending on the application requirements, the design may have different compilation flows and different design settings for each flow (for example, there can be a flat flow without partial reconfiguration support or a flow with partitions in the design to enable partial reconfiguration). Designers can control the flow and its settings using scripts. To allow selection of compile flow during application compile & to describe control of Intel\u00ae Quartus\u00ae software compilation as well as registration, automigration, the compile element of the board_spec.xml file and its associated attributes and parameters are used.
                                             
                                            Table 2-3: Attributes for compile Element
                                             Attribute Description name Name of the compilation flow. This name can be used to differentiate between flows at oneAPI kernel compilation time. oneAPI compiler allows selecting a compile flow using -Xsbsp-flow option. project Name of the Intel\u00ae Quartus\u00ae software project file (.qpf) that the Intel\u00ae Quartus\u00ae Prime Pro Edition Software intends to compile. revision Name of the revision within the Intel\u00ae Quartus\u00ae software project that the Intel\u00ae Quartus\u00ae Prime Pro Edition Software compiles to generate the final bitstream. qsys_file Name of the Platform Designer file into which the oneAPI kernel is embedded. You have the option to assign a value of \"none\" to qsys_file if you do not require the Intel\u00ae Quartus\u00ae Prime Pro Edition Software to create a top-level .qsys file for your design. In this scenario, oneAPI compiler adds a .qip file into the Intel\u00ae Quartus\u00ae software project. In this case, the custom oneAPI ASP must manually instantiate the generated HDL entity (generated entity is in the kernel_system.v file). generic_kernel Set this value to 1 if you want the offline compiler to generate a common Verilog interface for all compilations. This setting is necessary in situations where you must set up design partitions around the kernel, such as in the Configuration via Protocol (CvP) flow. generate_cmd Command required to prepare for full compilation, such as to generate the Verilog files for the Platform Designer system into which the oneAPI kernel is embedded. synthesize_cmd Command required to generate the fpga.bin file from the Custom Platform. Usually, this command instructs the Intel\u00ae Quartus\u00ae Prime Pro Edition Software to perform a full compilation. auto_migrate *platform_type\u2014Choose this value based on the value referenced in the Intel\u00ae FPGA Reference Platform from which you derive your Custom Platform. Valid values are a10_ref, s10_ref, and none.  *include fixes\u2014Comma-separated list of named fixes that you want to apply to the Custom Platform.  *exclude fixes\u2014Comma-separated list of named fixes that you do not want to apply to the Custom Platform. 
                                            Example below shows a populated compile element for a sample Intel\u00ae Quartus\u00ae software Project called ofs.qpf, the Intel\u00ae Quartus\u00ae software revision to be compiled is called asp (asp.qsf). In this example, the compiler generates the kernel system (entity is called kernel_system) and this entity is instantiated manually in the Intel\u00ae Quartus\u00ae software project (e.g. in a file called kernel_wrapper.v), hence qsys_file is set to \"none\". The synthesize_cmd points to a script \"compile.tcl\" located in the same directory as the board_spec.xml, compile script performs all necessary system generation and compile steps for generation of final bitstream. The project directory snippet below is for demonstration only. The compile flow is named \"demo_flow\".
                                             
                                            There can be multiple compile elements for the different compilation flows that a platform designer wishes to enable in their platform (e.g. different revisions with different Intel\u00ae Quartus\u00ae software settings or a PR revision).
                                             
                                            Figure 2-3: compile Element
                                             
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#213-device-element","title":"2.1.3 device Element","text":"
                                            A device model(DM) file is an XML file that has the total resources on the device (i.e. ALMs, FFs, DSPs, RAMs). This is required for any FPGA part used in a oneAPI design. Most device model files are provided as part of the Intel\u00ae oneAPI Base Toolkit (Base Kit) installation ($INTELFPGAOCLSDKROOT/share/models/dm, where INTELFPGAOCLSDKROOT is set by the setvars.sh environment setup script provided by oneAPI toolkit). If the device model file for your part number is not included in $INTELFPGAOCLSDKROOT/share/models/dm, it must be created and placed in the same folder as board_spec.xml. A new device model file can be created using existing files as reference. 
                                             
                                            The device model file name must be specified in the device_model attribute of device element. The used_resources attribute is used to specify the resources being utilized by the oneAPI ASP and peripheral IPs. The utilization by non-kernel logic is calculated during platform design. The compiler utilizes the total resources from device model file and utilized resources in used_resources section to estimate the available resources for application kernel.
                                             
                                            Table 2-4: Attributes for device Element
                                             Attribute Description device_model The file name of the device model file that describes the available FPGA resources on the accelerator board. used_resources Reports the number of adaptive logic modules (ALMs), flip-flops, digital signal processor (DSP) blocks and RAM blocks that the board design consumes in the absence of any kernel. If you create a defined partition around all the board logic, you can obtain the used resources data from the Partition Statistics section of the Fitter report. Extract the information from the following parameters: * alms num \u2014 The number of logic ALMs used, excluding the number of ALMs with only their registers used. The value should correspond to [a]+[b]+[d] from part [A] of the Fitter Partition Statistics. * ffs num \u2014 The number of flip flops. * dsps num \u2014 The number of DSP blocks. * rams num \u2014 The number of RAM blocks. 
                                            Example below shows the device element added for a Intel\u00ae Agilex\u00ae 7 FPGA based platform with device model file named \"agfb014r24a2e2vr0_dm.xml\". The number of used_resources are for demonstration purposes and are not to be used by oneAPI ASP developers.
                                             
                                            Figure 2-4: device Element
                                             
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#214-interface-attribute","title":"2.1.4 interface Attribute","text":"
                                            Note: This is different from the interfaces element discussed in upcoming sections. In the board_spec.xml file, each global memory, channel or kernel connection is comprised of individual interfaces. For the global_mem, channels, and interfaces XML elements, an interface attribute must be included to specify the corresponding parameters for each connection.
                                             
                                            Table 2-5: Parameters for interface attribute
                                             Parameter Description Applicable Interface name * For global_mem: instance name of the Platform Designer component.* For channels: instance name of the Platform Designer component that has the channel interface.* For interfaces: name of the entity in which the kernel interface resides (for example, board). All port port parameter can be defined either inline in interface attribute or as a separate element in interface attribute. See section 2.1.4.1 for more information. All type * For global_mem: set to agent. * For channels: - Set to streamsource for a stream source that provides data to the kernel.- Set to streamsink for a stream sink interface that consumes data from the kernel.* For interfaces: set to either host, irq, or streamsource. All width * For global_mem: width of the memory interface in bits.* For channels: number of bits in the channel interface.* For interfaces: width of the kernel interface in bits. All waitrequest_allowance * For global_mem: [Optional] Amount of Avalon\u00ae-MM waitrequest allowance supported on the agent interface (that is, kernel-facing interface) of the clock-crossing bridge that spans between the memory and the kernel clock domains.* For kernel_cra: [Optional] Amount of Avalon\u00ae-MM waitrequest allowance that the kernel_cra agent interface must support.This parameter defaults to 0 if you do not specify it in the board_spec.xml file. A value of 0 indicates that this waitrequest allowance feature is disabled. All maxburst Maximum burst size for the agent interface.Attention: The value of width \u00f7 8 x maxburst must be less than the value of interleaved_bytes. global_mem address Starting address of the memory interface that corresponds to the host interface-side address.For example, address 0 should correspond to the bank1 memory host from the Memory Bank Divider. In addition, any non-zero starting address must abut the end address of the previous memory. global_mem size Size of the memory interface in bytes.* For global_mem: The sizes of all memory interfaces should be equal.* For interfaces: interface can have variable sizes. global_meminterfaces > Note: Support for size parameter for interface attribute is available in Intel\u00ae oneAPI Base Toolkit (Base Kit) version 2024.0 and beyond. latency_type If the memory interface has variable latency, set this parameter to average to signify that the specified latency is considered the average case. If the complete kernel-to-memory path has a guaranteed fixed latency, set this parameter to fixed. global_mem chan_id A string used to identify the channel interface. The string may have up to 128 characters. channels clock For the streamsource kernel interface type, the parameter specifies the name of the clock that the snoop stream uses. Usually, this clock is the kernel clock. interfaces 
                                            Example for how the interface attribute is used in global_mem and interfaces elements is covered in section for these elements respectively.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#2141-port-parameter","title":"2.1.4.1 port Parameter","text":"
                                            As mentioned in Table 2-5, port parameter can be defined either inline in interface attribute or as a separate element in interface attribute. The definition method to use depends on the direction of the port. 
                                             
                                               
                                            • If the direction of the port is either read or write, it must be a separate element in interface attribute. 
                                              •  
                                            • If the direction is readwrite, port must be inline with the port name in interface attribute. No direction specification is required. 
                                              •  
                                             
                                            Table below shows the port element attributes.
                                             
                                            Table 2-6: port parameter
                                             
                                            | Parameter | Description | |---------|---------|---------| | name | * For global_mem: name of the Avalon\u00ae-MM interface in the Platform Designer component that corresponds to the interface attribute.* For channels: name of the streaming interface in the Platform Designer component.* For interfaces: name of the interface to the Kernel Interface Platform Designer component. For example, kernel_cra is the Avalon\u00ae-MM interface, and kernel_irq is an interrupt. | | direction | Direction of the port. Valid values are:  * \"r\" : Indicates read  * \"w\" : Indicates write |
                                             
                                            Snippet below shows the inline and separate element definitions of port parameter.
                                             
                                            Note: Direction specification for port is available in Intel\u00ae oneAPI Base Toolkit (Base Kit) versions 2024.0 and beyond. For versions prior to Intel\u00ae oneAPI Base Toolkit (Base Kit) version 2024.0, only the default inline definition of port parameter is supported. 
                                             
                                             
                                            Examples for global_mem and interfaces elements in sections below use the inline definition of port.  
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#215-global_mem-element","title":"2.1.5 global_mem Element","text":"
                                            The global_mem element of the board_spec.xml file is used to provide information on the memory interfaces that connect to the kernel.
                                             
                                            Note: For each global memory that the kernel accesses, you must include one interface element that describes its characteristics. The different attributes for global_mem element are discussed in table below.
                                             
                                            Table 2-7: Attributes for global_mem Element
                                             Attribute Description name The name FPGA application/kernel developer should use to identify the memory type. Each name must be unique and must comprise of less than 32 characters. max_bandwidth The maximum bandwidth, in megabytes per second (MB/s), of all global memory interfaces combined in their current configuration. The oneAPI compiler uses max_bandwidth to choose an architecture suitable for the application and the board.  Compute this bandwidth value from datasheets of memories on your board.  Example max_bandwidth calculation for a 64-bit DDR4 interface running at 1200 MHz:  max_bandwidth = 1200 MHz x 2 x 64 bits \u00f7 8-bits = 19200 MB/s  The max_bandwidth value will change based on global memory configuration, for example, if the memory configuration comprises of 4 banks of DDR4 configured as a single homogenous memory, the max_bandwidth will be 19200 x 4 (i.e. number of memory interfaces from kernel). Please see section 2.1.5.1 for more information on global memory configurations.  Designers have the option to use block RAM instead of or in conjunction with external memory as global memory. The formula for calculating max_bandwidth for block RAM is max_bandwidth = block RAM speed x (block RAM interface size \u00f7 8 bits).  Example max_bandwidth calculation for a 512-bit block RAM running at 100 MHz:  max_bandwidth = 100 MHz x 512 bits \u00f7 8 bits = 6400 MB/s interleaved_bytes Include the interleaved_bytes attribute in the board_spec.xml file when you instantiate multiple interfaces(i.e. memory banks) for a given global memory system. This attribute controls the size of data that the offline compiler distributes across the interfaces.  The offline compiler currently can interleave data across banks no finer than the size of one full burst. This attribute specifies this size in bytes and following are the recommended values:  For two or fewer global memory banks: maxburst x width_bytes  For four or more global memory banks: maxburst x width_bytes x 4  The interleaved_bytes value must be the same for the host interface and the kernels. Therefore, the configuration of the Memory Bank Divider must match the exported kernel agent interfaces in this respect (refer to section 3.1.1 for information about Memory Bank Divider)  For block RAM, interleaved_bytes equals the width of the interface in bytes. config_addr The address of the ACL Mem Organization Control Platform Designer component (mem_org_mode) that the host software uses to configure memory. You may omit this attribute if your board has homogeneous memory; the software uses the default address (0x18) for this component. If your board has heterogeneous memory, there is a mem_org_mode component in the board system for each memory type.  Enter the config_addr attribute and set it to the value of the base address of the mem_org_mode component(s). default Include this optional attribute and assign a value of 1 to set the global memory as the default memory interface. The default memory must start at address 0x0.  If you do not implement this attribute, the first memory type defined in the board_spec.xml file becomes the default memory interface. interface See the interface section above for the parameters you must specify for each interface. allocation_type A list that specifies which USM allocator is used to allocate from the global_mem element. Values allowed in this list are host, shared, and device. The following conditions apply:  If there are multiple global_mem elements with the same allocation_type attribute, the first allocation_type attribute in the board_spec.xml is assumed to be the one used by the specified allocator.  If there is a single global_mem element with multiple allocation_type attributes, this indicates that allocations of the specified types use this global_mem interface.  [Legacy support] If you have not specified the allocation_type attribute, it is assumed that all global memory interfaces have the device allocation_type. 
                                            Example below shows a global_mem element configuration for a kernel system connected to four 4GB DDR4 memory banks. The DDR4 interface is 64 bit operating at 1200MHz. Note that the name of the platform designer system name is board.qsys. As mentioned in description for interleaved_bytes in table above, the Memory Bank Divider configuration ensures that the host interface matches the interleaved_bytes setting (i.e. 512 bits x 64 burst size = 4096 bytes). For information on waitrequest_allowance, refer to section 2.1.4 on interface attribute.
                                             
                                            Note: More details on the Memory Bank Divider and the Clock Crossing Bridges is covered in section 3.0
                                             
                                            Figure 2-6: Memory Connection Example Block Diagram and Corresponding global_mem Element in board_spec.xml
                                             
                                             
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#2151-global-memory-configurations","title":"2.1.5.1 Global Memory Configurations","text":"
                                            A board can have a single memory bank, multiple memory banks of the same type (e.g. 4 banks of DDR4) or different banks of different types.
                                             
                                            The partitioning of memory for oneAPI kernel developers is explained in the FPGA Optimization Guide for Intel\u00ae oneAPI Toolkits. The global memory configuration required by an application kernel must match the configuration in board_spec.xml as the compiler uses this information to generate a suitable architecture for the application. The different memory configurations are
                                             
                                               
                                            • A single global memory region (possible with same type of memory banks)
                                              •  
                                            • Different global memories (heterogeneous memory)
                                              •  
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#21511-contiguous-global-memory","title":"2.1.5.1.1 Contiguous Global Memory","text":"
                                            For boards with multiple memory banks of the same type, designers can configure these as a single contiguous global memory region. This is done by specifying each memory interface within a single global_mem element. Figure 2-6 showed 4 DDR4 memory banks configured as a single global memory region.
                                             
                                            With this configuration, FPGA application developers have the option to use contiguous memory region in an interleaved or a non-interleaved fashion. Even with contiguous memory regions, kernel developers can partition data buffers across the banks/memory channels. Please refer to Global Memory Access Optimization section in FPGA Optimization Guide for Intel\u00ae oneAPI Toolkits for more details on these partitioning techniques.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#21512-heterogeneous-memory","title":"2.1.5.1.2 Heterogeneous Memory","text":"
                                            For boards with different memory technologies, designers must specify each type of memory that the kernel needs to access as a separate global memory.
                                             
                                            Figure 2-7 shows heterogeneous configurations and the global_mem element structure for two different types of memories (QDR, DDR4). The global_mem element in example below also demonstrates use of the default attribute. It is set to \"1\" for the DDR4 memory banks, indicating to the oneAPI compiler that the default global memory for the kernel is DDR4.
                                             
                                            Figure 2-7: Heterogeneous Memory Example Block Diagram and Corresponding global_mem Element in board_spec.xml
                                             
                                             
                                             
                                            Unified Shared Memory
                                             
                                            For applications that require USM support, the board_spec.xml must specify host and device memories in a heterogeneous manner. The allocation_type must be host for global memory region on the host processor. The allocation_type must be set to device for global memory on the FPGA board. Example below extends the board_spec.xml snippet in figure 2-6 to add a global_mem element for the kernel system to host processor memory interface.
                                             
                                            Figure 2-8: global_mem Element Example for Unified Shared Memory(USM)
                                             
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#216-host-element","title":"2.1.6 host Element","text":"
                                            The host element of the board_spec.xml file provides information on the interface from the host to the kernel. Figure below shows an example of host element.
                                             
                                            Figure 2-9: host Element Example
                                             
                                             
                                            Table 2-8: Attributes for the host Element
                                             Attribute Description kernel_config This attribute informs the oneAPI compiler at what offset the kernel resides, from the perspective of the kernel_cra host on the kernel_interface module.* start: the starting address of the kernel. Normally, this attribute has a value of 0 because the kernel_cra host should not host anything except kernels.* size: keep this parameter at the default value of 0x0100000."},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#217-interfaces-element","title":"2.1.7 interfaces Element","text":"
                                            The interfaces element of the board_spec.xml file describes the kernel interfaces that connect to application kernels and control kernel behavior. For this element, include one of each interface of types host, irq and streamsource. Refer to the interface section for the parameters you must specify for each interface. In addition to the host, irq, and streamsource interfaces, if your design includes a separate Platform Designer subsystem containing the board logic, the kernel clock and reset interfaces exported from it are also part of the interfaces element. Specify these interfaces with the kernel_clk_reset attribute and its corresponding parameters.
                                             
                                            Figure below shows example of interfaces element.
                                             
                                            Figure 2-10: interfaces Element Example
                                             
                                             
                                            Table 2-9:  Parameters for the kernel_clk_reset Attribute
                                             Attribute Description clk The Platform Designer name for the kernel clock interface clk2x The Platform Designer name for the 2xkernel clock interface reset The Platform Designer connection for the kernel reset 
                                            Note: Name the kernel clock and reset interfaces in the Platform Designer connection format (that is, .). For example: board.kernel_clk"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#218-channels-element","title":"2.1.8 channels Element","text":"
                                            The channels element provides channels for streaming data directly between kernel and I/O. Each channel (implemented using Avalon-ST specification) must be connected to the kernel via the interface attribute. The channel interface only supports data, and valid and ready Avalon-ST signals. The I/O channel defaults to 8-bit symbols and big-endian ordering at the interface level.
                                             
                                            Figure below shows an example of channels element for a single channel with a width of 64 bits. The chan_id attribute identified helps identify the port in the generated kernel_system. Refer to section 2.1.4 for more information about the interface attribute parameters. Additional interface attributes can be added for additional channels. 
                                             
                                            Figure 2-11: channels Element Example
                                             
                                             
                                            For more information about kernel development using channels, refer to I/O Pipes section in FPGA Optimization Guide for Intel\u00ae oneAPI Toolkits.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#22-board_envxml-file","title":"2.2 board_env.xml File","text":"
                                            The board_env.xml file is used by the oneAPI toolkit to set up the board installation that enables the compiler to target a specific accelerator platform. The board_env.xml file must be located in the top most level of the oneAPI ASP for each platform. For example, the board_env.xml for oneAPI ASP for OFS reference platforms is located in the oneapi-asp/Platform-Name folder, where Platform-Name is n6001 for Agilex OFS and d5005 for Stratix 10 OFS.
                                             
                                            A sample board_env.xml file is shown below. Table 2-10 explains the elements of this file.
                                             
                                            Figure 2-12: board_env.xml File Structure
                                             
                                             
                                            Table 2-10:  Specifications of XML Elements and Attributes in the board_env.xml File
                                             Element Attribute Description board_env * version: The oneAPI compiler version used to create oneAPI ASP* name: The runtime uses this as the name of the FPGA Client Driver(FCD) file name hardware * dir: Name of the subdirectory, within the oneAPI ASP directory, that contains the board variant directories for a platform * default: The default board variant that the compiler targets when a platform has multiple board variants and user does not specify an explicit argument using -Xstarget option platform name: Name of the operating system. A separate platform element must be specified for each supported OS for the oneAPI ASP platform mmdlib A string that specifies the path to the MMD library of your oneAPI ASP. To load multiple libraries, specify them in an ordered, comma-separated list. The host application will load the libraries in the order that they appear in the list> Note: You can use %b to reference your oneAPI ASP directory and provide path relative to oneAPI ASP directory, for example, if MMD library is located inside linux64/lib folder in oneAPI ASP, the path would be %b/linux64/lib/libintel_opae_mmd.so linkflags A string that specifies the linker flags necessary for linking with the MMD layer available with the board> Note: You can use %b to reference your oneAPI ASP directory and provide path relative to oneAPI ASP directory, for example, if MMD library is located inside linux64/lib folder in oneAPI ASP, the path would be %b/linux64/lib. linklibs A string that specifies the libraries the oneAPI runtime must link against to use the MMD layer available with the board utilbindir Directory in which the runtime expects to locate board utility executables (i.e. install, uninstall, program, diagnose, flash) > Note: You can use %b to reference your oneAPI ASP directory and provide path relative to oneAPI ASP directory, for example, if the utilities are located in linux64/libexec folder in oneAPI ASP, the path would be %b/linux64/libexec"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#30-oneapi-asp-hardware","title":"3.0 oneAPI ASP Hardware","text":"
                                            The oneAPI compiler generates the kernel system interfaces based on specifications provided by the oneAPI ASP developer in the board_spec.xml file. The kernel system interfaces with the rest of the oneAPI ASP RTL as shown in figure 1-3.
                                             
                                            Figure 1-3 shows 5 different paths, summarized below:
                                             
                                               
                                            • Host to EMIF: Consisting of RTL to handle data transfer between host and on-board memory (e.g. DDR4)
                                              •  
                                            • Host to Kernel: Consisting of RTL to handle control signals & interrupts between host and kernel
                                              •  
                                            • Kernel to EMIF: Consisting of RTL to handle data transfer between kernel and on-board memory
                                              •  
                                            • Kernel to Host memory: Required to support Unified Shared Memory. This requires some additional RTL to handle data transfer between kernel and host memory
                                              •  
                                            • Kernel to HSSI: Consisting of RTL to handle data streaming between kernel and I/O
                                              •  
                                             
                                            Please note that the kernel system generated by oneAPI compiler has Avalon\u00ae interfaces. OFS FIM has AXI interfaces. Additional logic blocks from Platform Interface Manager are used to handle protocol conversions. Please refer to section 5.3.1 for more details on PIM. The next few sections cover some of the important IP components  required to enable kernel communications with host and board peripherals. More design implementation details are covered in section 5.0.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#31-host-to-external-memory-interfaceemif","title":"3.1 Host to External Memory Interface(EMIF)","text":"
                                            The host to EMIF datapath consists of a PCIe Subsytem(SS), EMIF Subsystem located in the FIM and a Direct Memory Access(DMA) engine in the oneAPI ASP.
                                             
                                            PCIe Subsystem(SS) has the PCIe IP and additional logic to handle PCIe packet format and routing. FIM handles routing signals received from host to the user design located in a region referred to as Accelerator Functional Unit(AFU) (the Kernel system resides in the AFU).
                                             
                                            Note: For more information about the PCIe SS, please refer to Intel FPGA IP Subsystem for PCI Express IP User Guide
                                             
                                            The External Memory Interface Subsystem (EMIF SS) consists of EMIF IP and additional logic for handling transfers between AFU and on-board memories.
                                             
                                            Note: For more information about the EMIF SS, please refer to Memory Subsystem Intel FPGA IP User Guide
                                             
                                            Large buffers of data are usually transferred between host and on-board memory in oneAPI applications. This necessitates a Direct Memory Access(DMA) Engine between host and on-board memory. In oneAPI ASP designs for OFS reference platform, this DMA engine is placed in the AFU region. 
                                             
                                            As described in section 2.1.5.1, there are different configurations for memories on board. In addition to above, figure 1-3 also shows an additional IP in the host to memory datapath, called Memory Bank Divider. This IP is used for handling one of the most commonly used configurations, i.e. configuring multiple memory banks of same type as a contiguous memory region. In this case, the kernel has a contiguous view of the memory and data can be interleaved across the different memory channels. The host must also have the same view of the memory in order to ensure read and write transactions from correct addresses.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#311-memory-bank-divider","title":"3.1.1 Memory Bank Divider","text":"
                                            The OpenCL\u2122 Memory Bank Divider is a Platform Designer component that takes an incoming request from the host interface on the Avalon\u00ae-MM agent port and routes it to the appropriate bank host port. This component must reside on the path between the host and the global memory interfaces. In addition, it must reside outside of the path between the kernel and the global memory interfaces.
                                             
                                            Figure 3-1: Memory Bank Divider IP
                                             
                                             
                                             
                                            Table 3-1: Parameter Settings for the Memory Bank Divider Component
                                             Parameter Description Number of banks Number of memory banks for each of the global memory types included in your board system. Separate read/write ports Enable this parameter so that each bank has one port for read operation and one for write operation. Add pipeline stage to output Enable this parameter to allow for potential timing improvements. Data Width Width of the data bus to the memory in bits. Address Width (total addressable) Total number of address bits necessary to address all global memory. Burst size (maximum) Set to a value equal to interleaved_bytes/(width/8), where interleaved_bytes and width are defined in the interface attribute of the global_mem element in the board_spec.xml file. Maximum Pending Reads Maximum number of pending read transfers the component can process without asserting a waitrequest signal. Intel\u00ae recommended value is 64 if BSP has two global memory banks or fewer and 128 if BSP has four or more global memory banks.  CAUTION: A high Maximum Pending Reads value causes Platform Designer to insert a deep response FIFO buffer, between the component's host and agent, that consumes a lot of device resources. It also increases the achievable bandwidth between host and memory interfaces. Split read/write bursts on burst word boundary Enable splitting of read and write bursts on burst word boundary.  Enable this parameter if the Number of banks parameter value is greater than 1, and the burst reads and writes that the host controller sends to the agent port crosses burst word boundary. 
                                            Table 3-2: Signals and Ports for the Memory Bank Divider Component
                                             Signal or Port Description clk The bank divider logic uses this clock input. If the IP of your host and memory interfaces have different clocks, ensure that clk clock rate is not slower than the slowest of the two IP clocks. reset The reset input that connects to the board power-on reset. s The agent port that connects to the host interface controller. kernel_clk The kernel_clk drives this clock input kernel_reset The kernel_reset output from the Kernel Interface IP drives this reset input. acl_bsp_snoop Export this Avalon\u00ae Streaming (Avalon\u00ae-ST) source. In the board_spec.xml file, under interfaces, describe only the snoop interface for the default memory (acl_internal_snoop). If you have a heterogeneous memory design, perform these tasks only for the Memory Bank Divider component associated with the  default memory. Important: The memory system you build in Platform Designer alters the width of acl_bsp_snoop. You must update the width of the streamsource interface within the channels element in the board_spec.xml file to match the width of acl_bsp_snoop. In the board_spec.xml file, update the width of the snoop interface (acl_internal_snoop) specified with the streamsource kernel interface within the interfaces element. Updating the width ensures that the global_mem interface entries in board_spec.xml match the characteristics of the bankN Avalon\u00ae-MM hosts from corresponding Memory Bank Divider component for the default memory. acl_bsp_memorg_host This conduit connects to the acl_bsp_memorg_host interface of the OpenCL Kernel Interface`.> Note: Signal present if Number of banks > 1. bank1, bank2, ..., bank8 The number of memory hosts available in the Memory Bank Divider depends on the number of memory banks that were included when the unit was instantiated. Connect each bank with each memory interface in the same order as the starting address for the corresponding kernel memory interface specified in the board_spec.xml file.  For example, global_mem interface that begins at address 0 must correspond to the memory host in bank1 from the Memory Bank Divider."},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#32-host-to-kernel-interface","title":"3.2 Host to Kernel Interface","text":"
                                            The host exchanges control signals with kernel with the help of an additional IP . The control signals coming from the host are on a different clock domain (PCIe clock) while the kernel runs on different clock frequency . The Kernel Interface IP handles the clock domain crossing for these control signals as well as handles communication with kernel CSR, interrupts, generates the reset for kernel. All oneAPI ASP designs must instantiate Kernel Interface IPs to ensure the kernel functions correctly.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#321-kernel-interface","title":"3.2.1 Kernel Interface","text":"
                                            The Kernel Interface is a Platform Designer component that allows the host interface to access and control the oneAPI kernel.
                                             
                                            Figure 3-2: Kernel Interface IP
                                             
                                             
                                             
                                            Table 3-3: Parameter Settings for the Kernel Interface Component
                                             Parameter Description Number of global memory systems Number of global memory types in your board design. 
                                            Table 3-4: Signals and Ports for the Kernel Interface Component
                                             Signal or Port Description clk The clock input used for the host control interface. The clock rate of clk can be slow. reset This reset input resets the control interface. It also triggers the kernel_reset signal, which resets all kernel logic. ctrl Use this agent port to connect to the host interface. This interface is a low-speed interface with which you set kernel arguments and start the kernel's execution. kernel_clk kernel clock drives this clock input. kernel_cra This Avalon\u00ae-MM host interface communicates directly with the kernels generated by the oneAPI compiler. Export the Avalon\u00ae-MM interface to the Kernel Interface and name it in the board_spec.xml file. sw_reset_in When necessary, the host interface resets the kernel via the ctrl interface. If the board design requires a kernel reset, it can do so via this reset input. Otherwise, connect the interface to a global power-on reset. kernel_reset Use this reset output to reset the kernel and any other hardware that communicates with the kernel.  Warning: This reset occurs between the MMD open and close calls. Therefore, it must not reset anything necessary for the operation of your MMD. sw_reset_export This reset output is the same as kernel_reset, but it is synchronized to the clk interface. Use this output to reset logic that is not in the kernel_clk clock domain but should be reset whenever the kernel resets. acl_bsp_memorg_host The memory interfaces use these signals.  Based on the number of global memory systems you specify in the Kernel Interface component parameter editor, the Intel\u00ae Quartus\u00ae Prime Pro Edition Software creates the corresponding number of copies of this signal, each with a different hexadecimal suffix. Connect each signal to the Memory Bank Divider component associated with each global memory system (for example, DDR). Then, list the hexadecimal suffix in the config_addr attribute of the global_mem element in the board_spec.xml file. kernel_irq_from_kernel An interrupt input from the kernel. This signal is exported and named in the board_spec.xml file. kernel_irq_to_host An interrupt output from the kernel. This signal connects to the host interface."},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#33-kernel-to-external-memory-interface","title":"3.3 Kernel to External Memory Interface","text":"
                                            The kernel system masters the interface from kernel to external memory. oneAPI compiler generates kernel system memory interface logic (e.g. Load-Store Unit) according to the global memory configuration and interface specifications in board_spec.xml file. The kernel system operates at kernel clock(see next section for more information), hence, oneAPI ASP developers must handle clock domain crossing from kernel to EMIF clock domain.
                                             
                                            For implementation details for all datapaths discussed above, please refer to section 5.3.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#40-oneapi-asp-software","title":"4.0 oneAPI ASP Software","text":"
                                            The software components of oneAPI ASP consist of the Memory Mapped Device(MMD) layer and the board utility routine required by runtime.
                                             
                                            Section 4.1 introduces MMD layer and section 4.2 explains board utilities.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#41-memory-mapped-devicemmd-layer","title":"4.1 Memory Mapped Device(MMD) Layer","text":"
                                            The oneAPI ASP Memory Mapped Device (MMD) layer sits in between the oneAPI runtime and OPAE SDK and provides a set of API for device communication and control. The runtime calls into the MMD API for various operations like opening a handle to the device, allocating memory etc.
                                             
                                            Note: For more information about the FPGA runtime, please refer to FPGA Runtime documentation here.
                                             
                                            A header file, called aocl_mmd.h, has the list of MMD API calls that must be implemented by oneAPI ASPs. From the perspective of the caller, below is typical MMD API lifecycle:
                                             
                                               
                                            1. Open device to provide handle for further operations
                                              2.  
                                            2. Set interrupt and status handlers
                                              3.  
                                            3. Program device with kernel bitstream
                                              4.  
                                            4. Allocate memory if required
                                              5.  
                                            5. Perform Read, Write operations (DMA or MMIO)
                                              6.  
                                            6. Free memory if allocation done in step 4
                                              7.  
                                            7. Close device. No further operations permitted until subsequent open device call
                                              8.  
                                             
                                            Table below summarizes all APIs listed in aocl_mmd.h.  
                                             
                                            Table 4-1: Summary of MMD API from aocl_mmd.h
                                             API Purpose aocl_mmd_get_offline_info Obtain offline information about the board. This function is offline because it is device-independent and does not require a handle from the aocl_mmd_open() call aocl_mmd_get_info Obtain information about the board specified in the requested_info_id argument (refer to section 4.1.2 for more information) aocl_mmd_open Open and initialize the specified device aocl_mmd_close Close an opened device via its handle aocl_mmd_set_interrupt_handler Set the interrupt handler for the opened device aocl_mmd_set_device_interrupt_handler Sets the device interrupt handler for opened device, interrupt handler is called to notify runtime of any exceptions aocl_mmd_set_status_handler Set the operation status handler for the opened device aocl_mmd_yield The aocl_mmd_yield function is called when the host interface is idle. The host interface might be idle because it is waiting for the device to process certain events aocl_mmd_read Read operation on a single interface aocl_mmd_write Write operation on a single interface aocl_mmd_copy Copy operation on a single interface aocl_mmd_hostchannel_create Creates a channel interface aocl_mmd_hostchannel_destroy Destroys channel interface aocl_mmd_hostchannel_get_buffer Provides host with pointer used to read/write from channel interface aocl_mmd_hostchannel_ack_buffer Acknowledges read/write from channel aocl_mmd_program Reprogram operation for the specified device aocl_mmd_host_alloc Provide memory that is allocated on the host. Host allocations are accessible by the host and one or more devices aocl_mmd_free Free memory that has been allocated by MMD aocl_mmd_device_alloc Allocate memory that is owned by the device aocl_mmd_shared_alloc Allocate shared memory between the host and the FPGA aocl_mmd_shared_migrate Handle migration of non-concurrent shared allocations any time the accessor of the allocation changes 
                                            Sections below cover more details for each API (expected arguments, return values). Section 5.4 discusses more about the implementation of the MMD layer APIs in oneAPI ASPs for OFS reference platforms.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#411-aocl_mmd_get_offline_info","title":"4.1.1 aocl_mmd_get_offline_info","text":"
                                            The aocl_mmd_get_offline_info function obtains offline information about the board specified in the requested_info_id argument. This function is offline because it is device-independent and does not require a handle from the aocl_mmd_open() call.
                                             
                                            Syntax
                                             
                                            \n\n    int aocl_mmd_get_offline_info (\n        aocl_mmd_offline_info_t requested_info_id,\n        size_t param_value_size,\n        void* param_value,\n        size_t* param_size_ret )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1. requested_info_id: An enum value of type aocl_mmd_offline_info_t that indicates the offline device information returning to the caller.
                                              2.  
                                             
                                            Table 4-2: Possible Enum Values for the requested_info_id Argument
                                             Name Description Type AOCL_MMD_VERSION Version of MMD layer char* AOCL_MMD_NUM_BOARDS Number of candidate boards int AOCL_MMD_BOARD_NAMES Names of available boards > Note: Separate each board name by a semicolon (;) delimiter. char* AOCL_MMD_VENDOR_NAME Name of board vendor char* AOCL_MMD_VENDOR_ID An integer board vendor ID int AOCL_MMD_USES_YIELD A value of 0 instructs the runtime to suspend user's processes. The runtime resumes these processes after it receives an event update (for example, an interrupt) from the MMD layer.A value of 1 instructs the runtime to continuously call the aocl_mmd_yield function while it waits for events to complete.CAUTION: Setting AOCL_MMD_USES_YIELD to 1 might cause high CPU utilization if the aocl_mmd_yield function does not suspend the current thread. int 
                                               
                                            1.  
                                              param_value_size: Size of the param_value field in bytes. This size_t value should match the size of the expected return type that the enum definition indicates. For example, if AOCL_MMD_NUM_BOARDS returns a value of type int, set the param_value_size to sizeof (int). You should see the same number of bytes returned in the param_size_ret argument.
                                               
                                              2.  
                                            2.  
                                              param_value: A void* pointer to the variable that receives the returned information.
                                               
                                              3.  
                                            3.  
                                              param_size_ret: A pointer argument of type size_t* that receives the number of bytes of returned data.
                                               
                                              4.  
                                             
                                            Return Value
                                             
                                            A negative return value indicates an error.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#412-aocl_mmd_get_info","title":"4.1.2 aocl_mmd_get_info","text":"
                                            The aocl_mmd_get_info function obtains information about the board specified in the requested_info_id argument.
                                             
                                            Syntax
                                             
                                            \n\n    int aocl_mmd_get_info (\n        int handle,\n        aocl_mmd_info_t requested_info_id,\n        size_t param_value_size,\n        void* param_value,\n        size_t* param_size_ret )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1.  
                                              handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                               
                                              2.  
                                            2.  
                                              requested_info_id: An enum value of type aocl_mmd_info_t that indicates the device information returning to the caller.
                                               
                                              3.  
                                             
                                            Table 4-3: Possible Enum Values for the requested_info_id Argument
                                             Name Description Type AOCL_MMD_NUM_KERNEL_INTERFACES Number of kernel interfaces int AOCL_MMD_KERNEL_INTERFACES Kernel interfaces int* AOCL_MMD_PLL_INTERFACES Kernel clock handles int* AOCL_MMD_MEMORY_INTERFACE Global memory handle int AOCL_MMD_TERMPERATURE Temperature measurement float AOCL_MMD_PCIE_INFO PCIe\u00ae information char* AOCL_MMD_BOARD_NAME Board name char* AOCL_MMD_BOARD_UNIQUE_ID Unique board ID char* AOCL_MMD_CONCURRENT_READS Number of parallel readsA value of 1 indicates serial reads. int AOCL_MMD_CONCURRENT_WRITES Number of parallel writesA value of 1 indicates serial writes. int AOCL_MMD_CONCURRENT_READS_OR_WRITES Total number of concurrent read and write operations int AOCL_MMD_MIN_HOST_MEMORY_ALIGNMENT Minimum alignment that the oneAPI ASP supports for host allocations size_t AOCL_MMD_HOST_MEM_CAPABILITIES Capabilities of aocl_mmd_host_alloc() function unsigned int AOCL_MMD_SHARED_MEM_CAPABILITIES Capabilities of aocl_mmd_shared_alloc() function unsigned int AOCL_MMD_DEVICE_MEM_CAPABILITIES Capabilities of aocl_mmd_device_alloc() function unsigned int AOCL_MMD_HOST_MEM_CONCURRENT_GRANULARITY Granularity of concurrent host accesses size_t AOCL_MMD_SHARED_MEM_CONCURRENT_GRANULARITY Granularity of concurrent shared accesses size_t AOCL_MMD_DEVICE_MEM_CONCURRENT_GRANULARITY Granularity of concurrent device accesses size_t 
                                               
                                            1.  
                                              param_value_size: Size of the param_value field in bytes. This size_t value should match the size of the expected return type that the enum definition indicates. For example, if AOCL_MMD_TEMPERATURE returns a value of type float, set the param_value_size to sizeof (float). You should see the same number of bytes returned in the param_size_ret argument.
                                               
                                              2.  
                                            2.  
                                              param_value: A void* pointer to the variable that receives the returned information.
                                               
                                              3.  
                                            3.  
                                              param_size_ret: A pointer argument of type size_t* that receives the number of bytes of returned data.
                                               
                                              4.  
                                             
                                            Capability Values 
                                             
                                            Table 4-4: Capability Values for aocl_mmd_get_info Function
                                             Value Description AOCL_MMD_MEM_CAPABILITY_SUPPORTED If you do not set this value, allocation function is not supported even if other capabilities are set. AOCL_MMD_MEM_CAPABILITY_ATOMIC Supports atomic access to the memory by either the host or the device. AOCL_MMD_MEM_CAPABILITY_CONCURRENT Supports concurrent access to the memory either by the host or the device if the accesses are not on the same block. Block granularity is defined by AOCL_MMD_*_MEM_CONCURRENT_GRANULARITY. Blocks are aligned to this granularity. AOCL_MMD_MEM_CAPABILITY_P2P Memory can be accessed by multiple devices at the same time. 
                                            Return Value
                                             
                                            A negative return value indicates an error.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#413-aocl_mmd_open","title":"4.1.3 aocl_mmd_open","text":"
                                            The aocl_mmd_open function opens and initializes the specified device.
                                             
                                            Syntax
                                             
                                            \n\n    int aocl_mmd_open (const char *name)\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1. name: The function opens the board with a name that matches this const char* string. The name typically matches the one specified by the AOCL_MMD_BOARD_NAMES offline information.
                                              2.  
                                             
                                            The runtime first queries the AOCL_MMD_BOARD_NAMES offline information to identify the boards that it might be able to open. Then it attempts to open all possible devices by calling aocl_mmd_open and using each of the board names as argument.
                                             
                                            Note: The name must be a C-style NULL-terminated ASCII string.
                                             
                                            Return Value
                                             
                                            If aocl_mmd_open() executes successfully, the return value is a positive integer that acts as a handle to the board.
                                             
                                            If aocl_mmd_open() fails to execute, a negative return value indicates an error. In the event of an error, the runtime proceeds to open other known devices. Therefore, it is imperative that the MMD layer does not exit the application if an open call fails.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#414-aocl_mmd_close","title":"4.1.4 aocl_mmd_close","text":"
                                            The aocl_mmd_close function closes an opened device via its handle.
                                             
                                            Syntax
                                             
                                            \n\n    int aocl_mmd_close (int handle)\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                              2.  
                                             
                                            Return Value
                                             
                                            If the aocl_mmd_close() executes successfully, the return value is 0.
                                             
                                            If aocl_mmd_close() fails to execute, a negative return value indicates an error.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#415-aocl_mmd_set_interrupt_handler","title":"4.1.5 aocl_mmd_set_interrupt_handler","text":"
                                            The aocl_mmd_set_interrupt_handler function sets the interrupt handler for the opened device. When the device internals identify an asynchronous kernel event (for example, a kernel completion), the interrupt handler is called to notify the runtime of the event.
                                             
                                            Note: Ignore the interrupts from the kernel until this handler is set.
                                             
                                            Syntax
                                             
                                            \n\n    int aocl_mmd_set_interrupt_handler (\n        int handle, \n        aocl_mmd_interrupt_handler_fn fn,\n        void* user_data )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1.  
                                              handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                               
                                              2.  
                                            2.  
                                              fn: The callback function to invoke when a kernel interrupt occurs. The fn argument is of type aocl_mmd_interrupt_handler_fn, which is defined as follows:
                                               
                                              3.  
                                             
                                            \n    typedef void (*aocl_mmd_interrupt_handler_fn)( int handle, void* user_data );\n
                                             
                                               
                                            1. user_data: The void* type user-provided data that passes to fn when it is called.
                                              2.  
                                             
                                            Return Value
                                             
                                            If the function executes successfully, the return value is 0.
                                             
                                            If the function fails to execute, a negative return value indicates an error.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#416-aocl_mmd_set_device_interrupt_handler","title":"4.1.6 aocl_mmd_set_device_interrupt_handler","text":"
                                            The aocl_mmd_set_device_interrupt_handler function sets the device interrupt handler for the opened device. When the device internals identify an asynchronous exception event (for example, a bit correction event), the device interrupt handler is called to notify the runtime of the event.
                                             
                                            Note: Ignore the interrupts from the device until this handler is set.
                                             
                                            Syntax
                                             
                                            \n\n    int aocl_mmd_set_device_interrupt_handler (\n        int handle, \n        aocl_mmd_device_interrupt_handler_fn fn,\n        void* user_data )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1.  
                                              handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                               
                                              2.  
                                            2.  
                                              fn: The callback function to invoke when a kernel interrupt occurs. The fn argument is of type aocl_mmd_device_interrupt_handler_fn, which is defined as follows:
                                               
                                              3.  
                                             
                                             \n    typedef void (*aocl_mmd_device_interrupt_handler_fn)( int handle, aocl_mmd_interrupt_info* data_in, void* user_data );\n
                                             
                                            aocl_mmd_interrupt_info is defined as:
                                             
                                            \n\n    typedef struct {\n        unsigned long long int exception_type;\n        void *user_private_info;\n        size_t user_cb;\n    } aocl_mmd_interrupt_info;\n\n
                                             
                                            Where:
                                             
                                               
                                            • exception_type acts as a bitfield that contains exactly one bit, corresponding to an exception number.
                                              •  
                                            • user_private_info and user_cb represent pointers to binary data that the OpenCL implementation return. These pointers log additional information that is helpful for debugging the error.
                                              •  
                                             
                                               
                                            1. user_data: The void* type user-provided data that passes to fn when it is called.
                                              2.  
                                             
                                            Return Value
                                             
                                            If the function executes successfully, the return value is 0.
                                             
                                            If the function fails to execute, a negative return value indicates an error.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#417-aocl_mmd_set_status_handler","title":"4.1.7 aocl_mmd_set_status_handler","text":"
                                            The aocl_mmd_set_status_handler function sets the operation status handler for the opened device. The operation status handler is called under the following circumstances:
                                             
                                               
                                            • When the operation completes successfully and status is 0.
                                              •  
                                            • When the operation completes with errors and status is a negative value.
                                              •  
                                             
                                            Syntax
                                             
                                            \n\n    int aocl_mmd_set_status_handler (\n        int handle,\n        aocl_mmd_status_handler_fn fn,\n        void* user_data )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1.  
                                              handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                               
                                              2.  
                                            2.  
                                              fn: The callback function to invoke when a status update occurs. The fn argument is of type aocl_mmd_status_handler_fn, which is defined as follows:
                                               
                                              3.  
                                             
                                            \n    typedef void (*aocl_mmd_status_handler_fn)( int handle, void* user_data, aocl_mmd_op_t op, int status );\n
                                             
                                               
                                            1. user_data: The void* type user-provided data that passes to fn when it is called.
                                              2.  
                                             
                                            Return Value
                                             
                                            If the function executes successfully, the return value is 0.
                                             
                                            If the function fails to execute, a negative return value indicates an error.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#418-aocl_mmd_yield","title":"4.1.8 aocl_mmd_yield","text":"
                                            The aocl_mmd_yield function is called when the host interface is idle. The host interface might be idle because it is waiting for the device to process certain events.
                                             
                                            Syntax
                                             
                                            \n\n    int aocl_mmd_yield (int handle)\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1. handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                              2.  
                                             
                                            Return Value
                                             
                                            A nonzero return value indicates that the yield function performed work necessary for proper device functioning such as processing direct memory access (DMA) transactions.
                                             
                                            A return value of 0 indicates that the yield function did not perform work necessary for proper device functioning.
                                             
                                            Note: The yield function might be called continuously if it reports that it has necessary work to perform.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#419-aocl_mmd_read","title":"4.1.9 aocl_mmd_read","text":"
                                            The aocl_mmd_read function is the read operation on a single interface.
                                             
                                            Syntax
                                             
                                            \n\n    int aocl_mmd_read (\n        int handle,\n        aocl_mmd_op_t op,\n        size_t len,\n        void* dst,\n        int mmd_interface, size_t offset )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1.  
                                              handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                               
                                              2.  
                                            2.  
                                              op: The operation object of type aocl_mmd_op_t used to track the progress of the operation. If op is NULL, the call must block, and return only after the operation completes.
                                               
                                              3.  
                                             
                                            Note: aocl_mmd_op_t is defined as follows:
                                             
                                            \n    typedef void* aocl_mmd_op_t;\n
                                             
                                               
                                            1.  
                                              len: The size of the data, in bytes, that the function transfers. Declare len with type size_t.
                                               
                                              2.  
                                            2.  
                                              dst: The host buffer, of type void*, to which data is written.
                                               
                                              3.  
                                            3.  
                                              mmd_interface: the handle to the interface being accessed. For example, to access global memory this handle will be value obtained from aocl_mmd_get_info call with AOCL_MMD_MEMORY_INTERFACE as requested_info_id argument.
                                               
                                              4.  
                                            4.  
                                              offset: The size_t byte offset within the interface at which the data transfer begins.
                                               
                                              5.  
                                             
                                            Return Value
                                             
                                            If the read operation is successful, the return value is 0.
                                             
                                            If the read operation fails, a negative return value indicates an error.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4110-aocl_mmd_write","title":"4.1.10 aocl_mmd_write","text":"
                                            The aocl_mmd_write function is the write operation on a single interface.
                                             
                                            Syntax
                                             
                                            \n\n    int aocl_mmd_write (\n        int handle,\n        aocl_mmd_op_t op,\n        size_t len,\n        const void* src,\n        int mmd_interface, size_t offset )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1.  
                                              handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                               
                                              2.  
                                            2.  
                                              op: The operation object of type aocl_mmd_op_t used to track the progress of the operation. If op is NULL, the call must block, and return only after the operation completes.
                                               
                                              3.  
                                             
                                            Note: aocl_mmd_op_t is defined as follows:
                                             
                                            \n    typedef void* aocl_mmd_op_t;\n
                                             
                                               
                                            1.  
                                              len: The size of the data, in bytes, that the function transfers. Declare len with type size_t.
                                               
                                              2.  
                                            2.  
                                              src: The host buffer, of type const void*, from which data is read.
                                               
                                              3.  
                                            3.  
                                              mmd_interface: the handle to the interface being accessed. For example, to access global memory this handle will be value obtained from aocl_mmd_get_info call with AOCL_MMD_MEMORY_INTERFACE as requested_info_id argument.
                                               
                                              4.  
                                            4.  
                                              offset: The size_t byte offset within the interface at which the data transfer begins.
                                               
                                              5.  
                                             
                                            Return Value
                                             
                                            If the write operation is successful, the return value is 0.
                                             
                                            If the write operation fails, a negative return value indicates an error.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4111-aocl_mmd_copy","title":"4.1.11 aocl_mmd_copy","text":"
                                            The aocl_mmd_copy function is the copy operation on a single interface.
                                             
                                            Syntax
                                             
                                            \n\n    int aocl_mmd_copy (\n        int handle,\n        aocl_mmd_op_t op,\n        size_t len,\n        int mmd_interface, size_t src_offset, size_t dst_offset )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1.  
                                              handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                               
                                              2.  
                                            2.  
                                              op: The operation object of type aocl_mmd_op_t used to track the progress of the operation. If op is NULL, the call must block, and return only after the operation completes.
                                               
                                              3.  
                                             
                                            Note: aocl_mmd_op_t is defined as follows:
                                             
                                            \n    typedef void* aocl_mmd_op_t;\n
                                             
                                               
                                            1.  
                                              len: The size of the data, in bytes, that the function transfers. Declare len with type size_t.
                                               
                                              2.  
                                            2.  
                                              mmd_interface: the handle to the interface being accessed. For example, to access global memory this handle will be value obtained from aocl_mmd_get_info call with AOCL_MMD_MEMORY_INTERFACE as requested_info_id argument.
                                               
                                              3.  
                                            3.  
                                              src_offset: The size_t byte offset within the source interface at which the data transfer begins.
                                               
                                              4.  
                                            4.  
                                              dst_offset: The size_t byte offset within the destination interface at which the data transfer begins
                                               
                                              5.  
                                             
                                            Return Value
                                             
                                            If the copy operation is successful, the return value is 0.
                                             
                                            If the copy operation fails, a negative return value indicates an error.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4112-aocl_mmd_hostchannel_create","title":"4.1.12 aocl_mmd_hostchannel_create","text":"
                                            The aocl_mmd_hostchannel_create function creates a channel interface.
                                             
                                            Syntax
                                             
                                            \n\n    int aocl_mmd_hostchannel_create (\n        int handle,\n        char *channel_name,\n        size_t queue_depth,\n        int direction )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1.  
                                              handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                               
                                              2.  
                                            2.  
                                              channel_name: Name of the channel to be initialized. The channel name is same as that used in the board_spec.xml file.
                                               
                                              3.  
                                            3.  
                                              queue_depth: The size of pinned internal buffer in bytes. Pointer to the internal buffer is provided when the user calls the aocl_mmd_hostchannel_get_buffer() function.
                                               
                                              4.  
                                            4.  
                                              direction: The direction of the channel.
                                               
                                              5.  
                                             
                                            Return Value
                                             
                                            If the function executes successfully, the return value is positive and is handle to the channel.
                                             
                                            If the function fails to execute, a negative return value indicates an error.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4113-aocl_mmd_hostchannel_destroy","title":"4.1.13 aocl_mmd_hostchannel_destroy","text":"
                                            The aocl_mmd_hostchannel_destroy function destroys the channel interface.
                                             
                                            Syntax
                                             
                                            \n\n    int aocl_mmd_hostchannel_destroy (\n        int handle,\n        int channel )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1.  
                                              handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                               
                                              2.  
                                            2.  
                                              channel: A positive int value representing handle to the channel to close obtained from the aocl_mmd_hostchannel_create() call.
                                               
                                              3.  
                                             
                                            Return Value
                                             
                                            If the function executes successfully, the return value is 0.
                                             
                                            If the function fails to execute, a negative return value indicates an error.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4114-aocl_mmd_hostchannel_get_buffer","title":"4.1.14 aocl_mmd_hostchannel_get_buffer","text":"
                                            The aocl_mmd_hostchannel_get_buffer function provides a host with a pointer to the buffer they can access to write or read from the channel interface, along with the space or data available in the buffer, in bytes.
                                             
                                            Syntax
                                             
                                            \n\n    void *aocl_mmd_hostchannel_get_buffer (\n        int handle,\n        int channel,\n        size_t *buffer_size,\n        int *status )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1.  
                                              handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                               
                                              2.  
                                            2.  
                                              channel: A positive int value representing handle to the channel to close obtained from the aocl_mmd_hostchannel_create() call.
                                               
                                              3.  
                                            3.  
                                              buffer_size: A pointer to size_t that the function writes available buffer space or size to.
                                               
                                              4.  
                                            4.  
                                              status: A pointer to int that the function writes result of the call to.
                                               
                                              5.  
                                             
                                            Return Value
                                             
                                            If the function executes successfully, int pointed to by the status pointer is 0. Returned void* may still be NULL, in which case size_t pointed by the buffer_size is 0.
                                             
                                            If the function fails to execute, int pointed by the status pointer is a negative value.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4115-aocl_mmd_hostchannel_ack_buffer","title":"4.1.15 aocl_mmd_hostchannel_ack_buffer","text":"
                                            You can acknowledge write or read from the channel by calling aocl_mmd_hostchannel_ack_buffer.
                                             
                                            Syntax
                                             
                                            \n\n    size_t aocl_mmd_hostchannel_ack_buffer (\n        int handle,\n        int channel,\n        size_t send_size,\n        int *status )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1.  
                                              handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                               
                                              2.  
                                            2.  
                                              channel: A positive int value representing handle to the channel to close obtained from the aocl_mmd_hostchannel_create() call.
                                               
                                              3.  
                                            3.  
                                              send_size: The size in bytes that the user is acknowledging.
                                               
                                              4.  
                                            4.  
                                              status: A pointer to int that the function writes result of the call to.
                                               
                                              5.  
                                             
                                            Return Value
                                             
                                            If the function executes successfully, int pointed to by status pointer is 0. Also, there is no guarantee that the user's send_size is the actual size that gets acknowledged. The returned size_t is the amount of bytes that was actually acknowledged.
                                             
                                            If the function fails to execute, int pointed by status pointer is a negative value.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4116-aocl_mmd_program","title":"4.1.16 aocl_mmd_program","text":"
                                            The aocl_mmd_program function is the program operation for the specified device. The host must guarantee that no other operations are executing on the device during the program operation. 
                                             
                                            During aocl_mmd_program execution, the kernels are idle and no read, write, or copy operation can occur.
                                             
                                            Disable interrupts and program the FPGA with the data from user_data, which has a size specified by the size argument. The host then calls aocl_mmd_set_status_handler and aocl_mmd_set_interrupt_handler again, which enable the interrupts. If events such as interrupts occur during aocl_mmd_program execution, race conditions or data corruption might occur.
                                             
                                            Syntax
                                             
                                            \n\n int aocl_mmd_program (\n    int handle,\n    void * user_data,\n    size_t size,\n    aocl_mmd_program_mode_t program_mode )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1.  
                                              handle: A positive int value representing the handle to the board obtained from the aocl_mmd_open() call.
                                               
                                              2.  
                                            2.  
                                              user_data: The void* type binary contents of the fpga.bin file that is created during kernel compilation.
                                               
                                              3.  
                                            3.  
                                              size: The size of user_data in bytes. The size argument is of size_t.
                                               
                                              4.  
                                            4.  
                                              program_mode: The bit-field that specifies the mode of device programming.
                                               
                                              5.  
                                             
                                            Table 4-5: Possible Values for the program_mode Argument
                                             program_mode Argument Value Description AOCL_MMD_PROGRAM_PRESERVE_GLOBAL_MEMORY This flag specifies that during programming the global memory on the devices are preserved. 
                                            Return Value
                                             
                                            If aocl_mmd_program executes successfully, the return value is the pointer value that the host uses to access shared memory.
                                            "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4117-aocl_mmd_host_alloc","title":"4.1.17 aocl_mmd_host_alloc","text":"
                                            Host allocations provide memory that is allocated on the host. This memory must be deallocated with the aocl_mmd_free function. Host allocations are accessible by the host and one or more devices. The same pointer to a host allocation may be used on the host and all supported devices. They have address equivalence.
                                             
                                            Syntax
                                             
                                            Once the device has signaled completion through the aocl_mmd_interrupt_handler_fn function, the host can assume it has access to the latest contents of the memory, allocated by the aocl_mmd_host_alloc function call.
                                             
                                            \n\n    void* aocl_mmd_host_alloc (\n        int* handles,\n        size_t num_devices,\n        size_t size,\n        size_t alignment,\n        aocl_mmd_mem_properties_t *properties,\n        int* error )\n\n
                                             
                                            Function Arguments
                                             
                                               
                                            1.  
                                              handles: Handles for devices that needs access to this memory.
                                               
                                              2.  
                                            2.  
                                              num_devices: Number of devices in the handles.
                                               
                                              3.  
                                            3.  
                                              size: The size of the memory region.
                                               
                                              4.  
                                            4.  
                                              alignment: The alignment (in bytes) of the allocation.
                                               
                                              5.  
                                            5.  
                                              properties: Specifies additional information about the allocated memory, described by a property type name and its corresponding value. Each property type name is immediately followed by the corresponding desired value. The list is terminated with a zero.  For example, [, , , , 0] 
                                            6.  
                                              error: The error code defined by AOCL_MMD_ERROR*:
                                               
                                              7.  
                                                 
                                              • AOCL_MMD_ERROR_SUCCESS: No error occurred.
                                                •  
                                              • AOCL_MMD_ERROR_INVALID_HANDLE: The device handle provided is invalid.
                                                •  
                                              • AOCL_MMD_ERROR_OUT_OF_MEMORY: Ran out of memory.
                                                •  
                                              • AOCL_MMD_ERROR_UNSUPPORTED_ALIGNMENT: The device does not support the provided alignment.
                                                •  
                                              • AOCL_MMD_ERROR_UNSUPPORTED_PROPERTY: The device does not support the provided property.
                                                •  
                                               
                                              Return Value
                                               
                                              If the aocl_mmd_host_alloc function executes successfully, the return value is a valid pointer value. Otherwise, the return value is NULL.
                                              "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4118-aocl_mmd_free","title":"4.1.18 aocl_mmd_free","text":"
                                              Releases memory that was allocated by MMD.
                                               
                                              Syntax
                                               
                                              \n\n    int aocl_mmd_free (void* mem)\n\n
                                               
                                              Function Arguments
                                               
                                                 
                                              1. mem: The pointer to the memory region. Must be a pointer that is allocated by the MMD.
                                                2.  
                                               
                                              Return Value
                                               
                                              Returns one of the following error code:
                                               
                                                 
                                              • AOCL_MMD_ERROR_SUCCESS: No error occurred
                                                •  
                                              • AOCL_MMD_ERROR_INVALID_POINTER: Invalid pointer provided
                                                •  
                                              "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4119-aocl_mmd_device_alloc","title":"4.1.19 aocl_mmd_device_alloc","text":"
                                              Allocate memory that is owned by the device. This pointer can only be accessed by the kernel. It cannot be accessed by the host. The host is able to manipulate the pointer (for example, increment it) and not just access the underlying data. This memory must be deallocated by the aocl_mmd_free() function.
                                               
                                              Syntax
                                               
                                              \n\n    void * aocl_mmd_device_alloc (\n        int handle, \n        size_t size, \n        size_t alignment, \n        aocl_mmd_mem_properties_t *properties, \n        int* error )\n\n
                                               
                                              Function Arguments
                                               
                                                 
                                              1.  
                                                handle: Device that has access to this memory.
                                                 
                                                2.  
                                              2.  
                                                size: The size of the memory region.
                                                 
                                                3.  
                                              3.  
                                                alignment: The alignment (in bytes) of the memory region.
                                                 
                                                4.  
                                              4.  
                                                properties: Specifies additional information about the allocated memory, described by a property type name and its corresponding value. Each property type name is immediately followed by the corresponding desired value. The list is terminated with a zero.  For example, [, , , , 0] 
                                                Return Value
                                                 
                                                Returns one of the following error code:
                                                 
                                                   
                                                • AOCL_MMD_ERROR_SUCCESS: No error occurred
                                                  •  
                                                • AOCL_MMD_ERROR_INVALID_HANDLE: The device handle provided is invalid.
                                                  •  
                                                • AOCL_MMD_ERROR_OUT_OF_MEMORY: Ran out of memory.
                                                  •  
                                                • AOCL_MMD_ERROR_UNSUPPORTED_ALIGNMENT: The device does not support the provided alignment.
                                                  •  
                                                • AOCL_MMD_ERROR_UNSUPPORTED_PROPERTY: The device does not support the provided property.
                                                  •  
                                                "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4120-aocl_mmd_shared_alloc","title":"4.1.20 aocl_mmd_shared_alloc","text":"
                                                Shared allocations can migrate between the host and one or more associated device. The same pointer to a shared allocation can be used on the host and the supported device. They have address equivalence.
                                                 
                                                Syntax
                                                 
                                                \n\n    void * aocl_mmd_shared_alloc (\n        int handle, \n        size_t size, \n        size_t alignment, \n        aocl_mmd_mem_properties_t* properties, \n        int* error )\n\n
                                                 
                                                Function Arguments
                                                 
                                                   
                                                1.  
                                                  handle: Device that needs access to this memory.
                                                   
                                                  2.  
                                                2.  
                                                  size: The size of the memory region.
                                                   
                                                  3.  
                                                3.  
                                                  alignment: The alignment (in bytes) of the allocation.
                                                   
                                                  4.  
                                                4.  
                                                  properties: Specifies additional information about the allocated memory described by a property type name and its corresponding value. Each property type name is immediately followed by the corresponding desired value. The list is terminated with a zero.  For example, [, , , , 0] 
                                                5.  
                                                  error: The error code defined by AOCL_MMD_ERROR*.
                                                   
                                                  6.  
                                                     
                                                  • AOCL_MMD_ERROR_SUCCESS: No error occurred.
                                                    •  
                                                  • AOCL_MMD_ERROR_INVALID_HANDLE: The device handle provided is invalid.
                                                    •  
                                                  • AOCL_MMD_ERROR_OUT_OF_MEMORY: Ran out of memory.
                                                    •  
                                                  • AOCL_MMD_ERROR_UNSUPPORTED_ALIGNMENT: The device does not support the provided alignment.
                                                    •  
                                                  • AOCL_MMD_ERROR_UNSUPPORTED_PROPERTY: The device does not support the provided property.
                                                    •  
                                                   
                                                  Return Value
                                                   
                                                  If the aocl_mmd_shared_alloc function executes successfully, the return value is a valid pointer value. Otherwise, the return value is NULL.
                                                  "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#4121-aocl_mmd_shared_migrate","title":"4.1.21 aocl_mmd_shared_migrate","text":"
                                                  A call to the aocl_mmd_shared_migrate() function must be made for non-concurrent shared allocations any time the accessor of the allocation changes. For example, the aocl_mmd_shared_migrate() function should be called indicating that the allocation should be migrated to the device before a kernel accessing the allocation is launched on the device. Similarly, the aocl_mmd_shared_migrate() function should be called indicating that the allocation is migrated to the host before the host accesses the memory after kernel completion. For concurrent allocations, this call may be used as a performance hint, but it is not strictly required for functionality.
                                                   
                                                  Syntax
                                                   
                                                  \n\n    int aocl_mmd_shared_migrate (\n        int handle, \n        void* shared_ptr, \n        size_t size, \n        aocl_mmd_migrate_t destination )\n\n
                                                   
                                                  Function Arguments
                                                   
                                                     
                                                  1.  
                                                    handle: Device that has access to this memory.
                                                     
                                                    2.  
                                                  2.  
                                                    shared_ptr: Pointer allocated by the aocl_mmd_shared_alloc() function.
                                                     
                                                    3.  
                                                  3.  
                                                    size: Size (in bytes) of the migration. Must be a multiple of a page boundary that the oneAPI ASP supports.
                                                     
                                                    4.  
                                                  4.  
                                                    destination: The destination of the migration.
                                                     
                                                    5.  
                                                   
                                                  Return Value
                                                   
                                                  Returns one of the following error code:
                                                   
                                                     
                                                  • AOCL_MMD_ERROR_SUCCESS: No error occurred.
                                                    •  
                                                  • AOCL_MMD_ERROR_INVALID_HANDLE: The device handle provided is invalid.
                                                    •  
                                                  • AOCL_MMD_ERROR_INVALID_MIGRATION_SIZE: The migration size is not supported by the device.
                                                    •  
                                                  "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#42-board-utilities","title":"4.2 Board Utilities","text":"
                                                  oneAPI runtime provides a set of options for the aocl utility.
                                                   
                                                  Note: aocl is an utility available in the oneAPI runtime environment, please use aocl help command for more information on this.
                                                   
                                                  Table 4-6 shows the subcommands that aocl utility provides for FPGA platforms.
                                                   
                                                  Table 4-6: aocl Board Utilities
                                                   Subcommand Description Executable Call install Install board into the host system. This installs the FPGA Client Driver (FCD) for your FPGA platform. FCD allows runtime to find and load the FPGA platform libraries at runtime aocl install path-to-FPGA-platform-oneapi-asp uninstall Uninstall board from the host system. Removes FCD. aocl uninstall path-to-FPGA-platform-oneapi-asp initialize Configure a default FPGA image onto the board. For more information about initialization refer to Intel oneAPI FPGA Handbook Two methods are available to initalize the board: 1. aocl initialize device-name board-variant 2. aocl initialize device-name oneAPI fat binary > Note: The second option requires Intel\u00ae oneAPI Base Toolkit (Base Kit) version 2024.0 & above as well as 2023.3 OFS Release for oneAPI Accelerator Support Package and above program Configure a new FPGA image onto the board aocl program device-name aocx file diagnose Runs ICD and FCD diagnostics followed by querying devices in installed platforms. If a device-name is specified in the call, it run board vendor's test program for the FPGA platform * aocl diagnose : This queries the devices in FPGA platform and supplies a list of valid strings assigned to the list of devices  * aocl diagnose device-name : This runs full diagnostic test on the FPGA platform 
                                                  The runtime expects the routine for each of this utilities to be defined in the oneAPI ASP. It looks for the routine executables in the location specified by the utilbinder element in the board_env.xml file.
                                                   
                                                  install
                                                   
                                                  oneAPI runtime uses the information in board_env.xml file to create a FCD file. The FCD file name matches name attribute of board_env element and the FCD contents are the platform libraries specified in mmdlib element. Refer to section 2.2 for more information about board_env.xml file. The runtime adds the installed platform to the list of installed packages(file used by runtime to track installed platforms) and then invokes the install routine from oneAPI ASP.
                                                   
                                                  uninstall
                                                   
                                                  oneAPI runtime removes the FCD file and removes the platform from list of installed packages. It then invokes the uninstall routine for oneAPI ASP.
                                                   
                                                  initialize
                                                   
                                                  oneAPI runtime invokes initialize routine provided by oneAPI ASPs for installed platforms.
                                                   
                                                  program
                                                   
                                                  oneAPI runtime loads the programming file on the FPGA by invoking program routine provided by the oneAPI ASPs for the installed platform.
                                                   
                                                  diagnose
                                                   
                                                  oneAPI runtime runs ICD and FCD diagnostics to check the ICD and FCD files installed on the system. It then queries for available boards in the installed platform and lists boards matching every installed platform. If a device-name is specified in the call, runtime invokes the diagnostic routine provided in oneAPI ASP.
                                                   
                                                  For more information about the implementation of these routines in oneAPI ASPs for OFS reference platforms, please refer to section 5.5.
                                                  "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#50-oneapi-asp-implementation-details","title":"5.0 oneapi-asp Implementation Details","text":"
                                                  oneapi-asp in the OFS has two reference platform releases, one is based on Intel\u00ae Stratix 10\u00ae FPGA and the other is based on an Intel\u00ae Agilex\u00ae 7 FPGA. This chapter aims to explain the architecture and current implementation details of oneapi-asp for these platforms. The oneapi-asp repository is located here.
                                                   
                                                  The next section explains the oneapi-asp directory structure, followed by sections on hardware and MMD layers.
                                                  "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#51-oneapi-asp-directory-structure","title":"5.1 oneapi-asp Directory Structure","text":"
                                                  As described in section 2.0, oneAPI compiler & runtime use the board_env.xml and board_spec.xml files to get information about the FPGA platform. The compiler expects the board_env.xml file to be at the topmost level in the platform directory. The board_env.xml file describes the location of the hardware files & platform libraries.
                                                   
                                                  Figure 5-1: Sample board_env.xml File
                                                   
                                                   
                                                  Figure 5-1 shows a sample board_env.xml file, the corresponding oneAPI ASP directory structure must match the following format. Table 5-1 provides details on each folder. 
                                                   
                                                  In addition to below folders, oneAPI ASP for OFS reference platforms have another folder for the software layer (MMD & board utilities) source code located in common directory (oneapi-asp/common). This is because a common source code is utilized for both d5005 and n6001 reference platform ASPs. 
                                                   
                                                  \n    oneapi-asp_Platform-Name/\n    |--hardware/\n    |--|--Board-Variant-1/\n    |--|--Board-Variant-2/\n    |--linux64/\n    |--board_env.xml\n
                                                   
                                                  Note: In addition to above folders, oneAPI ASPs for OFS reference platforms have additional directories called scripts and bringup which contain helper scripts for platform generation & a sample for board bring up respectively. Please refer to the README for each reference platform in the oneASP-asp repository for more information on these additional folders.  * README for oneapi-asp targeting Intel\u00ae FPGA PAC D5005 reference platform: README  * README for oneapi-asp targeting Intel\u00ae FPGA SmartNIC N6001-PL Platform: README
                                                   
                                                  The Platform-Name is used for identifying the platform and can be alphanumeric value decided by the platform developer. For example, Intel uses the Platform-Name d5005 for oneapi-asp for Intel\u00ae Stratix 10\u00ae FPGA as the reference platform is Intel\u00ae FPGA PAC D5005.
                                                   
                                                  Table 5-1: Directory Contents
                                                   Files/Folder Descriptions hardware Contains hardware files (RTL, platform designer files, SDCs, compilation scripts, floorplan settings) and the board_spec.xml files for all board variants. See table 5-2 for more details common/source Source code for MMD layer as well as oneapi-asp board utilities linux64 Location for FPGA platform libraries and executables for oneapi-asp board utilities board_env.xml Contains platform installation information. Please refer to section 2.2 for more details on board_env.xml elements 
                                                  Tables 5-2 to 5-4 give more details on each of these folders for oneAPI ASPs for OFS reference platforms.
                                                  "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#511-hardware-folder","title":"5.1.1 hardware Folder","text":"
                                                  \nhardware/\n|--board-variant-1/\n|--|--build\n|--|--part-number_dm.xml (Please see note for this file in table 5-2)\n|--|--board_spec.xml\n|--|--quartus.ini\n
                                                   
                                                  Table 5-2: hardware Folder Contents
                                                   Files/Folder Description build Contains all hardware design files including Intel\u00ae Quartus\u00ae software project file (*.qpf), Intel\u00ae Quartus\u00ae software Settings Files (*.qsf), IP files,  *.sv, *.qsys, *.sdc as well as scripts to control compile flow board_spec.xml Defines compile flow, global memory, kernel interfaces. Please see section 2.1 for more information about board_spec.xml file part-number_dm.xml Device Model file for Intel\u00ae FPGA part on the target platform. The name must be of the format part-number_dm.xml. This file has the total resources available on the device. > Note: The device model files provided as part of the Intel\u00ae oneAPI Base Toolkit (Base Kit) installation are located in $INTELFPGAOCLSDKROOT/share/models/dm, where INTELFPGAOCLSDKROOT is set by the setvars.sh environment setup script provided by oneAPI toolkit. If the device model file for your part number is not included in $INTELFPGAOCLSDKROOT/share/models/dm, it must be created and placed in the same folder as board_spec.xml. quartus.ini Intel\u00ae Quartus\u00ae software ini settings file"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#512-commonsource-folder","title":"5.1.2 common/source Folder","text":"
                                                  \nsource/\n|--cmake/\n|--|--modules\n|--extra/\n|--|--intel-fpga-bbb\n|--host\n|--include\n|--util\n|--CMakeLists.txt\n
                                                   
                                                  Table 5-3: common/source Folder Contents
                                                   Files/Folder Description cmake/modules Contains Find***.cmake files to find packages required for building MMD library extra/intel-fpga-bbb oneAPI ASP for OFS reference platforms uses a library provided as part of Intel\u00ae FPGA Basic Building Blocks (BBB) repository. The oneapi-asp build scripts clone this repository in extra directory by default host Source code for MMD API include Contains MMD header files util Contains source code for oneapi-asp reference platform routines for diagnose and program utilities CMakeLists.txt Top-level CMakeLists.txt file for building MMD and other libraries required by oneapi-asp as well as the executables for diagnose and program utilities"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#513-linux64-folder","title":"5.1.3 linux64 Folder","text":"
                                                  \nlinux64/\n|--include (see note below)\n|--lib\n|--libexec\n
                                                   
                                                  Note: The include and lib folders do not exist in the oneapi-asp repository. These are added after the oneapi-asp build flow is complete.
                                                   
                                                  Table 5-4: linux64 Folder Contents
                                                   Files/Folder Description include Contains header files from Intel\u00ae FPGA BBB required by MMD (see section 5.4 for more information on use of MPF from Intel\u00ae FPGA BBB in MMD) lib Contains MMD and Intel\u00ae FPGA BBB libraries libexec Contains executables/scripts for oneapi-asp board utilities"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#52-oneapi-asp-build-flow","title":"5.2 oneapi-asp Build Flow","text":"
                                                  Figure 1-3 shows a high level overview of the hardware design and Figure 1-4 shows oneAPI ASP as part of the Open FPGA Stack. As shown in these images, all oneapi-asp hardware components reside in the AFU region in the PR slot.
                                                   
                                                  Note: The architecture of the FIM with PR slot is explained in the FIM technical reference manuals, please refer to section 1.3 for link to the manuals.
                                                   
                                                  The oneapi-asp repository contains source files for components that reside in the AFU region for each reference platform. oneapi-asp expects a compiled FIM netlist and a corresponding PR tree. The FIM database is copied to the oneapi-asp during ASP build flow (oneapi-asp/Platform-Name/scripts/build-bsp.sh). ASP compile scripts import the FIM database during oneAPI compilation.
                                                   
                                                  Notes:  1. FIM developer guide outlines steps to compile a FIM and generate PR tree, please refer to section 1.3 for links to FIM developer guides 2. The steps to build oneapi-asp using PR tree and build-bsp.sh script are covered in the oneAPI Accelerator Support Package (ASP): Getting Started User Guide
                                                   
                                                  The following figure shows the oneapi-asp build process.
                                                   
                                                  Figure 5-2: oneapi-asp Build Flow
                                                   
                                                   
                                                  Table 5-5 Environment Variables used in Build Flow
                                                   Variable Number Environment Variable Description 1 OFS_PLATFORM_AFU_BBB Should point to location where ofs-platform-afu-bbb repository is cloned, if this variable is not set, build-bsp.sh script clones the repository 2 OPAE_PLATFORM_ROOT Must point to the PR tree generated during FIM build, this is a required variable and build flow fails without this 3 LIBOPAE_C_ROOT Should point to the installation location for OPAE libraries (please see oneAPI Accelerator Support Package (ASP): Getting Started User Guide for more information on this variable setting), build-opae.sh script is used to clone & build OPAE library if this variable is not set 4 OPAE_SDK_REPO_BRANCH If LIBOPAE_C_ROOT is not set, it is recommended to set this variable to indicate the OPAE SDK branch to be used for building OPAE libraries (please see oneAPI Accelerator Support Package (ASP): Getting Started User Guide) 
                                                  All scripts required for oneapi-asp build are located in oneapi-asp/Platform-Name/scripts folder, where Platform-Name is:
                                                   
                                                     
                                                  • n6001 for Agilex OFS
                                                    •  
                                                  • d5005 for Stratix 10 OFS
                                                    •  
                                                   
                                                  The build flow generates the complete hardware directories for all board variants in oneapi-asp.
                                                   
                                                  Note: For more information about the build scripts, please refer to README in oneapi-asp/Platform-Name/scripts directory.
                                                  "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#53-oneapi-asp-hardware-implementation","title":"5.3 oneapi-asp Hardware Implementation","text":"
                                                  This section goes deeper into the current hardware architecture of the oneapi-asp.
                                                   
                                                  Figure 1-3 shows a high level overview of the hardware design. OFS reference platforms have different board variants enabling the different paths shown in Figure 1-3. Table below summarizes the board variants and paths supported in each. The Path numbers in the table match the ones in Figure 1-3. Figure 5-3 to 5-6 show the detailed diagram for oneapi-asp components in each of these board variants.
                                                   
                                                  Table 5-6: OFS Reference Platform Board Variants
                                                   # Device Board Variant Host to EMIF with DMA Engine (Path 1) Host to Kernel Interface (Path 2) Kernel to EMIF (Path 3) Kernel to Unified Shared Memory (Path 4) Kernel to HSSI (Path 5) Figure # 1 Agilex OFS  Stratix 10 OFS ofs_n6001  ofs_d5005 Yes Yes Yes No No 5-3 2 Agilex OFS  Stratix 10 OFS ofs_n6001_usm  ofs_d5005_usm Yes Yes Yes Yes No 5-4 3 Agilex OFS ofs_n6001_iopies Yes Yes Yes No Yes 5-5 4 Agilex OFS ofs_n6001_usm_iopipes Yes Yes Yes Yes Yes 5-6 
                                                  Note: Please see oneapi-asp/Platform-Name/hardware folder for the board variants for each OFS reference platform.
                                                   
                                                  Figure 5-3: oneapi-asp Reference Platform Hardware Design - Board Variant #1
                                                   
                                                   
                                                  Figure 5-4: oneapi-asp Reference Platform with USM Hardware Design - Board Variant #2
                                                   
                                                   
                                                  Figure 5-5: oneapi-asp Reference Platform with IO Pipes Hardware Design - Board Variant #3
                                                   
                                                   
                                                  Figure 5-6: oneapi-asp Reference Platform with IO Pipes and USM Hardware Design - Board Variant #4
                                                   
                                                   
                                                  The ofs_plat_afu.sv is the top level entity for the oneapi-asp.
                                                   
                                                  All hardware design files for the components inside ofs_plat_afu module are inside the oneapi-asp/Platform-Name/hardware/Board-Variant/build directory. The FIM database files imported during oneapi-asp build (section 5.2) are located in oneapi-asp/Platform-Name/hardware/Board-Variant/fim_platform directory.
                                                   
                                                  Table 5-7 gives a brief description of the important design files in all board variants.
                                                   
                                                  Table 5-7: Hardware Design Files in oneapi-asp/Platform-Name/hardware/Board-Variant
                                                   Files/Folder Description build/ip Contains all the *.ip files for IP components used in the hardware design build/rtl Contains the SystemVerilog and Verilog design files shown in figures 5-3 to 5-6 above build/scripts Contains scripts to control oneAPI kernel and asp Intel\u00ae Quartus\u00ae software compile flow (see section 5.3.4 for more information on compile flow) build/afu_ip.qsf Adds platform specific settings & design files, this file is sourced in afu_flat.qsf build/mpf_vtp.qsf Adds IP components from Intel\u00ae FPGA BBB (see section 5.3.2 below for information about use of MPF blocks from Intel\u00ae FPGA BBB) repository used in the design build/bsp_design_files.tcl Adds design files to the project, this file is source in afu_ip.qsf build/board.qsys Please refer to figures above for components in board.qsys system build/ddr_board.qsys Instantiated in the board.qsys system, contains IP components in the host to EMIF and kernel to EMIF datapath. Memory Bank Divider is instantiated in this platform designer system. Please refer to section 3.1.1 for more details on Memory Bank Divider build/ddr_channel.qsys Instantiated in ddr_board.qsys, contains bridges required for clock domain crossings between PCIe and EMIF clocks as well as kernel and EMIF clock fim_platform Contains Platform Interface Modules (PIM) modules and FIM database used in the design. See PIM subtopic below ofs_asp.sdc Contains clock group constraints for all clocks in oneAPI ASP 
                                                  The hardware implementation diagrams show a PF/VF Mux/De-mux module in the AFU region. The PF/VF mux routes packets to AFU component based on pf_num and vf_num information in PCIe TLP header. For more information about the PF/VF mux, please refer to the PF/VF Mapping details in FPGA Interface Manager Technical Reference Manual for your target device (links are available in section 1.3). 
                                                   
                                                  The oneapi-asp resides inside the AFU region and ,depending on the FIM configuration, connects to the lowest PF or lowest VF number that routes into the PR slot. Table below summazires the PF/VF mapping in oneapi-asp for some of the different FIM configurations.
                                                   
                                                  Table 5-8: PF/VF Mapping in oneapi-asp
                                                   Device FIM Configuration* PF/VF Mapping  in oneapi-asp Stratix 10 OFS Default PF0-VF1 Agilex OFS Default PF0-VF0 Agilex OFS Compiled using n6001_1pf_1vf.ofss, i.e. PCIe Subsystem configuration is set to PF0 with 1 VF PF0-VF0 Agilex OFS Compiled using n6001_2pf.ofss, i.e. PCIe Subsystem configuration is set to two physical functions PF0 and PF1 PF1 
                                                  *Note: For more information on different FIM configurations & how to compile these, please refer to FPGA Interface Manager Developer Guide for Open FPGA Stack for your target device (links are available in section 1.3).
                                                   
                                                  Sections below provide some more information on some blocks in the hardware design block diagram shown above. Refer to section 3.1 for more information about Memory Bank Divider and to section 3.2 for information about Kernel Interface. 
                                                  "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#531-platform-interface-modulespim","title":"5.3.1 Platform Interface Modules(PIM)","text":"
                                                  In addition to above files/folders, an important part of the design are the ofs_plat_* modules provided by Platform Interface Manager (PIM). oneAPI kernel system and oneapi-asp have Avalon\u00ae interfaces. FIM components have AXI interfaces. PIM modules are used for protocol translation. 
                                                   
                                                  Note: For more information about PIM, please refer to PIM README here.
                                                  "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#532-direct-memory-accessdma-module","title":"5.3.2 Direct Memory Access(DMA) Module","text":"
                                                  The Direct Memory Access module is located in the host to EMIF datapath in the oneapi-asp and provides a controller to execute transfers from host to DDR on the board and vice versa. The source files for the DMA module used in oneapi-asp for OFS reference platforms are located in oneapi-asp/Platform-Name/hardware/Board-Variant/build/rtl/dma directory. Figure below shows the DMA module interface.
                                                   
                                                  Figure 5-7: DMA Controller Block Diagram
                                                   
                                                   
                                                  Figure 5-3 shows the instantiation of this dma_top module as dma_controller_inst. Table 5-9 shows the signal names and descriptions. Section 5-5 covers the complete compilation flow.
                                                   
                                                  Table 5-9: DMA Module Signal Descriptions
                                                   Signal or Port Description mmio64_if Control signal from host, connects to DMA Control Status Registers(CSR) host_mem_rd_avmm_if Avalon\u00ae memory-mapped interface to read from host memory host_mem_wr_avmm_if Avalon\u00ae memory-mapped interface to write to host memory local_mem_rd_avmm_if Avalon\u00ae memory-mapped interface to read from on-board DDR4 memory local_mem_wr_avmm_if Avalon\u00ae memory-mapped interface to write to on-board DDR4 memory dma_irq_host2fpga Interrupt to indicate completion of host to FPGA DMA transaction (read from host, write to DDR4) dma_irq_fpga2host Interrupt to indicate completion of FPGA DDR4 to host DMA transaction (read from DDR4, write to host)> Note: The oneapi-asp for OFS reference platforms uses a host memory write to indicate completion of FPGA to host transaction 
                                                  The data transfer module manages data movement from source to destination addresses.
                                                   
                                                  To start a data transfer, the data transfer module requires following information, this is set by the dma_dispatcher:
                                                   
                                                     
                                                  • Source address
                                                    •  
                                                  • Destination address
                                                    •  
                                                  • Number of bytes to transfer
                                                    •  
                                                   
                                                  oneapi-asp for OFS reference platforms uses virtual addresses in the DMA controller for data transfer. The Memory Properties Factory(MPF) Virtual to Physical(VTP) blocks (i.e.mpf_vtp_* modules) shown in figure 5-3 translate virtual addresses to appropriate host addresses. 
                                                   
                                                  Note: Memory Properties Factory(MPF) is a part of Intel\u00ae FPGA Basic Building Blocks. Please refer to Intel\u00ae FPGA BBB repository for more information.
                                                  "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#533-user-datagram-protocoludp-engine","title":"5.3.3 User Datagram Protocol(UDP) Engine","text":"
                                                  I/O pipes allow kernel to stream data directly using HSSI. To demonstrate this functionality, reference design in oneapi-asp repository (refer to Figure 5-5 and 5-6) has a UDP protocol engine to allow transmitting UDP/IP packets over HSSI..
                                                   
                                                  Figure below shows a simple block diagram of the UDP engine.
                                                   
                                                  Figure 5-8: UDP Offload Engine
                                                   
                                                   
                                                  The UDP engine consists of a separate receive (rx) and trasmit (tx) path. The following functionalilty is performed by this reference design engine:
                                                   
                                                     
                                                  • Implements an Address Resolution Protocol (ARP) functionality to respond to be able to send & respond to ARP requests. This is needed for routing between different subnets using a gateway. 
                                                    •  
                                                  • Packetizes data from kernel to add the required header information (for MAC, IP & UDP layers) 
                                                    •  
                                                  • Extracts data from packets received by removing header information
                                                    •  
                                                  • Handles clock crossing between kernel clock and Ethernet MAC clock domains
                                                    •  
                                                   
                                                  The source files for UDP engine used in oneapi-asp for OFS reference platform are located in oneapi-asp/n6001/hardware/ofs_n6001_iopipes/build/rtl/udp_offload_engine directory.
                                                   
                                                  Note: The same engine is used in the board variant with USM shown in Figure 5-6 (source files are in oneapi-asp/n6001/hardware/ofs_n6001_usm_iopipes/build/rtl/udp_offload_engine). 
                                                  "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#534-hardware-compile-flow","title":"5.3.4 Hardware Compile Flow","text":"
                                                  Figure below shows the compile flow overview; the oneAPI compiler generated hardware circuit is compiled by Intel\u00ae Quartus\u00ae software along with design files for oneapi-asp.
                                                   
                                                  Figure 5-9: oneAPI Compile Flow Overview
                                                   
                                                   
                                                  The oneAPI compiler uses the board_spec.xml to get more information about the oneapi-asp configuration. board_spec.xml file has a compile element to allow control of the Intel\u00ae Quartus\u00ae software compilation flow. The attributes of this element are discussed in section 2.1.2. The oneapi-asp uses tcl scripts to control the Intel\u00ae Quartus\u00ae software compilation flow. Figure 5-10 shows the flow and scripts used. All compilation scripts are located in oneapi-asp/Platform-Name/hardware/Board-Variant/build/scripts folder.
                                                   
                                                  Figure 5-10: Compilation Scripts in oneapi-asp
                                                   
                                                   
                                                  Table 5-10 summarizes notes for reference numbers 1-5 marked in figure above.
                                                   
                                                  Table 5-10: Notes for Reference Numbers in Figure 5-10
                                                   Reference Number Note 1 revision_name is afu_flat for oneapi-asps for OFS reference platforms 2 gen-asp-quartus-report.tcl script generates a report (acl_quartus_report.txt) containing resource utilization and kernel clock frequency summary"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#54-oneapi-asp-memory-mapped-devicemmd-layer-implementation","title":"5.4 oneapi-asp Memory Mapped Device(MMD) Layer Implementation","text":"
                                                  As discussed in section 4.1, the MMD provides a set of API that allow the runtime to control the device and communicate with it.
                                                   
                                                  The source code for MMD layer is located in oneapi-asp/common/source/host folder. aocl_mmd.h is the header file for the implemented API and is located in oneapi-asp/common/source/include folder. Table below summarizes the APIs that have been implemented in oneapi-asp for OFS reference platforms.
                                                   
                                                  Note: For more details about the API, its arguments and enums please refer to the aocl_mmd.h file and to section 4.1.
                                                   
                                                  Table 5-11: MMD API Implemented in oneapi-asp for OFS Reference Platforms
                                                   API aocl_mmd_get_offline_info aocl_mmd_get_info aocl_mmd_open aocl_mmd_close aocl_mmd_set_interrupt_handler aocl_mmd_set_status_handler aocl_mmd_yield aocl_mmd_read aocl_mmd_write aocl_mmd_copy aocl_mmd_program aocl_mmd_host_alloc aocl_mmd_free aocl_mmd_shared_alloc aocl_mmd_shared_migrate 
                                                  The implementation of these APIs is in oneapi-asp/common/source/host/mmd.cpp. The functions used in the implementation are distributed across various source files. Table below provides details on source code files.
                                                   
                                                  Table 5-12: MMD Source Code Files
                                                   Files/Folder Description mmd.cpp This file has the implementation for all MMD API calls listed in table 5-11 fpgaconf.hfpgaconf.c Contains bitstream reconfiguration function declaration(.h) & definition(.c) kernel_interrupt.h Contains KernelInterrupt class declaration; the class consists of functions to handle kernel interrupts kernel_interrupt.cpp Contains KernelInterrupt class constructor and function definitions mmd_device.h Contains Device class declaration, which stores device data and has functions to interact with the device mmd_device.cpp Contains Device class constructor and function definitions mmd_dma.hmmd_dma.cpp Contain DMA functions declaration(.h) & definition(.cpp) mmd_iopipes.h Contains the iopipes class declaration(.h) mmd_iopipes.cpp Contains function definitions, these functions include iopipes class constructor as well as fucntions to detect and setup CSR space for IO pipes feature in board variants that support IO pipes zlib_inflate.hzlib_inflate.c Function declaration(.h) and definition(.c) for decompressing bitstream data CMakeLists.txt CMakeLists.txt file for building MMD source code 
                                                  The build flow scripts build the MMD library, i.e. libintel_opae_mmd, and place them in oneapi-asp/Platform-Name/linux64/lib folder. The MMD library is specified as part of mmdlib, linklibs element in board_env.xml and used at runtime (refer to figure 5-1 for sample board_env.xml file and section 2.2 for more information about board_env.xml elements).
                                                   
                                                  Use of OPAE library in MMD
                                                   
                                                  The MMD layer uses API from OPAE SDK for various device operations. Hence, the MMD layers requires OPAE library to be loaded to execute successfully. The mmdlib element specifies the libopae-c library to be loaded before the MMD library (demonstrated in sample board_env.xml in figure 5-1).
                                                   
                                                  Note: Please refer to Software Reference Manual: Open FPGA Stack for more information about OPAE SDK API. The document also has information about linux-dfl driver.
                                                   
                                                  Use of Memory Properties Factory(MPF) library in MMD
                                                   
                                                  In addition to OPAE, the MMD also uses API from Memory Properties Factory(MPF) software for memory operations. This The libMPF library is compiled as part of oneapi-asp build flow and placed in oneapi-asp/Platform-Name/linux64/lib. This library is also loaded before the MMD library by specifying before libintel_opae_mmd in mmdlib element.
                                                   
                                                  Note: Memory Properties Factory(MPF) is a part of Intel\u00ae FPGA BBB. Please refer to Intel\u00ae FPGA BBB wiki for more information about MPF.
                                                  "},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#55-oneapi-asp-utilities-implementation","title":"5.5 oneapi-asp Utilities Implementation","text":"
                                                  This section covers the implementation of board utilities (refer to section 4.2 for more information on board utilities) in oneapi-asp for OFS reference platforms.
                                                   
                                                  Table below shows the source code/script locations for the utilities.
                                                   
                                                  Table 5-13: oneapi-asp Utilities Source Code Locations
                                                   Utility Source Location diagnose oneapi-asp/common/source/util/diagnostic program oneapi-asp/common/source/util/reprogram installuninstallinitialize> Note: These are executable scripts oneapi-asp/Platform-Name/linux64/libexec/ 
                                                  diagnose and program are compiled as part of the oneapi-asp build flow and executables are placed in oneapi-asp/Platform-Name/linux64/libexec/. A CMakeLists.txt file is provided for building the utilities, located in oneapi-asp/common/source/util directory.
                                                   
                                                  The path to all of the above utility executables is used in utilbinder element in board_env.xml (demonstrated in sample board_env.xml in figure 5-1). The runtime uses this when the corresponding aocl utility is invoked.
                                                   
                                                  Brief descriptions for the source code files are given in table below.
                                                   
                                                  Table 5-14: Brief Descriptions of oneapi-asp Utility Routines for OFS Reference Platforms
                                                   File Description setup_permissions.sh Helper script to configure correct device port permissions, make changes to allow users to lock pages in memory and set the hugepages required for the software stack to function correctly. The helper script is used by install, initialize routines install install routine invokes the setup_permissions.sh script after the FPGA Client Driver (FCD) is setup by the runtime uninstall uninstall routine reverts the port permission, memory locking and hugepage setting changes performed by install routine and is invoked by runtime after the FCD is removed by runtime initialize initialize routine performs the following steps: * looks for the initialization binary for the board variant to be initialized * extracts the FPGA hardware configuration file from the oneAPI fat binary using clang-offload-extract command provided by Intel\u00ae oneAPI Base Toolkit (Base Kit) version 2024.0 and beyond  * invokes the setup_permissions.sh script to set correct device permissions * performs partial reconfiguration of the FPGA device by invoking program routine with the initialization bitstream as an argument  >Note: For more information about how initialize utility extracts FPGA hardware configuration file from oneAPI fat binary, refer to Intel oneAPI\u00ae FPGA Handbook program program routine allocates memory and loads the supplied initialization bitstream in memory followed by a call to reprogramming function provided by oneapi-asp's MMD library. The MMD library uses fpgaReconfigureSlot API provided by OPAE library to perform device reconfiguration> Note: Please refer to Software Reference Manual: Open FPGA Stack for more information about OPAE SDK API diagnose diagnose routine scans for the available devices for the installed platform and performs DMA transactions between host & device. It also reports the PCIe bandwidth. diagnose routine uses functions provided by the MMD library for scanning & opening connection to available devices"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#document-revision-history","title":"Document Revision History","text":"Date Release Changes May 26, 2023 2023.1 First release of oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack on https://ofs.github.io/ September 15, 2023 2023.2 1. Section 1: * Updated Figure 1-3 to add HSSI path 2. Section 2: * Added channels element in figure 2-1 and table 2-1, added information about board variants below this table * Added Section 2.1.8 on channels element 3. Section 4: * Added information about MMD API (table 4-1) and new sections 4.1.1 to 4.1.21 4. Section 5: * Moved oneapi-asp build flow information into new section 5.2 * Added new table 5-6 with oneAPI ASP board variants information * Added hardware design diagrams and information about new board variants with I/O pipes support (Hardware Design with IO Pipes and Hardware Design with IO Pipes and USM) * Updated hardware design diagrams to show PF/VF Mux/De-mux and added information about PF/VF mapping in section 5.3 * Added new section on UDP engine (section 5.3.3) * Updated figure 5-10 to remove import_opencl_kernel.tcl and add_bbb_to_pr_project.tcl * Updated table 5-12 to add mmd_iopipes.h and mmd_iopipes.cpp files Dec 13, 2023 2023.3-1 1. Section 2: * Update table 2-5 to add information about port parameter * Added new section 2.1.4.1 on port parameter * Update table & figure numbers * Changed \"master\" and \"slave\" ports to \"host\" & \"agent\" ports in figures 2. Section 4: * Added new executable call supported by initialize utility 3. Section 5: * Updated initialize utility information to add clang-offload-extractcommand usage * Updated table with hardware design files information, added information about ofs_asp.sdc * Updated compile flow diagram, added information about new gen-asp-quartus-report.tcl script * Updated hardware implementation diagrams for signal name and IP name changes, replaced mem_if_vtp block with host_mem_if_vtp * Updated OpenCL Memory Bank Divider to Memory Bank Divider * Updated OpenCL Kernel Interface to Kernel Interface 2023.3-2 1. Section 5: * Updated Table 5-8 to add PF/VF mapping for different FIM configurations"},{"location":"hw/common/reference_manual/oneapi_asp/oneapi_asp_ref_mnl/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                  Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                   
                                                  OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                  "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/","title":"AFU Development Guide: OFS for Intel\u00ae Intel\u00ae Agilex\u00ae 7 FPGA PCIe Attach FPGAs","text":"
                                                  Last updated: February 03, 2024 
                                                  "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#1-introduction","title":"1. Introduction","text":"
                                                  This document is a design guide for the creation of an Accelerator Functional Unit (AFU) using Open FPGA Stack (OFS) for Intel\u00ae Agilex\u00ae 7 FPGAs PCIe Attach. The AFU concept consists of separating out the FPGA design development process into two parts, the construction of the foundational FPGA Interface Manager (FIM), and the development of the Acceleration Function Unit (AFU), as shown in the diagram below.
                                                   
                                                   
                                                  This diagram shows the separation of FPGA board interface development from the internal FPGA workload creation.  This separation starts with the FPGA Interface Manager (FIM) which consists of the external interfaces and board management functions.  The FIM is the base system layer and is typically provided by board vendors. The FIM interface is specific to a particular physical platform.  The AFU makes use of the external interfaces with user defined logic to perform a specific application.  By separating out the lengthy and complicated process of developing and integrating external interfaces for an FPGA into a board allows the AFU developer to focus on the needs of their workload.  OFS for Intel\u00ae Agilex\u00ae 7 FPGAs PCIe Attach provides the following tools for rapid AFU development:
                                                   
                                                     
                                                  • Scripts for both compilation and simulation setup
                                                    •  
                                                  • Optional Platform Interface Manager (PIM) which is a set of SystemVerilog shims and scripts for flexible FIM to AFU interfacing
                                                    •  
                                                  • Acceleration Simulation Environment (ASE) which is a hardware/software co-simulation environment scripts for compilation and Acceleration
                                                    •  
                                                  • Integration with Open Programmable Acceleration Engine (OPAE) SDK for rapid software development for your AFU application
                                                    •  
                                                   
                                                  Please notice in the above block diagram that the AFU region consists of static and partial reconfiguration (PR) regions where the PR region can be dynamically reconfigured while the remaining FPGA design continues to function.  Creating AFU logic for the static region is described in FPGA Interface Manager Developer Guide for Intel\u00ae Agilex\u00ae 7 PCIe Attach.  This guide covers logic in the AFU Main region.
                                                  "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#11-document-organization","title":"1.1. Document Organization","text":"
                                                  This document is organized as follows:
                                                   
                                                     
                                                  • Description of design flow
                                                    •  
                                                  • Interfaces and functionality provided in the Intel\u00ae Agilex\u00ae 7 FPGAs PCIe Attach FIM
                                                    •  
                                                  • Setup of the AFU Development environment
                                                    •  
                                                  • Synthesize the AFU example
                                                    •  
                                                  • Testing the AFU example on the Intel\u00ae FPGA SmartNIC N6001-PL card
                                                    •  
                                                  • Hardware/Software co-simulation using ASE
                                                    •  
                                                  • Debugging an AFU with Remote Signal Tap
                                                    •  
                                                   
                                                  This guide provides theory followed by tutorial steps to solidify your AFU development knowledge.
                                                   
                                                  NOTE: 
                                                   
                                                  This guide uses the Intel\u00ae FPGA SmartNIC N6001-PL as the platform for all tutorial steps. Additionally, this guide and the tutorial steps can be used with other platforms, such as the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile).
                                                   
                                                  Some of the document links in this guide are specific to the  Intel\u00ae FPGA SmartNIC N6001-PL.   If using a different platform, please use the associated documentation for your platform as there could be differences in building the FIM and downloading FIM images. 
                                                   
                                                  If you have worked with previous Intel Programmable Acceleration products, you will find out that OFS for Intel\u00ae Agilex\u00ae 7 FPGAs PCIe Attach is similar. However, there are differences and you are advised to carefully read and follow the tutorial steps to fully understand the design tools and flow.
                                                  "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#12-prerequisite","title":"1.2. Prerequisite","text":"
                                                  This guide assumes you have the following FPGA logic design-related knowledge and skills:
                                                   
                                                     
                                                  • FPGA compilation flows including the Intel\u00ae Quartus\u00ae Prime Pro Edition design flow
                                                    •  
                                                  • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition software, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
                                                    •  
                                                  • RTL and coding practices to create synthesizable logic.
                                                    •  
                                                  • Understanding of AXI and Avalon memory mapped and streaming interfaces.
                                                    •  
                                                  • Simulation of complex RTL using industry standard simulators (Synopsys\u00ae VCS\u00ae or Siemens\u00ae QuestaSim\u00ae).
                                                    •  
                                                  • Signal Tap Logic Analyzer tool in the Intel\u00ae Quartus\u00ae Prime Pro Edition software.
                                                    •  
                                                   
                                                  You are strongly encouraged to review the FPGA Interface Manager Developer Guide for Intel\u00ae Agilex\u00ae 7 PCIe Attach.
                                                  "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#13-acceleration-functional-unit-afu-development-flow","title":"1.3. Acceleration Functional Unit (AFU) Development Flow","text":"
                                                  The AFU development flow is shown below: 
                                                  "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#131-understanding-platform-capabilities","title":"1.3.1. Understanding Platform Capabilities","text":"
                                                  The block diagram of the N6001 Board is shown below:
                                                   
                                                   
                                                  The N6001 FIM provided with this release is shown below:
                                                   
                                                   
                                                  This release FIM provides the following features:
                                                   
                                                     
                                                  • Host interface
                                                       
                                                    • PCIe Gen4 x 16
                                                      •  
                                                    • 5 - PF, 4 - VF, AXI-S TLP packets
                                                      •  
                                                    • MSI-X interrupts
                                                      •  
                                                    • Logic to demonstrate simple PCIe loopback
                                                      •  
                                                     
                                                    •  
                                                  • Network interface
                                                       
                                                    • 2 - QSFP28/56 cages
                                                      •  
                                                    • 2 x 4 x 25 GbE with exerciser logic demonstrating traffic generation/monitoring
                                                      •  
                                                     
                                                    •  
                                                  • External Memory - DDR4 - 2400
                                                       
                                                    • HPS - 1GB organized as 256 Mb x 32 with 256 Mb x 8 ECC
                                                      •  
                                                    • Channel 0, 1  -  4 GB organized as 1 Gb x 32
                                                      •  
                                                    • Channel 2, 3 - 4 GB organized as 1 Gb x 32 with 1 Gb x 8 ECC (ECC is not implemented in this release)
                                                      •  
                                                    • Memory exerciser logic demonstrating external memory operation
                                                      •  
                                                     
                                                    •  
                                                  • Board Management
                                                       
                                                    • SPI interface
                                                      •  
                                                    • FPGA configuration
                                                      •  
                                                    • Example logic showing DFH operation
                                                      •  
                                                     
                                                    •  
                                                  • Remote Signal Tap logic
                                                    •  
                                                  • Partial reconfiguration control logic
                                                    •  
                                                  • ARM HPS subsystem with embedded Linux
                                                       
                                                    • HPS Copy engine
                                                      •  
                                                     
                                                    •  
                                                  "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#132-high-level-data-flow","title":"1.3.2. High Level Data Flow","text":"
                                                  The OFS high level data flow is shown below:
                                                   
                                                  "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#133-considerations-for-pim-usage","title":"1.3.3. Considerations for PIM Usage","text":"
                                                  When creating an AFU, a designer needs to decide what type of interfaces the platform (FIM) should provide to the AFU.  The FIM can provide the native interfaces (i.e. PCIe TLP commands) or standard memory mapped interfaces (i.e. AXI-MM or AVMM) by using the PIM.  The PIM is an abstraction layer consisting of a collection of SystemVerilog interfaces and shims to enable partial AFU portability across hardware despite variations in hardware topology and native interfaces. The PIM adds a level of logic between the AFU and the FIM converting the native interfaces from the FIM to match the interfaces provided by the AFU.
                                                   
                                                   
                                                  The following resources are available to assist in creating an AFU:
                                                   
                                                  PIM Core Concepts provides details on using the PIM and its capabilities. 
                                                   
                                                  Connecting an AFU to a Platform using PIM guides you through the steps needed to connect a PIM Based AFU to the FIM. 
                                                   
                                                  The AFU Tutorial provides several AFU examples.  These examples can be run with the current OFS FIM package.  There are three AFU types of examples provided (PIM based, hybrid and native).  Each example provides the following:
                                                   
                                                     
                                                  • RTL, which includes the following interfaces: 
                                                       
                                                    • Host Channel: 
                                                         
                                                      • Host memory, providing a DMA interface.
                                                        •  
                                                      • MMIO, providing a CSR interface.  
                                                        •  
                                                       
                                                      •  
                                                    • Local Memory
                                                      •  
                                                     
                                                    •  
                                                  • Host software example interfacing to the CSR interface and host memory interface, using the OPAE C API.
                                                    •  
                                                  • Accelerator Description File .json file
                                                    •  
                                                  • Source file list
                                                    •  
                                                  "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#134-afu-interfaces-included-with-intel-fpga-smartnic-n6001-pl","title":"1.3.4. AFU Interfaces Included with Intel\u00ae FPGA SmartNIC N6001-PL","text":"
                                                  The figure below shows the interfaces available to the AFU in this architecture. It also shows the design hierarchy with module names from the fim (top.sv) to the PR region AFU (afu_main.sv). One of the main differences from the Stratix 10 PAC OFS architecture to this one is the presence of the static port gasket region (port_gasket.sv) that has components to facilitate the AFU and also consists of the PR region (afu_main.sv) via the PR slot. The Port Gasket contains all the PR specific modules and logic, e.g., PR slot reset/freeze control, user clock, remote STP etc. Architecturally, a Port Gasket can have multiple PR slots where user workload can be programmed into. However, only one PR slot is supported for OFS Release for Intel Agilex. Everything in the Port Gasket until the PR slot should be provided by the FIM developer. The task of the AFU developer is to add their desired application in the afu_main.sv module by stripping out unwanted logic and instantiating the target accelerator. As shown in the figure below, here are the interfaces connected to the AFU (highlighted in green) via the PCIe Attach FIM:
                                                   
                                                     
                                                  1. AXI Streaming (AXI-S) interface to the Host via PCIe Gen4x16
                                                    2.  
                                                  2. AXI Memory Mapped Channels (4) to the DDR4 EMIF interface
                                                    3.  
                                                  3. AXI Streaming (AXI-S) interface to the HSSI 25 Gb Ethernet
                                                    4.  
                                                   
                                                  "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#2-set-up-afu-development-environment","title":"2. Set Up AFU Development Environment","text":"
                                                  This section covers the setup of the AFU development environment.
                                                  "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#21-afu-development-environment-overview","title":"2.1. AFU development environment overview","text":"
                                                  A typical development and hardware test environment consists of a development server or workstation with FPGA development tools installed and a separate server with the target OFS compatible FPGA PCIe card installed.  The typical usage and flow of data between these two servers is shown below:
                                                   
                                                   
                                                  Note: both development and hardware testing can be performed on the same server if desired.
                                                   
                                                  This guide uses Intel\u00ae FPGA SmartNIC N6001-PL as the target OFS compatible FPGA PCIe card for demonstration steps.  The Intel\u00ae FPGA SmartNIC N6001-PL must be fully installed following the Getting Started User Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs. If using a different OFS FPGA PCIe card, contact your supplier for instructions on how to install and operate user developed AFUs.
                                                   
                                                  The following is a summary of the steps to set up for AFU development:
                                                   
                                                     
                                                  1. Install Quartus Prime Pro Version 23.3 for Linux with Agilex device support and required Quartus patches.
                                                    2.  
                                                  2. Make sure support tools are installed and meet version requirements.
                                                    3.  
                                                  3. Install OPAE SDK.
                                                    4.  
                                                  4. Download the Basic Building Blocks repository.
                                                    5.  
                                                  5. Build or download the relocatable AFU PR-able build tree based on your Intel\u00ae Agilex FPGA PCIe Attach FIM.
                                                    6.  
                                                  6. Download FIM to the Intel\u00ae Agilex FPGA PCIe Attach platform.
                                                    7.  
                                                   
                                                  Building AFUs with OFS for Agilex requires the build machine to have at least 64 GB of RAM. 
                                                  "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#22-installation-of-quartus-and-required-patches","title":"2.2. Installation of Quartus and required patches","text":"
                                                  Intel Quartus Prime Pro Version 23.3 is verified to work with the latest OFS release ofs-2023.3-2.  However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.
                                                   
                                                  Use RedHatEnterprise Linux\u00ae (RHEL) 8.6 for compatibility with your development flow and also testing your FIM design in your platform. 
                                                   
                                                  Prior to installing Quartus:
                                                   
                                                     
                                                  1.  
                                                    Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.
                                                     
                                                       
                                                    • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                                                      •  
                                                    • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.
                                                      •  
                                                     
                                                    2.  
                                                  2.  
                                                    Perform the following steps to satisfy the required dependencies.
                                                     
                                                    $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                                                     
                                                    Apply the following configurations.
                                                     
                                                    $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n
                                                     
                                                    3.  
                                                  3.  
                                                    Create the default installation path: /intelFPGA_pro/, where  is the default path of the Linux workstation, or as set by the system administrator and  is your Quartus version number. 
                                                    The installation path must satisfy the following requirements:
                                                     
                                                       
                                                    • Contain only alphanumeric characters
                                                      •  
                                                    • No special characters or symbols, such as !$%@^&*<>,
                                                      •  
                                                    • Only English characters
                                                      •  
                                                    • No spaces
                                                      •  
                                                     
                                                  4.  
                                                    Download your required Quartus Prime Pro Linux version here.
                                                     
                                                    5.  
                                                  5.  
                                                    Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe).
                                                     
                                                    6.  
                                                  6.  
                                                    After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                                                     
                                                    export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                                                     
                                                    For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:
                                                     
                                                    export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n
                                                     
                                                    7.  
                                                  7.  
                                                    Verify, Quartus is discoverable by opening a new shell:
                                                     
                                                    $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                                                     
                                                    8. "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#23-installation-of-support-tools","title":"2.3. Installation of Support Tools","text":"
                                                    Make sure support tools are installed and meet version requirements.
                                                     
                                                    The OFS provided Quartus build scripts require the following tools. Verify these are installed in your development environment.
                                                     Item Version Python 3.6.8 GCC 7.4.0 cmake 3.15 git 1.8.3.1 perl 5.8.8"},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#24-installation-of-opae-sdk","title":"2.4. Installation of OPAE SDK","text":"
                                                    Follow the instructions in the Getting Started Guide: Open FPGA Stack for Intel\u00ae FPGA SmartNIC N6001-PL, section 4.0 OPAE Software Development Kit to build and install the required OPAE SDK for the Intel\u00ae FPGA SmartNIC N6001-PL.
                                                     
                                                    Working with the Intel\u00ae Intel\u00ae FPGA SmartNIC N6001-PL card requires opae-2.10.0-1. Follow the instructions in the Getting Started Guide: Intel\u00ae Open FPGA Stack for Intel\u00ae FPGA SmartNIC N6001-PL section 4.0 OPAE Software Development Kit. Make sure to check out the cloned repository to tag 2.10.0-1 and branch release/2.10.0.
                                                     
                                                    $ git checkout tags/2.10.0-1 -b release/2.10.0\n
                                                     
                                                    Note: The tutorial steps provided in the next sections assume the OPAE SDK is installed in default system locations, under the directory, /usr. In most system configurations, this will allow the OS and tools to automatically locate the OPAE binaries, scripts, libraries and include files required for the compilation and simulation of the FIM and AFUs.
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#25-download-the-basic-building-blocks-repositories","title":"2.5. Download the Basic Building Blocks repositories","text":"
                                                    The ofs-platform-afu-bbb repository contains the PIM files as well as example PIM-based AFUs that can be used for testing and demonstration purposes. This guide will use the host_chan_mmio AFU example in the ofs-platform-afu-bbb repository and the hello_world sample accompanying the examples AFU repository to demonstrate how to synthesize, load, simulate, and test a PIM-based AFU using the Intel\u00ae FPGA SmartNIC N6001-PL card with the PCIe Attach FIM.
                                                     
                                                    Execute the next commands to clone the BBB repository.
                                                     
                                                      # Create top level directory for AFU development\n$ mkdir OFS_BUILD_ROOT\n$ cd OFS_BUILD_ROOT\n$ export OFS_BUILD_ROOT=$PWD\n\n  # Clone the ofs-platform-afu-bbb repository.\n$ cd $OFS_BUILD_ROOT\n$ git clone https://github.com/OFS/ofs-platform-afu-bbb.git\n\n  # Verify retrieval\n$ cd $OFS_BUILD_ROOT/ofs-platform-afu-bbb\n$ ls\nLICENSE  plat_if_develop  plat_if_release  plat_if_tests  README.md\n
                                                     
                                                    The documentation in the ofs-platform-afu-bbb repository further addresses   - The PIM concept.   - The structure of the PIM-based AFU examples.   - How to generate a release and configure the PIM.   - How to connect an AFU to an FIM.
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#26-build-or-download-the-relocatable-pr-build-tree","title":"2.6. Build or download the relocatable PR build tree","text":"
                                                    A relocatable PR build tree is needed to build the AFU partial reconfiguration area for the intended FIM. The tree is relocatable and may be copied to a new location. It does not depend on files in the original FIM build.
                                                     
                                                    You can use the  Intel\u00ae FPGA SmartNIC N6001-PL release package and download the PR build tree and FIM images, to develop your AFU.  These are located at OFS-N6001 release 
                                                     
                                                    Or you can build your own FIM and generate the PR build tree during the process.
                                                     
                                                    To download and untar the pr_build_template:
                                                     
                                                    $ cd $OFS_BUILD_ROOT\n$ wget https://github.com/OFS/ofs-agx7-pcie-attach/releases/download/ofs-2023.3-2/n6001-images_ofs-2023-3-2.tar.gz\n$ tar -zxvf n6001-images_ofs-2023-3-2.tar.gz\n$ cd n6001-images_ofs-2023-3-2/\n$ mkdir pr_build_template\n$ tar -zxvf pr_build_template-n6001.tar.gz -C ./pr_build_template\n$ cd pr_build_template\n$ export OPAE_PLATFORM_ROOT=$PWD\n
                                                     
                                                    To build your own FIM and generate the PR build tree for the Intel\u00ae FPGA SmartNIC N6001-PL platform, refer the FPGA Interface Manager Developer Guide for Intel\u00ae Agilex\u00ae 7 PCIe Attach and follow the Out-of-Tree PR FIM build flow.  If you are using a different platform, refer to the FPGA Interface Manager Developer Guide for your platform and follow the Out-of-Tree PR FIM build flow.
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#27-download-fim-to-fpga","title":"2.7. Download FIM to FPGA","text":"
                                                    The AFU requires that the FIM from which the AFU is derived be loaded onto the FPGA.   
                                                     
                                                    If you are using the Intel\u00ae FPGA SmartNIC N6001-PL release package downloaded in the previous section: 
                                                    $ cd $OFS_BUILD_ROOT/n6001-images_ofs-2023-3-2/\n
                                                     
                                                    If you are generating your own FIM, use the unsigned FPGA binary images from your FIM build. 
                                                     
                                                    Downlaod the FIM to the Intel\u00ae FPGA SmartNIC N6001-PL platform. If you are running on a Virtual Machine, refer to the [KVM User Guide] for passing the devices to the VM. 
                                                    $ sudo fpgasupdate ofs_top_page1_unsigned_user1.bin <N6001 SKU2 PCIe b:d.f>\n$ sudo fpgasupdate ofs_top_page2_unsigned_user2.bin <N6001 SKU2 PCIe b:d.f>\n$ sudo rsu fpga --page=user1 <N6001 SKU2 PCIe b:d.f>\n
                                                     
                                                    If you are using a different platform, refer to the documentation for your platform to download the FIM images onto your Intel\u00ae Agilex\u00ae PCIe Attach Platform.
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#28-set-up-required-environment-variables","title":"2.8. Set up required Environment Variables","text":"
                                                    Set the required environment variables as shown below. These environment variables must be set prior to simulation or compilation tasks. You can create a simple script to set these variables and save time going forward.
                                                     
                                                    # If not already done, export OFS_BUILD_ROOT to the top level directory for AFU development\n$ export OFS_BUILD_ROOT=<path to ofs build directory>\n\n# If not already done, export OPAE_PLATFORM_ROOT to the PR build tree directory\n$ export OPAE_PLATFORM_ROOT=<path to pr build tree>\n\n# Quartus Tools\n# Note, QUARTUS_HOME is your Quartus installation directory, e.g. $QUARTUS_HOME/bin contains Quartus executable.\n$ export QUARTUS_HOME=<user_path>/intelFPGA_pro/23.3/quartus\n$ export QUARTUS_ROOTDIR=$QUARTUS_HOME\n$ export QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\n$ export QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\n$ export IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\n$ export IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\n$ export QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys\n$ export PATH=$QUARTUS_HOME/bin:$QSYS_ROOTDIR/bin:$QUARTUS_HOME/../sopc_builder/bin/:$PATH\n\n# OPAE SDK release\n$ export OPAE_SDK_REPO_BRANCH=release/2.10.0\n\n# The following environment variables are required for compiling the AFU examples. \n\n# Location to clone the ofs-platform-afu-bbb repository which contains PIM files and AFU examples.\n$ export OFS_PLATFORM_AFU_BBB=$OFS_BUILD_ROOT/ofs-platform-afu-bbb \n\n# OPAE and MPF libraries must either be on the default linker search paths or on both LIBRARY_PATH and LD_LIBRARY_PATH.  \n$ export OPAE_LOC=/usr\n$ export LIBRARY_PATH=$OPAE_LOC/lib:$LIBRARY_PATH\n$ export LD_LIBRARY_PATH=$OPAE_LOC/lib64:$LD_LIBRARY_PATH\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#3-compiling-an-afu","title":"3. Compiling an AFU","text":"
                                                    In this section, you will use the relocatable PR build tree created in the previous steps from the FIM to compile an example PIM-based AFU. This section will be developed around the host_chan_mmio and hello_world AFU examples to showcase the synthesis of a PIM-based AFU.
                                                     
                                                    The build steps presented below demonstrate the ease in building and running an actual AFU on the board. To successfully execute the instructions in this section, you must have set up your development environment and have a relocateable PR Build tree as instructed in section 2 of this document.
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#31-creating-the-afu-synthesis-environment","title":"3.1. Creating the AFU Synthesis Environment","text":"
                                                    The PIM flow provides the script afu_synth_setup to create the synthesis environment to build the AFU examples. See how to use it below.
                                                     
                                                    usage: afu_synth_setup [-h] -s SOURCES [-p PLATFORM] [-l LIB] [-f] dst\n\nGenerate a Quartus build environment for an AFU. A build environment is\ninstantiated from a release and then configured for the specified AFU. AFU\nsource files are specified in a text file that is parsed by rtl_src_config,\nwhich is part of the OPAE base environment.\n\npositional arguments:\n  dst                   Target directory path (directory must not exist).\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -s SOURCES, --sources SOURCES\n                        AFU source specification file that will be passed to\n                        rtl_src_config. See \"rtl_src_config --help\" for the\n                        file's syntax. rtl_src_config translates the source\n                        list into either Quartus or RTL simulator syntax.\n  -p PLATFORM, --platform PLATFORM\n                        FPGA platform name.\n  -l LIB, --lib LIB     FPGA platform release hw/lib directory. If not\n                        specified, the environment variables OPAE_FPGA_HW_LIB\n                        and then BBS_LIB_PATH are checked.\n  -f, --force           Overwrite target directory if it exists.\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#32-building-and-running-host_chan_mmio-example-afu","title":"3.2. Building and Running host_chan_mmio example AFU","text":"
                                                    The $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio is a simple example demonstrating both hardware and software access to an AFU. The host_chan_mmio example AFU consists of the following files.  The hw directory contains the RTL to implement the hardware functionality using Avalon and AXI interfaces. However, this guide will use the AXI version of the host_chan_mmio AFU to go through the compilation steps. The sw directory of the AFU contains the source code of the host application that communicates with the actual AFU hardware.
                                                     
                                                    host_chan_mmio\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_avalon.sv\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_axi.sv\n\u2502       \u251c\u2500\u2500 host_chan_mmio.json\n\u2502       \u251c\u2500\u2500 test_mmio_avalon0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon1.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon2_512rw.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi1.txt\n\u2502       \u2514\u2500\u2500 test_mmio_axi2_512rw.txt\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 main.c\n    \u251c\u2500\u2500 Makefile\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#321-build-the-host_chan_mmio-example-afu","title":"3.2.1. Build the host_chan_mmio example AFU","text":"
                                                    Execute afu_synth_setup as follows to create the synthesis environment for a host_chan_mmio AFU that fits the PCIe Attach FIM previously constructed.
                                                     
                                                    $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio\n$ afu_synth_setup -s ./hw/rtl/test_mmio_axi1.txt afu_dev\n
                                                     Now, move into the synthesis environment afu_dev directory just created. From there, execute the afu_synth command. The successful completion of the command will produce the host_chan_mmio.gbs file under the synthesis environment directory, $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_dev.
                                                     
                                                    $ cd afu_dev\n$ $OPAE_PLATFORM_ROOT/bin/afu_synth\nCompiling ofs_top ofs_pr_afu\nGenerating host_chan_mmio.gbs\n==================================\n...\n...\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n Design meets timing\n===========================================================================\n
                                                     
                                                    The previous output indicates the successful compilation of the AFU and the compliance with the timing requirements. Analyze the reports generated in case the design does not meet timing. The timing reports are stored in the directory, $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_dev/build/syn/board/n6001/syn_top/output_files/timing_report.
                                                     
                                                    Once the compilation finishes successfully, load the new host_chan_mmio.gbs bitstream file into the partial reconfiguration region of the target Intel\u00ae FPGA SmartNIC N6001-PL board. Keep in mind, that the loaded image is dynamic - this image is not stored in flash and if the card is power cycled, then the PR region is re-loaded with the default AFU.
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#322-download-the-host_chan_mmio-example-afu","title":"3.2.2. Download the host_chan_mmio example AFU","text":"
                                                    To test the AFU in actual hardware, load the host_chan_mmio.gbs to the Intel\u00ae FPGA SmartNIC N6001-PL card. For this step to be successful, the PCIe Attach FIM must have already been loaded to the Intel\u00ae FPGA SmartNIC N6001-PL card following the steps described in Section 2 of this document. If you are running on a Virtual Machine, refer to the [KVM User Guide] for passing the devices to the VM.
                                                     
                                                    Verify Board and PCIe b.d.f. For the following example, the N6001 SKU2 PCIe b:d.f is B1:00.0, however this may be different in your system. 
                                                    $ fpgainfo fme\nIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\n...\n
                                                     
                                                    Download AFU. 
                                                    $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_dev\n$ sudo fpgasupdate host_chan_mmio.gbs B1:00.0\n[sudo] password for <<Your username>>: \n[2022-04-15 20:22:18.85] [WARNING ] Update starting. Please do not interrupt.\n[2022-04-15 20:22:19.75] [INFO    ] \nPartial Reconfiguration OK\n[2022-04-15 20:22:19.75] [INFO    ] Total time: 0:00:00.90\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#323-set-up-host-to-interface-with-example-afu","title":"3.2.3. Set up host to interface with example AFU","text":"
                                                    Set up host to interface with the newly loaded AFU.
                                                     
                                                    List the PFs available, the default N6001 FIM has 5 PFs. 
                                                    $ lspci -s B1:00\nB1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.2 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.3 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.4 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                                                     
                                                    Create the Virtual Functions (VFs) provided by the FIM, the default N6001 FIM has 3 VFs.  If your FIM uses only PFs, skip this step. 
                                                    $ sudo pci_device B1:00.0 vf 3\n\n# Verify the VFs have been added (device id: bccf)\n$ lspci -s B1:00\nB1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.2 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.3 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.4 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.5 Processing accelerators: Intel Corporation Device bccf\nB1:00.6 Processing accelerators: Intel Corporation Device bccf\nB1:00.7 Processing accelerators: Intel Corporation Device bccf\n
                                                     
                                                    Bind PFs and VFs to VFIO driver (except PF0/B1:00.0, which is the FME PF). 
                                                    # Enter your username.\n$ sudo opae.io init -d 0000:b1:00.1 <<Your username>>\n[sudo] password for <<Your username>>:\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 183\nAssigning /dev/vfio/183 to ceg\nChanging permissions for /dev/vfio/183 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.2 <<Your username>>\n[sudo] password for <<Your username>>:\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 184\nAssigning /dev/vfio/184 to ceg\nChanging permissions for /dev/vfio/184 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.3 <<Your username>>\n[sudo] password for <<Your username>>:\nUnbinding (0x1af4,0x1000) at 0000:b1:00.3 from virtio-pci\nBinding (0x1af4,0x1000) at 0000:b1:00.3 to vfio-pci\niommu group for (0x1af4,0x1000) at 0000:b1:00.3 is 185\nAssigning /dev/vfio/185 to ceg\nChanging permissions for /dev/vfio/185 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.4 <<Your username>>\n[sudo] password for <<Your username>>: \nUnbinding (0x8086,0xbcce) at 0000:b1:00.4 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.4 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.4 is 186\nAssigning /dev/vfio/186 to ceg\nChanging permissions for /dev/vfio/186 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.5 <<Your username>>\n[sudo] password for <<Your username>>:\nBinding (0x8086,0xbccf) at 0000:b1:00.5 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.5 is 315\nAssigning /dev/vfio/315 to ceg\nChanging permissions for /dev/vfio/315 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.6 <<Your username>>\n[sudo] password for <<Your username>>:\nBinding (0x8086,0xbccf) at 0000:b1:00.6 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.6 is 316\nAssigning /dev/vfio/316 to ceg\nChanging permissions for /dev/vfio/316 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.7 <<Your username>>\n[sudo] password for <<Your username>>:\nBinding (0x8086,0xbccf) at 0000:b1:00.7 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.7 is 317\nAssigning /dev/vfio/317 to ceg\nChanging permissions for /dev/vfio/317 to rw-rw----\n
                                                     
                                                    Verify the new AFU is loaded.  The host_chan_mmio AFU GUID is \"76d7ae9c-f66b-461f-816a-5428bcebdbc5\". 
                                                    $ fpgainfo port\n//****** PORT ******//\nObject Id                        : 0xEC00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0xE0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.7\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : d15ab1ed-0000-0000-0210-000000000000\n//****** PORT ******//\nObject Id                        : 0xC0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.6\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : d15ab1ed-0000-0000-0110-000000000000\n//****** PORT ******//\nObject Id                        : 0xA0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.5\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 76d7ae9c-f66b-461f-816a-5428bcebdbc5\n//****** PORT ******//\nObject Id                        : 0x80B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.4\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 44bfc10d-b42a-44e5-bd42-57dc93ea7f91\n//****** PORT ******//\nObject Id                        : 0x6098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 1aae155c-acc5-4210-b9ab-efbd90b970c4\n//****** PORT ******//\nObject Id                        : 0x40B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n//****** PORT ******//\nObject Id                        : 0x20B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 3e7b60a0-df2d-4850-aa31-f54a3e403501\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#324-run-the-host_chan_mmio-example-afu","title":"3.2.4. Run the host_chan_mmio example AFU","text":"
                                                    Now, navigate to the directory of the host_chan_mmio AFU containing the host application's source code, $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw. Once there, compile the host_chan_mmio host application and execute it on the host server to excercise the functionality of the AFU.
                                                     
                                                    Note: If OPAE SDK libraries were not installed in the default systems directories under /usr, you need to set the OPAE_LOC, LIBRARY_PATH, and LD_LIBRARY_PATH environment variables to the custom locations where the OPAE SDK libraries were installed.
                                                     
                                                    # Move to the sw directory of the the host_chan_mmio AFU. This directory holds the source for the host application.\n$ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw\n$ export OPAE_LOC=/usr\n$ export LIBRARY_PATH=$OPAE_LOC/lib:$LIBRARY_PATH\n$ export LD_LIBRARY_PATH=$OPAE_LOC/lib64:$LD_LIBRARY_PATH\n$ make\n\n# Run the application\n$  ./host_chan_mmio\nAFU ID:  76d7ae9cf66b461f 816a5428bcebdbc5\nAFU MMIO interface: AXI Lite\nAFU MMIO read bus width: 64 bits\n512 bit MMIO write supported: yes\nAFU pClk frequency: 470 MHz\n\nTesting 32 bit MMIO reads:\n  PASS - 4 tests\n\nTesting 32 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 64 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 512 bit MMIO writes:\n  PASS\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#33-building-and-running-the-hello_world-example-afu","title":"3.3. Building and running the hello_world example AFU","text":"
                                                    The platform-independent examples AFU repository also provides some interesting example AFUs. In this section, you will compile and execute the PIM based hello_world AFU. The RTL of the hello_world AFU receives from the host application an address via memory mapped I/O (MMIO) write and generates a DMA write to the memory line at that address. The content written to memory is the string \"Hello world!\". The host application spins, waiting for the memory line to be updated. Once available, the software prints out the string.
                                                     
                                                    The hello_world example AFU consists of the following files. The hw directory contains the RTL to implement the hardware functionality using CCIP, Avalon, and AXI interfaces. However, this guide will use the AXI version of the AFU to go through the compilation steps. The sw directory of the AFU contains the source code of the host application that communicates with the AFU hardware.
                                                     
                                                    hello_world\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u251c\u2500\u2500 ccip\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_ccip.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u2514\u2500\u2500 hello_world.json\n\u251c\u2500\u2500 README.md\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 hello_world\n    \u251c\u2500\u2500 hello_world.c\n    \u251c\u2500\u2500 Makefile\n    \u2514\u2500\u2500 obj\n        \u251c\u2500\u2500 afu_json_info.h\n        \u2514\u2500\u2500 hello_world.o\n
                                                     
                                                    The following instructions can be used to compile other AFU samples accompanying this repository.
                                                     
                                                    If not done already, download and clone the examples AFU repository. 
                                                    $ cd $OFS_BUILD_ROOT \n$ git clone https://github.com/OFS/examples-afu.git\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#331-build-the-hello_world-example-afu","title":"3.3.1. Build the hello_world example AFU","text":"
                                                    Compile the hello_word sample AFU.  
                                                    $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world\n$ afu_synth_setup --source hw/rtl/axi/sources.txt afu_dev\n$ cd afu_dev\n$ $OPAE_PLATFORM_ROOT/bin/afu_synth\nCompiling ofs_top ofs_pr_afu\nGenerating hello_world.gbs\n==================================\n.\n.\n.\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'hello_world.gbs'\n Design meets timing\n===========================================================================\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#332-download-the-hello_world-example-afu","title":"3.3.2. Download the hello_world example AFU","text":"
                                                    To test the AFU in actual hardware, load the hello_world.gbs to the Intel\u00ae FPGA SmartNIC N6001-PL card. For this step to be successful, the PCIe Attach FIM must have already been loaded to the Intel\u00ae FPGA SmartNIC N6001-PL card following the steps described in Section 2 of this document. If you are running on a Virtual Machine, refer to the [KVM User Guide] for passing the devices to the VM.
                                                     
                                                    Verify Board and PCIe b.d.f. For the following example, the N6001 SKU2 PCIe b:d.f is B1:00.0, however this may be different in your system. 
                                                    $ fpgainfo fme\nIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\n...\n
                                                     
                                                    Download AFU. 
                                                    $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/afu_dev\n$ sudo fpgasupdate hello_world.gbs B1:00.0\n  [sudo] password for <<Your username>>: \n[2022-04-15 20:22:18.85] [WARNING ] Update starting. Please do not interrupt.\n[2022-04-15 20:22:19.75] [INFO    ] \nPartial Reconfiguration OK\n[2022-04-15 20:22:19.75] [INFO    ] Total time: 0:00:00.90\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#333-set-up-host-to-interface-with-example-afu","title":"3.3.3. Set up host to interface with example AFU","text":"
                                                    Set up your Intel\u00ae FPGA SmartNIC N6001-PL board to work with the newly loaded hello_world.gbs file.  
                                                    # List the PF's available, the default N6001 FIM has 5 PF's\n$ lspci -s B1:00\nB1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.2 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.3 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.4 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                                                     
                                                    Download AFU. 
                                                    # Create the Virtual Functions (VFs) provided by the FIM, the default N6001 FIM has 3 VFs.  \n# If your FIM uses only PFs, skip this step.\n$ sudo pci_device B1:00.0 vf 3\n\n# Verify the VFs have been added (device id: bccf)\n$ lspci -s B1:00\nB1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.2 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.3 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.4 Processing accelerators: Intel Corporation Device bcce (rev 01)\nB1:00.5 Processing accelerators: Intel Corporation Device bccf\nB1:00.6 Processing accelerators: Intel Corporation Device bccf\nB1:00.7 Processing accelerators: Intel Corporation Device bccf\n
                                                     
                                                    Bind PFs and VFs to VFIO driver (except PF0/B1:00.0, which is the FME PF). 
                                                    #Enter your username\n$ sudo opae.io init -d 0000:b1:00.1 <<Your username>>\n[sudo] password for <<Your username>>:\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 183\nAssigning /dev/vfio/183 to ceg\nChanging permissions for /dev/vfio/183 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.2 <<Your username>>\n[sudo] password for <<Your username>>:\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 184\nAssigning /dev/vfio/184 to ceg\nChanging permissions for /dev/vfio/184 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.3 <<Your username>>\n[sudo] password for <<Your username>>:\nUnbinding (0x8086,0xbcce)  at 0000:b1:00.3 from virtio-pci\nBinding (0x8086,0xbcce)  at 0000:b1:00.3 to vfio-pci\niommu group for (0x8086,0xbcce)  at 0000:b1:00.3 is 185\nAssigning /dev/vfio/185 to ceg\nChanging permissions for /dev/vfio/185 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.4 <<Your username>>\n[sudo] password for <<Your username>>: \nUnbinding (0x8086,0xbcce) at 0000:v1:00.4 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.4 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.4 is 186\nAssigning /dev/vfio/186 to ceg\nChanging permissions for /dev/vfio/186 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.5 <<Your username>>\n[sudo] password for <<Your username>>:\nBinding (0x8086,0xbccf) at 0000:b1:00.5 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.5 is 315\nAssigning /dev/vfio/315 to ceg\nChanging permissions for /dev/vfio/315 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.6 <<Your username>>\n[sudo] password for <<Your username>>:\nBinding (0x8086,0xbccf) at 0000:b1:00.6 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.6 is 316\nAssigning /dev/vfio/316 to ceg\nChanging permissions for /dev/vfio/316 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.7 <<Your username>>\n[sudo] password for <<Your username>>:\nBinding (0x8086,0xbccf) at 0000:b1:00.7 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.7 is 317\nAssigning /dev/vfio/317 to ceg\nChanging permissions for /dev/vfio/317 to rw-rw----\n
                                                     
                                                    Verify the new AFU is loaded.  The hello_world AFU GUID is \"c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\". 
                                                    $ fpgainfo port\n\n//****** PORT ******//\nObject Id                        : 0xEE00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0xE098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.7\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : d15ab1ed-0000-0000-0210-000000000000\n//****** PORT ******//\nObject Id                        : 0xC098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.6\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : d15ab1ed-0000-0000-0110-000000000000\n//****** PORT ******//\nObject Id                        : 0xA098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.5\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\n//****** PORT ******//\nObject Id                        : 0x8098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.4\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 44bfc10d-b42a-44e5-bd42-57dc93ea7f91\n//****** PORT ******//\nObject Id                        : 0x6098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 1aae155c-acc5-4210-b9ab-efbd90b970c4\n//****** PORT ******//\nObject Id                        : 0x4098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n//****** PORT ******//\nObject Id                        : 0x2098000000000000\nPCIe s:b:d.f                     : 0000:B1:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 3e7b60a0-df2d-4850-aa31-f54a3e403501\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#334-run-the-hello_world-example-afu","title":"3.3.4. Run the hello_world example AFU","text":"
                                                    Compile and execute the host application of the hello_world AFU. You should see the application outputs the \"Hello world!\" message in the terminal.
                                                     
                                                    # Move to the sw directory of the hello_world AFU\n$ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/sw\n\n$ make\n\n# Launch the host application\n$ ./hello_world\n  Hello world!\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#34-modify-the-afu-user-clocks-frequency","title":"3.4. Modify the AFU user clocks frequency","text":"
                                                    An OPAE compliant AFU specifies the frequency of the uclk_usr and uclk_usr_div2 clocks through the JSON file for AFU configuration located under the <afu_example>/hw/rtl directory of an AFU design. For instance, the AFU configuration file of the host_chan_mmio example is $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/host_chan_mmio.json.
                                                     
                                                    The AFU specifies the frequency for uClk_usr in its platform configuration file using the following key:value pairs:
                                                     
                                                      \"clock-frequency-high\": [<float-value>|\u201dauto\u201d|\u201dauto-<float-value>\u201d]\n  \"clock-frequency-low\": [<float-value>|\u201dauto\u201d|\u201dauto-<float-value>\u201d]\n
                                                     
                                                    These key:value tuples are used to configure the PLL of the target platform that provides the user clocks through the AFU clocks interface. In addition, the specified frequency affects the timing closure process on the user clocks during AFU compilation. 
                                                     
                                                    Setting the value field to a float number (e.g., 315.0 to specify 315 MHz) drives the AFU generation process to close timing within the bounds set by the low and high values and sets the AFU's JSON metadata to specify the user clock PLL  frequency values.
                                                     
                                                    The following example shows the JSON file of the host_chan_mmio to set the AFU uClk to 500 MHz and uClk_div2 to 250 MHz.
                                                     
                                                    {\n   \"version\": 1,\n   \"afu-image\": {\n      \"power\": 0,\n      \"clock-frequency-high\": 500,\n      \"clock-frequency-low\": 250,\n      \"afu-top-interface\":\n         {\n            \"class\": \"ofs_plat_afu\"\n         },\n      \"accelerator-clusters\":\n         [\n            {\n               \"name\": \"host_chan_mmio\",\n               \"total-contexts\": 1,\n               \"accelerator-type-uuid\": \"76d7ae9c-f66b-461f-816a-5428bcebdbc5\"\n            }\n         ]\n   }\n}\n
                                                     
                                                    Save the changes to host_chan_mmio.json file, then execute the afu_synth_setup script to create a new copy of the AFU files with the modified user clock settigns.
                                                     
                                                    $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio\n$ afu_synth_setup -s ./hw/rtl/test_mmio_axi1.txt afu_clks\n\nCopying build from /home/<user_area>/ofs-agx7-pcie-attach/work_pr/pr_build_template/hw/lib/build...\nConfiguring Quartus build directory: build_n6001_afu_clks/build\nLoading platform database: /home/<user_area>/ofs-agx7-pcie-attach/work_pr/pr_build_template/hw/lib/platform/platform_db/ofs_agilex_adp.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting platform/platform_afu_top_config.vh\nWriting platform/platform_if_addenda.qsf\nWriting ../hw/afu_json_info.vh\n
                                                     Compile the host_chan_mmio AFU with the new frequency values.
                                                     
                                                    $ cd afu_clks\n$ $OPAE_PLATFORM_ROOT/bin/afu_synth\n
                                                     
                                                    During the compilation phase, you will observe the Timing Analyzer uses the specified user clock frequency values as the target to close timing.
                                                     
                                                     
                                                     
                                                    AFU developers must ensure the AFU hardware design meets timing. The compilation of an AFU that fails timing shows a message similar to the following.
                                                     
                                                    .\n.\n.\n\nWrote host_chan_mmio.gbs\n\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n\n  *** Design does not meet timing\n  *** See build/syn/board/n6001/syn_top/output_files/timing_report\n\n===========================================================================\n
                                                     
                                                    The previous output indicates the location of the timing reports for the AFU designer to identify the failing paths and perform the necessary design changes. Next, is a listing of the timing report files from a host_chan_mmio AFU that fails to meet timing after modifying the user clock frequency values.
                                                     
                                                    $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/n6001/afu_clks\n$ ls build/syn/board/n6001/syn_top/output_files/timing_report\n\nclocks.rpt  clocks.sta.fail.summary  clocks.sta.pass.summary\n
                                                     
                                                    Warning: AFU developers must inform software developers of the maximum operating frequency (Fmax) of the user clocks to avoid any unexpected behavior of the accelerator and potentially of the overall system.
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#4-simulating-an-afu-using-ase","title":"4. Simulating an AFU using ASE","text":"
                                                    The Application Simulation Environment (ASE) is a hardware/software co-simulation environment for your AFU. See diagram below illustrating ASE operation:
                                                     
                                                     
                                                    ASE uses the simulator Direct Programming Interface (DPI) to provide HW/SW connectivity.  The PCIe connection to the AFU under testing is emulated with a transactional model.
                                                     
                                                    The following list describes ASE operation:
                                                     
                                                       
                                                    • Attempts to replicate the transactions that will be seen in real system.
                                                      •  
                                                    • Provides a memory model to AFU, so illegal memory accesses can be identified early.
                                                      •  
                                                    • Not a cache simulator.
                                                      •  
                                                    • Does not guarantee synthesizability or timing closure.
                                                      •  
                                                    • Does not model system latency.
                                                      •  
                                                    • No administrator privileges are needed to run ASE.  All code is user level.
                                                      •  
                                                     
                                                    The remainder of this section is a tutorial providing the steps on how to run ASE with either Synopsys\u00ae VCS\u00ae or Siemens\u00ae QuestaSim\u00ae using an example AFU and the AFU build tree previously created in this guide.
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#41-set-up-steps-to-run-ase","title":"4.1. Set Up Steps to Run ASE","text":"
                                                    In this section you will set up your server to support ASE by independently downloading and installing OPAE SDK and ASE. Then, set up the required environment variables.
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#411-install-opae-sdk","title":"4.1.1. Install OPAE SDK","text":"
                                                    Follow the instructions documented in the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 FPGAs Targeting the Intel\u00ae FPGA SmartNIC N6001-PL, section 4.0 OPAE Software Development Kit to build and install the required OPAE SDK for the Intel\u00ae FPGA SmartNIC N6001-PL card.
                                                     
                                                    The N6001 SKU2 card requires 2.10.0-1. Follow the instructions provided in the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 FPGAs Targeting the Intel\u00ae FPGA SmartNIC N6001-PL, section 4.0 OPAE Software Development Kit. However, just make sure to check out the cloned repository to tag 2.10.0-1 and branch release/2.10.0.
                                                     
                                                    $ git checkout tags/2.10.0-1 -b release/2.10.0\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#412-install-ase-tools","title":"4.1.2 Install ASE Tools","text":"
                                                    ASE is an RTL simulator for OPAE-based AFUs. The simulator emulates both the OPAE SDK software user space API and the AFU RTL interface. The majority of the FIM as well as devices such as PCIe and local memory are emulated with simple functional models.
                                                     
                                                    ASE must be installed separatedly from the OPAE SDK. However, the recommendation is to install it in the same target directory as OPAE SDK.
                                                     
                                                       
                                                    1.  
                                                      If not done already, set the environment variables as described in section, Set Up AFU Development Environment.
                                                       
                                                      2.  
                                                    2.  
                                                      Clone the opae-sim repository. 
                                                      $ cd $OFS_BUILD_ROOT\n$ git clone https://github.com/OFS/opae-sim.git\n$ cd opae-sim  \n$ git checkout tags/2.10.0-1 -b release/2.10.0\n
                                                       
                                                      3.  
                                                    3.  
                                                      Create a build directory and build ASE to be installed under the default system directories along with OPAE SDK. 
                                                      $ mkdir build\n$ cd build\n$ cmake  -DCMAKE_INSTALL_PREFIX=/usr ..\n$ make\n
                                                       
                                                      4.  
                                                     
                                                    Optionally, if the desire is to install ASE binaries in a different location to the system's default, provide the path to CMAKE through the CMAKE_INSTALL_PREFIX switch, as follows. 
                                                    $ cmake -DCMAKE_INSTALL_PREFIX=<</some/arbitrary/path>> ..  \n
                                                     
                                                       
                                                    1. Install ASE binaries and libraries under the system directory /usr. 
                                                      $ sudo make install  \n
                                                      2.  
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#413-setup-required-ase-environment-variables","title":"4.1.3. Setup Required ASE Environment Variables","text":"
                                                    The values set to the following environment variables assume the OPAE SDK and ASE were installed in the default system directories below /usr. Setup these variables in the shell where ASE will be executed. You may wish to add these variables to the script you created to facilitate configuring your environment.
                                                     
                                                    $ export OPAE_PLATFORM_ROOT=<path to PR build tree>\n$ cd /usr/bin\n$ export PATH=$PWD:$PATH\n$ cd /usr/lib/python*/site-packages\n$ export PYTHONPATH=$PWD\n$ cd /usr/lib\n$ export LIBRARY_PATH=$PWD\n$ cd /usr/lib64\n$ export LD_LIBRARY_PATH=$PWD\n$ cd $OFS_BUILD_ROOT/ofs-platform-afu-bbb\n$ export OFS_PLATFORM_AFU_BBB=$PWD\n\n  ## For VCS, set the following:\n\n$ export VCS_HOME=<Set the path to VCS installation directory>\n$ export PATH=$VCS_HOME/bin:$PATH\n\n  ## For QuestaSIM, set the following:\n$ export MTI_HOME=<path to Modelsim installation directory>\n$ export PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#42-simulating-the-host_chan_mmio-afu","title":"4.2. Simulating the host_chan_mmio AFU","text":"
                                                    The $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio is a simple example demonstrating both hardware and software access to an AFU. The host_chan_mmio example AFU consists of the following files:
                                                     
                                                    host_chan_mmio\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_avalon.sv\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_axi.sv\n\u2502       \u251c\u2500\u2500 host_chan_mmio.json\n\u2502       \u251c\u2500\u2500 test_mmio_avalon0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon1.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon2_512rw.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi1.txt\n\u2502       \u2514\u2500\u2500 test_mmio_axi2_512rw.txt\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 main.c\n    \u251c\u2500\u2500 Makefile\n
                                                     
                                                    This example AFU contains examples using both Avalon and AXI interface buses. This guide will use the AXI version of the host_chan_mmio AFU.
                                                     
                                                    ASE uses client-server application architecture to deliver hardware/software co-simulation.  You require one shell for the hardware based simulation and another shell where the software application is running. The hardware is started first with a simulation compilation and simulator startup script, once the simulator has loaded the design, it will wait until the software process starts. Once the software process starts, the simulator proceeds.  Transaction logging and waveform capture is performed.
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#421-set-up-and-run-the-hw-simulation-process","title":"4.2.1 Set Up and Run the HW Simulation Process","text":"
                                                    You will run the afu_sim_setup script to create the scripts for running the ASE environment.  The afu_sim_setup script has the following usage:
                                                     
                                                    usage: afu_sim_setup [-h] -s SOURCES [-p PLATFORM] [-t {VCS,QUESTA,MODELSIM}]\n                     [-f] [--ase-mode ASE_MODE] [--ase-verbose]\n                     dst\n\nGenerate an ASE simulation environment for an AFU. An ASE environment is\ninstantiated from the OPAE installation and then configured for the specified\nAFU. AFU source files are specified in a text file that is parsed by\nrtl_src_config, which is also part of the OPAE base environment.\n\npositional arguments:\n  dst                   Target directory path (directory must not exist).\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -s SOURCES, --sources SOURCES\n                        AFU source specification file that will be passed to\n                        rtl_src_config. See \"rtl_src_config --help\" for the\n                        file's syntax. rtl_src_config translates the source\n                        list into either Quartus or RTL simulator syntax.\n  -p PLATFORM, --platform PLATFORM\n                        FPGA Platform to simulate.\n  -t {VCS,QUESTA,MODELSIM}, --tool {VCS,QUESTA,MODELSIM}\n                        Default simulator.\n  -f, --force           Overwrite target directory if it exists.\n  --ase-mode ASE_MODE   ASE execution mode (default, mode 3, exits on\n                        completion). See ase.cfg in the target directory.\n  --ase-verbose         When set, ASE prints each CCI-P transaction to the\n                        command line. Transactions are always logged to\n                        work/ccip_transactions.tsv, even when not set. This\n                        switch sets ENABLE_CL_VIEW in ase.cfg.\n
                                                     
                                                    Run afu_sim_setup to create the ASE simulation environment for the host_chan_mmio example AFU. The '-t VCS' option indicates to prepare the ASE simulation environment for Synopsys\u00ae VCS\u00ae.
                                                     
                                                    $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio\n\n$ afu_sim_setup -s $./hw/rtl/test_mmio_axi1.txt -t VCS afu_sim\n\nCopying ASE from /opae-sdk/install-opae-sdk/share/opae/ase...\n#################################################################\n#                                                               #\n#             OPAE Intel(R) Xeon(R) + FPGA Library              #\n#               AFU Simulation Environment (ASE)                #\n#                                                               #\n#################################################################\n\nTool Brand: VCS\nLoading platform database: /ofs-agx7-pcie-attach/work_pr/build_tree/hw/lib/platform/platform_db/ofs_agilex_adp.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting rtl/platform_afu_top_config.vh\nWriting rtl/platform_if_addenda.txt\nWriting rtl/platform_if_includes.txt\nWriting rtl/ase_platform_name.txt\nWriting rtl/ase_platform_config.mk and rtl/ase_platform_config.cmake\nASE Platform: discrete (FPGA_PLATFORM_DISCRETE)\n
                                                     
                                                    The afu_sim_setup creates the ASE scripts in the directory host_chan_mmio_sim where the afu_sim_setup script was run.  Start the simulator as shown below:
                                                     
                                                    $ cd afu_sim\n$ make\n$ make sim\n
                                                     
                                                    This process launches the AFU hardware simulator. Before moving to the next section, pay attention to the simulator output highlighted in the image below.
                                                     
                                                     
                                                    The simulation artifacts are stored in host_chan_mmio/work and consist of:
                                                     
                                                    log_ase_events.tsv\nlog_ofs_plat_host_chan.tsv \nlog_ofs_plat_local_mem.tsv \nlog_pf_vf_mux_A.tsv \nlog_pf_vf_mux_B.tsv \n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#422-set-up-and-run-the-sw-process","title":"4.2.2 Set Up and Run the SW Process","text":"
                                                    Open an additional shell to build and run the host application that communicates with the actual AFU hardware. Set up the same environment variable you have set up in the shell you have been working on until this point. 
                                                     
                                                    Additionally, as indicated by the hardware simulator output that is currently executing in the \"simulator shell\", copy and paste the line \"export ASE_WORKDIR=...\", into the new \"software shell\". See the last image of the previous section.
                                                     
                                                    $ export ASE_WORKDIR=$OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_sim/work\n
                                                     Then, go to the sw directory of the host_chan_mmio AFU example to compile the host application.
                                                     
                                                    $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw  \n$ make\n\nafu_json_mgr json-info --afu-json=../hw/rtl/host_chan_mmio.json --c-hdr=obj/afu_json_info.h\nWriting obj/afu_json_info.h\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c main.c -o obj/main.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c test_host_chan_mmio.c -o obj/test_host_chan_mmio.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/connect.c -o obj/connect.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/csr_mgr.c -o obj/csr_mgr.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/hash32.c -o obj/hash32.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/test_data.c -o obj/test_data.o\ncc -o host_chan_mmio obj/main.o obj/test_host_chan_mmio.o obj/connect.o obj/csr_mgr.o obj/hash32.o obj/test_data.o  -z noexecstack -z relro -z now -pie -luuid -lopae-c-ase\n
                                                     
                                                    Now, launch the host application to exercise the AFU hardware running on the simulator shell. The next image shows the AFU hardware simulation process on the left side shell. The right hand shell shows the host application's output of a successful simulation.
                                                     
                                                    $ with_ase ./host_chan_mmio\n  [APP]  Initializing simulation session ...\nRunning in ASE mode\nAFU ID:  76d7ae9cf66b461f 816a5428bcebdbc5\nAFU MMIO interface: AXI Lite\nAFU MMIO read bus width: 64 bits\n512 bit MMIO write supported: yes\nAFU pClk frequency: 470 MHz\n\nTesting 32 bit MMIO reads:\n  PASS - 4 tests\n\nTesting 32 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 64 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 512 bit MMIO writes:\n  PASS\n  [APP]  Deinitializing simulation session\n  [APP]         Took 1,003,771,568 nsec\n  [APP]  Session ended\n
                                                     
                                                     
                                                     
                                                    Finally, on the hardware simulation shell, you can view the wave forms by invoking the following command.
                                                     
                                                    $ make wave\n
                                                     
                                                    This brings up the VCS\u00ae simulator GUI and loads the simulation waveform files. Use the Hierarchy window to navigate to the afu instance located under, ase_top | ase_top_plat | ase_afu_main_pcie_ss | ase_afu_main_emul | afu_main | port_afu_instances | ofs_plat_afu | afu , as shown below.
                                                     
                                                     
                                                    Right click on the afu (afu) entry to display the drop-down menu. Then, click on Add to Waves | New Wave View to display the following waveforms window.
                                                     
                                                     
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#43-simulating-the-hello_world-afu","title":"4.3 Simulating the hello_world AFU","text":"
                                                    In this section you will quickly simulate the PIM-based hello_world sample AFU accompanying the examples-afu repository.
                                                     
                                                       
                                                    1.  
                                                      Set the environment variables as described in section 4.1. Set Up Steps to Run ASE.
                                                       
                                                      2.  
                                                    2.  
                                                      Prepare an RTL simulation environment for the AXI version of the hello_world AFU.
                                                       
                                                      3.  
                                                     
                                                    Simulation with ASE requires two software processes, one to simulate the AFU RTL and the other to run the host software that excercises the AFU. To construct an RTL simulation environment under the directory simulation, execute the following.
                                                     
                                                    $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world\n$ afu_sim_setup -s ./hw/rtl/axi/sources.txt -t VCS afu_sim\n\n\nCopying ASE from /usr/local/share/opae/ase...\n#################################################################\n#                                                               #\n#             OPAE Intel(R) Xeon(R) + FPGA Library              #\n#               AFU Simulation Environment (ASE)                #\n#                                                               #\n#################################################################\n\nTool Brand: VCS\nLoading platform database: /home/<user_area>/ofs-agx7-pcie-attach/work_pr/pr_build_template/hw/lib/platform/platform_db/ofs_agilex_adp.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting rtl/platform_afu_top_config.vh\nWriting rtl/platform_if_addenda.txt\nWriting rtl/platform_if_includes.txt\nWriting rtl/ase_platform_name.txt\nWriting rtl/ase_platform_config.mk and rtl/ase_platform_config.cmake\nASE Platform: discrete (FPGA_PLATFORM_DISCRETE)\n
                                                     
                                                    The afu_sim_setup script constructs an ASE environment in the hello_world_sim subdirectory. If the command fails, confirm that the path to the afu_sim_setup is on your PATH environment variable (in the OPAE SDK bin directory) and that your Python version is at least 2.7.
                                                     
                                                       
                                                    1. Build and execute the AFU RTL simulator.
                                                      2.  
                                                     
                                                    $ cd afu_sim\n$ make\n$ make sim  \n
                                                     
                                                    The previous commands will build and run the Synopsys\u00ae VCS\u00ae RTL simulator, which prints a message saying it is ready for simulation. The simulation process also prints a message instructing you to set the ASE_WORKDIR environment variable in a second shell.
                                                     
                                                       
                                                    1.  
                                                      Open a second shell where you will build and execute the host software. In this new \"software shell\", set up the environment variables you have set up so far in the \"hardware simulation\" shell.
                                                       
                                                      2.  
                                                    2.  
                                                      Also, set the ASE_WORKDIR environment variable following the instructions given in the \"hardware simulation\" shell.
                                                       
                                                      3.  
                                                     
                                                    $ export ASE_WORKDIR=$OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/afu_sim/work\n
                                                     6. Then, move to the sw directory of the hello_world AFU sample to build the host software.
                                                     
                                                    $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/sw\n$ make      \n
                                                     
                                                       
                                                    1. Run the hello_world host application to resume the work of the RTL simulation. The host software process and the RTL simulation execute in lockstep. If successful, you should see the Hello world! output.
                                                      2.  
                                                     
                                                    $ with_ase ./hello_world\n\n[APP]  Initializing simulation session ...\nHello world!\n[APP]  Deinitializing simulation session\n[APP]         Took 43,978,424 nsec\n[APP]  Session ended\n
                                                     
                                                    The image below shows the simulation of the AFU hardware and the execution of the host application side-by-side.
                                                     
                                                     
                                                     
                                                     
                                                       
                                                    1. Finally, on the hardware simulation shell, you can view the wave forms by invoking the following command.
                                                      2.  
                                                     
                                                    make wave\n
                                                     
                                                    This brings up the DVE GUI and loads the simulation waveform files. Use the Hierarchy window to navigate to the afu instance located under, ase_top | ase_top_plat | ase_afu_main_pcie_ss | ase_afu_main_emul | afu_main | port_afu_instances | ofs_plat_afu | hello_afu, as shown below.
                                                     
                                                     
                                                    Right click on the hello_afu entry to display the drop-down menu. Then, click on Add to Waves | New Wave View to display the following waveforms window.
                                                     
                                                     
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#5-adding-remote-signal-tap-logic-analyzer-to-debug-the-afu","title":"5. Adding Remote Signal Tap Logic Analyzer to debug the AFU","text":"
                                                    The OPAE SDK provides a remote Signal Tap facility. It also supports the following in system debug tools included with the Intel Quartus Prime Pro Edition:
                                                     
                                                       
                                                    • In-system Sources and Probes
                                                      •  
                                                    • In-system Memory Content Editor
                                                      •  
                                                    • Signal Probe
                                                      •  
                                                    • System Console
                                                      •  
                                                     
                                                    This section is a short guide on adding remote Signal Tap instances to an AFU for in system debugging. You can follow the steps in the following sections, in order of execution to create an instrumented AFU. The host_chan_mmio AFU is used in this guide as the target AFU to be instrumented.
                                                     
                                                    You need a basic understanding of Signal Tap. Please see the Signal Tap Logic Analyzer: Introduction & Getting Started Web Based Training for more information.
                                                     
                                                    You will run with a Signal Tap GUI running locally on the server with the Intel\u00ae FPGA SmartNIC N6001-PL as shown below:
                                                     
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#51-adding-rstp-to-the-host_chan_mmio-afu","title":"5.1. Adding RSTP to the host_chan_mmio AFU","text":"
                                                    RSTP is added to an AFU by:
                                                     
                                                       
                                                    1. Defining signals to be instrumented in Signal Tap.  This creates a new *.stp file.
                                                      2.  
                                                    2. Modify ofs_top.qpf to include the new *.stp file
                                                      3.  
                                                    3. Modify ofs_top.qsf
                                                      4.  
                                                    4. Modify ofs_pr_afu.qsf 
                                                      5.  
                                                    5. Run $OPAE_PLATFORM_ROOT/bin/afu_synth to build the PR-able image containing the RSTP instance
                                                      6.  
                                                     
                                                    You can use these detailed steps to add Signal Tap to your AFU.
                                                     
                                                       
                                                    1.  
                                                      Set path to platform root directory and create the host_chan_mmio AFU Quartus project for adding Signal Tap.: 
                                                      $ export OPAE_PLATFORM_ROOT=<path to PR build tree>\n\n # we will now build a new host_chahnel_mmio example based on Signal Tap\n\n$ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio\n$ afu_synth_setup -s ./hw/rtl/test_mmio_axi1.txt afu_stp\n
                                                       
                                                      2.  
                                                    2.  
                                                      Navigate to host_chan_mmio AFU Quartus project and open the project using Quartus GUI. 
                                                      $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp/build/syn/board/n6001/syn_top\n$ quartus ofs_top.qpf &\n
                                                       
                                                      3.  
                                                    3.  
                                                      Once the project is loaded in Quartus, run Analysis & Synthesis Processing | Start | Start Analysis & Synthesis. When complete, review the project hierarchy as shown in the Project Navigator.  This example will add Signal Tap probe points to the AFU region.  Reviewing the code will give insight into the function of this block.  You can bring up the code in the Project Navigator by expanding afu_top - port_gasket - pr_slot - afu_main - port_afu_instances - ofs_plat_afu, then select instance afu, right click, select Locacte Node - Locate in Design File as shown below. 
                                                       
                                                      4.  
                                                    4.  
                                                      Bring up Signal Tap to create the *.stp file.  In the Quartus GUI, go to Tools - Signal Tap Logic Analyzer.  In the New File from Template pop up, click Create to accept the default template.  The Signal Tap Logic Analyzer window comes up.
                                                       
                                                      5.  
                                                    5.  
                                                      Set up the clock for the Signal Tap logic instance by clicking ... button as shown below: 
                                                       
                                                      6.  
                                                    6.  
                                                      The Node Finder comes up and you will click ... as shown below to bring up the hierarchy navigator: 
                                                       
                                                      7.  
                                                    7.  
                                                      In the Select Hierarchy Level, navigate to top - afu_top - pg_afu.port_gasket - pr_slot - afu_main - port_afu_instances - ofs_plat_afu, then select instance afu and click Ok.
                                                       
                                                      8.  
                                                    8.  
                                                      Enter *clk* in the Named: box and click Search.  This brings up matching terms.  Click clk and >.  Verify your Node Finder is as shown below and then click Ok: 
                                                       
                                                      9.  
                                                    9.  
                                                      Double click the Double-click to add nodes and once again, click ... and navigate to top - afu_top - port_gasket - pr_slot - afu_main - port_afu_instances - ofs_plat_afu, then select instance afu and click Ok.  Enter mmio64_reg* and click Search. Then click >> to add these signals to the STP instance as shown below: 
                                                       
                                                      Then click Insert and Close. 
                                                       
                                                      10.  
                                                    10.  
                                                      Save the newly created STP by clicking File - Save As and in the save as navigate to $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp/build/syn/board/n6001/syn_top and save the STP file as host_chan_mmio.stp as shown below:
                                                       
                                                      11.  
                                                     
                                                     
                                                    Select Yes when asked to add host_chan_mmio.stp to current project.  Close Signal Tap window.
                                                     
                                                       
                                                    1. Edit ofs_top.qsf to add host_chan_mmio.stp file and enable STP.  Open $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp/build/syn/board/n6001/syn_top/ofs_top.qsf in an editor and add the lines shown below:
                                                      2.  
                                                     
                                                    set_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE host_chan_mmio.stp\nset_global_assignment -name SIGNALTAP_FILE host_chan_mmio.stp\n
                                                     
                                                    And also ensure \"INCLUDE_REMOTE_STP\" is enabled in ofs_top.qsf.
                                                     
                                                    # At most one of INCLUDE_REMOTE_STP and INCLUDE_JTAG_PR_STP should be\n# set. If both are defined, JTAG-based SignalTap takes precedence.\n# Remote STP uses mmlink. JTAG_PR_STP is on node 0 of the FPGA chain.\nset_global_assignment -name VERILOG_MACRO \"INCLUDE_REMOTE_STP\"       # Includes Remote SignalTap support in PR Region\n#set_global_assignment -name VERILOG_MACRO \"INCLUDE_JTAG_PR_STP\"      # Includes JTAG-based SignalTap via programming cable in the PR region\n
                                                     
                                                    Save the ofs_top.qsf.
                                                     
                                                       
                                                    1. Edit ofs_pr_afu.qsf to add host_chan_mmio.stp file and enable STP.  Open $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp/build/syn/board/n6001/syn_top/ofs_pr_afu.qsf in an editor and add the lines shown below:
                                                      2.  
                                                     
                                                    set_global_assignment -name VERILOG_MACRO \"INCLUDE_REMOTE_STP\"\nset_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE host_chan_mmio.stp\nset_global_assignment -name SIGNALTAP_FILE host_chan_mmio.stp\n
                                                     Save the ofs_pr_afu.qsf and close Quartus.
                                                     
                                                       
                                                    1. The host_chan_mmio AFU Quartus project is ready to be built.  In your original build shell enter the following commands:
                                                      2.  
                                                     
                                                    $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp\n$ $OPAE_PLATFORM_ROOT/bin/afu_synth\n\n...\n...\nWrote host_chan_mmio.gbs\n\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n  Design meets timing\n===========================================================================\n
                                                     
                                                       
                                                    1. Once compilation completes, the new host_chan_mmio.gbs file that contains the Signal Tap instance can be loaded.
                                                      2.  
                                                     
                                                     # For the following example, the N6001 SKU2 PCIe b:d.f is assumed to be b1:00.0,\n # however this may be different in your system\n\n# Enusre FIM is loading prior to loading AFU\n# Load AFU\n$ sudo fpgasupdate host_chan_mmio.gbs b1:00.0\n[2021-12-04 07:16:59,101] [WARNING ] Update starting. Please do not interrupt.\n[2021-12-04 07:16:59,740] [INFO    ] \nPartial Reconfiguration OK\n
                                                     
                                                       
                                                    1. Use the OPAE SDK mmlink tool to create a TCP/IP connection to your Intel Agilex card under test.  The mmlink command has the following format:
                                                      2.  
                                                     
                                                    Usage:\nmmlink\n<Segment>             --segment=<SEGMENT NUMBER>\n<Bus>                 --bus=<BUS NUMBER>           OR  -B <BUS NUMBER>\n<Device>              --device=<DEVICE NUMBER>     OR  -D <DEVICE NUMBER>\n<Function>            --function=<FUNCTION NUMBER> OR  -F <FUNCTION NUMBER>\n<Socket-id>           --socket-id=<SOCKET NUMBER>  OR  -S <SOCKET NUMBER>\n<TCP PORT>            --port=<PORT>                OR  -P <PORT>\n<IP ADDRESS>          --ip=<IP ADDRESS>            OR  -I <IP ADDRESS>\n<Version>             -v,--version Print version and exit\n
                                                     
                                                    Enter the command below to create a connection using port 3333:
                                                     
                                                    $ sudo mmlink -P 3333 -B 0xb1\n\n ------- Command line Input START ----\n\n Socket-id             : -1\n Port                  : 3333\n IP address            : 0.0.0.0\n ------- Command line Input END   ----\n\nPORT Resource found.\nServer socket is listening on port: 3333\n
                                                     
                                                    Leave this shell open with the mmlink connection.
                                                     
                                                       
                                                    1. In this step you will open a new shell and enable JTAG over protocol.  You must have Quartus 23.3 Programmer loaded on the N6001 server for local debugging.
                                                      2.  
                                                     
                                                    $ jtagconfig --add JTAG-over-protocol sti://localhost:0/intel/remote-debug/127.0.0.1:3333/0\n\n# Verify connectivity with jtagconfig --debug\n\n$ jtagconfig --debug\n1) JTAG-over-protocol [sti://localhost:0/intel/remote-debug/127.0.0.1:3333/0]\n   (JTAG Server Version 21.4.0 Build 67 12/06/2021 SC Pro Edition)\n    020D10DD   VTAP10 (IR=10)\n    Design hash    86099113E08364C07CC4\n    + Node 00406E00  Virtual JTAG #0\n\n  Captured DR after reset = (020D10DD) [32]\n  Captured IR after reset = (155) [10]\n  Captured Bypass after reset = (0) [1]\n  Captured Bypass chain = (0) [1]\n
                                                     
                                                       
                                                    1. Start Quartus Signal Tap GUI, connect to target, load stp file by navigating to $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp/build/syn/board/n6001/syn_top. The Quartus Signal Tap must be the same version of Quartus used to compile the host_chan_mmio.gbs. Quartus Prime Pro Version 23.3 is used in the steps below:
                                                      2.  
                                                     
                                                    $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_stp/build/syn/board/n6001/syn_top\n$ quartus_stpw host_chan_mmio.stp &\n
                                                     
                                                    This command brings up Signal Tap GUI. Connect to the Signal Tap over protocol by selecting the Hardware button on the right side of the GUI and click the \"Please Select\" pull down as shown below:
                                                     
                                                     
                                                    JTAG over protocol selected:
                                                     
                                                     
                                                    This connection process will take approximately 2-3 minutes for the Signal Tap instance to indicate \"Ready to acquire\".
                                                     
                                                       
                                                    1.  
                                                      Set the trigger condition for a rising edge on signal awvalid signal.
                                                       
                                                      2.  
                                                    2.  
                                                      In the Signal Tap window, enable acquisition by pressing key F5, the Signal Tap GUI will indicate \"Acquisition in progress\". Create and bind the VFs, then run the host_chan_mmio application following 3.2. Loading and Running host_chan_mmio example AFU, and observe that the Signal Tap instance has triggered. You should see signals being captured in the Signaltap GUI.
                                                       
                                                      3.  
                                                     
                                                    See captured image below:
                                                     
                                                     
                                                    To end your Signal Tap session, close the Signal Tap GUI, then in the mmlink shell, enter ctrl c to kill the mmlink process. To remove the JTAG over protocol connection:
                                                     
                                                    # This is assuming the JTAG over protocol is instance '1', as shown during jtagconfig --debug\n$ jtagconfig --remove 1\n
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#6-how-to-modify-the-pfvf-mux-configuration","title":"6. How to modify the PF/VF MUX configuration","text":"
                                                    For information on how to modify the PF/VF mapping for your own design, refer to the FPGA Interface Manager Developer Guide for Intel\u00ae Agilex\u00ae 7 PCIe Attach.
                                                    "},{"location":"hw/common/user_guides/afu_dev/ug_dev_afu_ofs_agx7_pcie_attach/ug_dev_afu_ofs_agx7_pcie_attach/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                    Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                     
                                                    OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                    "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/","title":"oneAPI Accelerator Support Package (ASP): Getting Started User Guide","text":""},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#10-introduction","title":"1.0 Introduction","text":""},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#11-about-this-document","title":"1.1 About This Document","text":"
                                                    This document serves as a quick start guide for setting up Intel\u00ae oneAPI Base Toolkit (Base Kit) on Open FPGA Stack (OFS) using oneapi-asp repository. Please see Table 1-1 for OFS reference platforms targeted in this guide.
                                                     
                                                    Table 1-1 HLD Tools
                                                     Target Device for the oneAPI ASP Target Platform for the oneAPI ASP Intel\u00ae Agilex\u00ae 7 FPGA Intel\u00ae FPGA SmartNIC N6001-PL Intel\u00ae Stratix 10\u00ae FPGA Intel\u00ae FPGA PAC D5005 
                                                    Attention: Intel is discontinuing the Intel FPGA SDK for OpenCL software product. Refer to the Product Discontinuation Notice PDN2219. Alternatively, Intel recommends using the Intel\u00ae oneAPI Base Toolkit (Base Kit), which provides core tools and libraries for developing high-performance data-centric applications across diverse architectures. It features an industry-leading C++ compiler that implements SYCL*, an evolution of C++ for heterogeneous computing. For more information, refer to the Intel\u00ae oneAPI Base Toolkit (Base Kit) web page. To migrate your OpenCL FPGA designs to SYCL, review Migrating OpenCL FPGA Designs to SYCL* guide that demonstrates important differences between OpenCL and SYCL for FPGA and provides steps to migrate your OpenCL designs.
                                                     
                                                    After reviewing the document you will be able to:
                                                     
                                                       
                                                    • Setup your host machine to develop HLD AFUs 
                                                      •  
                                                     
                                                       
                                                    • Compile and run sample HLD applications on OFS
                                                      •  
                                                    "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#12-terminology","title":"1.2 Terminology","text":"
                                                    This table defines some of the common terms used when discussing OFS.
                                                     
                                                    Table 1-2: Terminology
                                                     Term Abbreviation Description Open FPGA Stack OFS A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. Accelerator Functional Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside.  This AFU may or may not be a partial reconfiguration region. FPGA Interface Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs.  The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. High Level Design HLD For the purpose of this guide, this term refers to designing with Intel\u00ae oneAPI Base Toolkit (Base Kit). oneAPI Accelerator Support Package oneAPI ASP OR oneapi-asp A collection of hardware and software components that enable oneAPI kernels to communicate with oneAPI runtime as well as with OFS components. The hardware components of the oneAPI ASP along with the kernels lie in the AFU region. Open Programmable Acceleration Engine Software Development Kit OPAE SDK A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. Platform Interface Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Device Feature List DFL A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. Best Known Configuration BKC The exact hardware configuration Intel has optimized and validated the solution against. SYCL - SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL\u2122) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Installable Client Driver ICD Intel\u00ae FPGA Runtime for OpenCL\u2122 Software Technology supports the OpenCL ICD extension from the Khronos Group\u2122. The OpenCL ICD extension allows you to have multiple OpenCL implementations on your system. With the OpenCL ICD Loader Library, you may choose from a list of installed platforms and execute OpenCL API calls that are specific to your OpenCL implementation of choice. FPGA Client Driver FCD Intel\u00ae FPGA Runtime for OpenCL\u2122 Software Technology supports FPGA Client Driver(FCD) extension. FCD allows the runtime to automatically find and load the oneAPI ASP libraries at host run time"},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#13-introduction-to-high-level-design-on-ofs","title":"1.3 Introduction to High Level Design on OFS","text":"
                                                    Intel currently provides Intel\u00ae oneAPI Base Toolkit (Base Kit) for FPGA application development using high level languages like Data Parallel C++(DPC++).
                                                     
                                                    Figure 1-1 shows how OFS components can be used with Intel HLD tool.
                                                     
                                                    Figure 1-1 HLD Tool on OFS Platforms 
                                                     
                                                    For high level description and setup details for OFS components shown in figure above, please refer to the Getting Started guide for your target device.
                                                     
                                                       
                                                    • OFS Getting Started User Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
                                                      •  
                                                     
                                                       
                                                    • OFS Getting Started User Guide: For Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.
                                                      •  
                                                     
                                                    For a more detailed diagram and more information about the FPGA Interface Manager(FIM) shown in figure above, please refer to the FIM developer guides for your target device.
                                                     
                                                       
                                                    • Technical Reference Manual: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
                                                      •  
                                                     
                                                       
                                                    • FIM Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.
                                                      •  
                                                     
                                                       
                                                    • Intel\u00ae FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
                                                      •  
                                                     
                                                       
                                                    • FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.
                                                      •  
                                                     
                                                    The oneAPI ASP is required for compiling and running HLD application kernel on OFS platforms using Intel oneAPI. It is a collection of hardware and software components that enable oneAPI kernels to communicate with oneAPI runtime as well as with other OFS components. The hardware components of the oneAPI ASP along with the kernel lie in the AFU region shown in the figure above. For more details about the components of the oneAPI ASP, please refer to oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack.
                                                     
                                                    Figure 1-2 shows the setup steps to use oneAPI base toolkit on OFS platforms.
                                                     
                                                    Figure 1-2 Setup Steps for oneAPI base toolkit on OFS Platforms 
                                                     
                                                    The next section covers the setup steps in detail.
                                                     
                                                    Note: Administrative privileges are needed for multiple setup steps, ensure you have administrative/sudo privileges before proceeding.
                                                    "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#20-setup-flow-for-using-hld-tool-on-ofs","title":"2.0 Setup Flow for Using HLD Tool on OFS","text":""},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#21-setup-server-for-ofs","title":"2.1 Setup Server for OFS","text":"
                                                    As a first step, the server or host machine being used for developing HLD application needs to be setup for OFS. This involves setting up the FPGA card as well as installing OFS software stack including OPAE SDK and OFS DFL kernel driver.
                                                     
                                                    Please follow steps in Getting started guides for your target devices to setup Linux DFL kernel driver and install OPAE SDK.
                                                     
                                                       
                                                    • OFS Getting Started User Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs
                                                      •  
                                                    • OFS Getting Started User Guide: For Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.
                                                      •  
                                                    "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#22-clone-and-compile-fim","title":"2.2 Clone and Compile FIM","text":"
                                                    As shown in Figure 1-1, OFS components in the FPGA include the FIM and Accelerator Functional Unit(AFU). The oneAPI ASP is in the Partial Reconfiguration(PR) region of the AFU and relies on the compiled database of the static region(FIM) to interface with the host and board peripherals(e.g. on-board memory).
                                                     
                                                    Once the server is setup with OPAE SDK and DFL kernel driver, the next step is to clone and compile the static region of the design, i.e. FIM. You can use the default configuration of the FIM for both target platforms. Additionaly for Intel\u00ae FPGA SmartNIC N6001-PL for ofs_n6001 and ofs_n6001_usm board variants you have the option to create 2 different types of minimal FIM which removes the HSSI subsystem and host exercisers in the design. The difference between this two minimal FIM's, are the amount of VFs that must be created, in case of 1PF/1VF (built using n6001_1pf_1vf.ofss), only 1 VF is needed. In case of 2PF FIM (built using n6001_2pf.ofss) no VF creation is required. 2PF FIM could be used in FPGA development in virtual machines, see Section 3.0 OneAPI on OFS Running in a Virtual Machine. Please follow steps in the Intel\u00ae FPGA Interface Manager Developer Guides for your target device to compile FIM supporting PR release.
                                                     
                                                       
                                                    • Intel\u00ae FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
                                                      •  
                                                     
                                                       
                                                    • FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.
                                                      •  
                                                     
                                                    For more details on minimal FIM's for Intel\u00ae Agilex\u00ae 7 FPGA for ofs_n6001 and ofs_n6001_usm board variants and how to create them, refer to Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
                                                     
                                                    A pr_build_template directory will be generated in the work directory specified as part of the FIM compile command (using OFS/ofs-common/scripts/common/syn/build_top.sh script with the '-p' option enable to create an out-of-tree PR release). The pr_build_template directory is required for successful setup of the oneAPI ASP.
                                                     
                                                    Once the FIM compile is complete, please program FIM using fpgasupdate and Remote System Update(rsu) command. Use of these commands has been demonstrated in section named 5.3 Remote System Update in Intel\u00ae FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs refer to Test the hello_fim on a D5005 section in FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs for Intel\u00ae Stratix 10\u00ae FPGA.
                                                    "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#23-prerequisites","title":"2.3 Prerequisites","text":"
                                                    In addition to server setup and FIM compilation, a few linux packages are needed to setup the oneAPI ASP and develop HLD applications.
                                                     
                                                    1) Install the following packages:
                                                     
                                                        sudo dnf install numactl-devel ncurses-compat-libs\n
                                                     
                                                    2) Ensure that IOMMU is turned on as explained in section Building and Installing the OFS DFL Kernel Drivers from Source in Getting started guides for your target devices:
                                                     
                                                       
                                                    • OFS Getting Started User Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
                                                      •  
                                                     
                                                       
                                                    • OFS Getting Started User Guide: For Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.
                                                      •  
                                                     
                                                    You can verify this setting using cat /proc/cmdline command. The output must have intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200.
                                                     
                                                    $ cat /proc/cmdline\nBOOT_IMAGE=(hd1,gpt2)/vmlinuz-6.1.41-dfl root=/dev/mapper/rhel-root ro crashkernel=auto resume=/dev/mapper/rhel-swap rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\n
                                                     
                                                    3) Install HLD development software. Please see Table 2-1 below for download link. Install the latest version. Use sudo privileges to do the installation. 
                                                     
                                                    Table 2-1 Intel HLD Tool and Download Information
                                                     HLD Tool Target Platform for the oneAPI ASP Tool Download Information Intel\u00ae oneAPI Base Toolkit (Base Kit) 
                                                    • Intel\u00ae FPGA SmartNIC N6001-PL
                                                    • Intel\u00ae FPGA PAC D5005
                                                      •  Download here 
                                                      Tool installation guide for your reference:
                                                       
                                                         
                                                      • Intel\u00ae oneAPI Toolkits Installation Guide for Linux* OS
                                                        •  
                                                       
                                                      4) Ensure you have all the Quartus patches installed, refer to Table 2-3 for required Quartus version.
                                                       
                                                      Note: For Intel\u00ae Agilex\u00ae 7 FPGA ensure Quartus patch 0.13, 0.21 and 0.02iofs are installed. You can find them in a tar file under assets in the following link patch-agx7-2023-3.tar.gz. For Intel\u00ae Stratix 10\u00ae FPGA ensure Quartus patch 0.23 and 0.01iofs are installed. You can find them in a tar file under assets in the following link patch-s10-2023-3.tar.gz.For quartus patches installation to work properly, you must have Git Large File Storage (LFS) installed when cloning the ofs-fim repository. 
                                                       
                                                      Use following command to check Quartus version and installed patches.
                                                       
                                                          quartus_sh -v\n
                                                       
                                                      5) After completing the tool installation, set the following environment variables required to execute build scripts successfully:
                                                       
                                                          # Adding Quartus to PATH\n    export PATH=$PATH:path-to-quartus-installation-dir/bin\n    export QUARTUS_ROOTDIR=path-to-quartus-installation-dir/quartus\n    export QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\n    # Other OFS environment variables\n    export OFS_ROOTDIR=path-to-directory-containing-cloned-ofs-fim-repo/#ofs-agx7-pcie-attach for Intel\u00ae Agilex\u00ae 7 FPGA or ofs-d5005 for Intel\u00ae Stratix 10\u00ae FPGA\n    export WORKDIR=$OFS_ROOTDIR\n    export QUARTUS_HOME=$QUARTUS_ROOTDIR\n    export QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\n    export IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\n    export OPAE_SDK_REPO_BRANCH=release/branch-tag # Refer to Table 2-3, for OPAE SDK branch tag\n    export OFS_PLATFORM_AFU_BBB=path-to-cloned-ofs-platform-afu-bbb-repo\n    export OPAE_PLATFORM_ROOT=path-to-ofs-fim-pr_build_template-directory # (see section 2.2 for more details)\n    export  OFS_ASP_ROOT=path-to-directory-containing-oneapi-asp/oneapi-asp/platform-name #platform-name is n6001 for Intel\u00ae FPGA SmartNIC N6001-PL and d5005 for Intel\u00ae FPGA PAC D5005 \n    export LIBOPAE_C_ROOT=/usr # (OPAE libraries are installed in /usr/lib64 by default if you followed the OPAE SDK steps covered in section 2.1 as is and installed OPAE rpm packages. If you have a different OPAE installation path, please point LIBOPAE_C_ROOT to your OPAE installation location that you specified using -DCMAKE_INSTALL_PREFIX=installation-path in cmake command for building OPAE)\n
                                                       
                                                      Note: To re-use this environment setting, you can copy the above export statements to a shell script, update the paths to match your tool installations and source this script each time a new shell is started.
                                                       
                                                      6) Source initialization script for oneAPI, path is shown in table below.
                                                       
                                                      Table 2-2 Initialization Script for HLD tool
                                                       Tool Command to source initialization script Intel\u00ae oneAPI Base Toolkit (Base Kit) source path-to-intel-oneapi-toolkit-installation-directory/setvars.sh 
                                                      Once the environment variables are set, you can check the tool version using the following commands:
                                                       
                                                      # Printing all (Quartus, OpenCL SDK, GCC) versions for user info\nquartus_sh -v\nicpx --version (for Intel\u00ae oneAPI Base Toolkit (Base Kit))\ngcc --version\n
                                                       
                                                      Table 2-3 and 2-4 summarize the tool version/Best Known Configurations(BKC).
                                                       
                                                      Table 2-3 Best Known Configuration(BKC) for Intel\u00ae Agilex\u00ae 7 FPGA OFS
                                                       Component/Tool Version FPGA Platform Intel\u00ae FPGA SmartNIC N6001-PL Operating System RHEL 8.6 , Kernel : 6.1.41-dfl linux-dfl (DFL Kernel Driver) Tag: ofs-2023.3-6.1-3 opae-sdk Branch: release/2.10.0, Tag: 2.10.0-1 ofs-fim Tag: ofs-2023.3-2 oneapi-asp Tag: ofs-2023.3-2  > Note: Cloning and build of this repo is discussed in the section 2.4 Quartus Prime Pro Edition Version 23.3 Pro Edition with patches (0.13, 0.21 and 0.02iofs) under assets on this link patch-agx7-2023-3.tar.gz Intel\u00ae oneAPI Base Toolkit (Base Kit) Latest version GCC 7.4.0 cmake 3.15 
                                                      Table 2-4 Best Known Configuration(BKC) for Intel\u00ae Stratix 10\u00ae FPGA
                                                       Component/Tool Version FPGA Platform Intel\u00ae FPGA PAC D5005 Operating System RHEL 8.6 , Kernel : 6.1.41-dfl linux-dfl (DFL Kernel Driver) Tag: ofs-2023.3-6.1-1 opae-sdk Branch: release/2.10.0, Tag: 2.10.0-1 ofs-fim Tag: ofs-2023.3-1 oneapi-asp Tag: ofs-2023.3-2  > Note: Cloning and build of this repo is discussed in the section 2.4 Quartus Prime Pro Edition Version 23.3 Pro Edition  with patches( patch 0.23 and 0.01iofs) under assets on this link [patch-s10-2023-3.tar.gz ](https://github.com/OFS/ofs-d5005/releases/tag/ofs-2023.3-1) Intel\u00ae oneAPI Base Toolkit (Base Kit) Latest version GCC 7.4.0 cmake 3.15"},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#24-build-and-install-oneapi-asp","title":"2.4 Build and Install oneapi-asp","text":"
                                                      Once all pre-requisites are installed and the environment variables are set, next step is to clone and build oneapi-asp.
                                                       
                                                      1) Clone oneapi-asp repository and checkout tag matching the BKC for your target platform (see Tables 2-3 and 2-4 for the BKCs).
                                                       
                                                      Note: You will need a personal access token (use the classic mode) to be used as the password to clone successfully. Please see more information about token authentication requirements for Git operations here.
                                                       
                                                          git clone https://github.com/OFS/oneapi-asp.git\n    cd oneapi-asp\n    git checkout tags/ofs-2023.3-2\n
                                                       
                                                      Ensure the correct tag has ben checked out: 
                                                       
                                                          git describe --tags\n
                                                       
                                                         
                                                      • Output:
                                                        •  
                                                       
                                                          ofs-2023.3-2\n
                                                       
                                                      2) Ensure that OPAE_PLATFORM_ROOT and LIBOPAE_C_ROOT have been set as described in section 2.3. Generate the oneAPI ASP hardware and software using provided build-bsp.sh script. This script clones required repositories and builds the oneAPI ASP libraries required by HLD host application to run successfully.
                                                       
                                                          cd path-to-directory-containing-cloned-oneapi-asp-repo/oneapi-asp/platform-name\n    ./scripts/build-bsp.sh\n
                                                       
                                                      The generated directory structure is shown below. For more details refer to the oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack.
                                                       
                                                      \noneapi-asp/platform-name\n|--ase/\n|  |--base/\n|  |--hack_ip_files/\n|  |--compile-kernel.sh\n|  |--run-ase.sh\n|  |--setup.sh\n|  |--simulate-aocx.sh\n|--bringup/source/simple-add-buffers/\n|  |--simple-add-buffers.cpp\n|--hardware/\n|  |--ofs_platform-name/\n|  |--ofs_platform-name_iopipes/\n|  |--ofs_platform-name_usm/\n|  |--ofs_platform-name_usm_iopipes/\n|--linux64/libexec/\n|  |--flash\n|  |--initialize\n|  |--install\n|  |--setup_permissions.sh\n|  |--uninstall\n|--scripts/\n|  |--README.txt\n|  |--build-bsp-sw.sh\n|  |--build-bsp.sh\n|  |--build-default-binaries.sh\n|  |--build-mmd.sh\n|  |--build-opae.sh\n|  |--create-tarball.sh\n|  |--dedup-hardware.sh\n|  |--setup-bsp.py\n|--board_env.xml\n|--README.md\n
                                                       
                                                      3) Once the oneAPI ASP is generated, add the following to LD_LIBRARY_PATH. You can add it to your script for setting environment variables (if you created one as noted in step 5 in section 2.3)
                                                       
                                                          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:path-to-oneapi-asp/platform-name/linux64/lib\n
                                                       
                                                      4) Check if FPGA Client Driver(FCD) exists for any other version of oneAPI ASP or for any other board. You can check with  aocl list-devices command. It is recommended to run aocl list-devices as root user (login with sudo su) to see all installed ASPs on the host.
                                                       
                                                      A sample output when a oneAPI ASP FCD is installed is shown below:
                                                       
                                                      --------------------------------------------------------------------\nDevice Name:\nacl0\n\nBSP Install Location:\n/home/ofsuser/oneapi-asp/platform-name\n\nVendor: Intel Corp\n\nPhysical Dev Name   Status            Information\n\nofs_ee00000         Uninitialized     PR slot function not configured \n                                      Need to follow instructions to bind vfio-pci driver to PR slot function\n\nBSP DIAGNOSTIC_PASSED\n--------------------------------------------------------------------\n
                                                       
                                                      If a oneAPI ASP/BSP is installed, uninstall using aocl uninstall path-to-oneapi-asp-install-location, where path-to-oneapi-asp-install-location is provided under BSP Install Location: in the output of aocl list-devices. If you are prompted with a question to unset the FCD, type Y. If you are prompted with a question to remove OpenCL BSP configuration settings, type Y.
                                                       
                                                      Sample output for aocl uninstall command:
                                                       
                                                      $ aocl uninstall /home/ofsuser/oneapi-asp/platform-name\naocl uninstall: Removing the FPGA Client Driver (FCD) from the system\n[sudo] password for ofsuser:\naocl uninstall: Removing the board package /home/ofsuser/oneapi-asp/platform-name from the list of installed packages. This process may require admin privilege\naocl uninstall: Running uninstall from /home/ofsuser/oneapi-asp/platform-name/linux64/libexec\nDo you want to remove oneAPI-ASP configuration settings [Y/n] Y\nDeleting OPAE config files\nRemoving configuration files\nOFS oneAPI-ASP uninstall complete\n
                                                       
                                                      5) Install FPGA Client Driver(FCD) file for the oneAPI ASP using aocl install path-to-oneapi-asp/platform-name command as shown below. The host program uses FCD to find and link to the platform Memory Mapped Device (MMD) library. For more information about MMD library, refer to oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack.
                                                       
                                                          aocl install path-to-directory-containing-oneapi-asp/oneapi-asp/platform-name\n
                                                       
                                                      Notes:  1. Type Y when prompted to setup FCD at /opt/Intel/OpenCLFPGA/oneAPI/Boards (default location for Intel oneAPI). 
                                                       
                                                      Sample output aocl install command in  Intel\u00ae oneAPI Base Toolkit (Base Kit) environment is shown below.
                                                       
                                                      aocl install: Setting up the FPGA Client Driver (FCD) to the system. This process may require admin privilege\nInstall the FCD file to /opt/Intel/OpenCLFPGA/oneAPI/Boards\n[sudo] password for ofsuser:\naocl install: Adding the board package path-to-oneapi-asp/platform-name to the list of installed packages\nInstalling the board package driver to the system.\naocl install: Running install from path-to-oneapi-asp/platform-name/linux64/libexec\nConfiguring locked memory setting\nConfiguring udev rules for DFL FPGA device permission\nConfiguring system with 1024 2M hugepages\nSetting access permisions of /dev/uio to 666\nFinished setup_permissions.sh script. All configuration settings are persistent.\nIntel OFS oneAPI ASP install complete.\nRun 'aocl diagnose' to list devices or 'aocl initialize <dev_name> <board_variant> to load default image\n
                                                      "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#25-board-initialization","title":"2.5 Board Initialization","text":"
                                                      OFS software stack expects boards to be initialized with a bitstream for the board variant intended to be used for development. An oneAPI sample application, named simple-add-buffers, has been provided in the oneapi-asp repository for generating initialization bitstreams for included board variants. The sample is located in oneapi-asp/platform-name/bringup/source.
                                                       
                                                      oneapi-asp has four board variants for Intel\u00ae Agilex\u00ae 7 FPGA and two board variants for Intel\u00ae Stratix 10\u00ae FPGA (oneapi-asp/platform-name/hardware has the hardware design files for these). For more details on the architecture of the board variants, please refer to the oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack.
                                                       
                                                      Table 2-5 oneAPI Sample Applications
                                                       Board Variants Sample Application 
                                                      • ofs_platform-name
                                                       
                                                      • ofs_platform-name_usm
                                                      • ofs_n6001_iopipes
                                                       
                                                      • ofs_n6001_usm_iopipes
                                                       simple-add-buffers 
                                                      Note: platform-name is n6001 for Intel\u00ae Agilex\u00ae 7 FPGA OFS and d5005 for Intel\u00ae Stratix 10\u00ae FPGA OFS
                                                       
                                                      All samples are located in oneapi-asp/platform-name/bringup/source.
                                                      "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#251-compile-initialization-bitstreams","title":"2.5.1 Compile Initialization Bitstreams","text":"
                                                      A script is provided in repo to compile simple-add-buffers oneAPI sample application. The script is oneapi-asp/platform-name/scripts/build-default-binaries.sh.
                                                       
                                                      Script usage is as follows:
                                                       
                                                          ./build-default-binaries.sh -b name-of-board-variant\n
                                                       
                                                      Note: name-of-board-variant can be ofs_platform-name, ofs_platform-name_usm, ofs_n6001_iopipes or ofs_n6001_usm_iopipes where platform-name is n6001 for Intel\u00ae Agilex\u00ae 7 FPGA OFS and d5005 for Intel\u00ae Stratix 10\u00ae FPGA OFS. Refer to Table 2-5 for board variants available for each target platform. Compilation will take a few hours to complete.
                                                       
                                                      The output directory of the sample application is written to oneapi-asp/platform-name/build/bringup. The generated bitstreams are also copied to oneapi-asp/platform-name/bringup/binaries/. These are used in the initialization of the platform.
                                                       
                                                      Once the bitstreams are generated, create a VF and initialize the board as explained in following section. Ensure that the FIM has been programmed on the board as explained in section 2.2 Clone and Compile FIM
                                                      "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#252-pfvf-mapping","title":"2.5.2 PF/VF mapping","text":"
                                                      The oneAPI ASP is located in the PR region of the FIM and is accessed through PF/VF Mux. Refer to the FIM Reference Manual for your target platforms for more details about PF/VF mapping.
                                                       
                                                         
                                                      • Reference FIM for Intel\u00ae Agilex\u00ae 7 FPGA OFS: 
                                                        •  
                                                       
                                                      For Base_x16 FIM (default) and 1PF/1VF minimal FIM (built using n6001_1pf_1vf.ofss), VF0 is mapped to PR region and you can create 1 VF when using this FIM. Base_x16 FIM has 5 PF's and minimal FIM just 1 PF. 
                                                       
                                                      For 2PF FIM (built using n6001_2pf.ofss) PF1 is mapped to PR region and no VF creation is required.
                                                       
                                                      See Technical Reference Manual: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs for diagram showing PF/VF mapping.
                                                       
                                                         
                                                      • Reference FIM for Intel\u00ae Stratix 10\u00ae FPGA OFS: 
                                                        •  
                                                       
                                                      VF1 is mapped to PR region and you must create 2 VFs when using this FIM. This FIM has 1 PF. See FIM Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs for diagram showing PF/VF mapping.
                                                      "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#2521-create-vf","title":"2.5.2.1 Create VF","text":"
                                                      **Note:**This section only applies for Base_x16 FIM and 1PF/1VF minimal FIM for Intel\u00ae Agilex\u00ae 7 FPGA and default FIM for Intel\u00ae Stratix 10\u00ae FPGA. 
                                                       
                                                         
                                                      • Create a VF using PCIe ID obtained from the output of fpgainfo fme (PCIe s\\:b\\:d.f output)
                                                        •  
                                                       
                                                          sudo pci_device s:b:d.f vf num_vf  #num_vf is 1 for Intel\u00ae Agilex\u00ae 7 FPGA and 2 for Intel\u00ae Stratix 10\u00ae FPGA\n
                                                       
                                                         
                                                      • Check that the VF is created using sudo opae.io ls command and note the PCIe ID for the VF(s) (the function number in s\\:b\\:d.f will be different for the VF). Sample output for Intel\u00ae Agilex\u00ae 7 FPGA 1PF/1VF minimal FIM is shown below. Output for base_x16 FIM should display 5 PF's and the PCIe ID for VF0 will be s:b:d.5.
                                                        •  
                                                       
                                                          $ sudo opae.io ls\n    [0000:b1:00.0] (0x8086, 0xbcce) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n    [0000:b1:00.1] (0x8086, 0xbccf) Intel Acceleration Development Platform N6001 (Driver: None)\n
                                                       
                                                       Sample output for Intel\u00ae Stratix 10\u00ae FPGA is shown below.
                                                       
                                                          $ sudo opae.io ls\n    [0000:d8:00.0] (0x8086, 0xbcce)  Intel FPGA Programmable Acceleration Card D5005 (Driver: dfl-pci)\n    [0000:d8:00.1] (0x8086, 0xbccf)  Intel FPGA Programmable Acceleration Card D5005 (Driver: dfl-pci)\n    [0000:d8:00.2] (0x8086, 0xbccf)  Intel FPGA Programmable Acceleration Card D5005 (Driver: dfl-pci)\n
                                                       
                                                      Note:sudo opae.io ls will list the accelerators, respective PCIe ID as well as the driver it is currently bound to.
                                                       
                                                      Note: For more information about pci_device and opae.io utilities, refer to the OPAE FPGA tools page here.
                                                      "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#2522-bind-pf-and-vf","title":"2.5.2.2 Bind PF and VF","text":"
                                                      For Base_x16 FIM and 1PF/1VF minimal FIM for Intel\u00ae Agilex\u00ae 7 FPGA and default FIM for Intel\u00ae Stratix 10\u00ae FPGA :
                                                       
                                                         
                                                      • Bind the created VF(s) to vfio-pci driver, use the PCIe ID for the VF(s) for this step. Verify you are using the PCIe ID of the VFs you have created. For example:
                                                        •  
                                                      • From sample output for Agilex OFS target platform having 1PF/1VF minimal FIM programmed shown in Section 2.5.2.1 Create VF, s:b:d.vf will be 0000:b1:00.1 in command below. For base_x16 FIM should be s:b:d.5.
                                                        •  
                                                       
                                                          sudo opae.io init -d s:b:d.vf $USER\n
                                                       
                                                         
                                                      • Sample output for Intel\u00ae Agilex\u00ae 7 FPGA OFS target platform 1PF/1VF minimal FIM. Output for base_x16 FIM should be similar.
                                                        •  
                                                       
                                                          $ sudo opae.io init -d 0000:b1:00.1 $USER\n    Unbinding (0x8086,0xbccf) at 0000:b1:00.1 from dfl-pci\n    Binding (0x8086,0xbccf) at 0000:b1:00.1 to vfio-pci\n    iommu group for (0x8086,0xbccf) at 0000:b1:00.1 is 319\n    Assigning /dev/vfio/319 to ofsuser\n    Changing permissions for /dev/vfio/319 to rw-rw----\n\n    $ sudo opae.io ls\n    [0000:b1:00.0] (0x8086, 0xbcce) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n    [0000:b1:00.1] (0x8086, 0xbccf) Intel Acceleration Development Platform N6001 (Driver: vfio-pci)\n\n    $ ls -lt /dev/vfio\n    total 0\n    crw-rw----. 1 ofsuser root 509,   0 Dec 3 20:41 319\n    crw-rw-rw-. 1 root    root  10, 196 Dec 3 20:41 vfio\n
                                                       
                                                         
                                                      • Sample output for Intel\u00ae Stratix 10\u00ae FPGA OFS target platform:
                                                        •  
                                                       
                                                          $sudo opae.io init -d 0000:12:00.1 $USER\n    Unbinding (0x8086,0xbccf) at 0000:12:00.1 from dfl-pci\n    Binding (0x8086,0xbccf) at 0000:12:00.1 to vfio-pci\n    iommu group for (0x8086,0xbccf) at 0000:12:00.1 is 149\n    Assigning /dev/vfio/149 to ofsuser\n    Changing permissions for /dev/vfio/149 to rw-rw----\n\n    $sudo opae.io init -d 0000:12:00.2 $USER\n    Unbinding (0x8086,0xbccf) at 0000:12:00.2 from dfl-pci\n    Binding (0x8086,0xbccf) at 0000:12:00.2 to vfio-pci\n    iommu group for (0x8086,0xbccf) at 0000:12:00.2 is 152\n    Assigning /dev/vfio/152 to ofsuser\n    Changing permissions for /dev/vfio/152 to rw-rw----\n\n    $ sudo opae.io ls\n    [0000:12:00.0] (0x8086:0xbcce) Intel FPGA Programmable Acceleration Card D5005 (Driver: dfl-pci)\n    [0000:12:00.1] (0x8086:0xbccf) Intel FPGA Programmable Acceleration Card D5005 (Driver: vfio-pci)\n    [0000:12:00.2] (0x8086:0xbccf) Intel FPGA Programmable Acceleration Card D5005 (Driver: vfio-pci)\n\n    $ls -lt /dev/vfio\n    total 0\n    crw-rw----. 1 ofsuser  root  235,   3 Dec 3 16:25 149\n    crw-rw----. 1 ofsuser  root  235,   0 Dec 3 16:22 152\n    crw-rw-rw-. 1 root     root   10, 196 Dec 1 07:28 vfio\n
                                                       
                                                      For 2PF FIM for Intel\u00ae Agilex\u00ae 7 FPGA :
                                                       
                                                      Bind the PF1 to vfio-pci driver, use sudo opae.io ls command and note the PCIe ID (s\\:b\\:d.f) for the PF(s). Verify you are using the PCIe ID of the PF1.  
                                                       
                                                          $ sudo opae.io ls\n    [0000:b1:00.0] (0x8086:0xbcce) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n    [0000:b1:00.1] (0x8086:0xbcce) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                                                       
                                                      Output below shows the command to bind PF1, in this case s:b:d.f will be 0000:b1:00.1.
                                                       
                                                          sudo opae.io init -d s:b:d.f $USER\n
                                                       
                                                         
                                                      • Sample output for Intel\u00ae Agilex\u00ae 7 FPGA OFS target platform 2PF minimal FIM.
                                                        •  
                                                       
                                                          $ sudo opae.io init -d 0000:b1:00.1 $USER\n    Unbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\n    Binding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\n    iommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 13\n    Assigning /dev/vfio/13 to ofsuser\n    Changing permissions for /dev/vfio/13 to rw-rw----\n\n    $ sudo opae.io ls\n    [0000:b1:00.0] (0x8086:0xbcce) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n    [0000:b1:00.1] (0x8086:0xbcce) Intel Acceleration Development Platform N6001 (Driver: vfio-pci)\n\n    $ ls -lt /dev/vfio\n    total 0\n    crw-rw----. 1 ofsuser  root 511,   0 Feb  2 22:47 13\n    crw-rw-rw-. 1 root     root  10, 196 Feb  2 16:56 vfio\n
                                                       
                                                      If the driver fails to bind due to an error related to iommu_group (e.g. `No such file or directory: '/sys/bus/pci/devices/0000:b1:00.5/iommu_group'), ensure IOMMU is turned on as explained in step 2 in Section 2.3 Prerequisites.
                                                       
                                                      Note: For more information about opae.io utilities, refer to the OPAE FPGA tools page here.
                                                      "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#253-initialize-board-and-run-diagnostic-test","title":"2.5.3 Initialize Board and Run Diagnostic Test","text":"
                                                         
                                                      • Initialize the board with default bitstream using aocl initialize command
                                                        •  
                                                       
                                                          aocl initialize device-name name-of-board-variant\n
                                                       
                                                      Note: device-name in aocl initialize command is from the BSP Diagnostics section in the output of aocl list-devices (sample output is shown in Section 2.4). device-name is acl0 if there is only 1 board connected to the server. name-of-the-board-variant can be one of the supported board variants listed in Table 2-5 provided the sample application bitstream using steps in section above.
                                                       
                                                      Sample output for aocl initialize acl0 ofs_n6001 is shown below. Output for Intel\u00ae Stratix 10\u00ae FPGA and other board variants should be similar.
                                                       
                                                      $ aocl initialize acl0 ofs_n6001\naocl initialize: Running initialize from path-to-oneapi-asp/n6001/linux64/libexec\nInitializing with default ASP binary ofs_n6001.fpga\nSaving target image to \"ofs_n6001.aocx.0\"\nConfiguring locked memory setting\n[sudo] password for $USER:\nConfiguring udev rules for DFL FPGA device permission\nConfiguring system with 1024 2M hugepages\nSetting access permisions of /dev/uio to 666\nFinished setup_permissions.sh script. All configuration settings are persistent.\nProgram succeed.\n
                                                       
                                                      Notes:  1. aocl initialize command needs to be executed only one time unless you reboot the server. 
                                                       
                                                      Run aocl diagnose to check that the board Status under BSP Diagnostics is equal to Passed.
                                                       
                                                      platform-name in command output below is n6001 for Intel\u00ae Agilex\u00ae 7 FPGA OFS and d5005 for Intel\u00ae Stratix 10\u00ae FPGA OFS.
                                                       
                                                      $ aocl diagnose\n--------------------------------------------------------------------\nICD System Diagnostics\n--------------------------------------------------------------------\n\nUsing the following location for ICD installation:\n        /etc/OpenCL/vendors\n\nFound 3 icd entry at that location:\n        /etc/OpenCL/vendors/intel64.icd\n\nThe following OpenCL libraries are referenced in the icd files:\n        /opt/intel/oneapi/compiler/latest/lib/libintelocl.so\n\nChecking LD_LIBRARY_PATH for registered libraries:\n        /opt/intel/oneapi/compiler/latest/lib/libintelocl.so was registered on the system.\n\n\nChecking LD_LIBRARY_PATH for registered libraries:\n        /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga/host/linux64/lib/libalteracl.so was registered on the system.\n        /opt/intel/oneapi/compiler/latest/linux/lib/x64/libintelocl_emu.so\n\nChecking LD_LIBRARY_PATH for registered libraries:\n        /opt/intel/oneapi/compiler/latest/linux/lib/x64/libintelocl_emu.so was registered on the system.\n\nUsing OCL_ICD_FILENAMES to search for ICD clients, it is set to libintelocl_emu.so:libalteracl.so:/opt/intel/oneapi/compiler/2024.0/lib/libintelocl.so\n\nChecking LD_LIBRARY_PATH for registered libraries specified by OCL_ICD_FILENAMES\n    libintelocl_emu.so was registered on the system at \n    /opt/intel/oneapi/compiler/2024.0/lib\n    libalteracl.so was registered on the system at \n    /opt/intel/oneapi/compiler/2024.0/opt/oclfpga/host/linux64/lib\n    /opt/intel/oneapi/compiler/2024.0/lib/libintelocl.so was registered on the system.\n\nUsing the following location for fcd installations:\n        /opt/Intel/OpenCLFPGA/oneAPI/Boards\n\nFound 1 fcd entry at that location:\n        /opt/Intel/OpenCLFPGA/oneAPI/Boards/ofs_platform-name_shim.fcd\n\nThe following OpenCL libraries are referenced in the fcd files:\n        libopae-c.so\n/home/ofsuser/oneapi-asp/platform-name/linux64/lib/libMPF.so\n/home/ofsuser/oneapi-asp/platform-name/linux64/lib/libintel_opae_mmd.so\n\nChecking LD_LIBRARY_PATH for registered libraries:\n        libopae-c.so was registered on the system at /usr/lib64\n        /home/ofsuser/oneapi-asp/platform-name/linux64/lib/libMPF.so was registered on the system.\n        /home/ofsuser/oneapi-asp/platform-name/linux64/lib/libintel_opae_mmd.so was registered on the system.\n\nNumber of Platforms = 4\n        1. Intel(R) FPGA Emulation Platform for OpenCL(TM)    | Intel(R) Corporation           | OpenCL 1.2 Intel(R) FPGA SDK for OpenCL(TM), Version 20.3\n        2. Intel(R) FPGA SDK for OpenCL(TM)                   | Intel(R) Corporation           | OpenCL 1.0 Intel(R) FPGA SDK for OpenCL(TM), Version 2024.0\n        3. Intel(R) OpenCL                                    | Intel(R) Corporation           | OpenCL 3.0 LINUX\n        4. Intel(R) FPGA SDK for OpenCL(TM)                   | Intel(R) Corporation           | OpenCL 1.0 Intel(R) FPGA SDK for OpenCL(TM), Version 2024.0\n--------------------------------------------------------------------\nICD diagnostics PASSED\n--------------------------------------------------------------------\n--------------------------------------------------------------------\nBSP Diagnostics\n--------------------------------------------------------------------\n--------------------------------------------------------------------\nDevice Name:\nacl0\n\nBSP Install Location:\n/home/ofsuser/oneapi-asp/platform-name\n\nVendor: Intel Corp\n\nPhysical Dev Name   Status            Information\n\nofs_ee00001         Passed            Intel OFS Platform (ofs_ee00001)\n                                    PCIe b1:00.0\n                                    FPGA temperature = 59 degrees C.\n\nBSP DIAGNOSTIC_PASSED\n--------------------------------------------------------------------\n\nCall \"aocl diagnose <device-names>\" to run diagnose for specified devices\nCall \"aocl diagnose all\" to run diagnose for all devices\n
                                                       
                                                      Run complete diagnostic using aocl diagnose device-name command. device-name is acl0 if you have only 1 board in the server. The test reads and write data to the board to check the interfaces function correctly and report the measured bandwidth. The test must show BSP DIAGNOSTIC_PASSED message at the end.
                                                       
                                                          aocl diagnose acl0\n
                                                       
                                                      Next section you will build and run oneAPI host applications.
                                                       
                                                      Once you are done with your application testing, you can release the device from vfio-pci driver, the steps for this are provided in Section 2.7 Release VF.
                                                      "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#26-compile-and-run-oneapi-sample-applications","title":"2.6 Compile and Run oneAPI Sample Applications","text":"
                                                      Different code samples are provided for testing different board interfaces. Please refer to table below for more information.
                                                       
                                                      Table 2-6 DPC++ Sample Application
                                                       Board Variants Sample Application Description Link to Samples Repository Location of the Sample Samples Repository Tag 
                                                      • ofs_platform-name
                                                       
                                                      • ofs_platform-name_usm
                                                       board_test This sample measures kernel clock frequency, kernel launch latency and tests the different interfaces required by the oneAPI kernel to function correctly (e.g. host to kernel interface, kernel to EMIF as well as host to EMIF). oneAPI-samples path-to-oneAPI-samples/oneAPI-samples/DirectProgramming/C++SYCL_FPGA/ReferenceDesigns/board_test oneAPI compiler version you are using, this sample supports usm board variant from 2024.0 oneAPI version. 
                                                      • ofs_n6001_iopipes
                                                       
                                                      • ofs_n6001_usm_iopipes
                                                       io_streaming_one_pipe An FPGA code sample to perform a loopback test using SYCL* input/output (I/O) pipes to stream data through the FPGA I/O. examples-afu path-to-examples-afu/examples-afu/oneapi-samples/io_streaming_one_pipe ofs-2023.3-2 
                                                      First clone the repository and then checkout to the corresponding tag. 
                                                       
                                                          git clone link-to-samples-repository\n    cd oneAPI-samples # (oneapi-samples for examples-afu repository)\n    git checkout tags/samples-repository-tag\n
                                                       
                                                      Note: For link-to-samples-repository and samples-repository-tag refer to the table 2-6. To check your oneAPI compiler version use the command:
                                                       
                                                      \n    icpx --version\n
                                                       
                                                      Follow steps below to compile and run oneAPI board_test and io_streaming_one_pipe binaries. Use -DFPGA_DEVICE in cmake command to provide path to the oneAPI ASP and name of board variant being compiled. For board_test when targeting USM board variant add -DSUPPORTS_USM=1 flag in cmake command, for more information about this flag see the README file in the location of the sample, for this location refer to the table 2-6. 
                                                       
                                                      Ensure you have sourced setvars.sh script located in the root of your oneAPI installation as explained in Table 2-2 Initialization Script for HLD tool.
                                                       
                                                          cd path-to-sample-location\n    mkdir build\n    cd build\n
                                                       
                                                      Cmake for compiling board_test when targeting USM board variant.
                                                       
                                                          cmake -DFPGA_DEVICE=full-path-to-oneapi-asp/platform-name:board_variant -DSUPPORTS_USM=1 ..\n
                                                       
                                                      Cmake for compiling the rest of the board variants.  
                                                       
                                                          cmake -DFPGA_DEVICE=full-path-to-oneapi-asp/platform-name:board_variant ..\n
                                                       
                                                      Compile the design using the generated Makefile. The following build targets are provided:
                                                       
                                                         
                                                      • Generate the optimization report:
                                                        •  
                                                       
                                                          make report\n
                                                       
                                                         
                                                      • Compile for FPGA hardware (takes longer to compile, targets FPGA device):
                                                        •  
                                                       
                                                          make fpga\n
                                                       
                                                      Hardware compilation takes several hours to complete. Once complete, you should see sample-name.fpga executable generated, where sample-name could be board_test or io_streaming_one_pipe.
                                                       
                                                      For more information on additional environment settings required for running io_streaming_one_pipe sample see the README file in the location of the sample, for this location refer to the table 2-6 .
                                                       
                                                      Run the generated hardware executable as follows:
                                                       
                                                         ./sample-name.fpga\n
                                                       
                                                      Note: If your FPGA compile fails to meet timing requirements, the Intel oneAPI compiler prints an error message, returns an error code and deletes the generated binary. In case of timing failure, *.failing_clocks.rpt and *.failing_paths.rpt files are generated in compiled output directory sample-name.fpga.prj, where sample-name could be board_test or io_streaming_one_pipe. You can recompile with a different seed using -Xsseed option. You can pass this option using USER_HARDWARE_FLAGS=-Xsseed=seed_value in the cmake command above and recompile hardware image.
                                                       
                                                      To view test details and usage information using the binary, use the -help option.
                                                       
                                                          ./sample-name.fpga -help # sample-name could be board_test or io_streaming_one_pipe.\n
                                                      "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#27-release-vf","title":"2.7 Release VF","text":"
                                                      Once you are done with your application testing, you can release the device from vfio-pci driver using following command.
                                                       
                                                          $ sudo opae.io release -d s:b:d.vf\n
                                                       
                                                       Sample output for Intel\u00ae Agilex\u00ae 7 FPGA OFS target platform having programmed 1PF/1VF minimal FIM is shown below. The output for 2PF minimal FIM, base_x16 FIM for  Intel\u00ae Agilex\u00ae 7 FPGA and base_x16 FIM for Intel\u00ae Stratix 10\u00ae FPGA should be similar. 
                                                       
                                                      Note: For Intel\u00ae Stratix 10\u00ae FPGA you will need to release an extra VF as for this target 2 Vfs were created.
                                                       
                                                      $ sudo opae.io release -d 0000:b1:00.1\nReleasing (0x8086,0xbccf) at 0000:b1:00.1 from vfio-pci\nRebinding (0x8086,0xbccf) at 0000:b1:00.1 to dfl-pci\n\n$ sudo opae.io ls\n[0000:b1:00.0] (0x8086, 0xbcce) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n[0000:b1:00.1] (0x8086, 0xbccf) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                                                      "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#30-oneapi-on-ofs-running-in-a-virtual-machine","title":"3.0 OneAPI on OFS Running in a Virtual Machine","text":"
                                                      Virtual machines (VM's) can be used for FPGA development, 2PF minimal FIM (built using n6001_2pf.ofss) is provided to use oneAPI on OFS Intel\u00ae FPGA SmartNIC N6001-PL reference platform in a VM. For more information about 2PF FIM configuration refer to Intel\u00ae FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs. 
                                                       
                                                      The setup flow for your virtual machine could be found in the KVM User Guide: Open FPGA Stack, there are additional things for oneAPI flow listed below which you should ensure while configuring your VM:
                                                       
                                                         
                                                      • Assign the enough memory to the VM for running your oneAPI workloads.
                                                        •  
                                                       
                                                         
                                                      • In section 5.1 Passing Devices to the VM when adding the PCIe Host Devices to the VM, ensure to have PF0 and PF1 BDF adjacent (s\\:b\\:d.f, s\\:b\\:d.f+1). The following example shows the address element of the PCIe Host Device XML file of PF0 and PF1, keeping the same value for domain, bus and slot attributes and only changing the function attribute (increasing its value by one):
                                                        •  
                                                       
                                                       
                                                       
                                                       
                                                         
                                                      • Install libnsl.so.1 library with the following command:  
                                                          $sudo yum install libnsl.so.1\n
                                                         
                                                        •  
                                                       
                                                      Once this setup is done, follow Section 2.0 Setup Flow for Using HLD Tool on OFS to finish the configuration of the VM for oneAPI .
                                                      "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#31-further-development","title":"3.1 Further Development","text":"
                                                      Once you have completed running the oneAPI sample application, you can start developing your own applications.
                                                       
                                                      For more information about developing FPGA applications with Intel oneAPI, refer to Intel\u00ae oneAPI Programming Guide and FPGA Optimization Guide for Intel\u00ae oneAPI Toolkits.
                                                       
                                                      If you want to customize the oneAPI ASP, you can refer to oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack.
                                                      "},{"location":"hw/common/user_guides/oneapi_asp/ug_oneapi_asp/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                      Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                       
                                                      OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/","title":"Docker User Guide: Intel\u00ae Open FPGA Stack","text":"
                                                      Last updated: February 03, 2024 
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#1-introduction","title":"1 Introduction","text":"
                                                      This document is intended to help you get started in evaluating  Open FPGA Stack (Intel\u00ae OFS) using Docker for the Intel\u00ae Platforms. The Intel FPGA platforms  can be used as a starting point for evaluation and development. This document covers critical topics related to the initial setup of the Docker solution included with the OFS release.
                                                       
                                                      After reviewing the document, you shall be able to:
                                                       
                                                         
                                                      • Set up the Intel\u00ae Quartus\u2122 Prime Pro Edition Software in a host server
                                                        •  
                                                       
                                                         
                                                      • Set up the Docker engine
                                                        •  
                                                       
                                                         
                                                      • Build and load your Docker image to the Docker engine
                                                        •  
                                                       
                                                         
                                                      • Run a Docker container with OFS preloaded
                                                        •  
                                                       
                                                      The  Open FPGA Stack (OFS) Docker image has two main personas:
                                                       
                                                         
                                                      • Development: You can develop, simulate, and build any component of the OFS. The Docker image enables you to use your laptop or server without having drivers, FPGA  Platform, or specific Linux* distribution installed in your host computer. You can follow the development flow provided to run Docker on Linux.  
                                                        •  
                                                       
                                                         
                                                      • Deployment: You can program, load binaries, or execute real-time testing using the OPAE and OFS. To do so, the host computer must have the specified software distribution and drivers installed. 
                                                        •  
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#12-background-information","title":"1.2 Background Information","text":"
                                                      A container is a fully functional and portable cloud or non-cloud computing environment that includes an application, associated libraries, and other dependencies. Docker containers do not require a hardware hypervisor, instead using the application layer of the host computer, which means they tend to be smaller, faster to setup, and require fewer resources when compared to a virtual machine (VM).
                                                       
                                                      The OFS provides the flexibility to support various orchestration or management systems, including bare metal, VM, and Docker.
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#13-relevant-information","title":"1.3 Relevant information","text":"
                                                         
                                                      • What is a container?
                                                        •  
                                                      • Docker vs. Virtual Machines 
                                                        •  
                                                      • Does the Docker container have its own Kernel?
                                                           
                                                        • No, Docker image or Container uses the application layer of the host computer; this functionality is the main reason for docker having lightweight and fast applications.
                                                          •  
                                                         
                                                        •  
                                                      • Does Docker run on Linux, macOS, and Windows?
                                                        •  
                                                      • Intel Docker Image can use the PCIe card from the host server?
                                                           
                                                        • Yes, The drivers and additional information could be shared, but this could create potential security concerns (ensure your system is secure).
                                                          •  
                                                         
                                                        •  
                                                      • Docker security
                                                        •  
                                                      • Docker subscription
                                                        •  
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#20-prerequisites-and-scope","title":"2.0 Prerequisites and Scope","text":"
                                                      The OFS release targeting the compatible OFS Platform's is built upon tightly coupled software and firmware versions. Use this section as a general reference for the versions in this release.
                                                       
                                                      The following table highlights the hardware that comprises the Best-Known Configuration (BKC) for the OFS release. For a detailed explanation and safety information regarding the setup go to OFS Site select your desired platform and select Getting stated guide. This site walks you through the BIOS configuration changes needed to enable the OFS Platform's.
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#30-development-installation","title":"3.0 Development Installation","text":"
                                                      Docker engines have cross-compatibility with multiple systems, but the host server does not require any specific distribution. However, the Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 requires a specific version. For this guide, Red Hat Linux  is used for general instructions. 
                                                       
                                                      The OFS Docker image includes all the libraries and tools required by the OFS and OPAE SDK (Python, Perl, CMake, and so on).
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#31-intel-quartus-prime-software-installation","title":"3.1 Intel Quartus Prime Software Installation","text":"
                                                      Building AFUs with OFS for Intel Agilex FPGA requires the build machine to have at least 64 GB of RAM.
                                                       
                                                      Go to OFS Site select your desired platform and select Getting stated guide for a list of detailed steps for the Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 installation.  
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#32-docker-engine-installation","title":"3.2 Docker Engine installation","text":""},{"location":"hw/common/user_guides/ug_docker/ug_docker/#rhel-86","title":"RHEL 8.6","text":"
                                                      The Docker installation steps for RHEL 8.6 are the following:
                                                       
                                                         
                                                      1.  
                                                        Remove old versions; older versions of Docker were called docker or docker-engine. If these are installed, uninstall them, along with associated dependencies. Also, uninstall Podman and the related dependencies if installed already.
                                                         
                                                         sudo dnf remove docker \\\n                  docker-client \\\n                  docker-client-latest \\\n                  docker-common \\\n                  docker-latest \\\n                  docker-latest-logrotate \\\n                  docker-logrotate \\\n                  docker-engine \\\n                  podman \\\n                  runc\n
                                                         
                                                        2.  
                                                      2.  
                                                        Add the Docker repository to your system:
                                                         
                                                        sudo dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo\n
                                                         
                                                        3.  
                                                      3.  
                                                        Install the latest version of Docker Engine, containerd, and Docker Compose, or go to the next step to install a specific version.
                                                         
                                                        sudo dnf install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin\n
                                                         
                                                        4.  
                                                      4.  
                                                        Start the Docker daemon:
                                                         
                                                        sudo systemctl start docker\n
                                                         
                                                        5.  
                                                      5.  
                                                        Enable the Docker daemon to start on boot:
                                                         
                                                        sudo systemctl enable --now docker\nsudo systemctl enable --now containerd\n
                                                         
                                                        6.  
                                                      6.  
                                                        Verify that Docker is installed and running:
                                                         
                                                        sudo systemctl status docker\n
                                                         
                                                        You should see a message indicating that the Docker daemon is active and running.
                                                         
                                                        Note: If you want to use Docker as a non-root user, you should add your user to the docker group:
                                                         
                                                        sudo usermod -aG docker your-user\n
                                                         
                                                        You will need to log out and back in for the changes to take effect.
                                                         
                                                        7.  
                                                      7.  
                                                        Ensure your proxies are setup in case you needed
                                                         
                                                        sudo mkdir -p /etc/systemd/system/docker.service.d \n\nnano /etc/systemd/system/docker.service.d/http-proxy.conf\n\n[Service] \nEnvironment=\"HTTP_PROXY=http://proxy.example.com:80/\"\nEnvironment=\"HTTPS_PROXY=https://proxy.example.com:443/\"\n\n#save and close \n\nsudo systemctl daemon-reload\nsudo systemctl restart docker\n
                                                         
                                                        8.  
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#ubuntu-2204","title":"Ubuntu 22.04","text":"
                                                      The Docker installation steps for Ubuntu are the following:
                                                       
                                                         
                                                      1.  
                                                        Remove old versions; older versions of Docker were called docker or docker-engine. If these are installed, uninstall them, along with associated dependencies. 
                                                         
                                                        sudo apt-get remove docker docker-engine docker.io containerd runc\n
                                                         
                                                        2.  
                                                      2.  
                                                        Install packages to allow apt to use a repository
                                                         
                                                        sudo apt-get update\nsudo apt-get install \\\n    ca-certificates \\\n    curl \\\n    gnupg \\\n    lsb-release\n
                                                         
                                                        3.  
                                                      3.  
                                                        Add Docker's official GPG key:
                                                         
                                                        sudo mkdir -p /etc/apt/keyrings\ncurl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg\n
                                                         
                                                        4.  
                                                      4.  
                                                        The following command to set up the repository:
                                                         
                                                        echo \\\n  \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \\\n  $(lsb_release -cs) stable\" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null\n
                                                         
                                                        5.  
                                                      5.  
                                                        Update the package manager index again:
                                                         
                                                        sudo apt-get update\n
                                                         
                                                        6.  
                                                      6.  
                                                        Install Docker:
                                                         
                                                        sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin\n
                                                         
                                                        7.  
                                                      7.  
                                                        Start the Docker daemon:
                                                         
                                                        sudo systemctl start docker\n
                                                         
                                                        8.  
                                                      8.  
                                                        Enable the Docker daemon to start on boot:
                                                         
                                                        sudo systemctl enable --now docker\nsudo systemctl enable --now containerd\n
                                                         
                                                        9.  
                                                      9.  
                                                        Verify that Docker is installed and running:
                                                         
                                                        sudo systemctl status docker\n
                                                         
                                                        You should see a message indicating that the Docker daemon is active and running.
                                                         
                                                        Note: If you want to use Docker as a non-root user, you should add your user to the docker group:
                                                         
                                                        sudo usermod -aG docker your-user\n
                                                         
                                                        You will need to log out and back in for the changes to take effect.
                                                         
                                                        10.  
                                                      10.  
                                                        Ensure your proxies are setup in case you needed
                                                         
                                                        sudo mkdir -p /etc/systemd/system/docker.service.d \n\nnano /etc/systemd/system/docker.service.d/http-proxy.conf\n\n[Service] \nEnvironment=\"HTTP_PROXY=http://proxy.example.com:80/\"\nEnvironment=\"HTTPS_PROXY=https://proxy.example.com:443/\"\n\n#save and close \n\nsudo systemctl daemon-reload\nsudo systemctl restart docker\n
                                                         
                                                        11.  
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#33-load-docker-image-installation","title":"3.3 Load Docker Image installation","text":"
                                                      The Dockerfile is released in conjunction with the OFS stack release, and The file needs to be loaded into your host computer to start a docker container.
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#build-the-image","title":"Build the image","text":"
                                                         
                                                      1.  
                                                        You can download the Dockefile from OFS GitHub Docker.
                                                         
                                                        2.  
                                                      2.  
                                                        Inside the Dockerfile folder, you will find the DockerFile edit and modify the following lines:
                                                         
                                                        ENV no_proxy=   #you could use  github.com here\nENV http_proxy= #setup proxy\nENV https_proxy=  #setup proxy\nENV GITUSER= #setup github user\nENV GITTOKEN= #setup github token\nENV REDUSER= #redhat user \nENV REDPASS= #redhat password\nENV DW_LICENSE_FILE= #DW license\nENV SNPSLMD_LICENSE_FILE= #Synopsys license\nENV LM_LICENSE_FILE= #Quartus License\n
                                                         
                                                        Save the file 
                                                         
                                                        3.  
                                                      3.  
                                                        Create and load the image:
                                                         
                                                        cd Docker_file\ndocker build -t ofs:latest . --no-cache\n
                                                         
                                                        Note: Never remove --no-cache this could cause issues with your environmental variables inside of the container
                                                         
                                                        4.  
                                                      4.  
                                                        Use the following command to ensure the image is loaded correctly:
                                                         
                                                        sudo docker images\nREPOSITORY    TAG                       IMAGE ID       CREATED          SIZE\nofs           latest                    fc80175d13a0   \u221e seconds ago   2.55GB\n
                                                         
                                                        5.  
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#volumen-creation","title":"Volumen creation","text":"
                                                         
                                                      1.  
                                                        Docker requires a volume to move data from the host computer (Persistent data) to the docker container and vice versa. To create a docker volume, use the following command:
                                                         
                                                        docker volume create --name DataOFS\n
                                                         
                                                        For more information about Docker volume go here.
                                                         
                                                        Tip: Remember, The docker container has a limited lifecycle; the files and data are lost when the docker is Stopped-> Deleted.
                                                         
                                                        2.  
                                                      2.  
                                                        Check where the docker volume is mapped in your host server:
                                                         
                                                        docker volume inspect DataOFS\n[\n    {\n        \"CreatedAt\": \"xxxxxxxxxx\",\n        \"Driver\": \"local\",\n        \"Labels\": {},\n        \"Mountpoint\": \"/var/lib/docker/volumes/DataOFS/_data\",\n        \"Name\": \"DataOFS\",\n        \"Options\": {},\n        \"Scope\": \"local\"\n    }\n]\n
                                                         
                                                        3.  
                                                      3.  
                                                        Inside of your docker container, you can use cp command to copy from your docker to your host:
                                                         
                                                        cp /atmydocker/myfile.txt /dataofs\n
                                                         
                                                        The docker container path is /dataofs the host path is /var/lib/docker/volumes/DataOFS/_data.
                                                         
                                                        4.  
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#34-create-a-container","title":"3.4 Create a container","text":"
                                                      Now you are ready to start the container, and you should be prepared to run it: 1. First, Let's create the template for the run  command, copy your Quartus installation path and paste it under -v (Don't Run the command yet):
                                                       
                                                      docker run --rm -itd --name myOFS -v=<yourintallationfolder>:/home/intelFPGA_pro/:ro -v=DataOFS:/dataofs ofs:latest /bin/bash\n
                                                       
                                                      Tip: you can change myOFS with any other value. The value is the given name of the container.
                                                       
                                                         
                                                      1. Using the previous example now, you can execute the docker run command.    
                                                        docker run --rm -itd --name myOFS -v=/home/intelFPGA_pro/23.3:/home/intelFPGA_pro/:ro -v=DataOFS:/dataofs ofs:latest /bin/bash\nbdc1289fb0813bb325b55dd11df4eeec252143d6745a6e5772638fbc107d0949\n
                                                        2.  
                                                      2.  
                                                        Now the docker container should be available.
                                                         
                                                        # sudo docker ps\nCONTAINER ID   IMAGE                         COMMAND       CREATED          STATUS   PORTS     NAMES\nbdc1289fb081   ofs:latest                    \"/bin/bash\"   46 seconds ago   Up 45 seconds      myOFS\n
                                                         
                                                        3.  
                                                       
                                                      Your Container ID is bdc1289fb081. 
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#35-evaluate-ofs-container","title":"3.5 Evaluate OFS container","text":"
                                                      The OFS container has two possible ways to interact with the container:
                                                       
                                                         
                                                      •  
                                                        Interactive mode: 
                                                         
                                                        This mode it takes you straight inside the container and uses the command terminal as a regular Linux console. 
                                                         
                                                           
                                                        1.  
                                                          Enable the interactive mode:
                                                           
                                                          docker attach myOFS\n[root@bdc1289fb081 /]#\n
                                                           
                                                          The container id is shown when you are in interactive mode [root@bdc1289fb081 /]#.
                                                           
                                                          2.  
                                                        2.  
                                                          Now verify the variables and Quartus is appropriately set up and recognized:
                                                           
                                                          quartus_syn --version\n\nQuartus Prime Synthesis\nVersion Quartus Prime Pro Version 23.3\n
                                                           
                                                          3.  
                                                        3.  
                                                          Everything is set up correctly. Please go to the following link for more information related to the OFS Site select your desired platform and select Getting stated guide.
                                                           
                                                          Tip: If you need to de-attach without stopping the container, you can use Ctrl+P or Ctrl+Q. For custom combinations, for example, docker attach --detach-keys=\"ctrl-a\"  myOFS and if you press CTRL+A you will exit the container without killing it.
                                                           
                                                          4.  
                                                         
                                                        •  
                                                       
                                                         
                                                      •  
                                                        De-attach Mode:
                                                         
                                                        This mode runs your container in the background and allows you to run multiple commands without going inside of the docker container.
                                                         
                                                           
                                                        1.  
                                                          The OFS Docker image already includes the evaluation script.
                                                           
                                                          2.  
                                                        2.  
                                                          Let's use option 2   - Check versions of Operating System and Quartus Premier Design Suite (QPDS); remember multiple options could not be available if the DFL drivers and the FPGA Platform is not installed, This example uses the Intel\u00ae FPGA SmartNIC N6001-PL . 
                                                           
                                                          $ sudo docker exec -it myOFS /home/OFS_BUILD_ROOT/ofs-agx7-pcie-attach_eval.sh 2\n\nGo to selection: 2\n###########################################################################################\n#################### Check versions of Operation System, Quartus ##########################\n###########################################################################################\n\nChecking Linux release\nLinux version 6.1.41-dfl .....\n\n....\n\ncycle complete exiting...\n
                                                           
                                                          3.  
                                                        3.  
                                                          The Intel Docker image includes the script ofs_extratool.sh to allow you to change the seed value.    
                                                          sudo docker exec -it myOFS /home/OFS_BUILD_ROOT/ofs_extratool.sh -s 5\n
                                                               Now you can control and compile the design. You can use the interactive or de-attach mode.
                                                           
                                                          4.  
                                                        4.  
                                                          If you need to save the log file and output files use the following command 
                                                           
                                                          sudo docker exec -it myOFS /home/OFS_BUILD_ROOT/ofs_extratool.sh -e\n
                                                           
                                                          all the files are saved under the share volume, DataOFS , /var/lib/docker/volumes/DataOFS/_data
                                                           
                                                          5.  
                                                         
                                                        •  
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#40-deployment","title":"4.0  Deployment","text":"
                                                      The OFS docker image allows you to connect with your FPGA Platform. The main difference from the development installation process is that you are able to test with real hardware, but you must have a specific requirement to have a fully compatible system. 
                                                       
                                                      Information related to host setup please go to OFS Site select your desired platform and select Getting stated guide.
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#41-installation-of-deployment-server","title":"4.1 Installation of Deployment server","text":"
                                                      Once you ensure the DFL drivers are installed, follow the below steps:
                                                       
                                                         
                                                      1. Follow the steps listed in sections  2.1 to 2.3
                                                           
                                                        • 2.1 Quartus installation
                                                          •  
                                                        • 2.2 Docker Engine installation
                                                          •  
                                                        • 2.3 Load Docker Image installation
                                                          •  
                                                         
                                                        2.  
                                                      2. The steps required for DFL driver installation are documented OFS Site select your desired platform and select Getting stated guide.
                                                        3.  
                                                       
                                                      Now you should have all the steps required, and you can run the docker image directly.
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#42-create-a-container","title":"4.2 Create a container","text":"
                                                      Now you are ready to start the container, and should be prepared to run it (Note: now we are adding a new flag to allow us to access the PCIe devices \u201c\u2014privileged\u201d) :
                                                       
                                                         
                                                      1.  
                                                        First, copy your Quartus installation path and  paste it under -v:
                                                         
                                                        docker run --rm --privileged -itd --name myOFS -v=<yourintallationfolder>:/home/intelFPGA_pro/:ro -v=DataOFS:/dataofs ofs:latest /bin/bash\n
                                                         
                                                        Example, my Quartus installation is located at \"/home/intelFPGA_pro/23.3\" as a result, my  command should be 
                                                         
                                                        docker run --rm --privileged -itd --name myOFS -v=/home/intelFPGA_pro/23.3:/home/intelFPGA_pro/:ro -v=DataOFS:/dataofs ofs:latest /bin/bash\nbdc1289fb0813bb325b55dd11df4eeec252143d6745a6e5772638fbc107d0949\n
                                                         
                                                        Tip: you can change myOFS with any other value. The value is the given name of the container.
                                                         
                                                        Important: The --privileged flag gives all capabilities to the container. When the operator executes docker run --privileged, Docker will enable access to all devices on the host as well as set some configuration in AppArmor or SELinux to allow the container nearly all the same access to the host as processes running outside containers on the host. Additional information about running with --privileged is available on the Docker Blog. 
                                                         
                                                        2.  
                                                       
                                                      :warning: Only use --privileged under development infrastructure, never in production!
                                                       
                                                         
                                                      1.  
                                                        Execute the docker run command.
                                                         
                                                        docker run --rm --privileged -itd --name myOFS -v=/home/intelFPGA_pro/23.3:/home/intelFPGA_pro/:ro -v=DataOFS:/dataofs ofs:latest /bin/bash\n25b41eb4d232de9c750b52ddc6b92a3db612200e5993f55733b59068898623d7\n
                                                         
                                                        2.  
                                                      2.  
                                                        Now, the docker container should be available.
                                                         
                                                        # sudo docker ps\nCONTAINER ID   IMAGE                              COMMAND       CREATED     STATUS     PORTS     NAMES\n25b41eb4d232   ofs:latest                        \"/bin/bash\"   13 seconds ago   Up 12 seconds     myOFS\n
                                                         
                                                        3.  
                                                       
                                                      \u200b       Your Container ID is 25b41eb4d232. 
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#43-evaluate-ofs-container","title":"4.3 Evaluate OFS container","text":"
                                                      The OFS container has two possible ways to interact with the container:
                                                       
                                                         
                                                      •  
                                                        Interactive mode: 
                                                         
                                                        This mode it takes you straight inside the container and uses the command terminal as a regular Linux console. 
                                                         
                                                           
                                                        1.  
                                                          Enable the interactive mode:
                                                           
                                                          docker attach myOFS\n[root@25b41eb4d232 /]#\n
                                                           
                                                          The container id is shown when you are in interactive mode [root@25b41eb4d232 /]#.
                                                           
                                                          2.  
                                                        2.  
                                                          Now verify the variables and Quartus is appropriately setup and recognized:
                                                           
                                                          quartus_syn --version\n\nQuartus Prime Synthesis\nVersion 23.3\n
                                                           
                                                          3.  
                                                        3.  
                                                          Everything is set up correctly. Please go to the following link for more information related to the OFS Site select your desired platform and select User Guide,  Technical Reference Manual, Developer Guide, or Getting Started Guide.
                                                           
                                                          Tip: If you need to de-attach without stopping the container you can use Ctrl+P or Ctrl+Q. For custom, combinations use for example docker attach --detach-keys=\"ctrl-a\"  myOFS and if you press CTRL+A you will exit the container, without killing it.
                                                           
                                                          4.  
                                                         
                                                        •  
                                                       
                                                         
                                                      •  
                                                        De-attach Mode:
                                                         
                                                        This mode runs your container in the background and allows you to run multiple commands without going inside of the docker container.
                                                         
                                                           
                                                        1.  
                                                          The OFS Docker image already includes the eval script.
                                                           
                                                          2.  
                                                        2.  
                                                          Run the script and make a selection, you can directly execute with the following command:
                                                           
                                                          Let's use option 3   - Identify  Platform Hardware via PCIe; remember the DFL drivers need be installed. 
                                                           
                                                          3.  
                                                         
                                                        $ sudo docker exec -it myOFS /home/OFS_BUILD_ROOT/ofs-agx7-pcie-attach_eval.sh 3\n\nGo to selection: 3\n\n\nPCIe card detected as\n\n\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nb1:00.1 Processing accelerators: Intel Corporation Device bcce\nb1:00.2 Processing accelerators: Intel Corporation Device bcce\nb1:00.4 Processing accelerators: Intel Corporation Device bcce\n\nHost Server is connected to SINGLE card configuration\n\ncycle complete exiting...\n
                                                         
                                                           
                                                        1. The Intel Docker image includes the script ofs_extratool.sh to allow you to change the seed value.
                                                          ```sh\nsudo docker exec -it myOFS /home/OFS_BUILD_ROOT/ofs_extratool.sh -s 5\n```\n
                                                           
                                                          2.  
                                                         
                                                        Now you can control and compile the design using the interactive or de-attach mode.
                                                         
                                                        •  
                                                      "},{"location":"hw/common/user_guides/ug_docker/ug_docker/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                      Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                       
                                                      OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/","title":"Evaluation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach","text":""},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#1-overview","title":"1 Overview","text":""},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#11-about-this-document","title":"1.1 About this Document","text":"
                                                      This document serves as a set-up and user guide for the checkout and evaluation of an Intel\u00ae FPGA SmartNIC N6001-PL and Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) development platform using Open FPGA Stack (OFS). After reviewing the document, you will be able to:
                                                       
                                                         
                                                      • Use the script to run through the most common build and simulation flows when using OFS.
                                                        •  
                                                       
                                                         
                                                      • Run hardware and software tests to evaluate the complete OFS flow
                                                        •  
                                                       
                                                         
                                                      • Modify and leverage the script to the your environment and design
                                                        •  
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#table-1-2-software-version-summary","title":"Table 1-2: Software Version Summary","text":"Component Version Description FPGA Platform Intel\u00ae FPGA SmartNIC N6001-PL, Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) Intel platform you can use for your custom board development OFS FIM Source Code Branch: https://github.com/OFS/ofs-agx7-pcie-attach, Tag: ofs-2023.3-2 OFS Shell RTL for Intel Agilex FPGA (targeting Intel\u00ae FPGA SmartNIC N6001-PL) AFU Examples Branch: examples-afu , Tag:ofs-examples-https://github.com/OFS/examples-afu/releases/tag/ofs-2023.3-2 Tutorials and simple examples for the Accelerator Functional Unit region (workload region) OPAE SDK Branch: 2.10.0-1, Tag: 2.10.0-1 Open Programmable Acceleration Engine Software Development Kit Kernel Drivers Branch: ofs-2023.3-6.1-3, Tag: ofs-2023.3-6.1-3 OFS specific kernel drivers OPAE Simulation Branch: opae-sim, Tag: 2.10.0-1 Accelerator Simulation Environment for hardware/software co-simulation of your AFU (workload) Intel Quartus Prime Pro Edition Design Software 23.3 [Intel\u00ae Quartus\u00ae Prime Pro Edition Linux] Software tool for Intel FPGA Development Operating System RHEL 8.6 Operating system on which this script has been tested 
                                                      A download page containing the release and already-compiled FIM binary artifacts that you can use for immediate evaluation on the Intel\u00ae FPGA SmartNIC N6001-PL can be found on the OFS ofs-2023.3-2 official release drop on GitHub.
                                                       
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#2-introduction-to-ofs-evaluation-script","title":"2 Introduction to OFS Evaluation Script","text":"
                                                      By following the setup steps and using the OFS evaluation script you can quickly evaluate many features that the OFS framework provides and also leverage this script for your own development.  
                                                       
                                                      NOTE: 
                                                       
                                                      This guide uses the Intel\u00ae FPGA SmartNIC N6001-PL as the platform for all example steps. Additionally, this guide and the example steps can be used with other platforms, such as the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile). 
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#21-pre-requisites","title":"2.1 Pre-Requisites","text":"
                                                      This script uses the following set of software tools which should be installed using the directory structure below. Tool versions can vary.
                                                       
                                                         
                                                      • Intel Quartus\u00ae Prime Pro Software
                                                        •  
                                                      • Synopsys\u00ae VCS Simulator
                                                        •  
                                                      • Siemens\u00ae Questa\u00ae Simulator
                                                        •  
                                                       
                                                      Figure 2-1 Folder Hierarchy for Software Tools
                                                       
                                                       
                                                         
                                                      1.  
                                                        You must create a directory named \"ofs-X.X.X\" where the X represents the current release number, for example ofs-2023.3-2. 
                                                         
                                                        2.  
                                                      2.  
                                                        You must clone the required OFS repositories as per Figure 2-2.  Please refer to the BKC table 1-2 for locations. When cloning the FIM repository, please follow the instructions in the following guides
                                                         
                                                        3.  
                                                       
                                                         
                                                      • FPGA Interface Manager (FIM) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach
                                                        •  
                                                      • FPGA Interface Manager (FIM) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach (2xF-tile).
                                                        •  
                                                       
                                                      Additionally, please follow the instructions in the following guides for the BKC software installation.
                                                       
                                                         
                                                      • Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL)
                                                        •  
                                                      • Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 SoC Attach FPGAs (Intel Agilx 7 FPGA F-Series Development Kit (2xF-Tile)) 
                                                        •  
                                                       
                                                         
                                                      1. Once the repositories are cloned, The scripts that go with each user guide are found in the assets tab of the corresponding FIM RTL repository. Download the evaluation script (ofs-agx7-pcie-attach_eval.sh) and copy it to the ofs-2023.3-2 directory location as shown in the example below:
                                                        2.  
                                                       
                                                      Figure 2-2 Directory Structure for OFS Project
                                                       
                                                      ## ofs-2023.3-2\n##  -> examples-afu\n##  -> linux-dfl\n##  -> ofs-agx7-pcie-attach\n##  -> oneapi-asp\n##  -> oneAPI-samples\n##  -> opae-sdk\n##  -> opae-sim\n##  -> ofs-agx7-pcie-attach_eval.sh\n
                                                       
                                                         
                                                      1. The README file that accompanies each user guide are found in the assets tab of the corresponding FIM RTL repository. Download the README file (README_ofs-agx7-pcie-attach_eval.txt) and copy it to the ofs-2023.3-2 directory location. The README informs the user which sections to modify in the script prior to building the FIM and running hardware, software and simulation tests.
                                                        2.  
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#22-intel-agilex-7-pcie-attach-evaluation-script-modification","title":"2.2 Intel\u00ae Agilex\u00ae 7 PCIe Attach Evaluation Script modification","text":"
                                                      To adapt this script to the user environment please follow the instructions below which explains which line numbers to change in the ofs-agx7-pcie-attach.sh script 
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#user-directory-creation","title":"User Directory Creation","text":"
                                                      The user must create the top-level source directory and then clone the OFS repositories
                                                       
                                                      mkdir ofs-2023.3-2\n
                                                       
                                                      In the example above we have used ofs-2023.3-2 as the directory name
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#set-up-proxy-server-lines-67-69","title":"Set-Up Proxy Server (lines 67-69)","text":"
                                                      Please enter the location of your proxy server to allow access to external internet to build software packages.
                                                       
                                                      Note: Failing to add proxy server will prevent cloning of repositories and the user will be unable to build the OFS framework.
                                                       
                                                      export http_proxy=<user_proxy>\nexport https_proxy=<user_proxy>\nexport no_proxy=<user_proxy>\n
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#license-files-lines-72-74","title":"License Files (lines 72-74)","text":"
                                                      Please enter the the license file locations for the following tool variables
                                                       
                                                      export LM_LICENSE_FILE=<user_license>\nexport DW_LICENSE_FILE=<user_license>\nexport SNPSLMD_LICENSE_FILE=<user_license>\n
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#tools-location-line-87-88-89-90","title":"Tools Location (line 87, 88, 89, 90)","text":"
                                                      Set Location of Quartus, Synopsys, Questasim and oneAPI Tools
                                                       
                                                      export QUARTUS_TOOLS_LOCATION=/home\nexport SYNOPSYS_TOOLS_LOCATION=/home\nexport QUESTASIM_TOOLS_LOCATION=/home\nexport ONEAPI_TOOLS_LOCATION=/opt\n
                                                       
                                                      In the example above /home is used as the base location of Quartus, Synopsys and Questasim tools, /opt is used for the oneAPI tools
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#quartus-tools-version-line-95","title":"Quartus Tools Version (line 95)","text":"
                                                      Set version of Quartus
                                                       
                                                      export QUARTUS_VERSION=23.3\n
                                                       
                                                      In the example above \"23.3\" is used as the Quartus tools version
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#opae-tools-line-108","title":"OPAE Tools (line 108)","text":"
                                                      change OPAE SDK VERSION
                                                       
                                                      export OPAE_SDK_VERSION=2.10.0-1\n
                                                       
                                                      In the example above \"2.10.0-1\" is used as the OPAE SDK tools version
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#pcie-bus-number","title":"PCIe (Bus Number)","text":"
                                                      The Bus number must be entered by the user after installing the hardware in the chosen server, in the example below \"b1\" is the Bus Number for a single card as defined in the evaluation script.
                                                       
                                                      export OFS_CARD0_BUS_NUMBER=b1\n
                                                       
                                                      The evaluation script uses the bus number as an identifier to interrogate the card. The command below will identify the accelerater card plugged into a server. 
                                                       
                                                      lspci | grep acc\n\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)<br>\nb1:00.1 Processing accelerators: Intel Corporation Device bcce<br>\nb1:00.2 Processing accelerators: Intel Corporation Device bcce<br>\nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device<br>\nb1:00.4 Processing accelerators: Intel Corporation Device bcce<br>\n
                                                       
                                                      The result identifies the card as being assigned \"b1\" as the bus number so the entry in the script changes to
                                                       
                                                      export OFS_CARD0_BUS_NUMBER=b1\n
                                                       
                                                      The user can also run the following command on the ofs-agx7-pcie-attach_eval.sh script to automatically change the bus number to b1 in the ofs-agx7-pcie-attach_eval.sh script.
                                                       
                                                      grep -rli 'b1' * | xargs -i@ sed -i 'b1' @
                                                       
                                                      if the bus number is 85 for example 
                                                       
                                                      85:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)<br>\n85:00.1 Processing accelerators: Intel Corporation Device bcce<br>\n85:00.2 Processing accelerators: Intel Corporation Device bcce<br>\n85:00.3 Processing accelerators: Red Hat, Inc. Virtio network device<br>\n85:00.4 Processing accelerators: Intel Corporation Device bcce<br>\n
                                                       
                                                      the command to change to 85 in the evaluation script would be
                                                       
                                                      grep -rli 'b1' * | xargs -i@ sed -i '85' @
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#ofs-platform-example-n6000-n6001-fseries-dk-iseries-dk-custom_board-choice-line-173","title":"OFS Platform (Example:= n6000, n6001, fseries-dk, iseries-dk, custom_board) choice (line 173)","text":"
                                                      The script is designed to accomodate many OFS platform choices eg, n6000, n6001, fseries-dk, iseries-dk or a custom_board of your choice. By default the script defaults to n6001 as shown below and is set by a variable on line 173 of the ofs-agx7-pcie-attach_eval.sh script.
                                                       
                                                        export OFS_PLATFORM=n6001\n
                                                       
                                                      but the user can switch platform by changing the OFS_PLATFORM variable, so for example if the user wants to switch to the i-series development kit the command would be
                                                       
                                                        export OFS_PLATFORM=iseries-dk\n
                                                       
                                                      The ofs-agx7-pcie-attach_eval.sh script has now been modified to the server set-up and the user can proceed to build, compile and simulate the OFS stack 
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#3-ofs-agx7-pcie-attach-evaluation-script","title":"3 ofs-agx7-pcie-attach Evaluation Script","text":""},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#31-overview","title":"3.1 Overview","text":"
                                                      The figure below shows a snapshot of the full evaluation script menu showing all 62 options and each one of 11 sub-menus which focus on different areas of evaluation. Each of these menu options are described in the next section.
                                                       
                                                      Figure 3-1 ofs-agx7-pcie-attach_eval.sh Evaluation Menu
                                                       
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#311-ofs-tools-menu","title":"3.1.1 OFS TOOLS MENU","text":"
                                                      By selecting \"List of Documentation for OFS Project,\" a list of links to the latest OFS documentation appears. Note that these links will take you to documentation for the most recent release which may not correspond to the release version you are evaluating. To find the documentation specific to your release, ensure you clone the intel-ofs-docs tag that corresponds to your OFS version.
                                                       
                                                      By selecting \"Check Versions of Operating System and Quartus Premier Design Suite\", the tool verifies correct Operating System, Quartus version, kernel parameters, license files and paths to installed software tools.
                                                       
                                                       Menu Option Example Output 1 - List of Documentation for OFS PCI Attach Project Project  Open FPGA Stack Overview                  Guides you through the setup and build steps to evaluate the OFS solution                 https://ofs.github.io 2 - Check versions of Operating System and Quartus Premier Design Suite (QPDS) Checking Linux release                 Linux version 5.15.52-dfl (guest@hw-rae-svr4-l) (gcc (GCC) 8.5.0 20210514 (Red Hat 8.5.0-4), GNU ld version 2.30-79.el8) #1 SMP Fri Sep 23 17:19:37 BST 2022                  Checking RedHat release                 CentOS Linux release 8.3.2011                  Checking Ubuntu release                 cat: /etc/lsb-release: No such file or directory                  Checking Kernel parameters                 BOOT_IMAGE=(hd0,gpt2)/vmlinuz-5.15.52-dfl root=/dev/mapper/cl-root ro crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200                  Checking Licenses                 LM_LICENSE_FILE is set to port@socket number:port@socket number                 DW_LICENSE_FILE is set to port@socket number:port@socket number                 SNPSLMD_LICENSE_FILE is set to port@socket number:port@socket number                  Checking Tool versions                 QUARTUS_HOME is set to /home/intelFPGA_pro/23.3/quartus                 QUARTUS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus                 IMPORT_IP_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../ip                 QSYS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../qsys/bin                  Checking QPDS Patches                 Quartus Prime Shell                 Version 23.3 Build XXX XX/XX/XXXX Patches X.XX SC Pro Edition                 Copyright (C) XXXX  Intel Corporation. All rights reserved."},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#312-ofs-hardware-menu","title":"3.1.2 OFS HARDWARE MENU","text":"
                                                      Identifies card by PCIe number, checks power, temperature and current firmware configuration. 
                                                       
                                                       Menu Option Example Output 3 - Identify Platform Hardware via PCIe PCIe card detected as                 b1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)                 b1:00.1 Processing accelerators: Intel Corporation Device bcce                 b1:00.2 Processing accelerators: Intel Corporation Device bcce                 b1:00.4 Processing accelerators: Intel Corporation Device bcce                 Host Server is connected to SINGLE card configuration 4 - Identify the Board Management Controller (BMC) Version and check BMC sensors Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.2.0                 Board Management Controller Build version: 3.2.0                 //****** BMC SENSORS ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                  Note: This feature is not supported on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)  5 - Identify the FPGA Management Engine (FME) Version Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.2.0                 Management Controller Build version: 3.2.0                 //****** FME ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                 Boot Page                        : user1                 Factory Image Info               : a7c6879683182ce61084c420e51f50b6                 User1 Image Info                 : 8a7440ddff52e0e27dbb989d5eb954f4                 User2 Image Info                 : a7c6879683182ce61084c420e51f50b6 6 - Check Board Power and Temperature Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.2.0                 Board Management Controller Build version: 3.2.0                 //****** POWER ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                 ( 1) VCCRT_GXER_0V9 Voltage      : 0.91 Volts                 etc ......................                  Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.2.0                 Board Management Controller Build version: 3.2.0                 //****** TEMP ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                 ( 1) FPGA E-Tile Temperature [Remote] : 33.50 Celsius                 etc ......................                  Note: This feature is not supported on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)          7 - Check Accelerator Port status //****** PORT ******//                 Object Id                        : 0xED00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00 8 - Check MAC and PHY status Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.2.0                 Board Management Controller Build version: 3.2.0                 //****** MAC ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                 Number of MACs                   : 255                 mac info is not supported                  Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.2.0                 Board Management Controller Build version: 3.2.0                 //****** PHY ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                  //****** HSSI information ******//                 HSSI version                     : 1.0                 Number of ports                  : 8                 Port0                            :25GbE        DOWN                 Port1                            :25GbE        DOWN                 Port2                            :25GbE        DOWN                 Port3                            :25GbE        DOWN                 Port4                            :25GbE        DOWN                 Port5                            :25GbE        DOWN                 Port6                            :25GbE        DOWN                 Port7                            :25GbE        DOWN                  Note: This feature is not supported on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#313-ofs-pfvf-mux-menu","title":"3.1.3 OFS PF/VF MUX MENU","text":"
                                                      This menu reports the number of PF/VF functions in the reference example and also allows you to reduce the number to 1PF and 1VF to reduce resource utilisation and create a larger area for your workload development. This selection is optional and if the user wants to implement the default number of PF's and VF's then option 9, 10 and 11 should not be used.  Additionally the code used to make the PF/VF modification can be leveraged to increase or modify the number of PF/VFs in the existing design within the limits that the PCIe Subsystem supports (8PF/2K VFs)
                                                       
                                                       Menu Option Description 9 - Check PF/VF Mux Configuration  This menu selection displays the current configuration of all n6001 ofss files located in the following directory $OFS_ROOTDIR/tools/ofss_config               Check n6001 base config OFSS set up             [ip]             type = ofs              [settings]             platform = n6001             family = agilex             fim = base_x16             part = AGFB014R24A2E2V             device_id = 6001               Check n6001 hssi_2x100 OFSS set up             [ip]             type = hssi              [settings]             output_name = hssi_ss             num_channels = 2             data_rate = 100GCAUI-4               Check n6001 hssi_2x100_caui2 OFSS set up             [ip]             type = hssi              [settings]             output_name = hssi_ss             num_channels = 2             data_rate = 100GAUI-2               Check n6001 hssi_8x10 OFSS set up             [ip]             type = hssi              [settings]             output_name = hssi_ss             num_channels = 8             data_rate = 10GbE              Check n6001 hssi_8x25 OFSS set up             [ip]             type = hssi              [settings]             output_name = hssi_ss             num_channels = 8             data_rate = 25GbE               Check n6001 IOPLL OFSS set up             [ip]             type = iopll              [settings]             output_name = sys_pll             instance_name = iopll_0              [p_clk]             freq = 470              Check n6001 Memory OFSS set up             [ip]             type = memory              [settings]             output_name = mem_ss_fm             preset = n6001              Check n6001 PCIe Host OFSS set up             [ip]             type = pcie              [settings]             output_name = pcie_ss              [pf0]             num_vfs = 3             bar0_address_width = 20             vf_bar0_address_width = 20              [pf1]              [pf2]             bar0_address_width = 18              [pf3]              [pf4]              Check n6001 PCIe 1pf_1vf OFSS set up             [ip]             type = pcie              [settings]             output_name = pcie_ss              [pf0]             num_vfs = 1             bar0_address_width = 20             vf_bar0_address_width = 20   10 - Modify PF/VF Mux Configuration As an example this menu selection modifies the pcie_host.ofss file to 1 PF located in the following directory $OFS_ROOTDIR/tools/ofss_config             This option also displays the the modified pcie_host.ofss file 11 - Build PF/VF Mux Configuration If option 10 is not used then then the default number of PF's and VF's is used to build the FIM, if option 10 is selected then only 1 VF is built to reduce logic utilisation"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#314-ofs-fimpr-build-menu","title":"3.1.4 OFS FIM/PR BUILD MENU","text":"
                                                      Builds FIM, Partial Reconfiguration Region and Remote Signal Tap
                                                       
                                                       Menu Option Description 12 - Check OFS software version for OFS Project OFS_ROOTDIR is set to /home/user_area/ofs-X.X.X/ofs-agx7-pcie-attach                 OPAE_SDK_REPO_BRANCH is set to release/X.X.X                 OPAE_SDK_ROOT is set to /home/user_area/ofs-X.X.X/ofs-agx7-pcie-attach/../opae-sdk                 LD_LIBRARY_PATH is set to /home/user_area/ofs-X.X.X/ofs-agx7-pcie-attach/../opae-sdk/lib64: 13 - Build FIM for Hardware This option builds the FIM based on the setting for the $OFS_PLATFORM, $FIM_SHELL environment variable. Check this variable in the following file ofs-agx7-pcie-attach_eval.sh 14 - Check FIM Identification of FIM for Hardware The FIM is identified by the following file fme-ifc-id.txt located at $OFS_ROOTDIR/$FIM_WORKDIR/syn/board/$OFS_PLATFORM/syn_top/ 15 - Build Partial Reconfiguration Tree for Hardware This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development and also for the oneAPI build flow  16 - Build Base FIM Identification(ID) into PR Build Tree template This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU and oneAPI workloads 17 - Build Partial Reconfiguration Tree for Hardware with Remote Signal Tap This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development and also for the oneAPI build flow for the Remote Signal Tap flow 18 - Build Base FIM Identification(ID) into PR Build Tree template with Remote Signal Tap This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree for Remote Signal Tap to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU and oneAPI workloads"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#315-ofs-hardware-programmingdiagnostic-menu","title":"3.1.5 OFS HARDWARE PROGRAMMING/DIAGNOSTIC MENU","text":"
                                                      The following submenu allows you to: * Program and check flash  * Perform a remote system update (RSU) of the FPGA image into the FPGA * Bind virtual functions to VFIO PCIe driver  * Run host exerciser (HE) commands such as loopback to test interfaces VFIO PCI driver binding * Read the control and status registers (CSRs) for bound modules that are part of the OFS reference design.
                                                       
                                                       Menu Option Description 19 - Program BMC Image into Hardware The user must place a new BMC flash file in the following directory $OFS_ROOTDIR/bmc_flash_files. Once the user executes this option a new BMC image will be programmed. A remote system upgrade command is initiated to store the new BMC image 20 - Check Boot Area Flash Image from Hardware This option checks which location area in FLASH the image will boot from, the default is user1              Boot Page : user1              Note: This feature is not supported on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)          21 - Program FIM Image into user1 area for Hardware This option programs the FIM image \"ofs_top_page1_unsigned_user1.bin\" into user1 area in flash              Note: Please refer to the Getting Started Guide for details on flashing images for the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)          22 - Initiate Remote System Upgrade (RSU) from user1 Flash Image into Hardware This option initiates a Remote System Upgrade and soft reboots the server and re-scans the PCIe bus for the new image to be loaded              2022-11-10 11:26:24,307 - [[pci_address(0000:b1:00.0), pci_id(0x8086, 0xbcce)]] performing RSU operation             2022-11-10 11:26:24,310 - [[pci_address(0000:b0:02.0), pci_id(0x8086, 0x347a)]] removing device from PCIe bus             2022-11-10 11:26:24,357 - waiting 10 seconds for boot             2022-11-10 11:26:34,368 - rescanning PCIe bus: /sys/devices/pci0000:b0/pci_bus/0000:b0             2022-11-10 11:26:35,965 - RSU operation complete              Note: Please refer to the Getting Started Guide for details on flashing images for the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)          23 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 24 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 25 - Create Virtual Functions (VF) and bind driver to vfio-pci Hardware This option creates vfio-pci driver binding for the PF's and VF's             Once the VF's have been bound to the driver the user can select menu option 23 to check that the new drivers are bound 26 - Verify FME Interrupts with hello_events The hello_events utility is used to verify FME interrupts. This tool injects FME errors and waits for error interrupts, then clears the errors 27 - Run HE-LB Test This option runs 5 tests              1) checks and generates traffic with the intention of exercising the path from the AFU to the Host at full bandwidth             2) run a loopback throughput test using one cacheline per request             3) run a loopback read test using four cachelines per request             4) run a loopback write test using four cachelines per request             5) run a loopback throughput test using four cachelines per request 28 - Run HE-MEM Test This option runs 2 tests              1) Checking and generating traffic with the intention of exercising the path from FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host             2) run a loopback throughput test using one cacheline per request 29 - Run HE-HSSI Test This option runs 1 test              HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs, and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic              1) Send traffic through the 10G AFU              30 - Run Traffic Generator AFU Test This option runs 3 tests              TG AFU has an OPAE application to access & exercise traffic, targeting a specific bank              1) Run the preconfigured write/read traffic test on channel 0             2) Target channel 1 with a 1MB single-word write only test for 1000 iterations             3) Target channel 2 with 4MB write/read test of max burst length for 10 iterations 31 - Read from CSR (Command and Status Registers) for Hardware This option reads from the following CSR's             HE-LB Command and Status Register Default Definitions             HE-MEM Command and Status Register Default Definitions             HE-HSSI Command and Status Register Default Definitions"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#316-ofs-hardware-afu-testing-menu","title":"3.1.6  OFS HARDWARE AFU TESTING MENU","text":"
                                                      This submenu tests partial reconfiguration by building and loading an memory-mapped I/O example AFU/workload, executes software from host, and tests remote signal tap.
                                                       
                                                       Menu Option Description 32 - Build and Compile host_chan_mmio example This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS (Green Bit Stream) ready for hardware programming 33 - Execute host_chan_mmio example This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test 34 - Modify host_chan_mmio example to insert Remote Signal Tap This option inserts a pre-defined host_chan_mmio.stp Signal Tap file into the OFS code to allow a user to debug the host_chan_mmio AFU example 35 - Build and Compile host_chan_mmio example with Remote Signal Tap This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS(Green Bit Stream) ready for hardware programming with Remote Signal tap enabled 36 - Execute host_chan_mmio example with Remote Signal Tap This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test. The user must open the Signal Tap window when running the host code to see the transactions in the Signal Tap window"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#317-ofs-hardware-afu-bbb-testing-menu","title":"3.1.7 OFS HARDWARE AFU BBB TESTING MENU","text":"
                                                      This submenu tests partial reconfiguration using a hello_world example AFU/workload, executes sw from host
                                                       
                                                       Menu Option Description 37 - Build and Compile hello_world example This option builds the hello_ world example from the following repo $FPGA_BBB_CCI_SRC/samples/tutorial/afu_types/01_pim_ifc/$AFU_BBB_TEST_NAME, where AFU_BBB_NAME=hello_world. This produces a GBS(Green Bit Stream) ready for hardware programming 38 - Execute hello_world example This option builds the host code for hello_world example and programs the GBS file and then executes the test"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#318-ofs-oneapi-project-menu","title":"3.1.8 OFS ONEAPI PROJECT MENU","text":"
                                                      Builds oneAPI kernel, executes sw from host and runs diagnostic tests
                                                       
                                                       Menu Option Result 39 - Check oneAPI software versions for Project This option checks the setup of the oneAPI software and adds the relevant oneAPI environment variables to the terminal. This option also informs the user to match the oneAPI software version to the oneAPI-samples version 40 - Build and clone shim libraries required by oneAPI host This option builds the oneAPI directory structure 41 - Install oneAPI Host Driver This option Installs the oneAPI Host driver at the following location /opt/Intel/OpenCLFPGA/oneAPI/Boards/, and requires sudo permission 42 - Uninstall oneAPI Host Driver This option Uninstall's the oneAPI Host driver, and requires sudo permissions 43 - Diagnose oneAPI Hardware This option Checks ICD (Intel Client Driver) and FCD (FPGA Client Driver), oneAPI library locations and detects whether oneAPI BSP is loaded into the FPGA 44 - Build oneAPI BSP Default Kernel (hello_world) This option Builds the oneAPI BSP using simple-add_buffers kernel 45 - Build oneAPI MakeFile Environment This option Builds the oneAPI environment using a Makefile for kernel insertion  46 - Compile oneAPI Sample Application (board_test) for Emulation This option compiles the board_test kernel for Emulation 47 - Run oneAPI Sample Application (board_test) for Emulation This option executes the board_test kernel for Emulation 48 - Generate oneAPI Optimization report for (board_test) This option generates an optimization report for the board_test kernel 49 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 50 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 51 - Create Virtual Function (VF) and bind driver to vfio-pci Hardware This option creates vfio-pci driver binding for the PF's and VF's             Once the VF's have been bound to the driver the user can select menu option 45 to check that the new drivers are bound 52 - Program OpenCL BSP Default Kernel (hello_world) This option programs the FPGA with a aocf file based on the hello_world kernel 53 - Compile oneAPI Sample Application (board_test) for Hardware This option compiles the board_test kernel for Hardware 54 - Run oneAPI Sample Application (board_test) for Hardware This option builds the host code for board_test kernel and executes the program running through kernel and host bandwidth tests"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#319-ofs-unit-test-project-menu","title":"3.1.9 OFS UNIT TEST PROJECT MENU","text":"
                                                      Builds, compiles and runs standalone simulation block tests.  More unit test examples are found at the following location ofs_n6001/sim/unit_test 
                                                       
                                                       Menu Option Result 55 - Generate Simulation files for Unit Test This option builds the simulation file set for running a unit test simulation 56 - Simulate Unit Test dfh_walker and log waveform This option runs the dfh_walker based on the environment variable \"UNIT_TEST_NAME=dfh_walker\" in the evaluation script. A user can change the test being run by modifying this variable"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#3110-ofs-uvm-project-menu","title":"3.1.10 OFS UVM PROJECT MENU","text":"
                                                      Builds, compiles and runs full chip simulation tests.  The user should execute the options sequentially ie 68,69, 70 and 71
                                                       
                                                       Menu Option Description 57 - Check UVM software versions for Project DESIGNWARE_HOME is set to /home/synopsys/vip_common/vip_Q-2020.03A                 UVM_HOME  is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm                 VCS_HOME is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel                 VERDIR is set to /home/user_area/ofs-X.X.X/ofs-agx7-pcie-attach/verification                 VIPDIR is set to /home/user_area/ofs-X.X.X/ofs-agx7-pcie-attach/verification 58 - Compile UVM IP This option compiles the UVM IP  59 - Compile UVM RTL and Testbench This option compiles the UVM RTL and Testbench 60 - Simulate UVM dfh_walking_test and log waveform This option runs the dfh_walking test based on the environment variable \"UVM_TEST_NAME=dfh_walking_test\" in the evaluation script. A user can change the test being run by modifying this variable 61 - Simulate all UVM test cases (Regression Mode) This option runs the n6001 regression mode, cycling through all UVM tests defined in verification/tests/test_pkg.svh file"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#3111-ofs-build-all-project-menu","title":"3.1.11 OFS BUILD ALL PROJECT MENU","text":"
                                                      Builds the complete OFS flow, good for regression testing and overnight builds
                                                       
                                                      For this menu a user can run a sequence of tests (compilation, build and simulation) and executes them sequentially. After the script is successfully executed, a set of binary files is produced which a you can use to evaluate your hardware. Log files are also produced which checks whether the tests passed.
                                                       
                                                      A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 62 from the main menu the script will execute 24 tests ie (main menu options 2, 9, 12, 13, 14, 15, 16, 17, 18, 32, 34, 35, 37, 39, 40, 44, 45, 53, 55, 56, 57, 58, 59 and 60. These 24 menu options are chosen to build the complete OFS flow covering build, compile and simulation.
                                                       
                                                       Menu Option Result 62 - Build and Simulate Complete n6001 Project Generating Log File with date and timestamp                 Log file written to /home/guest/ofs-2.3.1/log_files/ofs-agx7-pcie-attach_log_2022_11_10-093649/ofs-agx7-pcie-attach_eval.log"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#definition-of-multi-test-set-up","title":"Definition of Multi-Test Set-up","text":"
                                                      Menu Option 62 above in the evaluation script can be refined to tailor it to the users need and is principally defined by the variable below
                                                       
                                                      MULTI_TEST[A,B]=C
                                                       
                                                      where
                                                       
                                                      A= Total Number of menu options in script B= Can be changed to a number to select the test order C= Menu Option in Script
                                                       
                                                      Example 1 MULTI_TEST[62,0]=2
                                                       
                                                      A= 62 is the total number of options in the script B= 0 indicates that this is the first test to be run in the script C= Menu option in Script ie 2- List of Documentation for OFS n6001 Project
                                                       
                                                      Example 2 MULTI_TEST[62,0]=2 MULTI_TEST[62,1]=9
                                                       
                                                      In the example above two tests are run in order ie 0, and 1 and the following menu options are executed ie 2- List of Documentation for OFS n6001 Project and 9 - Check OFS software versions for OFS n6001 Project
                                                       
                                                      The user can also modify the build time by de-selecting options they do not wish to use, see below for a couple of use-case scenarios.
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#default-user-case","title":"Default User Case","text":"
                                                      A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 62 from the main menu the script will execute 24 tests ie (main menu options 2, 9, 12, 13, 14, 15, 16, 17, 18, 32, 34, 35, 37, 39, 40, 44, 45, 53, 55, 56, 57, 58, 59 and 60. All other tests with an \"X\" indicates do not run that test.
                                                       
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#user-case-for-ofs-fimpr-build-menu","title":"User Case for OFS FIM/PR BUILD MENU","text":"
                                                      In the example below when the user selects option 62 from the main menu the script will only run options from the OFS FIM/PR BUILD MENU (7 options, main menu options 12, 13, 14, 15, 16, 17 and 18). All other tests with an \"X\" indicates do not run that test.
                                                       
                                                       
                                                      "},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#4-n6001-common-test-scenarios","title":"4 n6001 Common Test Scenarios","text":"
                                                      This section will describe the most common compile build scenarios if a user wanted to evaluate an acceleration card on their server. The Pre-requisite column indcates the menu comamnds that must be run befere executing the test eg To run Test 5 then a user needs to have run option 13, 15 and 16 before running options 23, 24, 25, 32 and 33.
                                                       Test Test Scenario Pre-Requisite Menu Option Menu Option Test 1 FIM Build - 13 Test 2 Partial Reconfiguration Build 13 15, 16 Test 3 Program FIM and perform Remote System Upgrade 13 21, 22 Test 4 Bind PF and VF to vfio-pci drivers - 23, 24, 25 Test 5 Build, compile and test AFU on hardware 13, 15, 16 23, 24, 25, 32, 33 Test 6 Build, compile and test AFU Basic Building Blocks on hardware 13, 15, 16 23, 24, 25, 37, 38 Test 7 Build, compile and test oneAPI on hardware 13, 15, 16 39, 40, 41, 44, 45, 49, 50, 51, 52, 53, 54 Test 8 Build and Simulate Unit Tests - 55, 56 Test 9 Build and Simulate UVM Tests - 57, 58, 59, 60 Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/common/user_guides/ug_eval_script_ofs_agx7_pcie_attach/ug_eval_script_ofs_agx7_pcie_attach/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                      Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                       
                                                      OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                      "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/","title":"FPGA Developer Journey Guide: Open FPGA Stack","text":"
                                                      Last updated: February 03, 2024 
                                                      "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#1-introduction","title":"1 Introduction","text":"
                                                      This document is intended to help you understand the FPGA developer flow using OFS as well as considerations you should take when creating your custom platform.  
                                                       
                                                      After reviewing the document, you shall be able to:
                                                       
                                                         
                                                      • Understand how to evaluate the OFS framework for your platform needs.
                                                        •  
                                                      • Select a starting shell or FIM to begin your work.
                                                        •  
                                                      • Understand what resources are available for guiding you through making modifications for custom design as well as simulation and debug.
                                                        •  
                                                       
                                                      The general development flow is depicted in the diagram below and discussed in more detail in each section of this document.
                                                       
                                                      Figure 1-1: FPGA Developer Flow
                                                       
                                                      "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#2-evaluate-ofs","title":"2 Evaluate OFS","text":"
                                                      The repositories in the OFS site are tagged based on year and release number.  For example, a tag of 2023.3-2-1 indicates the first release package of the quarter.  If there an updates to this release package, the last number will be incremented by 1, such as 2023.3-2-2.
                                                       
                                                      By clicking on the release link to the right of the RTL repositories, you will find the latest release, the tag number and release notes.  
                                                       
                                                      Figure 2-1: OFS Repository Release Page Link
                                                       
                                                       
                                                      By scrolling to the end of the release page, you will find assets attached to the release that you can leverage for quick evaluation of OFS, such as FPGA binary files, POF and SOF as well as a sample AFU and a partial reconfiguration directory for building your own custom workload.
                                                       
                                                      There are two ways to evaluate OFS depending on your needs:
                                                       
                                                      Option 1: Setup your card and software in a server using the steps provided in one of the corresponding Getting Started Guides and leverage the appended binaries in the FIM RTL repository release page \"Assets\" tab to preview the software and design functionality the OFS framework provides you out of the box.  This step will give you a good high-level overview of OFS. Getting Started Guides are available for the following FIM(shell) designs:
                                                       
                                                         
                                                      • Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL)
                                                        •  
                                                      • Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile))
                                                        •  
                                                      • Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 SoC Attach FPGAs
                                                        •  
                                                      • Getting Started Guide: Open FPGA Stack for Intel\u00ae Stratix\u00ae 10 PCIe Attach FPGAs
                                                        •  
                                                       
                                                      Option 2: After your card and software are installed using the steps provided in one of the corresponding Getting Started Guides listed above, use a corresponding Evaluation Guide and provided evaluation script to run through all the capabilities of the OFS framework by selecting one of the choices in the evaluation menu.  The evaluation script gives you familiarity of the entire design, build, simulation, programming and test flow for OFS, including a OneAPI flow.
                                                       
                                                         
                                                      • Evaluation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach
                                                        •  
                                                      • Evaluation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach
                                                        •  
                                                      • Evaluation User Guide: OFS for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach
                                                        •  
                                                       
                                                      The scripts that go with each user guide are found in the assets tab of the corresponding FIM RTL repository's latest tag release page.
                                                       
                                                      To use the full functionality of the script you will want to ensure you clone all of the appropriate repositories below at the same level just under an OFS directory you create, such as ofs-2023.3-2, similar to the figure below. 
                                                       
                                                      Figure 2-2: Directory Structure of Cloned Repositories
                                                       
                                                      ##|-- ofs-2023.3-2\n##|    |--examples-afu\n##|    |--linux-dfl\n##|    |--ofs-agx7-pcie-attach\n##|    |--ofs-oneapi-asp\n##|    |--opae-sdk\n##|    |--opae-sim\n##|    |--ofs-agx7-pcie-attach_eval.sh\n
                                                       
                                                      You can access the repositories from ofs.github.io by clicking on the GitHub icon in the right corner of the site.
                                                       
                                                      Figure 2-3: OFS Site Link from ofs.github.io
                                                       
                                                       
                                                      After making your top level directory, initializing the repository and installing git lfs, clone one of the FIM RTL repositories you intend to use:
                                                       
                                                      #Make top level directory\nmkdir OFS\ncd OFS\n\n#Initialize repository and install git lfs\ngit init\n\ncurl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\nsudo dnf install git-lfs\ngit lfs install\n\n#Select a FIM RTL repository to clone\n#To clone Intel Agilex 7 PCIe Attach FIM RTL repository\n\ngit clone --recurse-submodules https://github.com/OFS/ofs-agx7-pcie-attach.git\ncd ofs-agx7-pcie-attach\ngit checkout --recurse-submodules tags/ofs-2023.3-2\n\n\n#To clone Intel Agilex 7 SoC Attach FIM RTL Repository\ngit clone --recurse-submodules https://github.com/OFS/ofs-f2000x-pl.git\ncd ofs-f2000x-pl\ngit checkout --recurse-submodules tags/ofs-2023.3-1\n\n#To clone Intel Stratix 10 PCIe Attach FIM RTL Repository\ngit clone --recurse-submodules https://github.com/OFS/ofs-d5005.git\ncd ofs-d5005\ngit checkout --recurse-submodules tags/ofs-2023.3-1\n
                                                       
                                                      After cloning your FIM RTL repository, clone the other necessary repositories:
                                                       
                                                      #All other repositories below should be cloned to use the evaluation script:\n\ngit clone https://github.com/OFS/linux-dfl\ncd /home/OFS/linux-dfl\ngit checkout tags/ofs-2023.3-6.1-3\n\n#Use this OPAE clone command for all PCIe Attach Cards\ngit clone https://github.com/OFS/opae-sdk\ncd /home/OFS/opae-sdk\ngit checkout tags/2.10.0-1\n\n#Use this OPAE clone command for the SoC Attach Card\ngit clone https://github.com/OFS/opae-sdk\ncd /home/OFS/opae-sdk\ngit checkout tags/2.10.0-1\n\ngit clone https://github.com/OFS/oneapi-asp.git \ncd /home/OFS/oneapi-asp\ngit checkout tags/ofs-2023.3-2\n\ngit clone https://github.com/OFS/ofs-platform-afu-bbb.git\ncd /home/OFS/ofs-platform-afu-bbb\ngit checkout tags/ofs-2023.2-1\n\ngit clone https://github.com/OFS/opae-sim.git\ncd /home/OFS/opae-sim\ngit checkout tags/2.10.0-1\n\ngit clone https://github.com/OFS/examples-afu.git\ncd /home/OFS/examples-afu\ngit checkout tags/ofs-2023.2-1\n
                                                       
                                                      You will also want to ensure you install the correct version of Intel Quartus Prime Pro as directed in the release notes in addition to any Quartus patches.  Note that Quartus Prime Pro software can be downloaded from the downloads tab on intel.com.  Quartus Prime Pro patches required are attached to the assets tab at the bottom of the tagged RTL repository release page. Simulator tools as listed corresponding Simulation User Guides:
                                                       
                                                         
                                                      • Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach
                                                        •  
                                                      • Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach
                                                        •  
                                                      • Simulation User Guide: OFS for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach
                                                        •  
                                                      "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#3-select-a-starting-shellfim","title":"3 Select a Starting Shell/FIM","text":"
                                                      To begin your development, start with an existing shell that most closely matches your device and end solution.  The OFS site has both Intel Intel Agilex 7 and Stratix 10 FPGA reference designs you can use as a starting point for your own design. These designs can be ported to different device OPNs if a different resource utilization is required.   
                                                       
                                                      To begin you may want to learn more about Intel Stratix 10 and Intel Agilex 7 family offerings.  Please refer to the following links:
                                                       
                                                         
                                                      • Intel Agilex 7 Product Page 
                                                        •  
                                                      • Intel Stratix 10 Product Page
                                                        •  
                                                       
                                                      Note that each reference design provides an integrated shell, called the FPGA Interface Manager (FIM), which is encompasses in blue in the diagrams below.  This shell provides standard AXI interfaces to the Accelerator Functional Unit (AFU) region, shown in green, which depicts a region where a workload or partial reconfiguration slot resides.  The regions are not drawn to scale.  The figures and tables below give an overview of the available starting points for your design.
                                                       
                                                      Figure 3-1: OFS FIM Targeting Intel\u00ae Agilex\u00ae7 PCIe Attach (P-tile, E-tile)
                                                       
                                                       Key Feature Description Target OPN AGFB014R24A2E2V PCIe P-tile PCIe* Gen4x16 Virtualization 5 physical functions/3 virtual functions with ability to expand Memory 5 DDR Channels:* One HPS DDR4 bank, x40 (x32 Data and x8 ECC), 1200 MHz, 1GB each* Four Fabric DDR4 banks, x32 (no ECC), 1200 MHz, 4GB Ethernet 2x4x25GbE, 2x4x10GbE or 2x100GbE Hard Processor System 64-bit quad core Arm\u00ae Cortex\u00ae-A53 MPCore with integrated peripherals. Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration)* Platform Controller Management Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported OneAPI OneAPI Acceleration Support Package (ASP) provided with compiled FIM to support OneAPI Runtime Software Support * Linux DFL drivers targeting OFS FIMs* OPAE Software Development Kit* OPAE Tools Target Board Intel\u00ae FPGA SmartNIC N6001-PL 
                                                      Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae PCIe Attach Reference FIM documentation collection.
                                                       
                                                       
                                                      Figure 3-2: OFS FIM Targeting Intel\u00ae Agilex\u00ae7 PCIe Attach (2xF-tile)
                                                       
                                                       Key Feature Description Target OPN AGFB027R24C2E2VR2 PCIe P-tile PCIe* Gen4x16 (currently downtrains to Gen4x8 in the ES version of the development kit) Virtualization 5 physical functions/3 virtual functions with ability to expand Memory 3 DDR Channels:* One HPS DDR4 bank, x40 (x32 Data and x8 ECC), 2400 MHz, 1GB each* Two Fabric DDR4 banks, x64 (no ECC), 2400 MHz, 8GB Ethernet 2x4x25GbE Hard Processor System 64-bit quad core Arm\u00ae Cortex\u00ae-A53 MPCore with integrated peripherals. Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration)* Platform Controller Management Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported OneAPI OneAPI Acceleration Support Package (ASP) provided with compiled FIM to support OneAPI Runtime Software Support * Linux DFL drivers targeting OFS FIMs* OPAE Software Development Kit* OPAE Tools Target Board Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) 
                                                      Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae PCIe Attach Reference FIM documentation collection.
                                                       
                                                       
                                                      Figure 3-3: OFS FIM Features Targeting Intel\u00ae Agilex\u00ae 7 SoC Attach
                                                       
                                                       Key Feature Description Device OPN AGFC023R25A2E2VR0 PCIe P-tile PCIe* Gen4x16 to the HostP-tile PCIe* Gen4x16 to the SoC (IceLake-D) Virtualization Host: 2 physical functions SoC:   1 physical function and 3 Virtual functions Memory Four Fabric DDR4 banks, x40 (optional ECC, be configured as x32 and ECC x8 ), 1200 MHz, 4GB Ethernet 2x4x25GbE Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration) * Platform Controller Management Interface (PMCI) Module for Board Management Controller Partial Reconfiguration Supported Software Support * Linux DFL drivers targeting OFS FIMs * OPAE Software Development Kit * OPAE Tools Target Board Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL 
                                                      Note: Source code for BMC RTL/Firmware that works with this reference FIM can be obtained by contacting your Intel Sales Representative.
                                                       
                                                      Click here for the OFS Collateral for Intel\u00ae Agilex\u00ae SoC Attach Reference FIM documentation collection.
                                                       
                                                       
                                                      Figure 3-4: OFS FIM Targeting Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach
                                                       
                                                       Key Feature Intel Stratix 10 Reference FIM Device OPN 1SX280HN2F43E2VG Ethernet Configuration 1x10GbE example with 2x100GbE capability PCIe Gen3x16 EMIF Up to four DDR channels PF/VF 1 PF/3 VFs Management FPGA Management Engine (FME) with FIM management registers Interface Arm\u00ae AMBA\u00ae4 AXI Interface HLD support oneAPI Software Kernel code upstreamed to Linux.org Target Board Intel\u00ae FPGA Programmable Acceleration Card D5005 
                                                      Click here for the OFS Collateral for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach Reference FIM documentation.
                                                      "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#4-review-release-notes","title":"4 Review Release Notes","text":"
                                                      Before beginning your design, read the release notes for each repository you are using by selecting the appropriate release tag found by clicking on the \"Release\" link to the right of the corresponding repository.  The release page may also have assets appended such as useful binaries, patches or scripts.
                                                       
                                                       
                                                       
                                                      "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#5-setup-your-environment-for-development","title":"5 Setup Your Environment For Development","text":"
                                                      When you are ready to begin development you will want to ensure you have any other setup requirements satisfied by reviewing instructions in the corresponding FIM Developer Guide and if you are implementing a OneAPI Board Support Package, the oneAPI ASP Getting Started User Guide as well.
                                                       
                                                         
                                                      • FPGA Interface Manager (FIM) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach (2xF-tile)     * Can be used with Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)
                                                        •  
                                                      • FPGA Interface Manager (FIM)  Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach (P-tile, E-tile)     * Can be used with Intel FPGA SmartNIC N6001-PL
                                                        •  
                                                      • FPGA Interface Manager (FIM) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach     * Can be used with Intel Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL 
                                                        •  
                                                      • FPGA Interface Manager (FIM) Developer Guide: OFS for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach     * Can be used with Intel FPGA PAC D5005
                                                        •  
                                                       
                                                      For oneAPI setup: 
                                                       
                                                         
                                                      • oneAPI Accelerator Support Package (ASP): Getting Started User Guide     * Can be used with:             - Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)             - Intel FPGA SmartNIC N6001-PL
                                                        •  
                                                      "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#6-customize-your-shell-build-and-debug","title":"6 Customize Your Shell, Build and Debug","text":"
                                                      For your own custom design, you may need to:
                                                       
                                                         
                                                      • Target the OFS reference design to a different device
                                                        •  
                                                      • Modify the pin assignments
                                                        •  
                                                      • Modify or remove existing peripheral IP
                                                        •  
                                                      • Add new interface IP
                                                        •  
                                                      • Repartition the partial reconfiguration region 
                                                        •  
                                                       
                                                      The FIM Developer Guides for each reference FIM show you how to make these changes and provide steps on how to run unit simulation or add SignalTap to your design for debugging.
                                                       
                                                      If you are also interested in testing different examples for the Acceleration Functional Unit (AFU) or workload region of your design or creating your own AFU design, you can refer to the corresponding AFU Developer Guides:
                                                       
                                                         
                                                      • Accelerator Functional Unit (AFU) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach
                                                        •  
                                                      • Accelerator Functional Unit (AFU) Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach
                                                        •  
                                                      • [Accelerator Functional Unit (AFU) Developer Guide: OFS for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach]
                                                        •  
                                                      "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#7-simulate-and-debug","title":"7 Simulate and Debug","text":"
                                                      Setup and test files to perform system-level Universal Verification Methodology (UVM) testing are provided in each FIM RTL repository.  Please refer to the corresponding Simulation User Guide for details on test bench architecture, setup and testing.
                                                       
                                                         
                                                      • Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach
                                                        •  
                                                      • Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae 7 SoC Attach
                                                        •  
                                                      • Simulation User Guide: OFS for Intel\u00ae Stratix\u00ae 10 FPGA PCIe Attach
                                                        •  
                                                      "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#8-optional-build-oneapi-accelerator-support-package-asp","title":"8 Optional: Build OneAPI Accelerator Support Package (ASP)","text":"
                                                      If you are considering providing oneAPI support for your custom board design, you must integrate the oneAPI ASP hardware and software components into your design.  Reference the following documents to learn about the architecture and implementation flow for the oneAPI ASP with an OFS design.
                                                       
                                                         
                                                      • oneAPI Accelerator Support Package (ASP): Getting Started User Guide
                                                        •  
                                                      • oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack
                                                        •  
                                                      "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#9-optional-driver-or-ofs-software-tool-modifications","title":"9 Optional: Driver or OFS Software Tool Modifications","text":"
                                                      As you add or remove interfaces to your custom design, you may need to modify or enhance existing drivers that accompany the OFS reference design.  Additionally, you may decide you want to create additional utilities or plugins leveraging the OFS software infrastructure.  In this case, refer to the OFS Software tab to learn more about the underlying driver and software architecture and how to make modifications.
                                                       
                                                      Additionally, for guidance on using a Kernel-based Virtual Machine with OFS, refer to our KVM User Guide. 
                                                       
                                                      KVM User Guide: Open FPGA Stack
                                                      "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#10-test-and-deploy","title":"10 Test and Deploy","text":"
                                                      When testing and deploying your platform you are encouraged to modify and tailor the evaluation scripts you used in Section 2 to assist in automating your continuous integration flow.  You may also want to refer to our Docker User Guide to understand how to use Docker for Development and Deployment.
                                                       
                                                         
                                                      • Docker User Guide: Intel Open FPGA Stack
                                                        •  
                                                      "},{"location":"hw/common/user_guides/ug_fpga_developer/ug_fpga_developer/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                      Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                       
                                                      OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/","title":"Virtual machine User Guide: Open FPGA Stack + KVM","text":"
                                                      Last updated: February 03, 2024 
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#document-scope","title":"Document scope","text":"
                                                      The document describes setting up and configuring a virtual machine to use PCIe devices. Here are the steps that the document may include:
                                                       
                                                         
                                                      1. Install the necessary tools, such as virt-manager, on the host machine. This may involve downloading and installing the software from the internet.
                                                        2.  
                                                      2. Enable the virtualization feature on the host machine. This may involve going into the BIOS settings and enabling hardware-assisted virtualization or using a command-line tool to enable it in the operating system.
                                                        3.  
                                                      3. Use virt-manager to create a new virtual machine and configure its settings. This may involve choosing a name and operating system for the virtual machine and setting the amount of memory and storage it will use.
                                                        4.  
                                                      4. Install the OPAE (Open Programmable Acceleration Engine) tool on the virtual machine. This may involve downloading and installing the OPAE software.
                                                        5.  
                                                      5. Install the DFL (Data Field Level) drivers on the virtual machine. These drivers allow the virtual machine to access and use the PCIe devices on the host machine. This may involve downloading and installing the drivers from the internet.
                                                        6.  
                                                      6. Once all of the steps have been completed, you should be able to use the virtual machine to access and use the PCIe devices on the host machine. You may need to configure the virtual machine's settings to enable it to use the PCIe devices, such as by assigning a specific device to the virtual machine.
                                                        7.  
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#1-modes-of-operation","title":"1. Modes of Operation","text":"
                                                      Our current operational framework stipulates two distinct modes of operation for PF/VF configurations. When using a 2 PF enabled FIM design, both the workload and management ports can be interchangeably passed through to a VM or run on bare-metal.
                                                       
                                                         
                                                      1.  
                                                        Management Mode: This mode necessitates the passthrough of only the FME device (use fpgainfo fme to discover your port number, normally .0). The reason for this is that the Open FPGA Stack (OFS) depends on this address for management. Under this mode, the use of the exerciser and virtual functions is not feasible.
                                                         
                                                        2.  
                                                      2.  
                                                        Virtual Function Mode: This mode comes into effect when a user needs to utilize the Virtual Functions (VF). The user will convert (example) Physical Function 0 (PF0) to three Virtual Functions (VF). This means the PF will cease to function for management purposes. Once the VFs are set up, they essentially take over the role of the PF in communicating with the Virtual Machines (VMs).
                                                         
                                                        However, this mode is subject to a limitation. If the user needs to execute 'fpgainfo fme' or 'fpgaupdate', they will need to transition from Virtual Function Mode to Management Mode. Conversely, if the user intends to utilize the Virtual Functions, they would need to switch from Management Mode to Virtual Function Mode. It is imperative to bear this limitation in mind when operating within these modes.
                                                         
                                                        3.  
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#2-enable-virtualization","title":"2. Enable Virtualization","text":"
                                                      To check if virtualization is enabled on a Red Hat system using lscpu and grep, you can use the following command:
                                                       
                                                      lscpu -e | grep Virtualization\n
                                                       
                                                      This command will run lscpu with the -e or --extended option, which displays information about the CPU and its available virtualization capabilities. Then, it pipes the output to grep with the search pattern \"Virtualization\". If the system has support for virtualization, the output will show the \"Virtualization\" field and its value, for example:
                                                       
                                                      Virtualization: VT-x\n
                                                       
                                                      In this example, the output shows that the system supports Intel VT-x virtualization technology. If the \"Virtualization\" field is empty, the system does not have support for virtualization. Keep in mind that even if the system has support for virtualization, it may not be enabled in the BIOS or the operating system itself. 
                                                       
                                                      Check the following for the bios configuration, Enabling Intel VT-d Technology
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#3-verify-environment-setup","title":"3. Verify Environment Setup","text":"
                                                         
                                                      1. Open a terminal window and log in as a user with sudo privileges.
                                                        2.  
                                                      2. Check if the virtualization kernel modules are loaded by running the following command:
                                                        3.  
                                                       
                                                      lsmod | grep kvm\n
                                                       
                                                         
                                                      1.  
                                                        If the command outputs a list of modules, the virtualization kernel modules are loaded, and virtualization is enabled on your system.
                                                         
                                                        2.  
                                                      2.  
                                                        The virtualization kernel modules are not loaded if the command does not output anything. You can try loading them manually by running the following command:
                                                         
                                                        3.  
                                                       
                                                      sudo modprobe kvm\n
                                                       
                                                         
                                                      1. If the kernel modules are not loaded, and you cannot load them manually, it may be because virtualization is not supported or enabled in your system's BIOS or UEFI settings. You must reboot your system and enter the BIOS or UEFI settings menu to enable virtualization. The exact steps for doing this may vary depending on your system's hardware and BIOS/UEFI version, so consult your motherboard or system documentation for specific instructions.
                                                        2.  
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#4-install-virtual-machine-manager","title":"4. Install Virtual Machine Manager","text":"
                                                      Virtual Machine Manager (also known as libvirt) can be installed by following the below steps:
                                                       
                                                         
                                                      1. Open a terminal window and log in as a user with sudo privileges.
                                                        2.  
                                                      2.  
                                                        Update your system package index by running the following command:
                                                         
                                                           
                                                        • Redhat
                                                          •  
                                                         
                                                        sudo dnf update\n
                                                          * Ubuntu  
                                                        sudo apt update\n
                                                         
                                                        3.  
                                                      3.  
                                                        Install the libvirt package and any required dependencies by running the following command:
                                                         
                                                           
                                                        • Redhat 
                                                          •  
                                                         
                                                        sudo dnf install @virtualization\n
                                                         
                                                           
                                                        • Ubuntu
                                                          •  
                                                         
                                                        sudo apt install qemu-kvm libvirt-bin bridge-utils virt-manager\n
                                                         
                                                        4.  
                                                      4.  
                                                        Start the libvirtd service and enable it to start automatically at boot time by running the following commands:
                                                         
                                                        5.  
                                                       
                                                      sudo systemctl start libvirtd\nsudo systemctl enable libvirtd\n
                                                       
                                                         
                                                      1. Optional: Install the virt-manager package, which provides a GUI application for managing virtual machines, by running the following command:
                                                        2.  
                                                       
                                                      sudo dnf install virt-manager\n
                                                       
                                                         
                                                      1. Optional: If you want to be able to run virtual machines as a non-root user, add your user to the libvirt group by running the following command, replacing \"USERNAME\" with your username:
                                                        2.  
                                                       
                                                      sudo usermod -a -G libvirt USERNAME\n
                                                       
                                                         
                                                      1. You can now launch virt-manager by running the command virt-manager as the non-root user.
                                                        2.  
                                                       
                                                      Note: By default, virt-manager will only allow non-root users to create and manage virtual machines with limited resources, such as a limited amount of memory and CPU cores. To allow non-root users to create and manage virtual machines with more resources, you need to edit the /etc/libvirt/qemu.conf configuration file and set the user and group values for the dynamic_ownership option to 1. For example:
                                                       
                                                      # Set user and group ownership of dynamic /dev/kvm device nodes\ndynamic_ownership = 1\nuser = \"root\"\ngroup = \"root\"\n
                                                       
                                                      You will also need to restart the libvirtd service for the changes to take effect. You can do this by running the command. 
                                                       
                                                      sudo systemctl restart libvirtd\n
                                                       
                                                         
                                                      1. Reboot your server to apply the changes 
                                                        2.  
                                                       
                                                      reboot\n
                                                       
                                                      After completing these steps, you should be able to use the virt-manager GUI application to manage virtual machines on your system. 
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#5-create-a-vm-using-virt-manager","title":"5. Create a VM Using Virt-Manager","text":"
                                                      Before creating the virtual machine, ensure the DFL drivers are installed in your host machine; the instructions are located here, OFS Site select your desired platform and select Getting stated guide.
                                                       
                                                      To create a Red Hat 8.2 or Ubuntu 22.04  virtual machine (VM) using virt-manager and share PCI devices with the VM, you will need to perform the following steps:
                                                       
                                                         
                                                      1. Start the virt-manager GUI by running the following command:
                                                        2.  
                                                       
                                                      sudo virt-manager&\n
                                                       
                                                       
                                                         
                                                      1.  
                                                        Create a new connection from the menu File-> \"Add Connection,\" Use the default options and click \"Connect.\"
                                                         
                                                         
                                                         
                                                        2.  
                                                      2.  
                                                        In the virt-manager window, click the \"New virtual machine\" button.
                                                         
                                                         
                                                        3.  
                                                      3.  
                                                        In the \"New VM\" wizard, select \"Local install media (ISO image or CDROM)\" as the installation source, and then click \"Forward.\"
                                                         
                                                         
                                                           
                                                        •  
                                                          Get the Red Hat image from the following link.
                                                           
                                                          https://developers.redhat.com/content-gateway/file/rhel-8.2-x86_64-dvd.iso
                                                           
                                                          •  
                                                         
                                                           
                                                        •  
                                                          Get the Ubuntu image from the following link.
                                                           
                                                          https://releases.ubuntu.com/22.04/ubuntu-22.04.1-desktop-amd64.iso
                                                           
                                                          •  
                                                         
                                                        4.  
                                                      4.  
                                                        In the next step, Click Browse -> Browse local, select the Red Hat 8.2 ISO image as the installation source and click \"Forward\".
                                                         
                                                         
                                                         
                                                        Note: if the system is not detected, disable \"Automatic detected from the installation media/source\" and type ubuntu and select 19.10 (this should be fine for the 22.04); this step is necessary to copy the default values for the specific OS
                                                         
                                                         
                                                        5.  
                                                      5.  
                                                        In the next step, specify a name and location for the VM, and select the desired memory and CPU configuration. in our case, 16 cores and 64 GB of RAM; Click \"Forward\" to continue.
                                                         
                                                         
                                                        6.  
                                                      6.  
                                                        Select \"enable storage for this virtual machine,\" Select \"Create a new disk for the virtual machine,\" and enter a size for the virtual disk (at least 200~300GB in case you need to compile the design) or create a custom storage.
                                                         
                                                         
                                                           
                                                        1.  
                                                          If you need to create custom storage, select \"Select or Create custom storage\" and click \"Manage.\"
                                                           
                                                           
                                                          2.  
                                                        2.  
                                                          Click on the \"+\" icon (Bottom left) to create the storage pool.
                                                           
                                                           
                                                          3.  
                                                        3.  
                                                          In the \"Create a new storage pool\" dialog, enter a name for the storage pool and select the type of storage pool you want to create; select the Target Path and Click \"Finish.\"
                                                           
                                                           
                                                          4.  
                                                        4.  
                                                          Select the pool and later click on the \"+\" icon (The Icon is on the right side of the Volume label) to create the New Storage Volume.
                                                           
                                                           
                                                          5.  
                                                        5.  
                                                          In the \"Create Storage Volume\" dialog, Define the name and format (keep with the default qcow2) and select the Max Capacity (at least 200~300GB  in case you need to compile the design); click \"Finish\" to create the disk.
                                                           
                                                           
                                                          6.  
                                                        6.  
                                                          Once the disk is created, it will appear in your virtual machine's list of storage devices. You can now use this disk just like any other disk. Select from the list and Click \"Choose Volume.\"
                                                           
                                                           
                                                          7.  
                                                         
                                                        7.  
                                                      7.  
                                                        In the next step, select the \"Customize configuration before install\" option and click \"Finish.\"
                                                         
                                                         
                                                        8.  
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#51-passing-devices-to-the-vm","title":"5.1 Passing Devices to the VM","text":"
                                                      In the \"Overview\" tab, select \"Add Hardware,\" choose \"PCI Host Device\" from the drop-down menu and choose the PCI device you want to share with the VM. Click \"Apply\" to apply the changes, and then click \"Finish\" to create the VM.
                                                       
                                                      Depending on the FIM currently loaded onto your FPGA device, you have access to a few modes of operation. Management Mode and Deployment mode can be used on any FIM that supports a PF/VF split architecture. When using the 2 PF FIM, see 2 PF Mode.
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#511-management-mode","title":"5.1.1 Management Mode","text":"
                                                      This will only allow you to load the binaries to the FPGA, you only need to add the PF listed at the fpgainfo fme command.
                                                       
                                                      fpgainfo fme\n\nfpgainfo fme\nIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: xxxx \nBoard Management Controller Build version: xxxx \n//****** FME ******//\nObject Id                        : 0xEE00000\nPCIe s:b:d.f                     : 0000:b1:00.0\n
                                                       
                                                       
                                                       
                                                       
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#512-deployment-mode","title":"5.1.2 Deployment Mode","text":"
                                                      The main idea of this mode is enable the Virtual function used by the Agilex PCIe Attach OFS under the Physical Function 0, This option will allow us to use the Host Exercises.
                                                       
                                                      Note: assigning multiple devices to the same VM on a guest IOMMU, you may need to increase the hard_limit option in order to avoid hitting a limit of pinned memory. The hard limit should be more than (VM memory size x Number of PCIe devices)
                                                       
                                                         
                                                      1.  
                                                        Create 3 VFs in the PR region.
                                                         
                                                        sudo pci_device b1:00.0 vf 3 \n
                                                         
                                                        2.  
                                                      2.  
                                                        Verify all 3 VFs were created.
                                                         
                                                        sh lspci -s b1:00  b1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)  b1:00.1 Processing accelerators: Intel Corporation Device bcce  b1:00.2 Processing accelerators: Intel Corporation Device bcce  b1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device  b1:00.4 Processing accelerators: Intel Corporation Device bcce  b1:00.5 Processing accelerators: Intel Corporation Device bccf  b1:00.6 Processing accelerators: Intel Corporation Device bccf  b1:00.7 Processing accelerators: Intel Corporation Device bccf
                                                         
                                                        3.  
                                                      3.  
                                                        Bind all of the PF/VF endpoints to the vfio-pci driver.
                                                         
                                                        sudo opae.io init -d 0000:b1:00.1 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 187\nAssigning /dev/vfio/187 to DCPsupport\nChanging permissions for /dev/vfio/187 to rw-rw----\n\nsudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188\nAssigning /dev/vfio/188 to DCPsupport\nChanging permissions for /dev/vfio/188 to rw-rw----\n\n...\n\nsudo opae.io init -d 0000:b1:00.7 user:user\nBinding (0x8086,0xbccf) at 0000:b1:00.7 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.7 is 319\nAssigning /dev/vfio/319 to DCPsupport\nChanging permissions for /dev/vfio/319 to rw-rw----\n
                                                         
                                                        4.  
                                                      4.  
                                                        Check that the accelerators are present using fpgainfo. Note your port configuration may differ from the below.
                                                         
                                                        sudo fpgainfo port \n//****** PORT ******//\nObject Id                        : 0xEC00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0xE0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.7\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 4dadea34-2c78-48cb-a3dc-5b831f5cecbb\n//****** PORT ******//\nObject Id                        : 0xC0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.6\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 823c334c-98bf-11ea-bb37-0242ac130002\n//****** PORT ******//\nObject Id                        : 0xA0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.5\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n//****** PORT ******//\nObject Id                        : 0x80B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.4\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 44bfc10d-b42a-44e5-bd42-57dc93ea7f91\n//****** PORT ******//\nObject Id                        : 0x40B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n//****** PORT ******//\nObject Id                        : 0x20B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 3e7b60a0-df2d-4850-aa31-f54a3e403501\n
                                                         
                                                        5.  
                                                       
                                                      The following table contains a mapping between each VF, Accelerator GUID, and component.
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#table-16-accelerator-pfvf-and-guid-mappings","title":"Table 16: Accelerator PF/VF and GUID Mappings","text":"Component VF Accelerator GUID Intel N6001-PL FPGA SmartNIC Platform base PF XXXX:XX:XX.0 N/A VirtIO Stub XXXX:XX:XX.1 3e7b60a0-df2d-4850-aa31-f54a3e403501 HE-MEM Stub XXXX:XX:XX.2 56e203e9-864f-49a7-b94b-12284c31e02b Copy Engine XXXX:XX:XX.4 44bfc10d-b42a-44e5-bd42-57dc93ea7f91 HE-MEM XXXX:XX:XX.5 8568ab4e-6ba5-4616-bb65-2a578330a8eb HE-HSSI XXXX:XX:XX.6 823c334c-98bf-11ea-bb37-0242ac130002 MEM-TG XXXX:XX:XX.7 4dadea34-2c78-48cb-a3dc-5b831f5cecbb 
                                                         
                                                      1.  
                                                        Ensure you add the desired VF in your PCIE devices list.
                                                         
                                                         
                                                         
                                                        2.  
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#513-2-pf-mode","title":"5.1.3 2 PF Mode","text":"
                                                      For FIMs that support the dual PF architecture, you have the option to pass through any number of PFs into the VM. The VM's software will recognize any management / workload ports and probe them appropriately. This assumes you have the OPAE SDK and Linux DFL drivers installed on both the VM and host.
                                                       
                                                         
                                                      1.  
                                                        Bind all endpoints you wish to pass through to the VM to the vfio-pci driver on the host.
                                                         
                                                        sudo opae.io init -d 0000:b1:00.0 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 187\nAssigning /dev/vfio/187 to user\nChanging permissions for /dev/vfio/187 to rw-rw----\nsudo opae.io init -d 0000:b1:00.1 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 187\nAssigning /dev/vfio/187 to user\nChanging permissions for /dev/vfio/187 to rw-rw----\n
                                                         
                                                        2.  
                                                      2.  
                                                        Pass through any required hardware endpoints, select \"Add Hardware\" -> \"PCI Host Device\".
                                                         
                                                         
                                                         
                                                        3.  
                                                      3.  
                                                        Run the following command on the host and VM to allocate hugepages for workload testing:
                                                         
                                                        echo 4194304 | sudo tee /sys/module/vfio_iommu_type1/parameters/dma_entry_limit\n
                                                         
                                                        4.  
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#52-virt-manager-configuration-changes","title":"5.2 Virt-Manager Configuration Changes","text":"
                                                         
                                                      1.  
                                                        Edit the XML file for your machine and include the following
                                                         
                                                           
                                                        1.  
                                                          < ioapic driver='qemu'/> inside of features:
                                                           
                                                          <features>\n  <acpi/>\n  <apic/>\n  <ioapic driver='qemu'/>\n</features>\n
                                                           
                                                          2.  
                                                        2.  
                                                          Inside of devices
                                                           
                                                          <devices>\n    ........\n    ......\n    <iommu model='intel'>\n      <driver intremap='on' caching_mode='on'/>\n    </iommu>\n</devices>\n
                                                           
                                                          3.  
                                                        3.  
                                                          Ensure the hard limit is setup correctly otherwise you can only pass one device:
                                                           
                                                          <memtune>\n  <hard_limit unit='G'>64</hard_limit>\n</memtune>\n
                                                           
                                                          Note: assigning multiple devices to the same VM on a guest IOMMU, you may need to increase the hard_limit option in order to avoid hitting a limit of pinned memory. The hard limit should be more than (VM memory size x Number of PCIe devices)
                                                           
                                                          4.  
                                                        4.  
                                                          Save the changes \"Apply\"
                                                           
                                                          5.  
                                                         
                                                        2.  
                                                      2.  
                                                        On the host machine append intel_iommu=on to the end of the GRUB_CMDLINE_LINUX line in the grub configuration file.
                                                         
                                                        nano /etc/default/grub\n......\nGRUB_CMDLINE_LINUX=\"....... ... intel_iommu=on\"\n...\n#Refresh the grub.cfg file for these changes to take effect\n\ngrub2-mkconfig -o /boot/grub2/grub.cfg\nshutdown -r now\n
                                                         
                                                        3.  
                                                      3.  
                                                        Ensure your devices are enumerated properly.
                                                         
                                                           
                                                        1.  
                                                          Example in you host system should look like this:
                                                           
                                                          1. Management Mode:
                                                           
                                                          B1:00.0\n
                                                           
                                                          2. Deployment Mode:
                                                           
                                                           B1:00.5\n
                                                           
                                                          2.  
                                                        2.  
                                                          Under the virtual machine (The PCIe Address is an example you could get a different  number):
                                                           
                                                          1. Management Mode:
                                                           
                                                          177:00.0\n
                                                           
                                                          2. Deployment Mode:
                                                           
                                                           177:00.0\n
                                                           
                                                          3.  
                                                         
                                                        4.  
                                                      4.  
                                                        Click on \"Begin Installation.\" and follow the wizard installation of the OS.
                                                         
                                                         
                                                        5.  
                                                      5.  
                                                        Once the VM is created, you can start it by selecting it in the virt-manager window and clicking the \"Run\" button. This will boot the VM and start the Red Hat 8.2/Ubuntu installation process. Follow the on-screen instructions to complete the installation.
                                                         
                                                         
                                                         
                                                        6.  
                                                      6.  
                                                        Under your virtual machine, configure your VM proxy:
                                                         
                                                           
                                                        • Redhat How to apply a system-wide proxy?
                                                          •  
                                                        • Ubuntu Define proxy settings
                                                          •  
                                                        • Configure Git to use a proxy
                                                          •  
                                                         
                                                        7.  
                                                      7.  
                                                        To include OPAE in your virtual machine, follow the instructions from the following link  OFS Site select your desired platform and select Getting stated guide. To install the DFL drivers, please follow the instructions from the following link  OFS Site select your desired platform and select Getting stated guide.
                                                         
                                                        8.  
                                                      8.  
                                                        Use the OPAE SDK tool opae.io (under your virtual machine) to check default driver binding using your card under test PCIe B:D.F (Management mode).
                                                         
                                                        sudo fpgainfo fme\n\nIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: xxx \nBoard Management Controller Build version: xxx\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:177:00.0\n
                                                         
                                                        9.  
                                                      9.  
                                                        Use the Virtual function (Not supported at management mode)
                                                         
                                                           
                                                        1.  
                                                          Ensure the DFL kernel drivers is install in your VM system
                                                           
                                                          2.  
                                                        2.  
                                                          Bind VFs to VFIO driver
                                                           
                                                          $ sudo opae.io init -d 0000:177:00.0\n[sudo] password for dcpsupport: \nopae.io 0.2.3\nBinding (0x8086,0xbccf) at 0000:177:00.0 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:177:00.0 is 13\n
                                                           
                                                          3.  
                                                        3.  
                                                          Verify the binding is correct.
                                                           
                                                          $ opae.io ls\nopae.io 0.2.3\n[0000:177:00.0] (0x8086, 0xbccf) Intel N6001 ADP VF (Driver: vfio-pci)\n
                                                           
                                                          4.  
                                                        4.  
                                                          Test the  HE mem
                                                           
                                                           host_exerciser mem\n starting test run, count of 1\n API version: 1\n Frequency of AFU clock unknown. Assuming 350 MHz.\n Allocate SRC Buffer\n Allocate DST Buffer\n Allocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 6737\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1022\n    Bandwidth: 3.405 GB/s\n    Test mem(1): PASS\n
                                                           
                                                          5.  
                                                         
                                                        10.  
                                                       
                                                      After the installation, you can use virt-manager to manage and configure the VM to move from Management mode to Deployment or vice versa, including setting up networking, attaching additional storage, and installing additional software. The shared PCI device will be available to the VM, allowing it to use it as if it were connected directly to the physical system.
                                                      "},{"location":"hw/common/user_guides/ug_kvm/ug_kvm/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                      Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                       
                                                      OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/","title":"Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach**","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel Max10 or Intel Cyclone10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implemented on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Management Component Transport Protocol MCTP A standardized model for communication with management controllers. Defines the transport protocol carrying PLDM messages through the BMC. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE-SDK The OPAE-SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Level Data Model PLDM A specification for reporting telemetry data to the host, such as board temperature, voltage, and current. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#1-overview","title":"1 Overview","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#11-about-this-document","title":"1.1 About this Document","text":"
                                                      This document serves as a set-up and user guide for the UVM simulation tool using Open FPGA Stack (OFS) for Intel\u00ae Agilex\u00ae 7 FPGAs PCIe Attach and the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile). After reviewing the document, you will be able to:
                                                       
                                                         
                                                      • Set-up the UVM verification tool suite
                                                        •  
                                                       
                                                         
                                                      • Run pre-existing UVM unit tests and also create new UVM tests for your design
                                                        •  
                                                       
                                                       
                                                      NOTE: 
                                                       
                                                      This guide uses the Intel\u00ae FPGA SmartNIC N6001-PL as the platform for all example steps. Additionally, this guide and the example steps can be used with other platforms, such as the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile). 
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#2-introduction-to-uvm","title":"2 Introduction to UVM","text":"
                                                      The Functional Verification Environment for OFS is UVM (Universal Verification Methodology) compliant and provides configurable setup for verifying various FIM features in simulation.
                                                       
                                                      The purpose of this document is to demonstrate a full chip level and unit level UVM based verification environment for the current base shell FIM architecture as well as providing examples to extend the setup for different OFS variants by reusing the existing architecture and infrastructure for UVM based verification.
                                                       
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#3-universal-testbench-architecture","title":"3 Universal Testbench Architecture","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#31-overview","title":"3.1 Overview","text":"
                                                      The main idea behind UVM is to develop modular, reusable, and a scalable testbench structure by providing an API framework that can be deployed across multiple projects.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#32-core-verification-concepts","title":"3.2 Core Verification Concepts","text":"
                                                      The following is the list of verification components that will be used to design a UVM testbench architecture:
                                                       
                                                      \u2022   Sequencer \u2022   Driver \u2022   Monitor \u2022   Scoreboard \u2022   Virtual Sequencer \u2022   Interface \u2022   Verification Environment \u2022   TOP Testbench
                                                       
                                                      Figure 1 provides the general UVM Testbench and the verification components involved in the top-level architecture.
                                                       
                                                       
                                                      Figure 1 Typical UVM Testbench
                                                       
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4-ofs-testbench-architecture","title":"4 OFS Testbench Architecture","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#41-overview","title":"4.1 Overview","text":"
                                                      OFS (Open FPGA Stack) provides a UVM (Universal Verification Methodology) environment for the FIM with a modular, reusable, and scalable testbench structure via an API framework.
                                                       
                                                      The framework consists of a FIM Testbench which is UVM compliant and integrates third party VIPs from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this Testbench. UVM RAL(Register Abstraction Level) is used for CSR (Command and Status Registers) verification. 
                                                       
                                                      The qualified verification IPs will help to detect incorrect protocol behavior, help to focus on FIM features and accelerate the verification process.
                                                       
                                                      Verification components include:
                                                       
                                                      \u2022 FIM monitor to detect correct design behavior\n\u2022 FIM assertions for signal level integrity testing\n\u2022 Arm AMBA Arm\u00ae AMBA\u00ae 4 AXI4 scoreboards to check data integrity\n\u2022 FIM coverage to collect functional data\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#42-base-fim-dut","title":"4.2 Base FIM DUT","text":"
                                                      The hardware architecture of an Agilex FIM is based on the OFS hardware architecture. The following is the list of features and subsystems supported in the base shell.
                                                       
                                                      \u2022   PCIe Subsystem\n\u2022   HSSI Subsystem\n\u2022   Memory Subsystem\n\u2022   HPS Subsystem\n\u2022   FME\n\u2022   AFU with PR support\n\u2022   QSFP Controllers\n\u2022   PMCI Controller, MCTP\n
                                                       
                                                       
                                                      Figure 2 DUT Base Shell Diagram
                                                       
                                                      Figure 2 shows the high level architecture of an Agilex Base Shell. It has a Gen4x16, 100G Ethernet Datapath in a 2x4x25G configuration. The Agilex Base Shell is a shell that will enable a user to build other shell variants for a custom configuration. For the N6001 board there is one shell variant
                                                       
                                                      base_x16
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#43-full-chip-level-verification-archiecture-for-fim","title":"4.3 Full Chip Level Verification Archiecture for FIM","text":"
                                                      Figure 3 shows a graphical representation a full chip testbench that includes major RTL blocks depicted in a OFS Agilex based UVM environment 
                                                       
                                                       
                                                      Figure 3 OFS FIM Testbench
                                                       
                                                      The major connection is the interface between the Xeon CPU and the FPGA where the PCIe Verification IP is connected to PCIe Subsystem. Therefore, as a full chip simulation environment, PCIe host VIP is the sole VIP/BFM used. PCIe host VIP connects to PCIe device which resides in FPGA in serial mode.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#431-testbench-components","title":"4.3.1 Testbench components","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4311-tb-top","title":"4.3.1.1 TB TOP","text":"
                                                      TOP is the top level testbench and consists of a FIM DUT instance and top-level UVM Verification Environment instance. It binds RTL inputs with the verification environmnet interfaces to drive stimulus. It also has clock generation and reset driving logic.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4312-fim-verification-environment","title":"4.3.1.2 FIM Verification Environment","text":"
                                                      This is the top most verification environment class and consists of the protocol specific PCI Express and AXI UVM environment VIP instances, Monitors, Scoreboards, Assertions, Functional coverage Modules and other verification components. It instantiates Virtual sequencers to control stimuli for FIM traffic from different sequencers of the VIPs.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4313-synopsys-vips","title":"4.3.1.3 Synopsys VIPs","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#43131-pci-vip-as-host","title":"4.3.1.3.1 PCI VIP as Host","text":"
                                                      This is Synopsys Licensed PCI Express Gen4 VIP and acts as Root Port. The PCI Express link is connected to the DUT using TX-RX lanes. Agent is an active component and includes a sequencer to generate TLPs, Driver to drive it on a PCI Express link and Monitor to check the protocol correctness.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#43132-axi-streaming-vip-monitors","title":"4.3.1.3.2 AXI-Streaming VIP Monitors","text":"
                                                      This is Synopsys Licensed AXI streaming interface Verification IP used as a Passive Agent to monitor AXI-ST links at various points. Please refer to Figure 3 to see all the AXI-ST monitors connected to different modules.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#43133-axi4-memory-mapped-vip-monitors","title":"4.3.1.3.3 AXI4-Memory Mapped VIP Monitors","text":"
                                                      This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 memory mapped interface Verification IP used in passive mode to observe memory requests. For each master-slave pair, the verification environment has a VIP instantiated in passive mode for monitoring protocol violations and performing data integrity checks.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#43134-axi4-lite-vip-monitors","title":"4.3.1.3.4 AXI4-Lite VIP Monitors","text":"
                                                      This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 Lite interface Verification IP used in passive mode to observe MMIO requests. For each master-slave pair, the verification environment has a VIP instantiated in passive mode for monitoring protocol violations and perfoming data integrity checks. Please refer to Figure 3 to see all the Arm\u00ae AMBA\u00ae 4 AXI4-Lite monitors connected to different modules.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#43135-axi4-lite-vip-as-pmci-master","title":"4.3.1.3.5 AXI4-Lite VIP as PMCI Master","text":"
                                                      This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 Lite interface Verification IP used to drive and observe MMIO requests as PMCI master. For each master-slave pair, the verification environment has a VIP instantiated in active mode and includes a sequencer to generate MMIO requests, driver to drive these requests on AXI-4 Lite interface to BPF and a monitor for observing protocol violations and data integrity checks. 
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4314-axi4-s-scoreboard","title":"4.3.1.4 AXI4-S Scoreboard","text":"
                                                      The Arm\u00ae AMBA\u00ae 4 AXI4-S scoreboard checks data integrity of source and sink components. It has input transactions from VIP monitors and a TLP to AXI converter for PCIe TLP packets. It makes sure the valid TLPs or AXI transactions are not missed while traversing from Host to AFU and reverse. The scoreboard will be disabled for error testing especially for invalid TLP requests and UR responses.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4315-virtual-sequencer","title":"4.3.1.5 Virtual Sequencer","text":"
                                                      The virtual sequencer is the top-level sequencer which controls Enumeration, MMIO Requests, downstream and Upstream traffic as well as HSSI and Memory transactions. It makes sure that the transactions are ordered correctly as per the design specification while running a UVM test simulation.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4316-fim-monitor","title":"4.3.1.6 FIM Monitor","text":"
                                                      The FIM monitor is used for checking the correctness of a specific FIM feature. As an example, a user can add interrupt related checks, error generation and clearing checks, Protocol checker impacts etc. in this component. 
                                                       
                                                      This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4317-fim-assertions","title":"4.3.1.7 FIM Assertions","text":"
                                                      The assertion component is used for checking the signal level integrity of a specific FIM feature or behavior. As an example, we can add interrupt signal triggering and clear assertions, Error detection to error register bit assertions, protocol checker and clearing logic, FLR behavior assertion etc. in this top-level module. There are alos assertions to make sure the inputs are driven correctly to catch errors in a users simulation. 
                                                       
                                                      This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#4318-fim-functional-coverage","title":"4.3.1.8 FIM Functional Coverage","text":"
                                                      The FIM functional coverage component will have the functional coverage of each feature like interrupts, CSR's, MMIO requests, Byte align transactions, error reporting etc. to demonstrate a variety of FIM features. This will help us to find holes in the design as well as wide verification coverage to make sure thorough testing of a design. It will also provide a user what the FIM is capable of and how much code coverage they are testing.
                                                       
                                                      This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                                                       
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#5-uvm-verification-set-up","title":"5 UVM Verification set-up","text":"
                                                      To run the tutorial steps in this guide requires the following development environment:
                                                       Item Version Intel Quartus Prime Pro Intel Quartus Prime Pro 23.3 Simulator Synopsys VCS P-2019.06-SP2-5 or newer for UVM simulation of top level FIM Simulator (Questasim) Questasim 2023.2 or newer for UVM simulation of top level FIM"},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#51-uvm-prerequisite","title":"5.1 UVM Prerequisite","text":"
                                                      Retrieve OFS repositories.
                                                       
                                                      The OFS FIM source code is included in the OTCShare GitHub repository. Create a new directory to use as a clean starting point to store the retrieved files.  The following is a short description of each repository, followed by the git commands for cloning.  The instructions section uses the HTTPS git method for cloning repositories.  Cloning the repo using the HTTPS method requires a personal access token.  Please see this blog post for information about obtaining a personal access token Token authentication requirements for Git operations.
                                                       
                                                      Navigate to location for storage of OFS source, create the top-level source directory and clone OFS repositories.
                                                       
                                                      $ mkdir ofs-2023.3-2\n$ cd ofs-2023.3-2\n$ export OFS_BUILD_ROOT=$PWD\n$ git clone --recurse-submodules https://github.com/OFS/ofs-agx7-pcie-attach.git\n\nCloning into 'ofs-agx7-pcie-attach'...'\nUsername for 'https://github.com': <<Enter your git hub username>>\nPassword for 'https://<<Your username>>': <<Enter your personal access token>>\nremote: Enumerating objects:  ....\n...\n...\nResolving deltas  ..., done.\n\n$ cd ofs-agx7-pcie-attach\n$ git checkout tags/ofs-2023.3-2\n
                                                       
                                                      Verify that the correct tag/branch have been checked out        
                                                       
                                                      $ git describe --tags\n\n$ ofs-2023.3-2\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#52-license-requirements","title":"5.2 License Requirements","text":"
                                                      The FIM Testbench is UVM compliant and integrates third party VIP's from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this TB. UVM RAL (Register Abstraction Layer) is used for CSR Verification.
                                                       
                                                      The Qualified Verification IPs will help to detect incorrect protocol behavior easily, help to focus on FIM features and accelerate the verification process.
                                                       
                                                      \u2022 VCS & DVE\n\u2022 SNPS-Assertions\n\u2022 Verdi\n\u2022 VerdiCoverage\n\u2022 VerdiSimDB\n\u2022 VerdiTransactionDebugUltra\n\u2022 VIP-AMBA-AXI-SVT\n\u2022 VIP-AMBA-STREAM-SVT\n\u2022 VIP-PCIE-SVT\n\u2022 VIP-PCIE-TS-SVT\n\u2022 VIP-PCIE-G3-OPT-SVT\n\u2022 VIP-Ethernet-SVT\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#53-software-tools-requirements","title":"5.3 Software Tools Requirements","text":"
                                                      The following tools are required for successful UVM set-up
                                                       
                                                         
                                                      • Python 3.6.8
                                                        •  
                                                      • Synopsys PCIE and AMBA AXI UVM VIP Q-2020.03A License
                                                        •  
                                                      • Synopsys Verdi R-2020.12-SP2 License      Note: Makefile can be modified to use DVE instead of Verdi
                                                        •  
                                                       
                                                         
                                                      • VCS R-2020.12-SP2 License 
                                                        •  
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#54-creating-a-software-tools-script","title":"5.4 Creating a Software Tools Script","text":"
                                                      The UVM tool set-up is best done by creating a simple set-up script so all applicable tools are sourced before running the tests.
                                                       
                                                      The following environment variables can be pasted into a script and used prior to running the UVM verification environment
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#license-files","title":"License Files","text":"
                                                      export LM_LICENSE_FILE=\nexport SNPSLMD_LICENSE_FILE=\n
                                                       
                                                      The license environment variables LM_LICENSE_FILE and SNPSLMD_LICENSE_FILE can point to a server license on your system.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#general-environment-variables","title":"General Environment Variables","text":"
                                                      export IOFS_BUILD_ROOT=$PWD\nexport OFS_ROOTDIR=<user_path>/ofs-agx7-pcie-attach\nexport WORKDIR=$OFS_ROOTDIR\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#quartus-tools","title":"Quartus Tools","text":"
                                                      export QUARTUS_HOME=<user_path>/intelFPGA_pro/23.3/quartus\nexport QUARTUS_ROOTDIR=$QUARTUS_HOME\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$PATH\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-verification-tools","title":"Synopsys Verification Tools","text":"
                                                      export DESIGNWARE_HOME=<user_path>/synopsys/vip_common/vip_Q-2020.03A\nexport PATH=$DESIGNWARE_HOME/bin:$PATH\nexport UVM_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport PATH=$VCS_HOME/bin:$PATH\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-verification-tools","title":"QuestaSIM Verification Tools","text":"
                                                      export MTI_HOME=<user_path>/mentor/questasim/2023.2/linux64\nexport PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\nexport QUESTA_HOME=$MTI_HOME\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#6-running-a-uvm-simulation-test-and-analysing-results","title":"6 Running a UVM Simulation Test and Analysing Results","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#61-simulation","title":"6.1 Simulation","text":"
                                                      The default simulator used in the simulation script is Synopsys VCS-MX. Users can refer to the options and adopt the options for other simulators. The script is a makefile that calls vlogan, vcs and simv for compilation, elaboration and simulation, respectively.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#62-file-structure","title":"6.2 File Structure","text":"
                                                      After cloning the repo, the verification and ofs-common directories contain all UVM verification related files. The directory structure is shown in Figure 4 below.
                                                       
                                                       
                                                      Figure 4 UVM Verification Directory File Structure
                                                       
                                                      ofs-agx7-pcie-attach/verification/testbench has a testbench, uvm env, virtual sequencer, RAL etc. 
                                                       
                                                      ofs-agx7-pcie-attach/tests contains all uvm tests and sequences. 
                                                       
                                                      Users can run the simulation under \"ofs-agx7-pcie-attach/verification/scripts\" directory and the simulation result is outputted to a \"sim\" directory for Synopsys VCS or sim_msim for Questasim.
                                                       
                                                      The simulation result folder is named after the test name with increasing suffix number. If user runs the same test multiple times, the suffix is incremented by 1 each time.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#63-uvm-test-suite","title":"6.3 UVM Test Suite","text":"
                                                      The UVM environment contains a variety of tests that have been developed to test out the FIM portion of OFS.
                                                       
                                                      The table below has four columns which describe the \"Test Name\", \"DUT Scope\", \"Test Scenario\" and the \"Checking Criteria\".
                                                       
                                                      Tests are located at ofs-agx7-pcie-attach/verification/tests
                                                       Test Name DUT Scope Test Scenario Checking Criteria afu_mmio_flr_pf0_test PF0 FLR Reset Apply FLR Reset for PF0 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF0 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf0_vf0_test PF0_VF0_FLR Reset Apply FLR Reset for PF0_VF0 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR       access are working fine afu_mmio_flr_pf0_vf1_test PF0_VF1_FLR Reset Apply FLR Reset for PF0_VF1 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf0_vf2_test PF0_VF2_FLR Reset Apply FLR Reset for PF0_VF2 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf2_test PF2 FLR Reset Apply FLR Reset for PF2 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF2 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf3_test PF3 FLR Reset Apply FLR Reset for PF3 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF3 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf4_test PF4 FLR Reset Apply FLR Reset for PF4 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF4 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_stress_5bit_tag_test AFU-Stress To check the AFU Stress by sending traffic from all PF/VF. i.e Send traffic on HE-LPK/HE-MEM and initiating MMIO access to other PF/VFs Data checking afu_stress_8bit_tag_test AFU-Stress To check the AFU Stress by sending traffic from all PF/VF. i.e Send traffic on HE-LPK/HE-MEM and initiating MMIO access to other PF/VFs Data checking afu_stress_test AFU-Stress 1. Initiate transactions to all the supported PF/VF from PCIE VIP and ensure that traffic is sent to all blocks of the AFU.                 2. Ensure that CE/HE-LB/HE-MEM/HSSI/BPF/FME are seeing traffic.                 3. Ensure that HE-LB/HE-MEM/CE sends DMWR/DMRD requests to PCIE VIP.                 4. Ensure the Mux/DeMux blocks is able to handle the traffic based on the PF's/VF's and proper muxing/demuxing happens. Data checking bar_32b_test PCIe MMIO Path Set the BAR values to 32bit and test mmio access BAR address, Register Base Offset bar_64b_test PCIe MMIO Path Set the BAR values to 64bit and test mmio access BAR address, Register Base Offset dfh_walking_test DFH DFH walking offset checking, eol checking -> tb emif_csr_test EMIF CSR access data checking fme_csr_test FME CSR CSR accesses data checking fme_err_intr_test Interrupt FME Interrupt request using FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read fme_intr_test Interrupt FME interrupt request using RAS ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read fme_multi_err_intr_test Interrupt FME interrupt request using RAS ERROR and FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests generated from FME and RAS ERROR bits he_hssi_csr_test HE-HSSI CSR accesses data checking he_hssi_rx_lpbk_25G_10G_test HE-HSSI Sending back to back ethernet data traffic with 25G speed on HSSI RX Port0-7 lanes using Ethernet VIPs Enable the loopback mode in HE-HSSI and compare the pkts recived on HSSI TX Port(DATA CHECKING) he_hssi_tx_lpbk_P0_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port0  lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P1_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port1 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P2_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port2 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P3_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port3 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P4_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port4 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P5_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port5 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P6_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port6 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P7_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port7 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_lpbk_cont_test HE-LPBK Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking he_lpbk_csr_test HE-LPBK CSR accesses data checking he_lpbk_flr_rst_test HE-LPBK Set HE_LPK in  all the modes randomly and iterate the transactions in loop. At the end of every loop assert the Soft reset in the middle of the transactions data checking, counter checking he_lpbk_long_rst_test HE-LPBK Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_lpbk_long_test HE-LPBK Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_lpbk_multi_user_intr_test HE-LPBK generate user HE_LB interrupt interrupt checking he_lpbk_rd_cont_test HE-LPBK Read only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking he_lpbk_rd_test HE-LPBK Read only mode. Randomize num_lines, addresses, req_len counter checking he_lpbk_reqlen16_test HE-LPBK To check the behavior of HE_LPK block when req_length 16 and num_lines set to 1024 Cache lines in Loopback mode data checking, counter checking he_lpbk_reqlen1_test HE-LPBK Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_lpbk_reqlen2_test HE-LPBK Loopback mode. 128 CLs, req_len = 2CL, random addresses data checking, counter checking he_lpbk_reqlen4_test HE-LPBK Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_lpbk_reqlen8_test HE-LPBK Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_lpbk_rst_in_middle_test PCIe MMIO Path Set HE_LPK in  all the modes randomly and iterate the transactions in loop. At the end of every loop assert the Soft reset in the middle of the transactions Register Base Offset he_lpbk_test HE-LPBK Loopback mode.  Randomize num_lines, addresses, req_len data checking, counter checking he_lpbk_thruput_contmode_test HE-LPBK Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking he_lpbk_thruput_test HE-LPBK generate user HE_LB interrupt counter checking he_lpbk_user_intr_test HE-LPBK Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_lpbk_wr_cont_test HE-LPBK Write only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking he_lpbk_wr_test HE-LPBK Write only mode. Randomize num_lines, addresses, req_len counter checking he_mem_cont_test HE-MEM Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking, counter checking he_mem_csr_test HE-MEM CSR accesses data checking he_mem_lpbk_long_rst_test HE-MEM Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_mem_lpbk_long_test HE-MEM Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_mem_lpbk_reqlen16_test HE-MEM To check the behavior of HE_MEM block when req_length 16 and num_lines set to 1024 Cache lines in Loopback mode data checking, counter checking he_mem_lpbk_reqlen1_test HE-MEM Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_mem_lpbk_reqlen2_test HE-MEM Loopback mode. 128 CLs, req_len = 2CL, random addresses data checking, counter checking he_mem_lpbk_reqlen4_test HE-MEM Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_mem_lpbk_reqlen8_test HE-MEM Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_mem_lpbk_test HE-MEM Loopback mode.  Randomize num_lines, addresses, req_len data checking, counter checking he_mem_multi_user_intr_test Interrupt Back to back multiple User interrupt request from HE MEM Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read for multiple back to back request he_mem_rd_cont_test HE-MEM Read only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_rd_test HE-MEM Read only mode. Randomize num_lines, addresses, req_len counter checking he_mem_thruput_contmode_directed_test HE-MEM Set HE_LPK in thruput mode and send traffic with req len 1 and num_lines set to 40 data checking, counter checking he_mem_thruput_contmode_test HE-MEM Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_thruput_test HE-MEM Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_mem_user_intr_test Interrupt FME interrupt request using RAS ERROR and FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests generated from FME and RAS ERROR bits he_mem_wr_cont_test HE-MEM Write only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_wr_test HE-MEM Write only mode. Randomize num_lines, addresses, req_len counter checkingt he_random_test All HEs Enable all HEs and randomize modes data checking if in lpbk mode, counter checking helb_rd_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Read only mode Measure the performance helb_rd_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Read only mode Measure the performance helb_rd_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Read only mode Measure the performance helb_thruput_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Thruput mode Measure the performance helb_thruput_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Thruput mode Measure the performance helb_thruput_4cl_5bit_tag_test Performance Set HE_LPK in thruput mode and send traffic with req len 4 and num_lines set to 1024. Measure the Read/Write performance Measure the performance helb_thruput_4cl_8bit_tag_test Performance Set HE_LPK in thruput mode and send traffic with req len 4 and num_lines set to 1024. Measure the Read/Write performance Measure the performance helb_thruput_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Thruput mode Measure the performance helb_wr_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Write only mode Measure the performance helb_wr_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Write only mode Measure the performance helb_wr_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Write only mode Measure the performance hssi_ss_test HSSI CSR accesses data checking malformedtlp_pcie_rst_test Protocol Checker generate malformed TLP protocol error and initiate pcie reset to clear the error Check for error malformedtlp_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MaxTagError_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transactions are completing. mem_tg_csr_test MEM-TG CSR access data checking mem_tg_traffic_gen_test MEM-TG This test checks the MEM_TG traffic generator flow for 1 bank data checking mini_smoke_test All HEs shorter simpler version of random test for turn-in sanity check data checking if in lpbk mode, counter checking mix_intr_test Interrupt Mix interrupt testcase to send multiple FME and User interrupt request simultaneously Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests through different sources - FME and HE-MEM modules mmio_pcie_mrrs_128B_mps_128B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_128B_mps_256B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_256B_mps_128B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_256B_mps_256B_test PCIe - Max Payload/Max Read Req Size Random length mmio Read Checking valid possible combination of MPS & MRRS mmio_stress_nonblocking_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux with non-blocking MMIO reads data checking mmio_stress_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux data checking mmio_test PCIe MMIO Path MMIO targeting PF0(ST2MM, FME, PMCI, QSFP, HSSI SS), PF1, PF2,PF3, PF4, PF1.VF1, PF1.VF2 data checking mmio_unimp_test PCIe MMIO Path MMIO acccess to unimplemented addresses data checking MMIODataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MMIOInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MMIOTimedout_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. pcie_csr_test PCIESS Earlier msix registers were in fme block but now it has moved from fme to pciess. Hence coded a seperate test for msix data checking pcie_pmci_mctp_multi_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM multiple packets received from PMCI controller over APF and BPF to PCIe HIA subsystem pcie_pmci_mctp_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM (Vendor Defined Message) single packet received from PCIe HIA subsystem to PMCI controller over APF and BPF via AXI4-lite memory write request pmci_csr_test PMCI CSR CSR accesses data checking pmci_fme_csr_test PMCI FME CSR CSR accesses data checking pmci_pcie_mctp_multi_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM multiple packets received from PMCI controller over APF and BPF to PCIe HIA subsystem pmci_pcie_mctp_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM single packet received from PMCI controller over APF and BPF to PCIe HIA subsystem pmci_pciess_csr_test PMCI PCIESS CSR CSR accesses data checking pmci_qsfp_csr_test PMCI QSFP CSR CSR accesses data checking port_gasket_csr_test PORT GASKET CSR accesses data checking qsfp_csr_test QSFP CSR accesses data checking TxMWrDataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. TxMWrInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. uart_intr_test UART Checking Generates UART interrupt Check interrupt UnexpMMIORspErr_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. vdm_err_vid_test Vendor ID check generate a packet with undefined Vendor-ID from Host to PMCI_SS ID check 
                                                       
                                                      The next section describes how to compile and build the UVM environment prior to running each UVM test and analyinsg the results in the log files 
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#64-ip-compile","title":"6.4 IP Compile","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-vcs","title":"Synopsys VCS","text":"
                                                      To compile all IPs for the Synopsys VCS simulater targetting the Intel\u00ae FPGA SmartNIC N6001-PL:
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk cmplib_adp\n
                                                       
                                                      To compile all IPs for the Synopsys VCS simulater targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk cmplib_adp FTILE_SIM=1\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-tbd","title":"Questasim (TBD)","text":"
                                                      To compile all IPs for the Questasim simulater targetting the Intel\u00ae FPGA SmartNIC N6001-PL:
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk cmplib_adp\n
                                                       
                                                      To compile all IPs for the Questasim simulater targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk cmplib_adp FTILE_SIM=1\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#65-rtl-test-bench-compile","title":"6.5 RTL & Test Bench Compile","text":"
                                                      The RTL file list for compilation is located here: verification/scripts/rtl_comb.f
                                                       
                                                      The TB file list for compilation is located here: verification/scripts/ver_list.f
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-vcs_1","title":"Synopsys VCS","text":"
                                                      To compile RTL and Testbench for the Synopsys VCS simulater targetting the Intel\u00ae FPGA SmartNIC N6001-PL:
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_adp DUMP=1\n
                                                       
                                                      To compile RTL and Testbench for the Synopsys VCS simulater targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_adp FTILE_SIM=1 DUMP=1\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-tbd_1","title":"Questasim (TBD)","text":"
                                                      To compile RTL and Testbench for the Questasim simulater targetting the Intel\u00ae FPGA SmartNIC N6001-PL:
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_adp DUMP=1\n
                                                       
                                                      To compile RTL and Testbench for the Questasim simulater targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_adp FTILE_SIM=1 DUMP=1\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#66-ip-and-rtl-test-bench-compile","title":"6.6 IP and RTL & Test Bench Compile","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-vcs_2","title":"Synopsys VCS","text":"
                                                      If the user wants to compile all IPs and RTL Testbench in one command for Synopsys VCS targetting the Intel\u00ae FPGA SmartNIC N6001-PL then follow the procedure below
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_all DUMP=1\n
                                                       
                                                      If the user wants to compile all IPs and RTL Testbench in one command for Synopsys VCS targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) then follow the procedure below
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_all FTILE_SIM=1 DUMP=1\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-tbd_2","title":"Questasim (TBD)","text":"
                                                      If the user wants to compile all IPs and RTL Testbench in one command for Questasim targetting the Intel\u00ae FPGA SmartNIC N6001-PL then follow the procedure below
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_all DUMP=1\n
                                                       
                                                      If the user wants to compile all IPs and RTL Testbench in one command for Questasim targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) then follow the procedure below
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_all FTILE_SIM=1 DUMP=1\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-vcs_3","title":"Synopsys VCS","text":"
                                                      To run a simulation for Synopsys VCS targetting the Intel\u00ae FPGA SmartNIC N6001-PL:
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk run TESTNAME=ofs_mmio_test DUMP=1\n
                                                       
                                                      To run a simulation for Synopsys VCS targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk run TESTNAME=ofs_mmio_test FTILE_SIM=1 DUMP=1\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-tbd_3","title":"Questasim (TBD)","text":"
                                                      To run a simulation for Questasim targetting the Intel\u00ae FPGA SmartNIC N6001-PL:
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=ofs_mmio_test DUMP=1\n
                                                       
                                                      To run a simulation for Questasim targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):
                                                       
                                                          cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=ofs_mmio_test FTILE_SIM=1 DUMP=1\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-vcs_4","title":"Synopsys VCS","text":"
                                                      To dump the waveform, \"DUMP=1\" must be added into the command line for Synopsys VCS build and simulation targetting the Intel\u00ae FPGA SmartNIC N6001-PL:
                                                       
                                                          gmake -f Makefile_VCS.mk build_adp DUMP=1\n\n    gmake -f Makefile_VCS.mk run TESTNAME=<test_case_name> DUMP=1\n
                                                       
                                                      Or
                                                       
                                                          gmake -f Makefile_VCS.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                                                       
                                                      To dump the waveform, \"DUMP=1\" must be added into the command line for Synopsys VCS build and simulation targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):
                                                       
                                                          gmake -f Makefile_VCS.mk build_adp FTILE_SIM=1 DUMP=1\n\n    gmake -f Makefile_VCS.mk run TESTNAME=<test_case_name> FTILE_SIM=1 DUMP=1\n
                                                       
                                                      Or
                                                       
                                                          gmake -f Makefile_VCS.mk build_run TESTNAME=<test_case_name> FTILE_SIM=1 DUMP=1\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-tbd_4","title":"Questasim (TBD)","text":"
                                                      To dump the waveform, \"DUMP=1\" must be added into the command line for Questasim build and simulation targetting the Intel\u00ae FPGA SmartNIC N6001-PL: 
                                                       
                                                          gmake -f Makefile_MSIM.mk build_adp DUMP=1\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=<test_case_name> DUMP=1\n
                                                       
                                                      Or
                                                       
                                                          gmake -f Makefile_MSIM.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                                                       
                                                      To dump the waveform, \"DUMP=1\" must be added into the command line for Questasim build and simulation targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile):
                                                       
                                                          gmake -f Makefile_MSIM.mk build_adp FTILE_SIM=1 DUMP=1\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=<test_case_name> FTILE_SIM=1 DUMP=1\n
                                                       
                                                      Or
                                                       
                                                          gmake -f Makefile_MSIM.mk build_run TESTNAME=<test_case_name> FTILE_SIM=1 DUMP=1\n
                                                       
                                                      There are some optimizations in the Table below for convenience if you want to bypass some commands for both Synopsys VCS and Questasim:
                                                       Command (Synopsys VCS) for n6001 Command (Questasim) for n6001 Command (Synopsys VCS) for fseries-dk Command (Questasim) for fseries-dk Details gmake -f Makefile_VCS.mk build_all DUMP=1 gmake -f Makefile_MSIM.mk build_all DUMP=1 gmake -f Makefile_VCS.mk build_all FTILE_SIM=1 DUMP=1 gmake -f Makefile_MSIM.mk build_all FTILE_SIM=1 DUMP=1 compile IP + compile RTL gmake -f Makefile_VCS.mk build_run TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk build_run TESTNAME= DUMP=1 gmake -f Makefile_VCS.mk build_run TESTNAME= FTILE_SIM=1 DUMP=1 gmake -f Makefile_MSIM.mk build_run TESTNAME= FTILE_SIM=1 DUMP=1 compile RTL + run test gmake -f Makefile_VCS.mk do_it_all TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk do_it_all TESTNAME= DUMP=1 gmake -f Makefile_VCS.mk do_it_all TESTNAME= FTILE_SIM=1 DUMP=1 gmake -f Makefile_MSIM.mk do_it_all TESTNAME= FTILE_SIM=1 DUMP=1 compile IP, RTL and run test gmake -f Makefile_VCS.mk rundb TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk rundb TESTNAME= DUMP=1 gmake -f Makefile_VCS.mk rundb TESTNAME= FTILE_SIM=1 DUMP=1 gmake -f Makefile_MSIM.mk rundb TESTNAME= FTILE_SIM=1 DUMP=1 run test in sim dir + over-writes content"},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#67-uvm-regression-test","title":"6.7 UVM Regression Test","text":"
                                                      If the user wants to run the complete set of UVM tests in one command for VCS and Questasim targetting the Intel\u00ae FPGA SmartNIC N6001-PL then follow the procedure below
                                                       
                                                      cd $VERDIR/scripts\n\npython uvm_regress.py -l -n 8 -p adp -k top_pkg -s vcs -c none\n\nFor Regression in VCS with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -p adp -k top_pkg -s vcs -c none\n
                                                       
                                                      Results are created in a sim directory ($VERDIR/sim) with individual testcase log dir
                                                       
                                                      For Regression in MSIM with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -p adp -k top_pkg -s msim -c none\n
                                                       
                                                      Results are created in a sim directory ($VERDIR/sim_msim) with individual testcase log dir
                                                       
                                                      If the user wants to run the complete set of UVM tests in one command for VCS and Questasim targetting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) then follow the procedure below
                                                       
                                                      TBC\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#68-uvm-waveform-and-transcript-analysis","title":"6.8 UVM Waveform and Transcript Analysis","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#synopsys-vcs_5","title":"Synopsys VCS","text":"
                                                      Running Synopsys VCS UVM tests will generate a ofs-agx7-pcie-attach/verification/sim directory
                                                       
                                                      \u2022 All build time logs are located at ofs-agx7-pcie-attach/verification/sim\n\n\u2022 Each testcase will have separate directory inside sim ofs-agx7-pcie-attach/verification/sim/<test_case_name>\n
                                                       
                                                      There are two tracker or log files that are available: runsim.log and trans.log. 
                                                       
                                                      runsim.log is the simulation log file generated from Synopsys VCS. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 5
                                                       
                                                       
                                                      Figure 5 runsim.log
                                                       
                                                      trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 6
                                                       
                                                       
                                                      Figure 6 trans.log
                                                       
                                                      The waveform generated is named as \"inter.vpd\". To open the waveform, go to simulation result directory and run 
                                                       
                                                          dve -full64 -vpd inter.vpd &\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#questasim-tbd_5","title":"Questasim (TBD)","text":"
                                                      Running Questasim UVM tests will generate a ofs-agx7-pcie-attach/verification/sim_msim directory
                                                       
                                                      \u2022 All build time logs are at ofs-agx7-pcie-attach/verification/sim_msim\n\n\u2022 Each testcase will have separate directory inside sim ofs-agx7-pcie-attach/verification/sim_msim/<test_case_name>\n
                                                       
                                                      There are two tracker or log files that are available: runsim.log and trans.log. 
                                                       
                                                      runsim.log is the simulation log file generated from Questasim. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 7
                                                       
                                                       
                                                      Figure 7 runsim.log
                                                       
                                                      trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 8
                                                       
                                                       
                                                      Figure 8 trans.log
                                                       
                                                      The waveform generated is named as \"vsim.wlf\". To open the waveform, go to simulation result directory and run 
                                                       
                                                          vsim -view vsim.wlf &\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#69-uvm-coverage-analysis","title":"6.9 UVM Coverage Analysis","text":"
                                                      For Intel\u00ae FPGA SmartNIC N6001-PL
                                                       
                                                      The following command allows to run a single testcase with coverage enabled
                                                       
                                                          gmake -f Makefile_VCS.mk cmplib_adp && gmake -f Makefile_VCS.mk build_adp DUMP=1 DEBUG=1 COV_FUNCTIONAL=1&& gmake -f Makefile_VCS.mk run TESTNAME=<TESTCASE-NAME> DUMP=1 DEBUG=1 COV_FUNCTIONAL=1 &\n
                                                       
                                                      The following command shows how to merge and generate the coverage report
                                                       
                                                          urg -dir <$VERDIR/sim/simv.vdb> <$VERDIR/sim/regression.vdb> -format both -dbname <regression_database_name>\n
                                                       
                                                      This will generate both urgreport directory and .vdb file Multiple regression.vdb from different regressions can be merged with the same command. 
                                                          e.g \"urg -dir <path1_till_simv.vdb> <path1_till_regression.vdb> <path2_till_regression.vdb> -report <dir> -format both -dbname <dirname>\"\n
                                                       
                                                      The following commands shows how to launch DVE and check the coverage reports
                                                       
                                                      To open DVE of a single regression or testcase, execute:\n\n    dve -full64 -cov -covdir simv.vdb regression.vdb &\n\nTo open DVE of a merged regression test, execute:\n\n    dve -full64 -cov -covdir <dirname.vdb> &\n
                                                       
                                                      Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)
                                                       
                                                      TBC\n
                                                       
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#7-csr-verification-using-uvm-ral","title":"7 CSR Verification using UVM RAL","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#71-overview","title":"7.1 Overview","text":"
                                                      The UVM Register Layer provides a standard base class library that enable users to implement the object-oriented model to access the DUT registers and memories. The UVM Register Layer is also referred to as UVM Register Abstraction Layer (UVM RAL). Design registers can be accessed independently of the physical bus interface. i.e. by calling read/write methods. This is shown in Figure 9 below.
                                                       
                                                       
                                                      Figure 9 RAL UVM Testbench
                                                       
                                                      The RAL register models for different CSR's mimics the design registers. All RAL files are located here.
                                                       
                                                          ofs-agx7-pcie-attach/verification/testbench/ral\n
                                                       
                                                      The RAL model is generated through the Synopsys RALGEN tool and is used for CSR verification.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#72-ral-integration","title":"7.2 RAL Integration","text":"
                                                      For UVM RAL model integration to the environment, adapters for each CSR is implemented and integrated into the Testbench Environment. It is used to convert the PCIe bus sequence items into uvm_reg_bus_op and vice versa. The CSR test cases pick up all the registers from the respective CSR blocks and perform a default value, wr/rd check on them. 
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#73-ral-model-generation","title":"7.3 RAL Model Generation","text":"
                                                      Steps for RAL model generation
                                                       
                                                      Excel(xls) file containing the registers is required. Make sure there are separate xls files for each CSR and the worksheet name must contain the name \"reg_fields\".  
                                                       
                                                      Excel sheet snapshot example below for EMIF_CSR.xls located at /ipss/mem/rtl/adp
                                                       
                                                       
                                                      \u2022 Navigate to ofs-agx7-pcie-attach/ofs-common/verification/common/scripts/ral\n\u2022 Copy the excel file (xls) to the above area\n\u2022 In the bash terminal run mk_ral.sh <Excel sheet name without extension > <output *.sv file name without ral_  prepended >\n\u2022 The above steps generate two ral *.sv files. File with _cov suffix is a coverage enabled ral model. \n\u2022 Copy *.sv files to ofs-agx7-pcie-attach/verification/testbench/ral\n
                                                       
                                                       
                                                      \u2022 As an example to generate ral_ac_ce.sv from AC_CE_CSR.xls file the command is\n\n    mk_ral.sh AC_CE_CSR ac_ce\n
                                                       
                                                      This generates two ral models (ral_ac_ce.sv and ral_ac_ce_cov.sv)
                                                       
                                                      To add new registers
                                                       
                                                      \u2022 To create new registers, copy existing ones and modify the cells in the xls. Make sure the last line is also a copied blank line\n\u2022 Follow all the steps of RAL model generation\n
                                                       
                                                      To Generate a RAL model when a new xls sheet is created for a new component
                                                       
                                                      \u2022 Copy the relevant xls sheet to ofs-agx7-pcie-attach/ofs-common/verification/common/scripts/ral\n\u2022 Follow all the steps of RAL model generation\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#74-top-level-verification-architecture-for-csr-testing","title":"7.4 Top Level Verification Architecture for CSR testing","text":""},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#741-testbench-components","title":"7.4.1 Testbench components","text":"
                                                      The testbench components for RAL are defined below
                                                       
                                                      \u2022 ral_reg_iofs_* (uvm_reg) generated by the steps as mentioned in section 5.3\n
                                                       
                                                      The uvm register class is written by extending the uvm_reg. A register represents a set of fields that are accessible as a single entity Each register contains any number of fields, which mirror the values of the corresponding elements in hardware
                                                       
                                                      \u2022 ral_block_iofs_* (uvm_block) generated in the same register file\n
                                                       
                                                      A register model is an instance of a register block, which may contain any number of registers, register files, memories, and other blocks
                                                       
                                                      \u2022 ral_block_ofs (uvm_block) \u2013 Contains all the CSR block instantiations\n\u2022 Reg2vip_*_adapter (uvm_reg_adapter) \u2013 This class defines an interface for converting between uvm_reg_bus_op and a specific bus transaction. For each CSR a respective adapter is present\n
                                                       
                                                      All the components are defined in ofs-agx7-pcie-attach/ofs-common/verification/testbench
                                                       
                                                      Integration of components in testbench
                                                       
                                                      \u2022 The RAL blocks and adapters for each CSR is instantiated in tb_env\n\u2022 The bar range for each CSR is also set in the tb_env\n
                                                       
                                                      Sample Environment Integration snippets 
                                                       
                                                       
                                                       
                                                       
                                                       
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#8-modifying-uvm-testbench","title":"8 Modifying UVM Testbench","text":"
                                                      The next sections describe what needs to be considered when modifying the UVM, adding a new interface to the testbench and creating a new UVM test for a customised OFS Accelerator platform.
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#81-modifying-uvm-environment-for-new-shell-variant","title":"8.1 Modifying UVM environment for new Shell Variant","text":"
                                                      OFS n6001 comprises a shell based on PCIe Gen4x16 and is named base_x16
                                                       
                                                      This base_x16 shell is described by an RTL file list, IP File lists and setup scripts to complete the build flow
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#82-modifying-uvm-environment-and-setting-up-compile-and-run-flow","title":"8.2 Modifying UVM environment and setting up Compile and Run flow","text":"
                                                      All the variants can mostly reuse the existing UVM infrastructure to setup the build and run flow
                                                       
                                                      \u2022 Create directory under $OFS_BUILD_ROOT new variant e.g ofs-n9000\n\u2022 Change directory to $OFS_BUILD_ROOT/ofs-n9000/verification/scripts\n\u2022 modify Makefile it to point to the new RTL, IP and script files.\n
                                                       
                                                      Following these three steps above will enable the build and sim flow to run the existing UVM TB and tests with new IOFS n9000 variant.
                                                       
                                                      Adding a new interface requires signal connections in the testbench. An additional BFM or verification IP is needed to drive the new interface. The main testbench file tb_top.sv is found at the following location
                                                       
                                                          ofs-agx7-pcie-attach/verification/testbench\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#83-adding-a-new-ral-directory","title":"8.3 Adding a new RAL directory","text":"
                                                      In case the design has many register changes and the user decides to generate all the new RAL models instead of reusing from existing base RAL models, the following steps will help to create and integrate a new RALDIR in the UVM environment.
                                                       
                                                      \u2022 Generate the new RAL files in desired directory. Preferably under the \"ofs-agx7-pcie-attach/verification/common/testbench\" \n\u2022 By default, the Makefile points to base FIM RAL so set the RALDIR path in the Makefile to the new generated RAL file directory\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#84-modifying-tbtest-files-for-new-variant","title":"8.4 Modifying TB/Test files for new Variant","text":"
                                                      Create a define for the new variant. e.g 'define FIM_NEW. If you are modifying common files then add new logic under this define so other projects will not get affected with variant specific change.
                                                       
                                                      If there are more changes, please create separate \"testbench\" and \"test\" folders under this new variant. 
                                                       
                                                      \u2022 Extend variant specific env file from base env file to add variant specific changes.\n\u2022 Create new test/seq lib files in \"tests\" folder.\n\u2022 Create new variant package, add new TB/Tests/Sequence lib files and also import the base package files.\n
                                                       
                                                      If you are adding new files then make sure it's included in Makefile for the build+run flow. 
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#85-uvm-pcie-drivers","title":"8.5 UVM PCIe Drivers","text":"
                                                      The \"svt_pcie_driver_app_transaction_base_sequence\" is part of Synopsys PCIe VIP library. You can find the sequence definition in the following two directories
                                                       
                                                      \u2022 Navigate to \"$DESIGNWARE_HOME/vip/svt/pcie_svt/Q-2020.03/sverilog/src/vcs/svt_pcie_driver_app_transaction_sequence_collection.svp\" file. All the base and PCIe sequences are defined in this file.\n\n\u2022 When the OFS UVM build command is executed, it creates \"vip\" directory under \"$OFS_BUILD_ROOT/ofs-agx7-pcie-attach/verification\". You can also find the same sequence file at \"$IOFS_BUILD_ROOT/ofs-agx7-pcie-attach/verification/vip/pcie_vip/src/sverilog/vcs/svt_pcie_driver_app_transaction_sequence_collection.sv\"\n
                                                      "},{"location":"hw/common/user_guides/ug_sim_ofs_agx7_pcie_attach/ug_sim_ofs_agx7_pcie_attach/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                      Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                       
                                                      OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                      "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/","title":"Accelerator Functional Unit Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae","text":""},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#1-introduction","title":"1. Introduction","text":"
                                                      This document is a design guide for creating an Accelerator Functional Unit (AFU) using Open FPGA Stack (OFS) for Intel\u00ae Stratix 10\u00ae FPGA. The AFU concept consists of separating the FPGA design development process into two parts, the FIM and AFU, as shown in the diagram below:
                                                       
                                                       
                                                       
                                                      This diagram shows the FPGA board interface development separation from the internal FPGA workload creation. This separation starts with the FPGA Interface Manager (FIM), which consists of the external interfaces and board management functions. The FIM is the base system layer typically provided by board vendors. The FIM interface is specific to a particular physical platform. The AFU uses the external interfaces with user-defined logic to perform a specific application. Separating the lengthy and complicated process of developing and integrating external interfaces for an FPGA into a board allows the AFU developer to focus on their workload needs.  Intel\u00ae OFS for Intel\u00ae Stratix 10\u00ae FPGA provides the following tools for rapid AFU development:
                                                       
                                                         
                                                      • Scripts for both compilation setup
                                                        •  
                                                      • Integration with Open Programmable Acceleration Engine (OPAE) SDK for rapid software development for your AFU application
                                                        •  
                                                       
                                                      Please notice that the AFU region consists of both static and PR logic in the above block diagram. Creating AFU logic for the static region is described in FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs. This guide covers logic in the AFU Main (PR) region.
                                                      "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#11-document-organization","title":"1.1 Document Organization","text":"
                                                      This document is organized as follows:
                                                       
                                                         
                                                      • Description of design flow
                                                        •  
                                                      • Interfaces and functionality provided in the Intel\u00ae FPGA PAC D5005 FIM
                                                        •  
                                                      • Downloading and installing Intel\u00ae OFSand OPAE SDK
                                                        •  
                                                      • Hardware/Software co-simulation using ASE
                                                        •  
                                                      • Testing the AFU example in Intel\u00ae FPGA PAC D5005
                                                        •  
                                                      • Debugging an AFU with Remote Signal Tap
                                                        •  
                                                       
                                                      This guide provides theory followed by tutorial steps to solidify your AFU development knowledge.
                                                       
                                                      This guide uses the Intel\u00ae FPGA PAC D5005 as the platform for all tutorial steps. Additionally, this guide and the tutorial steps can be used with other platforms; However, please consult the board and FIM supplier of other platforms for specific instructions on the use of custom FIM to develop  AFU design.
                                                       
                                                      If you have worked with previous Intel\u00ae Programmable Acceleration products, you will find OFS for Intel\u00ae Stratix 10\u00ae FPGA is similar; however, there are differences, and you are advised to carefully read and follow the tutorial steps to understand the design tools and flow fully.
                                                      "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#12-prerequisite","title":"1.2 Prerequisite","text":"
                                                      This guide assumes you understand the following FPGA logic design-related knowledge and skills:
                                                       
                                                         
                                                      • FPGA compilation flows, including the  Intel\u00ae Quartus\u00ae Prime Pro Edition design flow.
                                                        •  
                                                      • Static Timing closure, including familiarity with the Timing Analyzer tool in  Intel\u00ae Quartus\u00ae Prime Pro Edition software, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on critical timing paths.
                                                        •  
                                                      • RTL and coding practices to create synthesized logic.
                                                        •  
                                                      • High-level synthesis (HLS) and Platform Designer design entry tools are supported.
                                                        •  
                                                      • RTL simulation tools.
                                                        •  
                                                      • Signal Tap Logic Analyzer tool in the  Intel\u00ae Quartus\u00ae Prime Pro Edition software.
                                                        •  
                                                      "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#121-development-environment","title":"1.2.1 Development Environment","text":"
                                                      To run the tutorial steps in this guide requires this development environment:
                                                       Item Version Operating System RHEL 8.6 Python 3.6.8 cmake 3.15 GCC 7.4.0 git 1.8.3.1 perl 5.8.8 
                                                      Verify your development has the above tools installed.
                                                       
                                                      The following server and Intel\u00ae PAC card are required to run the examples in this guide:
                                                       
                                                         
                                                      1. Intel\u00ae FPGA PAC D5005 with root entry hash erased (Please contact Intel\u00ae for root entry hash erase instructions). The standard Intel\u00ae FPGA PAC D5005 card is programmed only to allow the FIM binary files signed by Intel\u00ae to be loaded. The root entry hash erases process will allow unsigned FIM binary files to be loaded.
                                                        2.  
                                                      2. Qualified Server Models see Qualified Servers.
                                                        3.  
                                                      3. Intel\u00ae FPGA PAC D5005 installed in the qualified server following instructions in OFS Getting Started User Guide: For Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.
                                                        4.  
                                                      "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#13-acceleration-functional-unit-afu-development-flow","title":"1.3 Acceleration Functional Unit (AFU) Development Flow","text":"
                                                      OFS Stack provides a rapid design methodology for creating complex FPGA applications. In addition, you are provided with the following:
                                                       
                                                         
                                                      • Hardware shell layer, known as FIM 
                                                        •  
                                                      • Software stack including tools for debug/diagnostics
                                                        •  
                                                      • FPGA design flow  with full-stack simulation support
                                                        •  
                                                      • AFU code samples demonstrating all interfaces
                                                        •  
                                                       
                                                      For any non-Intel\u00ae platform, contact your board vendor for the above components specific to the platform. To start with AFU development, the first step should be to understand your platform capabilities. For example, what interface is the FPGA connected to the Host machine over PCI-E, if it is AXI like the Intel\u00ae Stratix 10\u00ae FPGA Platform, or CCIP or CXL. Does the platform provide an External Memory Interface or the HSSI interface? Once you know what the platform offers, you can develop your AFU requirements and architecture as the next step.  This document will cover example AFU architecture and things that will help build AFU for Intel\u00ae Stratix 10\u00ae FPGA reference platform and others coming in the future. In addition, this knowledge can be relatively applied for AFU development on other vendor-provided platforms.
                                                       
                                                      The figure below shows a typical AFU development process independent of the platform used. 
                                                       
                                                       
                                                      flowchart  TB;\n    A[Understand platform capabilities with OFS]-->B[Review AFU requirements and code samples provided];\n    B[Review AFU requirements and code samples provided]-->C[Define AFU architecture];\n    C[Define AFU architecture]-->D[Design AFU hardware];\n    D[Design AFU hardware]-->E[Develop AFU software to control hardware];\n    E[Develop AFU software to control hardware]-->F{\"Simulate in AFU Simulation Enviroment (ASE)\"};\n    F:::if -- Pass --> H[\"Compile AFU for synthesis, place & route and timing (uses Quartus)\"];\n    H[\"Compile AFU for synthesis, place & route and timing (uses Quartus)\"] --> I[\"Analyze Quartus Compile reports\"];\n    I --> J{\"Quartus reports clean? (e.g. timing closed)\"};\n    J:::if -- No --> P2;\n    J -- Yes --> K[\"Run/Validate design on OFS Platform\"];\n    K --> L{\"Hardware validation pass?\"};\n    L == Yes ==> M[\"AFU ready to deploy\"];\n    L -- No --> N[\"Debug on hardware using traditional FPGA tools (e.g. SignalTab\"];\n    N --> P2[\"Fix AFU design (e.g Design changes, timing closure constraints)\"];\n    P2 --> O{\"Need functional validation?\"};\n    O:::if -- Yes -->P[\"Fix AFU design (e.g Functional design changes, bug fixes)\"];\n    O -- No -->H;    \n    F -- Fail --> P;\n    P -->D;      \n\n    classDef default color:#fff,fill:#0071c5,stroke:#71c5,stroke-width:1px\n    classDef if color:#0071c5,fill:#fff,stroke:#0071c5,stroke-width:2px
                                                      "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#131-high-level-data-flow","title":"1.3.1. High Level Data Flow","text":"
                                                      The OFShigh-level data flow is shown below:  The control and data are composed of the following:
                                                       
                                                         
                                                      • Host Interface Adapter (PCIe)
                                                        •  
                                                      • Low-Performance Peripherals
                                                           
                                                        • Slow speed peripherals (I2C, Smbus, etc)
                                                          •  
                                                        • Management peripherals (FME)
                                                          •  
                                                         
                                                        •  
                                                       
                                                         
                                                      • High-Performance Peripherals
                                                           
                                                        • Memory peripherals
                                                          •  
                                                        • Acceleration Function peripherals
                                                          •  
                                                        • HPS Peripheral
                                                          •  
                                                         
                                                        •  
                                                       
                                                         
                                                      • Fabrics
                                                           
                                                        • Peripheral Fabric (multi-drop)
                                                          •  
                                                        • AFU Streaming fabric (point to point)
                                                          •  
                                                         
                                                        •  
                                                       
                                                      Peripherals are connected using AXI or Avalon:
                                                       
                                                         
                                                      • Via the peripheral fabric (AXI4-Lite, multi-drop)
                                                        •  
                                                      • Via the AFU streaming fabric (AXI-S, point to point)
                                                        •  
                                                       
                                                      Peripherals are presented to software as:
                                                       
                                                         
                                                      • OFS managed peripherals that implement DFH CSR structure.  
                                                        •  
                                                      • Native driver managed peripherals (i.e., Exposed via an independent PF, VF)
                                                        •  
                                                       
                                                      The peripherals connected to the peripheral fabric are primarily OPAE managed resources, whereas the peripherals connected to the AFU are \"primarily\" driven by native OS drivers. The word \"primarily\" is used since the AFU is not mandated to expose all its peripherals to Intel\u00ae OPAE. Instead, it can be connected to the peripheral fabric but can choose to expose only a subset of its capability to OPAE.
                                                       
                                                      OFS uses a defined set of CSRs to expose the functionality of the FPGA to the host software. These registers are described in Open FPGA Stack Reference Manual - MMIO Regions section.
                                                       
                                                      If you make changes to the FIM that affect the software operation, Intel\u00ae OFS provides a mechanism to communicate that information to the proper software driver. The Device Feature Header (DFH) structure provides a mechanism to maintain compatibility with OPAE software. Please see FPGA Device Feature List (DFL) Framework Overview for an excellent description of DFL operation from the driver perspective.
                                                       
                                                      When planning your address space for your FIM updates, please be aware OFS FIM targeting Intel\u00ae FPGA PAC D5005, 256KB of MMIO region is allocated for external FME features, and 128kB of MMIO region is given for external port features. Each external feature must implement a feature DFH, and the DFH needs to be placed at the 4KB boundary. The last feature in the external feature list must have the EOL bit in its DFH set to 1 to mark the end of the external feature list. Since the FPGA address space is limited, consider using an indirect addressing scheme to conserve address space.
                                                      "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#132-considerations-for-pim-usage","title":"1.3.2. Considerations for PIM Usage","text":"
                                                      When creating an AFU, a designer needs to decide what type of interfaces the platform (FIM) should provide to the AFU.  The FIM can provide the native interfaces (i.e. PCIe TLP commands) or standard memory mapped interfaces (i.e. AXI-MM or AVMM) by using the PIM.  The PIM is an abstraction layer consisting of a collection of SystemVerilog interfaces and shims to enable partial AFU portability across hardware despite variations in hardware topology and native interfaces. The PIM adds a level of logic between the AFU and the FIM converting the native interfaces from the FIM to match the interfaces provided by the AFU.
                                                       
                                                       
                                                      The following resources are available to assist in creating an AFU:
                                                       
                                                      PIM Core Concepts provides details on using the PIM and its capabilities. 
                                                       
                                                      Connecting an AFU to a Platform using PIM guides you through the steps needed to connect a PIM Based AFU to the FIM. 
                                                       
                                                      The AFU Tutorial provides several AFU examples.  These examples can be run with the current OFS FIM package.  There are three AFU types of examples provided (PIM based, hybrid and native).  Each example provides the following:
                                                       
                                                         
                                                      • RTL, which includes the following interfaces: 
                                                           
                                                        • Host Channel: 
                                                             
                                                          • Host memory, providing a DMA interface.
                                                            •  
                                                          • MMIO, providing a CSR interface.  
                                                            •  
                                                           
                                                          •  
                                                        • Local Memory
                                                          •  
                                                         
                                                        •  
                                                      • Host software example interfacing to the CSR interface and host memory interface, using the OPAE C API.
                                                        •  
                                                      • Accelerator Description File .json file
                                                        •  
                                                      • Source file list
                                                        •  
                                                      "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#133-afu-interfaces-included-with-intel-fpga-pac-d5005","title":"1.3.3 AFU Interfaces Included with Intel\u00ae FPGA PAC D5005","text":"
                                                      The figure below shows the interfaces available to the AFU in this architecture. It also shows the design hierarchy with module names from the FIM (top.sv) to the PR  region AFU (afu_main.sv). One of the main differences from the previous Intel\u00ae Stratix 10\u00ae FPGA OFS architecture is a static port gasket region (port_gasket.sv) that has components to facilitate the AFU and also consists of the GBS region (afu_main.sv) via the PR  slot. The Port Gasket contains all the PR -specific modules and logic, e.g., PR  slot reset/freeze control, user clock, remote STP etc. Architecturally, a Port Gasket can have multiple PR  slots to which user workload can be programmed. However, only one PR  slot is supported for Intel\u00ae OFS Release for Intel\u00ae Stratix 10\u00ae FPGA. Therefore, everything in the Port Gasket until the PR  slot should be provided by the FIM developer. The task of the AFU developer is to add their desired application in the afu_main.sv module by stripping out unwanted logic and instantiating the target accelerator. As shown in the figure below, here are the interfaces connected to the AFU (highlighted in green) via Intel\u00ae FPGA PAC D5005 FIM:
                                                       
                                                         
                                                      • AXI Streaming (AXI-S) interface to the Host via PCIe Gen3x16
                                                        •  
                                                      • Avalon Memory-Mapped Channels (4) to the DDR4 EMIF interface
                                                        •  
                                                      • AXI Streaming (AXI-S) interface to the HSSI 10G Ethernet
                                                        •  
                                                       
                                                      "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#134-platform-capabilities","title":"1.3.4. Platform Capabilities","text":"
                                                      The FIM targets operation in the Intel\u00ae FPGA PAC D5005 card.  The block diagram of the Intel\u00ae FPGA PAC D5005 is shown below: 
                                                       
                                                      The key Intel\u00ae FPGA PAC D5005 FPGA interfaces are:
                                                       
                                                         
                                                      • Host interface      - PCIe Gen3 x 16
                                                        •  
                                                      • Network interface
                                                           
                                                        • 2 - QSFP28 cages
                                                          •  
                                                        • Current FIM supports 1 x 10 GbE, other interfaces can be created  
                                                          •  
                                                         
                                                        •  
                                                      • External Memory
                                                           
                                                        • 2 or 4 channels of DDR4-2400 to RDIMM modules
                                                          •  
                                                        • RDIMM modules =  8GB organized as 1 Gb X 72
                                                          •  
                                                         
                                                        •  
                                                      • Board Management
                                                           
                                                        • SPI interface
                                                          •  
                                                        • FPGA configuration
                                                          •  
                                                         
                                                        •  
                                                      "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#135-top-level-fpga","title":"1.3.5. Top Level FPGA","text":"
                                                      The internal FPGA architecture is shown below:
                                                       
                                                      "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2-set-up-afu-development-environment","title":"2. Set Up AFU Development Environment","text":"
                                                      This section covers:
                                                       
                                                         
                                                      • Development environment set up
                                                        •  
                                                      • Retrieving and installing OFS, OPAE SDK
                                                        •  
                                                      • Building theIntel\u00ae FPGA PAC D5005 FIM
                                                        •  
                                                      • Building a relocatable AFU tree
                                                        •  
                                                      • Compiling the host_chan_mmio example AFU
                                                        •  
                                                       
                                                      Additionally, this section includes steps to demonstrate loading and running the host_chan_mmio example AFU in an Intel\u00ae FPGA PAC D5005 equipped Linux server.
                                                      "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#21-prepare-afu-development-environment","title":"2.1. Prepare AFU development environment","text":"
                                                      Typical development and hardware test environments consist of a development server or workstation with installed FPGA development tools and a separate server installed with the target OFS-compatible FPGA PCIe card. The typical usage and flow of data between these two servers are shown below:
                                                       
                                                       
                                                      Please refer to Unit Level Simulation if you would like to make any simulation Unit Level Simulation.
                                                       
                                                      Note that both development and hardware testing can be performed on the same server if desired.
                                                       
                                                      This guide uses Intel\u00ae FPGA PAC D5005 as the target OFS-compatible FPGA PCIe card platform for demonstration steps. The Intel\u00ae FPGA PAC D5005 must be fully installed following OFS Getting Started User Guide: For Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs. If using a different OFS FPGA PCIe card, contact your supplier for instructions on how to install and operate a user-developed AFU.
                                                       
                                                      NOTE: 
                                                       
                                                      The following chapters assume you use the same server for development and Deployment (Run the FIM/AFU/SW over the Intel\u00ae FPGA PAC D5005):
                                                       
                                                      Development: Modify the FIM/AFU/SW run simulation and compile the design (Generate the binaries).  Deployment: Program the binaries under the Intel\u00ae FPGA PAC D5005 and exercise the Hardware and Sw with real hardware
                                                      "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#211-installation-of-quartus-and-ofs","title":"2.1.1. Installation of Quartus and OFS","text":"
                                                      Building AFU with OFS forIntel\u00ae Stratix 10\u00ae FPGA requires the build machine to have at least 64 GB of RAM.  
                                                       
                                                      The following is a summary of the steps to set up for AFU development:
                                                       
                                                         
                                                      1. Install  Intel\u00ae Quartus\u00ae Prime Pro Edition  23.3 Linux with Intel\u00ae Stratix 10\u00ae FPGA device support.
                                                        2.  
                                                      2. Make sure support tools are installed and meet version requirements.
                                                        3.  
                                                      3. Clone the repository.
                                                        4.  
                                                      4. Review the files provided in the repository.
                                                        5.  
                                                      5. Build a relocatable PR  tree - this will be the base FIM for your AFU.
                                                        6.  
                                                       
                                                      Intel\u00ae Quartus\u00ae Prime Pro Edition version  23.3 is the currently verified version of  Intel\u00ae Quartus\u00ae Prime Pro Edition 23.3 used for building the AFU images. The recommended Best Known Configuration (BKC) OFS Version 2023.3:
                                                       Item Version Intel\u00ae Quartus\u00ae Prime Pro Edition 23.3 Operating System RHEL 8.6 OPAE SDK 2.10.0-1 OFS Release ofs-2023.3-2 Python 3.6.8 cmake 3.15 GCC 7.4.0 git 1.8.3.1 perl 5.8.8"},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2111-installation-of-quartus","title":"2.1.1.1 Installation of Quartus","text":"
                                                      Intel Quartus Prime Pro Version 23.3 is verified to work with the latest OFS release ofs-2023.3.  However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.
                                                       
                                                      Use RedHatEnterprise Linux\u00ae (RHEL) for compatibility with your development flow and also testing your FIM design in your platform. 
                                                       
                                                      Prior to installing Quartus:
                                                       
                                                         
                                                      1.  
                                                        Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.
                                                         
                                                           
                                                        • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                                                          •  
                                                        • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.
                                                          •  
                                                         
                                                        2.  
                                                      2.  
                                                        Perform the following steps to satisfy the required dependencies.
                                                         
                                                        $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                                                         
                                                        Apply the following configurations.
                                                         
                                                        $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n
                                                         
                                                        3.  
                                                      3.  
                                                        Create the default installation path: /intelFPGA_pro/, where  is the default path of the Linux workstation, or as set by the system administrator and  is your Quartus version number. 
                                                        The installation path must satisfy the following requirements:
                                                         
                                                           
                                                        • Contain only alphanumeric characters
                                                          •  
                                                        • No special characters or symbols, such as !$%@^&*<>,
                                                          •  
                                                        • Only English characters
                                                          •  
                                                        • No spaces
                                                          •  
                                                         
                                                      4.  
                                                        Download your required Quartus Prime Pro Linux version here.
                                                         
                                                        5.  
                                                      5.  
                                                        Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.23.
                                                         
                                                        6.  
                                                      6.  
                                                        After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                                                         
                                                        export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                                                         
                                                        For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:
                                                         
                                                        export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n
                                                         
                                                        7.  
                                                      7.  
                                                        Verify, Quartus is discoverable by opening a new shell:
                                                         
                                                        $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                                                         
                                                        8. "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2112-install-ofs","title":"2.1.1.2. Install OFS","text":"
                                                           
                                                        1.  
                                                          Retrieve OFS repositories:
                                                           
                                                          The Intel\u00ae OFS FIM source code is included in the OFS GitHub repository. First, create a new directory to store the retrieved files as a clean starting point. The following is a short description of each repository, followed by the git commands for cloning. The instructions section uses the HTTPS git method for cloning repositories.  
                                                           
                                                          2.  
                                                        2.  
                                                          Navigate to the location for storage of OFS source, create the top-level source directory, and clone OFS repositories.
                                                           
                                                          3.  
                                                         
                                                        mkdir ofs_fim_build_root\ncd ofs_fim_build_root\n
                                                         
                                                        export OFS_BUILD_ROOT=$PWD\n
                                                         
                                                        git clone --recurse-submodules  https://github.com/OFS/ofs-d5005.git\n
                                                         
                                                        Console Output: 
                                                        Cloning into 'ofs-d5005' ...\nUsername for 'https://github.com': <<Enter your git hub username>>\nPassword for 'https://<<Your username>>': <<Enter your personal access token>>\nremote: Enumerating objects:  ....\n...\n...\nResolving deltas  ..., done.\n
                                                         
                                                         
                                                        Edit your bashrc file ~/.bashrc to add the following lines: 
                                                        export OFS_ROOTDIR=$OFS_BUILD_ROOT/ofs-d5005\nexport WORKDIR=$OFS_ROOTDIR\nexport VERDIR=$OFS_ROOTDIR/verification\n
                                                         
                                                        cd ofs-d5005\n
                                                         
                                                        Select the latest OFS Release 
                                                        git checkout tags/ofs-2023.3-2\n
                                                         
                                                        Console Output: ```sh You are in 'detached HEAD' state. You can look around, make experimental changes and commit them, and you can discard any commits you make in this state without impacting any branches by performing another checkout.
                                                         
                                                        If you want to create a new branch to retain commits you create, you may do so (now or later) by using -b with the checkout command again. Example:
                                                         
                                                        git checkout -b  HEAD is now at 7e4dc70 ofs-2023.3-2"},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2113-directory-structure-of-ofs","title":"2.1.1.3. Directory Structure of OFS","text":"
                                                        Verify the following directories in the $OFS_BUILD_ROOT directory with the following command.  
                                                         
                                                        cd  $OFS_ROOTDIR\nls\n
                                                         
                                                        Console Output: 
                                                         eval_scripts ipss ofs-common license  LICENSE.md  README.md  sim  src  syn  verification\n
                                                         
                                                         
                                                        The directories are arranged as shown below:
                                                         
                                                        \u251c\u2500\u2500 eval_scripts\n\u2502   \u251c\u2500\u2500 ofs_d5005_eval.sh\n\u2502   \u251c\u2500\u2500 README_ofs_d5005_eval.txt\n|\n\u251c\u2500\u2500 ofs-common\n\u2502   \u251c\u2500\u2500 scripts\n\u2502   \u251c\u2500\u2500 src\n\u2502   \u251c\u2500\u2500 verification\n|   \u251c\u2500\u2500 LICENSE.txt   \n\u2502   \u2514\u2500\u2500 README.md\n\u251c\u2500\u2500 ipss        **Directory ipss consists of Platform Designer subsystems used in FIM**\n\u2502   \u251c\u2500\u2500 hssi\n\u2502   \u251c\u2500\u2500 mem\n\u2502   \u251c\u2500\u2500 pcie\n|   \u251c\u2500\u2500 pmic \n|   \u251c\u2500\u2500 spi  \n\u2502   \u2514\u2500\u2500 README.md\n\u251c\u2500\u2500 license\n\u2502   \u2514\u2500\u2500 quartus-0.0-0.01iofs-linux.run    ** Quartus Patch with IP licenses.  \n\u2502                                         ** Note, these licenses are not used for Intel\u00ae FPGA PAC D5005** \n\u251c\u2500\u2500 sim             **Unit level simulation files**\n\u2502   \u251c\u2500\u2500 unit_test\n\u2502   \u251c\u2500\u2500 scripts\n\u2502   \u251c\u2500\u2500 bfm\n\u2502   \u251c\u2500\u2500 rp_bfm   \n\u2502   \u2514\u2500\u2500 readme.txt     \n\u2502    \n\u251c\u2500\u2500 LICENSE.txt\n\u251c\u2500\u2500 README.md\n|\n\u251c\u2500\u2500 src             **Source RTL files**\n\u2502   \u251c\u2500\u2500 afu_top\n\u2502   \u251c\u2500\u2500 includes\n\u2502   \u251c\u2500\u2500 pd_qsys\n\u2502   \u251c\u2500\u2500 top\n|   \u2514\u2500\u2500 README.md\n\u251c\u2500\u2500 external\n\u2502   \u2514\u2500\u2500 ofs-platform-afu-bbb\n|\n\u251c\u2500\u2500 syn              **Quartus compilation settings**\n\u2502   \u251c\u2500\u2500 scripts\n\u2502   \u251c\u2500\u2500 setup\n\u2502   \u251c\u2500\u2500 syn_top\n\u2502   \u251c\u2500\u2500 readme.txt\n\u2502   \u2514\u2500\u2500 README\n
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2114-license-installation-for-ofs","title":"2.1.1.4 License Installation for OFS","text":"
                                                        The required setup Intel\u00ae OFS License quartus-0.0-0.01iofs-linux.run, follow the following steps :
                                                         
                                                        cd $OFS_ROOTDIR/license\nchmod +x quartus-0.0-0.01iofs-linux.run\nsudo ./quartus-0.0-0.01iofs-linux.run\n# Confirm the license instaltion using below command.\nquartus_syn --version\n
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2115-retrieve-pim-files","title":"2.1.1.5. Retrieve PIM Files","text":"
                                                        The ofs-platform-afu-bbb repository contains the PIM files and example AFU that can be used for testing and demonstration purposes. This guide will use the host_chan_mmio example in the remaining sections to demonstrate OFS capabilities.
                                                         
                                                        cd $OFS_BUILD_ROOT\n
                                                         
                                                        git clone https://github.com/OFS/ofs-platform-afu-bbb.git\n
                                                         Edit your bashrc file ~/.bashrc to add the following lines: 
                                                        export OFS_PLATFORM_AFU_BBB=$OFS_BUILD_ROOT/ofs-platform-afu-bbb\n
                                                         
                                                        Verify the following directories are present in $OFS_BUILD_ROOT directory. 
                                                        cd $OFS_PLATFORM_AFU_BBB\n
                                                         
                                                        ls\n
                                                         
                                                        Console Output: 
                                                         COPYING  plat_if_develop  plat_if_release  plat_if_tests  README.md\n
                                                         
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#212-compiling-the-ofs-fim","title":"2.1.2. Compiling the OFS FIM","text":"
                                                        Intel\u00ae OFS provides a build script with the following FPGA image creation options:
                                                         
                                                           
                                                        • Flat compile, which combines the FIM and AFU into one FPGA image loaded into the entire FPGA device as a static image.
                                                          •  
                                                        • A PR  compile that creates an FPGA image consisting of the FIM that is loaded into the static region of the FPGA and a default AFU that is loaded into dynamic region. Additional AFU may be loaded into the dynamic region using partial reconfiguration.  
                                                          •  
                                                         
                                                        The build scripts included with OFS are verified to run in a bash shell. Other shells have not been tested. Each build script step will take several hours to complete. Building in Quartus GUI is not supported - you must build with the provided scripts.
                                                         
                                                        The following sections describe how to set up the environment and build the provided FIM with a relocatable tree supporting PR . You will use this relocatable PR  tree for all example AFU simulation and compilation steps in this guide.
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2121-setting-up-required-environment-variables","title":"2.1.2.1. Setting Up Required Environment Variables","text":"
                                                        Set required environment variables as shown below. These environment variables must be set before simulation or compilation tasks, so creating a simple script to set these variables saves time.
                                                         
                                                        Edit your bashrc file ~/.bashrc to add the following lines: 
                                                        export OPAE_SDK_REPO_BRANCH=release/2.10.0\n
                                                         
                                                        Check point : Ensure you file ~/.bashrc have all the following lines:
                                                         
                                                        export QUARTUS_MAINPATH=<Quartus install directory>\nexport QUARTUS_ROOTDIR=$QUARTUS_MAINPATH/quartus\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ipexport\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport INTELFPGAOCLSDKROOT=$QUARTUS_MAINPATH/hld\nexport QSYS_ROOTDIR=$QUARTUS_MAINPATH/qsys/bin\nexport PATH=$PATH:$QUARTUS_ROOTDIR/bin\nexport OFS_BUILD_ROOT=<root location> ** Here should be located your ofs-d5005 and ofs-platform-afu-bbb\nexport OFS_ROOTDIR=$OFS_BUILD_ROOT/ofs-d5005\nexport WORKDIR=$OFS_ROOTDIR\nexport VERDIR=$OFS_ROOTDIR/verification\nexport OFS_PLATFORM_AFU_BBB=$OFS_BUILD_ROOT/ofs-platform-afu-bbb\nexport OPAE_SDK_REPO_BRANCH=release/2.10.0\n
                                                         
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2122-compiling-your-base-fim","title":"2.1.2.2. Compiling Your Base FIM","text":"
                                                        The usage of the compile build script is shown below:
                                                         
                                                        ofs-common/scripts/common/syn/build_top.sh [-p] target_configuration work_dir \n\n      * target_configuration - Specifies the project  \n         For example: d5005\n\n      * work_dir - Work Directory for this build in the form a directory name. It is created in the <local repo directory>/ofs-d5005/<work_dir> \n          - NOTE: The directory name must start with \"work\". If the working directory exists, the script stops and asks if you want to overwrite the directory.\n            - e.g.\n                - ofs-common/scripts/common/syn/build_top.sh d5005 work_d5005\n\n                work directory as a name will be created in <local repo directory>/ofs-d5005/work_d5005\n\n\n                The obmission of <work_dir> results in a default work directory (<local repo  directory>/ofs-d5005/work)\n\n        - compile reports and artifacts (.rpt, .sof, etc) are stored in <work_dir>/syn/<OFS_PROJECT>/<OFS_FIM>/<OFS_BOARD>/syn_top/output_files\n\n        - There is a log file created in ofs-d5005 directory.  \n        - [-p]  Optional switch for creating a relocatable PR  build tree supporting the creation of a PR -able AFU workload.   \n        The \"-p\" switch invokes generate_pr_release.sh at the end of the FIM build and writes the PR  build tree to the top of the working directory. More information on this option is provided below. \n
                                                         In the following example, you will build the provided example design using a flat, non-PR  build flow. If you use the -p, you could avoid the section.
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#21221-relocatable-pr-directory-tree","title":"2.1.2.2.1. Relocatable PR Directory Tree.","text":"
                                                        Build the provided base example design:
                                                         
                                                        cd $OFS_ROOTDIR\n
                                                         
                                                        ofs-common/scripts/common/syn/build_top.sh d5005 work_d5005\n
                                                         
                                                        Console Output: 
                                                            ... build takes ~5 hours to complete\n\nCompile work directory:     <$OFS_BUILD_ROOT>/work_d5005/syn/syn_top\nCompile artifact directory: <$OFS_BUILD_ROOT>/work_d5005/syn/syn_top/output_files\n\n\n***********************************\n***\n***        OFS_PROJECT: d5005\n***        Q_PROJECT:  d5005\n***        Q_REVISION: d5005\n***        SEED: 03\n***        Build Complete\n***        Timing Passed!\n***\n
                                                         
                                                         
                                                        Pro Tip: if the timing report fails, try to go into the iofs_pr_afu.qsf and modify the seed number from 3 to 4, it will create multiple seed/starting points of your design to find the best timing/fit. /home/<myuser>/<mainfolderforOFS>/ofs-d5005/work_d5005/syn/syn_top/iofs_pr_afu.qsf 
                                                        set_global_assignment -name SEED 0 #0-4\n
                                                         
                                                         
                                                        The build script copies the ipss, sim, src, and syn directories to the specified work directory, and then these copied files are used in the Quartus compilation process. Therefore, do not edit the files in the work directory; these files are copies of source files.
                                                         
                                                        Some of the critical output files are described below:
                                                         
                                                        $OFS_ROOTDIR//syn/syn_top 
                                                        \u251c\u2500\u2500 syn_top                    //Intel\u00ae FPGA PAC D5005 Quartus build area with Quartus files used this build\n\u2502  \u251c\u2500\u2500 d5005.ipregen.rpt       // IP regeneration report states the output of IP upgrade\n\u2502  \u251c\u2500\u2500 d5005.qpf               // Quartus Project File (qpf) mentions about Quartus version and project revision\n\u2502  \u251c\u2500\u2500 d5005.qsf               // Quartus Settings File (qsf) lists current project settings and entity level assignments\n\u2502  \u251c\u2500\u2500 d5005.stp               // Signal Tap file included in the d5005.qsf. This file can be modified as required\n\u2502  \u251c\u2500\u2500 fme_id.mif              // the fme id hex value is stored in a mif file format\n\u2502  \u251c\u2500\u2500 iofs_pr_afu.json        // PR JSON file\n\u2502  \u251c\u2500\u2500 iofs_pr_afu.qsf                // PR AFU qsf file\n\u2502  \u251c\u2500\u2500 iofs_pr_afu_sources.tcl        // AFU source file list\n
                                                         
                                                        $OFS_ROOTDIR//syn/syn_top/output_files == Directory with build reports and FPGA programming files. 
                                                        The programming files consist of the Quartus generated d5005.sof and d5005.pof. The Intel\u00ae FPGA PAC D5005 board hardware provides a 2 Gb flash device to store the FPGA programming files and a BMC CARD that reads this flash and programs the Intel\u00ae FPGA PAC D5005 Intel\u00ae Stratix 10\u00ae FPGA. The ./ofs-common/scripts/common/syn/build_top.sh script runs script file ./ofs-common/scripts/common/syn/build_top.sh which takes the Quartus generated d5005.sof and creates binary files in the proper format to be loaded into the 2 Gb flash device.  You can also run build_flash.sh by itself if needed. 
                                                         
                                                        The build script will run PACSign and create an unsigned FPGA programming file for both user1 and user2 locations of the Intel\u00ae FPGA PAC D5005 flash. Please note, if the Intel\u00ae FPGA PAC D5005 has the root entry hash key loaded, then PACsign must be run to add the proper key to the FPGA binary file. Please see Security User Guide: Intel\u00ae Open FPGA Stack for Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs for details on the security aspects of Intel\u00ae Open FPGA Stack and refer to Board Management User Guide for Flash partition.
                                                         
                                                        The following table provides further detail on the generated bin files.
                                                         File Description d5005.sof This is the Quartus generated programming file created by Quartus synthesis and place and route. This file can be used to program the FPGA using a JTAG  programmer. This file is the source file for the binary files used to program the FPGA flash. d5005.bin This is an intermediate raw binary image of the FPGA d5005_page1.bin This is the binary file created from the input file, d5005.sof. This file is used as the input file to the PACSign utility to generate d5005_page1_unsigned.bin binary image file. d5005_page1_unsigned.bin This is the unsigned PACSign output which can be programmed into the FPGA flash of an unsigned Intel\u00ae FPGA PAC D5005 using the OPAE SDK utility fpgasupdate mfg_d5005_reversed.bin A particular programming file for a third-party device used in board manufacturing. This file is typically not used. 
                                                        build/output_files/timing_report == Directory containing clocks report, failing paths and passing margin reports
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#213-relocatable-pr-directory-tree","title":"2.1.3. Relocatable PR Directory Tree","text":"
                                                        If you are developing FIM to be used by another team developing the AFU workload, scripts are provided that create a relocatable PR  directory tree. ODM and board developers will use this capability to enable a broad set of AFU to be loaded on a board using PR . The relocatable PR  directory contains the Quartus *.qdb file that goes the FIM.
                                                         
                                                        Creating the relocatable PR directory tree requires a clone of the Intel\u00ae Basic Building Blocks (BBB) repository. The OFS_PLATFORM_AFU_BBB environment variable must point to the repository.  If not done previously, clone the Intel\u00ae Basic Building Blocks repository and create OFS_PLATFORM_AFU_BBB environment variable.
                                                         
                                                        cd $OFS_BUILD_ROOT\n
                                                         
                                                        git clone https://github.com/OFS/ofs-platform-afu-bbb.git\n
                                                         
                                                        export OFS_PLATFORM_AFU_BBB=$OFS_BUILD_ROOT/ofs-platform-afu-bbb\n
                                                         
                                                        cd $OFS_ROOTDIR\n
                                                         
                                                        You can create this relocatable PR  directory tree by either:
                                                         
                                                           
                                                        • Build FIM and AFU using ofs-common/scripts/common/syn/build_top.sh followed by running /syn/common/scripts/generate_pr_release.sh (section 2.1.3. Relocatable PR Directory Tree)
                                                          •  
                                                        • Build FIM and AFU using ofs-common/scripts/common/syn/build_top.sh with optional -p switch included
                                                          •  
                                                         
                                                        The generate_pr_release.sh has the following command structure:
                                                         
                                                        ofs-common/scripts/common/syn/generate_pr_release.sh -t <path to generated release tree> *Board Build Target* <work dir from build_top.sh>\nWhere:\n-t <path to generated release tree> = location for your relocatable PR  directory tree\n*Board Build Target* is the name of the board target/FIM e.g. d5005\n<work dir from build_top.sh> \n
                                                         Here is an example of running the generate_pr_release.sh script in user mode:
                                                         
                                                        ofs-common/scripts/common/syn/generate_pr_release.sh -t work_d5005/build_tree d5005  work_d5005\n
                                                         
                                                        Console Output: 
                                                        **********************************\n********* ENV SETUP **************\nFIM Project:\n  OFS_PROJECT = d5005\n  OFS_FIM     = .\n  OFS_BOARD   = .\n  Q_PROJECT   = d5005\n  Q_REVISION  = d5005\n  Fitter SEED = 03\nFME id\n  BITSTREAM_ID = 040100022c164db1\n  BITSTREAM_MD = 0000000002212053\n\n...\n...\n
                                                         
                                                         
                                                        The resulting relocatable build tree has the following structure:
                                                         
                                                        .\n\u251c\u2500\u2500 bin\n\u2502   \u251c\u2500\u2500 afu_synth\n\u2502   \u251c\u2500\u2500 build_env_config\n\u2502   \u251c\u2500\u2500 run.sh -> afu_synth\n\u2502   \u2514\u2500\u2500 update_pim\n\u251c\u2500\u2500 hw\n\u2502   \u251c\u2500\u2500 blue_bits\n\u2502   \u2502   \u251c\u2500\u2500 d5005_page1_unsigned.bin\n\u2502   \u2502   \u2514\u2500\u2500 d5005.sof -> ../lib/build/syn/syn_top/   output_files/d5005.sof\n\u2502   \u2514\u2500\u2500 lib\n\u2502       \u251c\u2500\u2500 build\n\u2502       \u251c\u2500\u2500 fme-ifc-id.txt\n\u2502       \u251c\u2500\u2500 fme-platform-class.txt\n\u2502       \u2514\u2500\u2500 platform\n\u251c\u2500\u2500 README\n
                                                         
                                                        Edit your bashrc file ~/.bashrc to add the following line: 
                                                        export OPAE_PLATFORM_ROOT=$OFS_ROOTDIR/work_d5005/build_tree\n
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#214-programing-the-fim","title":"2.1.4. Programing the FIM","text":"
                                                           
                                                        1. Run the following command to find the PCIe address for your card.
                                                          2.  
                                                         
                                                        sudo fpgainfo fme\n
                                                         
                                                        Console Output: 
                                                        Board Management Controller, MAX10 NIOS FW version: 2.0.14\nBoard Management Controller, MAX10 Build version: 2.0.8\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nDevice Id                        : 0xBCCE\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 288511863935352239\nBitstream Version                : 4.0.1\nPr Interface Id                  : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98\nBoot Page                        : user\n
                                                         
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#2141-load-fim-into-the-flash-of-the-intel-fpga-pac-d5005","title":"2.1.4.1. Load FIM into the Flash of the Intel\u00ae FPGA PAC D5005","text":"
                                                        The base FIM used in AFU compilation must be loaded on the board. In this step, you will load the generated FIM binary into the Intel\u00ae FPGA PAC D5005 FPGA flash. By performing this step, subsequent AFU developed in this guide will use this base FIM and allow your newly created AFU to match the base FIM loaded on the board.
                                                         
                                                        More information related to fpgaupdate is located OFS Getting Started User Guide: For Intel\u00ae Stratix 10\u00ae PCIe Attach FPGAs.
                                                         
                                                        Run fpgasupdate to load the image into the user location of the Intel\u00ae FPGA PAC D5005 FPGA flash and the RSU  command to reboot the PCIE Card:
                                                         
                                                        sudo fpgasupdate $OFS_ROOTDIR/work_d5005/syn/syn_top/output_files/d5005_page1_unsigned.bin 3b:00.0\n
                                                         Run rsu command to re-configure FPGA on Intel\u00ae FPGA PAC D5005. 
                                                        sudo rsu bmcimg 3b:00.0\n
                                                         
                                                        sudo fpgainfo fme\n
                                                         
                                                        Console Output: ```sh
                                                         
                                                        Board Management Controller, MAX10 NIOS FW version: 2.0.14 Board Management Controller, MAX10 Build version: 2.0.8 //****** FME ******// Object Id                        : 0xF000000 PCIe s:b:d.f                     : 0000:3b:00.0 Device Id                        : 0xBCCE Socket Id                        : 0x00 Ports Num                        : 01 Bitstream Id                     : 288511863935352239 Bitstream Version                : 4.0.1 Pr Interface Id                  : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98 Boot Page                        : user 
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#30-opae-software-development-kit","title":"3.0 OPAE Software Development Kit","text":"
                                                        The OPAE SDK software stack sits in user space on top of the Intel\u00ae OFS kernel drivers. It is a common software infrastructure layer that simplifies and streamlines the integration of programmable accelerators such as FPGAs into software applications and environments. OPAE consists of a set of drivers, user-space libraries, and tools to discover, enumerate, share, query, access, manipulate, and re-configure programmable accelerators. OPAE is designed to support a layered, common programming model across different platforms and devices. To learn more about OPAE, its documentation, code samples, an explanation of the available tools, and an overview of the software architecture, please visit the OPAE.io page.
                                                         
                                                        The OPAE SDK source code is contained within a single GitHub repository hosted at the OPAE GitHub. This repository is open source and should not require any permissions to access.
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#31-opae-sdk-build-environment-setup","title":"3.1 OPAE SDK Build Environment Setup","text":"
                                                        This installation process assumes the user has access to an internet connection to pull specific GitHub repositories and satisfy package dependencies. If an offline install process is required, please reach out to your Intel\u00ae representative.
                                                         
                                                        1. Before OPAE SDK installation, the user must remove any prior OPAE frameworks. To remove these packages:
                                                         
                                                        sudo dnf remove opae*\n
                                                         
                                                        2. The user must enable the following repository changes in order to install all dependencies on CentOS 8.3:
                                                         
                                                        sudo dnf config-manager --set-enabled powertools\nsudo dnf install epel-release\n
                                                         
                                                        3. The following package dependencies must be satisfied by the user. Double check that all packages have been found and installed:
                                                         
                                                        sudo dnf install autoconf automake bison boost boost-devel cmake doxygen dwarves elfutils-libelf-devel \\\nflex gcc gcc-c++ git hwloc-devel json-c-devel libarchive libedit libedit-devel libpcap libpng12 libuuid libuuid-devel libxml2 libxml2-devel make ncurses  \\\nncurses-devel ncurses-libs openssl-devel python2-pip python3-devel python3-jsonschema rsync tbb-devel libudev-devel\n
                                                         
                                                        All steps in this installation will use a generic top-level directory at $OFS_BUILD_ROOT. If the user has created a different top-level directory, replace this path with the user's custom path.
                                                         
                                                        4. Initialize an empty git repository and clone the tagged OPAE SDK source code:
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#32-install-opae-sdk","title":"3.2 Install OPAE SDK","text":"
                                                        Perform the following steps to install OPAE SDK:
                                                         
                                                        cd $OFS_BUILD_ROOT\ngit clone https://github.com/OFS/opae-sdk.git\ncd opae-sdk\ngit checkout tags/2.10.0-1 -b release/2.10.0\n
                                                         Verify proper branch is selected
                                                         
                                                        git describe\n  2.10.0-1\n\ngit branch\n  master\n  * release/2.10.0\n
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#33-building-and-installing-the-opae-sdk","title":"3.3 Building and Installing the OPAE SDK","text":"
                                                        1. Build the OPAE SDK source code, and pack it into several local RPM packages. Building the code into packages allows for easier installation and removal. This build script can use multiple processors to parallelize the build process. Display how many processors are available with the nproc command, and then specify how many make threads to utilize with the -j option. Note that the number of threads can exceed the number of processors. In this case, the number of threads is set to the number of processors in the system.
                                                         
                                                        cd $OFS_BUILD_ROOT/opae-sdk\nmkdir install-opae-sdk\ncd install-opae-sdk\ncmake .. -DCPACK_GENERATOR=RPM -DOPAE_BUILD_FPGABIST=ON -DOPAE_BUILD_PYTHON_DIST=ON -DCMAKE_BUILD_PREFIX=/install-opae-sdk \nmake -j `nproc`\nmake -j `nproc` package_rpm\n
                                                         
                                                        The install-opae-sdk directory location was selected for ease of use. If the user wishes to build the OPAE SDK in a different location, they will need to replace the '..' in the above command with the direct or relative path to the opae-sdk repository.
                                                         
                                                        2. After a successful compile, there should be eight packages present:
                                                         
                                                        cd $OFS_BUILD_ROOT/opae-sdk/install-opae-sdk\nls | grep rpm\nopae-2.10.0-1.x86_64.rpm                                                                                                    \nopae-PACSign-2.10.0-1.x86_64.rpm                                                                                            \nopae-devel-2.10.0-1.x86_64.rpm                                                                                              \nopae-libs-2.10.0-1.x86_64.rpm                                                                                               \nopae-opae.admin-2.10.0-1.x86_64.rpm                                                                                         \nopae-packager-2.10.0-1.x86_64.rpm                                                                                           \nopae-tests-2.10.0-1.x86_64.rpm                                                                                              \nopae-tools-2.10.0-1.x86_64.rpm                                                                                              \nopae-tools-extra-2.10.0-1.x86_64.rpm\n
                                                         
                                                        3. Install the OPAE SDK packages:
                                                         
                                                        cd $OFS_BUILD_ROOT/opae-sdk/install-opae-sdk\nsudo dnf localinstall -y opae*.rpm\n
                                                         
                                                        4. check that all packages have been installed:
                                                         
                                                        rpm -qa | grep opae\nopae-devel-2.10.0-1.x86_64                                                                                                  \nopae-packager-2.10.0-1.x86_64                                                                                               \nopae-2.10.0-1.x86_64                                                                                                        \nopae-tools-2.10.0-1.x86_64                                                                                                  \nopae-PACSign-2.10.0-1.x86_64                                                                                                \nopae-tools-extra-2.10.0-1.x86_64                                                                                            \nopae-opae.admin-2.10.0-1.x86_64                                                                                             \nopae-tests-2.10.0-1.x86_64                                                                                                  \nopae-libs-2.10.0-1.x86_64\n
                                                         
                                                        5. Setup required environment variables
                                                         
                                                        export PATH=$PATH:$OFS_BUILD_ROOT/opae-sdk/install-opae-sdk/bin\nexport LIBRARY_PATH=$OFS_BUILD_ROOT/opae-sdk/install-opae-sdk/lib\nexport LD_LIBRARY_PATH=$OFS_BUILD_ROOT/opae-sdk/install-opae-sdk/lib64\n
                                                         
                                                        cd ../lib/python*/site-packages\nexport PYTHONPATH=$PWD\n
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#4-compiling-an-afu","title":"4. Compiling An AFU","text":"
                                                        This section will use the FIM build tree created in the previous steps to compile an example AFU. This section will continue the work with the host_chan_mmio AFU.. You can perform the build steps listed below to demonstrate the ease in building and running a real example on the Intel\u00ae FPGA PAC D5005.
                                                         
                                                        To run the steps in this section, you must complete all steps in section 2. Set Up AFU Development Environment, and ensure the OPAE_PLATFORM_ROOT \"environment variable that points to the directory of the PR  build tree generated previously. 
                                                         
                                                        Ensure your bashrc file ~/.bashrc have the following line: 
                                                        export OPAE_PLATFORM_ROOT=$OFS_ROOTDIR/work_d5005/build_tree\n
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#41-set-afu-synthesis-environment","title":"4.1. Set AFU Synthesis Environment","text":"
                                                        Here, you will create the synthesis environment to build the host_chan_mmio example. The PIM flow includes the synthesis environment creation script afu_synth_setup for this task. The usage of afu_synth_setup is shown below:
                                                         
                                                        usage: afu_synth_setup [-h] -s SOURCES [-p PLATFORM] [-l LIB] [-f] dst\nGenerate a Quartus build environment for an AFU. A build environment is\ninstantiated from a release and configured for the specified AFU. AFU\nsource files are specified in a text file parsed by rtl_src_config,\nwhich is part of the OPAE base environment.\npositional arguments:\n  dst                   Target directory path (directory must not exist).\noptional arguments:\n  -h, --help            show this help message and exit\n  -s SOURCES, --sources SOURCES\n                        AFU source specification file that will be passed to\n                        rtl_src_config. See \"rtl_src_config --help\" for the\n                        file's syntax. rtl_src_config translates the source\n                        list into either Quartus or RTL simulator syntax.\n  -p PLATFORM, --platform PLATFORM\n                        FPGA platform name.\n  -l LIB, --lib LIB     FPGA platform release hw/lib directory. If not\n                        specified, the environment variables OPAE_FPGA_HW_LIB\n                        and then BBS_LIB_PATH are checked.\n  -f, --force           Overwrite target directory if it exists.\n
                                                         
                                                        Execute afu_synth_setup \"as follows to create the synthesis environment for a host_chan_mmio \"AFU that fits the Intel\u00ae FPGA PAC D5005 FIM previously constructed.
                                                         
                                                        cd $OFS_ROOTDIR/work_d5005/\nafu_synth_setup -s $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/test_mmio_axi1.txt build_d5005_x16\n
                                                         
                                                        Now, execute the afu_synth command that resides inside the $OFS_ROOTDIR/work_d5005/build_tree/bin directory, to actually build the host_chan_mmio AFU.
                                                         
                                                        cd $OFS_ROOTDIR/work_d5005/build_d5005_x16\n$OPAE_PLATFORM_ROOT/bin/afu_synth\n...\n...\nWrote host_chan_mmio.gbs\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n Design meets timing\n===========================================================================\n
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#411-loading-and-running-the-host_chan_mmio-example-afu","title":"4.1.1. Loading and Running the host_chan_mmio example AFU","text":"
                                                        Once the compilation completes successfully, load the new bitstream file, host_chan_mmio.gbs, into the partial reconfiguration region of the target Intel\u00ae FPGA PAC D5005. Keep in mind, that the loaded image is dynamic - this image is not stored in flash, and if the card is power cycled, then the PR  region is re-loaded with the default AFU.
                                                         
                                                        To load the image, perform the following steps:
                                                         
                                                        cd $OFS_ROOTDIR/work_d5005/build_d5005_x16\nsudo fpgasupdate host_chan_mmio.gbs 3b:00.0\n[sudo] password for <<Your username>>: \n[WARNING ] Update starting. Please do not interrupt.\n[INFO    ] \nPartial Reconfiguration OK\n[INFO    ] Total time: 0:00:01.88\n
                                                         
                                                        Determine the BDF of the Intel\u00ae FPGA PAC D5005.
                                                         
                                                        The PCIe BDF address is initially determined when the server powers on. The user can determine the addresses of all Intel\u00ae FPGA PAC D5005 using lspci:
                                                         
                                                        lspci -d :bcce\n3b:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                                                         
                                                        Set up your board to work with the newly loaded host_chan_mmio.gbs
                                                         
                                                           
                                                        1.  
                                                          Create the Virtual Functions (VFs): 
                                                          sudo pci_device 3b:00.0 vf 3\n
                                                           
                                                          2.  
                                                        2.  
                                                          Verify that all three VFs have been created.
                                                           
                                                          3.  
                                                         
                                                        $ lspci -s 3b:00\n3b:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n3b:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\n3b:00.2 Processing accelerators: Intel Corporation Device bcce (rev 01)\n3b:00.3 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                                                         
                                                           
                                                        1. Bind the 3 VFs to the vfio-pci driver.
                                                          2.  
                                                         
                                                        sudo opae.io init -d  , e.g. 
                                                        $ sudo opae.io init -d 0000:3b:00.1 user:user\nBinding (0x8086,0xbccf) at 0000:3b:00.1 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:3b:00.1 is 142\nAssigning /dev/vfio/142 to <local user>\nChanging permissions for /dev/vfio/142 to rw-rw----\n\n$ sudo opae.io init -d 0000:3b:00.2 user:user\nBinding (0x8086,0xbccf) at 0000:3b:00.2 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:3b:00.2 is 143\nAssigning /dev/vfio/143 to <local user>\nChanging permissions for /dev/vfio/143 to rw-rw-----\n\n$ sudo opae.io init -d 0000:3b:00.3 user:user\nBinding (0x8086,0xbccf) at 0000:3b:00.3 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:3b:00.3 is 144\nAssigning /dev/vfio/144 to <local user>\nChanging permissions for /dev/vfio/144 to rw-rw----\n
                                                         
                                                           
                                                        1. Verify the new AFU is loaded.  The host_chan_mmio AFU GUID is 76d7ae9c-f66b-461f-816a-5428bcebdbc5.
                                                          2.  
                                                         
                                                        $ fpgainfo port\n//****** PORT ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0x603B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : d15ab1ed-0000-0000-0210-000000000000\n//****** PORT ******//\nObject Id                        : 0x403B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : 76d7ae9c-f66b-461f-816a-5428bcebdbc5\n//****** PORT ******//\nObject Id                        : 0x203B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n
                                                         
                                                        Run the host_chan_mmio software application to demonstrate the newly loaded AFU image. You navigate to $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw and compile the software application and then run.
                                                         
                                                        If OPAE SDK libraries were not installed in the default systems directory /usr/lib64/ \", define the OPAE_LOC environment variable to point to the directory where the OPAE SDK libraries were installed.
                                                         
                                                        $ export OPAE_LOC=/usr\n$ export LIBRARY_PATH=$OPAE_LOC/lib64:$LIBRARY_PATH\n$ export LD_LIBRARY_PATH=$OPAE_LOC/lib64:$LD_LIBRARY_PATH\n
                                                         
                                                        cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw\nmake \n./host_chan_mmio\n
                                                         
                                                        Console Output: 
                                                        AFU ID:  76d7ae9cf66b461f 816a5428bcebdbc5\nAFU MMIO interface: AXI Lite\nAFU MMIO read bus width: 64 bits\n512 bit MMIO write supported: yes\nAFU pClk frequency: 250 MHz\n\nTesting 32 bit MMIO reads:\n  PASS - 4 tests\n\nTesting 32 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 64 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 512 bit MMIO writes:\n  PASS\n
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#412-loading-and-running-the-hello_world-example-afu","title":"4.1.2. Loading and running the hello_world example AFU","text":"
                                                        The platform-independent example AFUs repository provides some interesting examples AFU's. In this section, you will compile and execute the PIM-based hello_world AFU. The RTL of the hello_world AFU receives from the host application an address via memory-mapped I/O (MMIO) write and generates a DMA write to the memory line at that address. The content written to memory is the string \"Hello world!\". The host application spins, waiting for the memory line to be updated. Once available, the software prints out the string.
                                                         
                                                        The hello_world example AFU consists of the following files. 
                                                         
                                                        hello_world\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u251c\u2500\u2500 ccip\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_ccip.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u2514\u2500\u2500 hello_world.json\n\u251c\u2500\u2500 README.md\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 hello_world.c\n    \u2514\u2500\u2500  Makefile\n
                                                         The hw directory contains the RTL to implement the hardware functionality using CCIP, Avalon, and AXI interfaces. However, this guide will use the AXI version of the AFU to go through the compilation steps. The sw directory of the AFU contains the source code of the host application that communicates with the AFU hardware.
                                                         
                                                        The following instructions can be used to compile other AFU samples accompanying this repository.
                                                         
                                                           
                                                        1. If not done already, download and clone the repository.
                                                          2.  
                                                         
                                                           cd $OFS_BUILD_ROOT \n   git clone https://github.com/OFS/examples-afu.git\n
                                                         
                                                           
                                                        1. Make sure to set the next environment variables.
                                                          2.  
                                                         
                                                          # OPAE and MPF libraries must either be on the default linker search paths or on both LIBRARY_PATH and LD_LIBRARY_PATH.  \n  $ export OPAE_LOC=/usr\n  $ export LIBRARY_PATH=$OPAE_LOC/lib:$LIBRARY_PATH\n  $ export LD_LIBRARY_PATH=$OPAE_LOC/lib64:$LD_LIBRARY_PATH\n\n  # OPAE_PLATFORM_ROOT points to a release tree that has been configured with the Platform Interface Manager (PIM).  \n  $ export OPAE_PLATFORM_ROOT=$OFS_ROOTDIR/work_d5005/build_tree\n
                                                         
                                                           
                                                        1.  
                                                          Compile the hello_word sample AFU.  
                                                            $ cd $OFS_ROOTDIR/work_d5005\n  $ afu_synth_setup -s $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/hw/rtl/axi/sources.txt hello_world_synth\n  $ cd hello_world_synth\n  $ ${OPAE_PLATFORM_ROOT}/bin/afu_synth\n\n\n.\n.\n.\nInfo (19538): Reading SDC files took 00:00:06 cumulatively in this process.\nWrote hello_world.gbs\n\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'hello_world.gbs'\n Design meets timing\n===========================================================================\n
                                                           
                                                          2.  
                                                        2.  
                                                          To test the AFU in actual hardware, load the hello_world.gbs to the Intel\u00ae FPGA PAC D5005 card. For this step to be successful, the Intel\u00ae FPGA PAC D5005 FIM must have already been loaded to the Intel\u00ae FPGA PAC D5005 card following the steps described in Section 2 of this document.
                                                           
                                                          3.  
                                                         
                                                          $ cd $OFS_ROOTDIR/work_d5005/hello_world_synth\n  $ sudo fpgasupdate hello_world.gbs 3b:00.0\n  [sudo] password for <<Your username>>: \n[2022-12-06 13:25:10.22] [WARNING ] Update starting. Please do not interrupt.\n[2022-12-06 13:25:12.06] [INFO    ] \nPartial Reconfiguration OK\n[2022-12-06 13:25:12.06] [INFO    ] Total time: 0:00:01.83\n
                                                         
                                                        Set up your Intel\u00ae FPGA PAC D5005 board to work with the newly loaded hello_world.gbs file.
                                                         
                                                        #  Create the Virtual Functions (VFs):\n\n $ sudo pci_device 3b:00.0 vf 3\n\n # Verify:\n $ lspci -s 3b:00\n3b:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n3b:00.1 Processing accelerators: Intel Corporation Device bccf (rev 01)\n3b:00.2 Processing accelerators: Intel Corporation Device bccf (rev 01)\n3b:00.3 Processing accelerators: Intel Corporation Device bccf (rev 01)\n\n # Bond VFs to VFIO driver.  Enter <<Your username>>\n\nsudo opae.io init -d 0000:3b:00.1 <Your username>\n Unbinding (0x8086,0xbcce) at 0000:3b:00.1 from dfl-pci\n Binding (0x8086,0xbccf) at 0000:3b:00.1 to vfio-pci\n iommu group for (0x8086,0xbccf) at 0000:3b:00.1 is 142\n Assigning /dev/vfio/142 to <Your username>\n Changing permissions for /dev/vfio/142 to rw-rw----\n\nsudo opae.io init -d 0000:3b:00.2 <Your username>\n Unbinding (0x8086,0xbccf) at 0000:3b:00.2 from dfl-pci\n Binding (0x8086,0xbccf) at 0000:3b:00.2 to vfio-pci\n iommu group for (0x8086,0xbccf) at 0000:3b:00.2 is 143\n Assigning /dev/vfio/143 to <Your username>\n Changing permissions for /dev/vfio/143 to rw-rw----\n\nsudo opae.io init -d 0000:3b:00.3 <Your username>\n Unbinding (0x8086,0xbccf) at 0000:3b:00.3 from dfl-pci\n Binding (0x8086,0xbccf) at 0000:3b:00.3 to vfio-pci\n iommu group for (0x8086,0xbccf) at 0000:3b:00.3 is 144\n Assigning /dev/vfio/144 to <Your username>\n Changing permissions for /dev/vfio/144 to rw-rw----\n\n# < Verify the new AFU is loaded.  The hello_world AFU GUID is \"c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\".\n\n$ fpgainfo port\n//****** PORT ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0x603B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\n//****** PORT ******//\nObject Id                        : 0x403B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\n//****** PORT ******//\nObject Id                        : 0x203B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n
                                                         
                                                           
                                                        1. Compile and execute the host application of the hello_world AFU. You should see the application outputs the \"Hello world!\" message in the terminal.
                                                          2.  
                                                         
                                                          # Move to the sw directory of the hello_world AFU and run the following commands in user mode\n   cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/sw/\n\n   make\n\n  # Launch the host application\n   ./hello_world\n   Hello world TLP!\n
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#413-modify-the-afu-user-clocks-frequency","title":"4.1.3. Modify the AFU user clocks frequency","text":"
                                                        An OPAE compliant AFU specifies the frequency of the uclk_usr and uclk_usr_div2 clocks through the JSON file for AFU configuration located under the <afu_example>/hw/rtl directory of an AFU design. For instance, the AFU configuration file of the host_chan_mmio example is $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/host_chan_mmio.json.
                                                         
                                                        The AFU specifies the frequency for uClk_usr in its platform configuration file using the following key:value pairs:
                                                         
                                                          \"clock-frequency-high\": [<float-value>|\u201dauto\u201d|\u201dauto-<float-value>\u201d]\n  \"clock-frequency-low\": [<float-value>|\u201dauto\u201d|\u201dauto-<float-value>\u201d]\n
                                                         
                                                        These key:value tuples are used to configure the PLL of the target platform that provides the user clocks through the AFU clocks interface. In addition, the specified frequency affects the timing closure process on the user clocks during AFU compilation. 
                                                         
                                                        Setting the value field to a float number (e.g., 315.0 to specify 315 MHz) drives the AFU generation process to close timing within the bounds set by the low and high values and sets the AFU's JSON metadata to specify the user clock PLL  frequency values.
                                                         
                                                        The following example shows the JSON file of the host_chan_mmio to set the AFU uClk to 300 MHz and uClk_div2 to 150 MHz.
                                                         
                                                        {\n   \"version\": 1,\n   \"afu-image\": {\n      \"power\": 0,\n      \"clock-frequency-high\": 300,\n      \"clock-frequency-low\": 150,\n      \"afu-top-interface\":\n         {\n            \"class\": \"ofs_plat_afu\"\n         },\n      \"accelerator-clusters\":\n         [\n            {\n               \"name\": \"host_chan_mmio\",\n               \"total-contexts\": 1,\n               \"accelerator-type-uuid\": \"76d7ae9c-f66b-461f-816a-5428bcebdbc5\"\n            }\n         ]\n   }\n}\n
                                                         
                                                        Save the changes to host_chan_mmio.json file, then execute the afu_synth_setup script to create a new copy of the AFU files with the modified user clock settigns.
                                                         
                                                          $ cd $OFS_ROOTDIR/work_d5005\n  $ afu_synth_setup -s $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/test_mmio_axi1.txt build_d5005_afu_clks\n\nLoading platform database: /home/<Your username>/<Your localpath>/ofs-d5005/work_d5005/build_tree/hw/lib/platform/platform_db/ofs_d5005.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting platform/platform_afu_top_config.vh\nWriting platform/platform_if_addenda.qsf\nWriting ../hw/afu_json_info.vh\n
                                                         Compile the host_chan_mmio AFU with the new frequency values.
                                                         
                                                           cd $OFS_ROOTDIR/work_d5005/build_d5005_afu_clks\n   $OFS_ROOTDIR/work_d5005/build_tree/bin/afu_synth\n
                                                         
                                                        During the compilation phase, you will observe the Timing Analyzer uses the specified user clock frequency values as the target to close timing.
                                                         
                                                         
                                                        AFU developers must ensure the AFU hardware design meets timing. The compilation of an AFU that fails timing shows a message similar to the following.
                                                         
                                                        .\n.\n.\n\nWrote host_chan_mmio.gbs\n\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n\n  *** Design does not meet timing\n  *** See build/syn/syn_top/output_files/timing_report\n\n===========================================================================\n
                                                         
                                                        The previous output indicates the location of the timing reports for the AFU designer to identify the failing paths and perform the necessary design changes. Next, is a listing of the timing report files from a host_chan_mmio AFU that fails to meet timing after modifying the user clock frequency values.
                                                         
                                                          $ cd $OFS_ROOTDIR/work_d5005/build_d5005_afu_clks\n  $ ls build/syn/syn_top/output_files/timing_report\n\nclocks.rpt  clocks.sta.fail.summary  clocks.sta.pass.summary iofs_pr_afu_2_slow_900mv_0c_recovery.rpt\niofs_pr_afu_2_slow_900mv_0c_setup.rpt\niofs_pr_afu_2_slow_900mv_100c_recovery.rpt\niofs_pr_afu_2_slow_900mv_100c_setup.rpt\niofs_pr_afu_2_slow_vid2_0c_recovery.rpt\niofs_pr_afu_2_slow_vid2_0c_setup.rpt\niofs_pr_afu_2_slow_vid2_100c_recovery.rpt\niofs_pr_afu_2_slow_vid2_100c_setup.rpt\niofs_pr_afu_MIN_fast_900mv_0c_recovery.rpt\niofs_pr_afu_MIN_fast_900mv_0c_setup.rpt\niofs_pr_afu_MIN_fast_900mv_100c_recovery.rpt\niofs_pr_afu_MIN_fast_900mv_100c_setup.rpt\n
                                                         
                                                        Warning: AFU developers must inform software developers of the maximum operating frequency (Fmax) of the user clocks to avoid any unexpected behavior of the accelerator and potentially of the overall system.
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#5-simulating-an-afu-using-ase","title":"5. Simulating an AFU using ASE","text":"
                                                        The AFU Simulation Environment (ASE) is a hardware/software co-simulation environment for your AFU. See diagram below illustrating ASE operation:
                                                         
                                                         
                                                        ASE uses the simulator Direct Programming Interface (DPI) to provide HW/SW connectivity.  The PCIe connection to the AFU under testing is emulated with a transactional model.
                                                         
                                                        The following list describes ASE operation:
                                                         
                                                           
                                                        • Attempts to replicate the transactions that will be seen in real system.
                                                          •  
                                                        • Provides a memory model to AFU, so illegal memory accesses can be identified early.
                                                          •  
                                                        • Not a cache simulator.
                                                          •  
                                                        • Does not guarantee synthesizability or timing closure.
                                                          •  
                                                        • Does not model system latency.
                                                          •  
                                                        • No administrator privileges are needed to run ASE.  All code is user level.
                                                          •  
                                                         
                                                        The remainder of this section is a tutorial providing the steps on how to run ASE with either VCS or QuestaSim using an example AFU and the AFU build tree previously created in this guide.
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#51-set-up-steps-to-run-ase","title":"5.1. Set Up Steps to Run ASE","text":"
                                                        In this section you will set up your server to support ASE by independently downloading and installing OPAE SDK and ASE. Then, set up the required environment variables.
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#511-install-opae-sdk","title":"5.1.1. Install OPAE SDK","text":"
                                                        Follow the instructions documented in the Getting Started Guide: Intel\u00ae Open FPGA Stack for Intel\u00ae FPGA PAC D5005, section 5.0 OPAE Software Development Kit to build and install the required OPAE SDK for the Intel\u00ae FPGA PAC D5005 PAC card.
                                                         
                                                        The Intel\u00ae FPGA PAC D5005 PAC card requires opae-2.10.0-1. Follow the instructions provided in the Getting Started Guide: Intel\u00ae Open FPGA Stack for Intel\u00ae FPGA PAC D5005 section 5.0 OPAE Software Development Kit. However, just make sure to check out the cloned repository to tag 2.10.0-1 and branch release/2.10.0.
                                                         
                                                        git checkout tags/2.10.0-1 -b release/2.10.0\n
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#512-install-ase-tools","title":"5.1.2 Install ASE Tools","text":"
                                                        ASE is an RTL simulator for OPAE-based AFUs. The simulator emulates both the OPAE SDK software user space API and the AFU RTL interface. The majority of the FIM as well as devices such as PCIe and local memory are emulated with simple functional models.
                                                         
                                                        ASE must be installed separatedly from the OPAE-SDK. However, the recommendation is to install it in the same target directory as OPAE-SDK.
                                                         
                                                           
                                                        1.  
                                                          If not done already, set the environment variables as described in section, Set Up AFU Development Environment.
                                                           
                                                          2.  
                                                        2.  
                                                          Clone the ase-sim repository.
                                                           
                                                          3.  
                                                         
                                                          $ cd $OFS_BUILD_ROOT\n  $ git clone https://github.com/OFS/opae-sim.git\n  $ cd opae-sim  \n
                                                         2. Building ASE requires the include file mock/opae_std.h. If the OPAE-SDK was installed under the default system directories, the C_INCLUDE_PATH variable must be set as follows. 
                                                         
                                                        export C_INCLUDE_PATH=\"/usr/src/debug/opae-2.10.0-1.el8.x86_64/tests/framework\"\n
                                                         
                                                           
                                                        1. Create a build directory and build ASE to be installed under the default system directories along with OPAE SDK.
                                                          2.  
                                                         
                                                           mkdir build\n   cd build\n   cmake  -DCMAKE_INSTALL_PREFIX=/usr ..\n   make\n
                                                         
                                                        Optionally, if the desire is to install ASE binaries in a different location to the system's default, provide the path to CMAKE through the CMAKE_INSTALL_PREFIX switch, as follows.
                                                         
                                                           cmake -DCMAKE_INSTALL_PREFIX=<</some/arbitrary/path>> ..  \n
                                                         
                                                           
                                                        1. Install ASE binaries and libraries under the system directory /usr.
                                                          2.  
                                                         
                                                           sudo make install  \n
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#513-setup-required-ase-environment-variables","title":"5.1.3. Setup Required ASE Environment Variables","text":"
                                                        The values set to the following environment variables assume the OPAE SDK and ASE were installed in the default system directories below /usr. Setup these variables in the shell where ASE will be executed. You may wish to add these variables to the script you created to facilitate configuring your environment.
                                                         
                                                           cd /usr/bin\n   export PATH=$PWD:$PATH\n   cd ../lib/python*/site-packages\n   export PYTHONPATH=$PWD\n   cd /usr/lib\n   export LIBRARY_PATH=$PWD\n   cd /usr/lib64\n   export LD_LIBRARY_PATH=$PWD\n   cd $OFS_BUILD_ROOT/ofs-platform-afu-bbb\n   export OFS_PLATFORM_AFU_BBB=$PWD\n   cd $OFS_ROOTDIR/work_d5005/build_tree\n   export OPAE_PLATFORM_ROOT=$PWD\n\n  ## For VCS, set the following:\n\n   export VCS_HOME=<Set the path to VCS installation directory>\n   export PATH=$VCS_HOME/bin:$PATH\n\n  ## For QuestaSIM, set the following:\n   export MTI_HOME=<path to Modelsim installation directory>\n   export PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\n
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#52-simulating-the-host_chan_mmio-afu","title":"5.2. Simulating the host_chan_mmio AFU","text":"
                                                        The $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio is a simple example demonstrating both hardware and software access to an AFU. The host_chan_mmio example AFU consists of the following files:
                                                         
                                                        host_chan_mmio\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon_from_ccip.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon_from_axi.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_avalon.sv\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_axi.sv\n\u2502       \u251c\u2500\u2500 host_chan_mmio.json\n\u2502       \u251c\u2500\u2500 test_mmio_avalon0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon1.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon2_512rw.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi1.txt\n\u2502       \u2514\u2500\u2500 test_mmio_axi2_512rw.txt\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 main.c\n    \u251c\u2500\u2500 Makefile\n
                                                         
                                                        This example AFU contains examples using both Avalon and AXI interface buses. This guide will use the AXI version of the host_chan_mmio AFU.
                                                         
                                                        ASE uses client-server application architecture to deliver hardware/software co-simulation.  You require one shell for the hardware based simulation and another shell where the software application is running. The hardware is started first with a simulation compilation and simulator startup script, once the simulator has loaded the design, it will wait until the software process starts. Once the software process starts, the simulator proceeds.  Transaction logging and waveform capture is performed.
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#521-set-up-and-run-the-hw-simulation-process","title":"5.2.1 Set Up and Run the HW Simulation Process","text":"
                                                        You will run the afu_sim_setup script to create the scripts for running the ASE environment.  The afu_sim_setup script has the following usage:
                                                         
                                                        usage: afu_sim_setup [-h] -s SOURCES [-p PLATFORM] [-t {VCS,QUESTA,MODELSIM}]\n                     [-f] [--ase-mode ASE_MODE] [--ase-verbose]\n                     dst\n\nGenerate an ASE simulation environment for an AFU. An ASE environment is\ninstantiated from the OPAE installation and then configured for the specified\nAFU. AFU source files are specified in a text file that is parsed by\nrtl_src_config, which is also part of the OPAE base environment.\n\npositional arguments:\n  dst                   Target directory path (directory must not exist).\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -s SOURCES, --sources SOURCES\n                        AFU source specification file that will be passed to\n                        rtl_src_config. See \"rtl_src_config --help\" for the\n                        file's syntax. rtl_src_config translates the source\n                        list into either Quartus or RTL simulator syntax.\n  -p PLATFORM, --platform PLATFORM\n                        FPGA Platform to simulate.\n  -t {VCS,QUESTA,MODELSIM}, --tool {VCS,QUESTA,MODELSIM}\n                        Default simulator.\n  -f, --force           Overwrite target directory if it exists.\n  --ase-mode ASE_MODE   ASE execution mode (default, mode 3, exits on\n                        completion). See ase.cfg in the target directory.\n  --ase-verbose         When set, ASE prints each CCI-P transaction to the\n                        command line. Transactions are always logged to\n                        work/ccip_transactions.tsv, even when not set. This\n                        switch sets ENABLE_CL_VIEW in ase.cfg.\n
                                                         
                                                        Run afu_sim_setup to create the ASE simulation environment for the host_chan_mmio example AFU. The '-t VCS' option indicates to prepare the ASE simulation environment for VCS.
                                                         
                                                        cd $OFS_ROOTDIR/work_d5005/\n\nafu_sim_setup -s $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/test_mmio_axi1.txt -t VCS host_chan_mmio_sim\n\nCopying ASE from /opae-sdk/install-opae-sdk/share/opae/ase...\nCopying ASE from /usr/share/opae/ase...\n#################################################################\n#                                                               #\n#             OPAE Intel(R) Xeon(R) + FPGA Library              #\n#               AFU Simulation Environment (ASE)                #\n#                                                               #\n#################################################################\n\nTool Brand: VCS\nLoading platform database: /home/<Your username>/<Your localpath>/ofs-d5005/work_d5005/build_tree/hw/lib/platform/platform_db/ofs_d5005.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting rtl/platform_afu_top_config.vh\nWriting rtl/platform_if_addenda.txt\nWriting rtl/platform_if_includes.txt\nWriting rtl/ase_platform_name.txt\nWriting rtl/ase_platform_config.mk and rtl/ase_platform_config.cmake\nASE Platform: discrete (FPGA_PLATFORM_DISCRETE)\n
                                                         
                                                        The afu_sim_setup creates the ASE scripts in the directory host_chan_mmio_sim where the afu_sim_setup script was run.  Start the simulator as shown below in user mode:
                                                         
                                                           cd host_chan_mmio_sim\n   make\n   make sim\n
                                                         
                                                        This process launches the AFU hardware simulator. Before moving to the next section, pay attention to the simulator output highlighted in the image below.
                                                         
                                                         
                                                        The simulation artifacts are stored in host_chan_mmio/work and consist of:
                                                         
                                                        log_ase_events.tsv\nlog_ofs_plat_host_chan.tsv \nlog_ofs_plat_local_mem.tsv \nlog_pf_vf_mux_A.tsv \nlog_pf_vf_mux_B.tsv \n
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#522-set-up-and-run-the-sw-process","title":"5.2.2 Set Up and Run the SW Process","text":"
                                                        Open an additional shell to build and run the host application that communicates with the actual AFU hardware. Set up the same environment variable you have set up in the shell you have been working on until this point. 
                                                         
                                                        Additionally, as indicated by the hardware simulator output that is currently executing in the \"simulator shell\", copy and paste the line \"export ASE_WORKDIR=...\", into the new \"software shell\". See the last image of the previous section.
                                                         
                                                           export ASE_WORKDIR= <<as directed in HW simulation shell>>\n
                                                         Then, go to the sw directory of the host_chan_mmio AFU example to compile the host application.
                                                         
                                                        cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw  \nmake\n\nafu_json_mgr json-info --afu-json=../hw/rtl/host_chan_mmio.json --c-hdr=obj/afu_json_info.h\nWriting obj/afu_json_info.h\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c main.c -o obj/main.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c test_host_chan_mmio.c -o obj/test_host_chan_mmio.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/connect.c -o obj/connect.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/csr_mgr.c -o obj/csr_mgr.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/hash32.c -o obj/hash32.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/test_data.c -o obj/test_data.o\ncc -o host_chan_mmio obj/main.o obj/test_host_chan_mmio.o obj/connect.o obj/csr_mgr.o obj/hash32.o obj/test_data.o  -z noexecstack -z relro -z now -pie -luuid -lopae-c\n
                                                         
                                                        Now, launch the host application to exercise the AFU hardware running on the simulator shell. The next image shows the AFU hardware simulation process on the left side shell. The right hand shell shows the host application's output of a successful simulation.
                                                         
                                                         
                                                        Finally, on the hardware simulation shell, you can view the wave forms by invoking the following command.
                                                         
                                                           make wave\n
                                                         
                                                        This brings up the VCS simulator GUI and loads the simulation waveform files. Use the Hierarchy window to navigate to the afu instance located under, ase_top | ase_top_plat | ofs_plat_afu | afu , as shown below.
                                                         
                                                         
                                                        Right click on the afu (afu) entry to display the drop-down menu. Then, click on Add to Waves | New Wave View to display the following waveforms window.
                                                         
                                                         
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#53-simulating-the-hello_world-afu","title":"5.3 Simulating the hello_world AFU","text":"
                                                        In this section, you will quickly simulate the PIM-based hello_world sample AFU accompanying the example_afu repository.
                                                         
                                                           
                                                        1.  
                                                          Set the environment variables as described in section 5.1. Set Up Steps to Run ASE.
                                                           
                                                          2.  
                                                        2.  
                                                          Prepare an RTL simulation environment for the AXI version of the hello_world AFU.
                                                           
                                                          Simulation with ASE requires two software processes, one to simulate the AFU RTL and the other to run the host software that exercises the AFU. To construct an RTL simulation environment under the directory $OFS_ROOTDIR/work_d5005, execute the following.
                                                           
                                                            $ cd $OFS_ROOTDIR/work_d5005\n  $ afu_sim_setup -s $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/hw/rtl/axi/sources.txt -t VCS hello_world_sim\n\nCopying ASE from /usr/local/share/opae/ase...\n#################################################################\n#                                                               #\n#             OPAE Intel(R) Xeon(R) + FPGA Library              #\n#               AFU Simulation Environment (ASE)                #\n#                                                               #\n#################################################################\n\nTool Brand: VCS\nLoading platform database: /home/<Your username>/<Your localpath>/ofs-d5005/work_d5005/build_tree/hw/lib/platform/platform_db/ofs_d5005.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting rtl/platform_afu_top_config.vh\nWriting rtl/platform_if_addenda.txt\nWriting rtl/platform_if_includes.txt\nWriting rtl/ase_platform_name.txt\nWriting rtl/ase_platform_config.mk and rtl/ase_platform_config.cmake\nASE Platform: discrete (FPGA_PLATFORM_DISCRETE)\n
                                                           
                                                          The afu_sim_setup script constructs an ASE environment in the hello_world_sim subdirectory. If the command fails, confirm that the path to the afu_sim_setup is on your PATH environment variable (in the OPAE SDK bin directory) and that your Python version is at least 3.7.
                                                           
                                                          3.  
                                                        3.  
                                                          Build and execute the AFU RTL simulator in user mode.
                                                           
                                                             cd $OFS_ROOTDIR/work_d5005/hello_world_sim\n   make\n   make sim  \n
                                                           
                                                          The previous commands will build and run the VCS RTL simulator, which prints a message saying it is ready for simulation. The simulation process also prints a message instructing you to set the ASE_WORKDIR environment variable in a second shell.
                                                           
                                                             
                                                          1.  
                                                            Open a second shell where you will build and execute the host software. In this new \"software shell\", set up the environment variables you have set up so far in the \"hardware simulation\" shell.
                                                             
                                                            2.  
                                                          2.  
                                                            Also, set the ASE_WORKDIR environment variable following the instructions given in the \"hardware simulation\" shell.
                                                             
                                                            3.  
                                                           
                                                          export ASE_WORKDIR=$OFS_ROOTDIR/work_d5005/hello_world_sim/work\n
                                                           6. Then, move to the sw directory of the hello_world AFU sample to build the host software.
                                                           
                                                          cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/sw\n   make      \n
                                                           
                                                             
                                                          1. Run the hello_world host application to resume the work of the RTL simulation. The host software process and the RTL simulation execute in lockstep. If successful, you should see the Hello world! output.
                                                            2.  
                                                           
                                                            $ with_ase ./hello_world\n\n  [APP]  Initializing simulation session ...\nHello world!\n  [APP]  Deinitializing simulation session\n  [APP]         Took 43,978,424 nsec\n  [APP]  Session ended\n
                                                           
                                                          The image below shows the simulation of the AFU hardware and the execution of the host application side-by-side.
                                                           
                                                           
                                                           
                                                           
                                                             
                                                          1. Finally, on the hardware simulation shell, you can view the wave forms by invoking the following command.
                                                            2.  
                                                           
                                                             make wave\n
                                                           
                                                          This brings up the DVE GUI and loads the simulation waveform files. Use the Hierarchy window to navigate to the AFU instance located under, ase_top | ase_top_plat | ofs_plat_afu, as shown below.
                                                           
                                                           
                                                          Right click on the ofs_plat_afu entry to display the drop-down menu. Then, click on Add to Waves | New Wave View to display the following waveforms window.
                                                           
                                                           
                                                           
                                                          4.  
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#6-adding-remote-signal-tap-logic-analyzer-to-debug-the-afu","title":"6. Adding Remote Signal Tap Logic Analyzer to debug the AFU","text":"
                                                        The OPAE SDK provides a remote Signal Tap facility. It also supports the following in system debug tools included with the Intel\u00ae  Intel\u00ae Quartus\u00ae Prime Pro Edition:
                                                         
                                                           
                                                        • In-system Sources and Probes
                                                          •  
                                                        • In-system Memory Content Editor
                                                          •  
                                                        • Signal Probe
                                                          •  
                                                        • System Console
                                                          •  
                                                         
                                                        This section is a short guide on adding remote Signal Tap instances to an AFU for in-system debugging. In order of execution, you can follow the steps in the following sections to create an instrumented AFU. The host_chan_mmio AFU is used in this guide as the target AFU to be instrumented.
                                                         
                                                        You need a basic understanding of Signal Tap. Please see the Signal Tap Logic Analyzer: Introduction & Getting Started Web-Based Training for more information.
                                                         
                                                        You will run with a Signal Tap GUI running locally on the server with the Intel\u00ae FPGA PAC D5005 as shown below:
                                                         
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#61-adding-rstp-to-the-host_chan_mmio-afu","title":"6.1. Adding RSTP to the host_chan_mmio AFU","text":"
                                                        RSTP is added to an AFU by:
                                                         
                                                           
                                                        1. Defining signals to be instrumented in Signal Tap. Create a new *.stp file.
                                                          2.  
                                                        2. Modify ofs_top.qpf to include the new *.stp file
                                                          3.  
                                                        3. Modify ofs_top.qsf
                                                          4.  
                                                        4. Modify ofs_pr_afu.qsf 
                                                          5.  
                                                        5. Re-run afu_synth_setup to update project settings
                                                          6.  
                                                        6. Re-run $OPAE_PLATFORM_ROOT/bin/afu_synth to build the PR -able image containing the RSTP instance
                                                          7.  
                                                         
                                                        The following steps use the previously built host_chan_mmio AFU example. You can use these detailed steps to instrument your AFU.
                                                         
                                                           
                                                        1.  
                                                          Navigate to host_chan_mmio AFU Quartus project and open the project using Quartus GUI.
                                                           
                                                           cd $OFS_ROOTDIR/work_d5005/build_d5005_x16/build/syn/syn_top\n $ quartus d5005.qpf &\n
                                                           
                                                          2.  
                                                        2.  
                                                          Once the project is loaded in Quartus, review the project hierarchy as shown in the Project Navigator. This example will add Signal Tap probe points to the AFU region. Reviewing the code will give insight into the function of this block. You can up the code in the Project Navigator by expanding afu_top - port_gasket - pr_slot - afu_main - ofs_plat_afu, then select instance afu, right-click, select Locate Node - Locate in Design File as shown below.
                                                           
                                                           
                                                          3.  
                                                        3.  
                                                          Bring up Signal Tap to create the *.stp file. In the Quartus GUI, go to Tools - Signal Tap Logic Analyzer. Click Create to accept the default template in the New File from Template pop-up. The Signal Tap Logic Analyzer window comes up.
                                                           
                                                          4.  
                                                        4.  
                                                          Set up the clock for the Signal Tap logic instance by clicking ... button as shown below:
                                                           
                                                           5. The Node Finder comes up, and you will click ... as shown below to bring up the hierarchy navigator or copy-paste the following location at Look in:
                                                           
                                                          5.  
                                                         
                                                        iofs_top|afu_top|port_gasket|pr_slot|afu_main|ofs_plat_afu|afu\n
                                                         
                                                         
                                                           
                                                        1.  
                                                          In the Select Hierarchy Level, navigate to top - afu_top - port_gasket - pr_slot - afu_main - ofs_plat_afu, then select instance afu and click Ok.
                                                           
                                                          2.  
                                                        2.  
                                                          Enter *clk* in the Named: box and click Search.  This brings up matching terms.  Click mmio64_if.clk and >.  Verify your Node Finder is as shown below and then click Ok:    
                                                           
                                                          3.  
                                                        3.  
                                                          Double click the Double-click to add nodes and once again, click ... and navigate to top - afu_top - port_gasket - pr_slot - afu_main - ofs_plat_afu, then select instance afu and click Ok.  Enter mmio64 then click >> to add these signals to the STP instance as shown below:     
                                                           
                                                          Then click Insert and Close.
                                                           
                                                          4.  
                                                        4.  
                                                          Save the newly created STP by clicking File - Save As in the save as navigate to $OFS_ROOTDIR/work_d5005/build_d5005_x16/build/syn/syn_top and save the STP file as host_chan_mmio.stp as shown below:       
                                                           
                                                          5.  
                                                        5. Edit ofs_top.qsf to add host_chan_mmio.stp file and enable STP.  Open $OFS_ROOTDIR/work_d5005/build_d5005_x16/build/syn/syn_top/d5005.qpf in an editor and modify lines as shown below:
                                                          6.  
                                                         
                                                        set_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE host_chan_mmio.stp\nset_global_assignment -name SIGNALTAP_FILE host_chan_mmio.stp\n
                                                         
                                                        Save the d5005.qpf and close Quartus.
                                                         
                                                           
                                                        1. Edit iofs_pr_afu.qsf to add host_chan_mmio.stp file and enable STP.  Open $OPAE_PLATFORM_ROOT/hw/lib/build/syn/syn_top/iofs_pr_afu.qsf in an editor and ensure the lines are included as  below (note: the verilog macro INCLUDE_REMOTE_STP will already be present), also copy and paste the file host_chan_mmio.stp in this location:
                                                          2.  
                                                         
                                                        The updated lines are: 
                                                        set_global_assignment -name VERILOG_MACRO \"INCLUDE_REMOTE_STP\"\nset_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE host_chan_mmio.stp\nset_global_assignment -name SIGNALTAP_FILE host_chan_mmio.stp\n
                                                         Save the iofs_pr_afu.qsf and ensure Quartus is closed.
                                                         
                                                           
                                                        1. The afu_synth script is run to create a new copy of AFU files. In your original build shell, enter the following commands:
                                                          2.  
                                                         
                                                            $ cd $OFS_ROOTDIR/build_d5005_x16\n    $ afu_synth_setup -s $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/test_mmio_axi1.txt build_d5005_x16_stp\n\n    Notice that your previous build_d5005_x16_stp directory is preserved, and a new build_d5005_x16_stp directory is created. You will use build_d5005_x16_stp to build the STP-enabled image.\n\n    $ cd build_d5005_x16_stp\n    $ $OPAE_PLATFORM_ROOT/bin/afu_synth\n\n...\n...\nWrote host_chan_mmio.gbs\n\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n  Design meets timing\n===========================================================================\n
                                                         
                                                           
                                                        1. Once compilation completes, the new host_chan_mmio.gbs file that contains the Signal Tap instance can be loaded.
                                                          2.  
                                                         
                                                        $ sudo fpgasupdate host_chan_mmio.gbs 3b:00.0\n[sudo] password for <myuser>: \n[WARNING ] Update starting. Please do not interrupt.\n [INFO    ] \nPartial Reconfiguration OK\n[INFO    ] Total time: 0:00:01.87\n
                                                         
                                                           
                                                        1. Use the OPAE SDK mmlink tool to create a TCP/IP connection to your Intel\u00ae Stratix 10\u00ae FPGA card under test. The mmlink command has the following format:
                                                          2.  
                                                         
                                                        Usage:\nmmlink\n<Segment>             --segment=<SEGMENT NUMBER>\n<Bus>                 --bus=<BUS NUMBER>           OR  -B <BUS NUMBER>\n<Device>              --device=<DEVICE NUMBER>     OR  -D <DEVICE NUMBER>\n<Function>            --function=<FUNCTION NUMBER> OR  -F <FUNCTION NUMBER>\n<Socket-id>           --socket-id=<SOCKET NUMBER>  OR  -S <SOCKET NUMBER>\n<TCP PORT>            --port=<PORT>                OR  -P <PORT>\n<IP ADDRESS>          --ip=<IP ADDRESS>            OR  -I <IP ADDRESS>\n<Version>             -v,--version Print version and exit\n
                                                         
                                                        ProTip: 
                                                         
                                                        Open a new shell session for mmlink; this console needs to remain open to allow mmlink connection.
                                                         
                                                        Enter the command below to create a connection using port 3333:
                                                         
                                                        $ sudo mmlink -P 3333 -B 0x3b\n\n ------- Command line Input START ----\n\n Socket-id             : -1\n Port                  : 3333\n IP address            : 0.0.0.0\n ------- Command line Input END   ----\n\nPORT Resource found.\nServer socket is listening on port: 3333\n
                                                         
                                                        Leave this shell open with the mmlink connection.
                                                         
                                                           
                                                        1. In this step, you will open a new shell and enable JTAG over protocol. You must have Quartus Prime Pro \u00ae  23.3 Programmer loaded on the Intel\u00ae FPGA PAC D5005 server for local debugging.
                                                          2.  
                                                         
                                                        $ jtagconfig --add JTAG-over-protocol sti://localhost:0/intel/remote-debug/127.0.0.1:3333/0\n\nVerify connectivity with jtagconfig --debug\n\n$ jtagconfig --debug\n1) JTAG-over-protocol [sti://localhost:0/intel/remote-debug/127.0.0.1:3333/0]\n   (JTAG Server Version 23.3.0 Build 104 09/14/2022 SC Pro Edition)\n  020D10DD   VTAP10 (IR=10)\n    Design hash    D41D8CD98F00B204E980\n    + Node 00406E00  Virtual JTAG #0\n\n  Captured DR after reset = (020D10DD) [32]\n  Captured IR after reset = (155) [10]\n  Captured Bypass after reset = (0) [1]\n  Captured Bypass chain = (0) [1]\n
                                                         
                                                           
                                                        1. Start Quartus Signal Tap GUI, connect to target, load stp file by navigating to  $OPAE_PLATFORM_ROOT/hw/lib/build/syn/syn_top/ . The Quartus Signal Tap must be the same version of Quartus used to compile the host_chan_mmio.gbs. Quartus Prime Pro \u00ae  23.3 Pro is used in the steps below:
                                                          2.  
                                                         
                                                        cd $OPAE_PLATFORM_ROOT/hw/lib/build/syn/syn_top/\nquartus_stpw host_chan_mmio.stp\n
                                                         
                                                        This command brings up Signal Tap GUI. Connect to the Signal Tap over protocol by selecting the Hardware button on the right side of the GUI and clicking the \"Please Select\" pull-down as shown below:
                                                         
                                                        JTAG  over protocol selected:
                                                         
                                                         
                                                        This connection process will take approximately 2-3 minutes for the Signal Tap instance to indicate \"Ready to acquire\".
                                                         
                                                        8) Set the trigger condition for a rising edge on signal valid signal. 9) In the Signal Tap window, enable acquisition by pressing key F5. The Signal Tap GUI will indicate \"Acquisition in progress\". Run the hello_world application and observe that the Signal Tap instance has triggered. You should see signals being captured in the Signaltap GUI.
                                                         
                                                        See captured image below:
                                                         
                                                         
                                                        To end your Signal Tap session, close the Signal Tap GUI, then in the mmlink shell, enter ctrl c to kill the mmlink process.
                                                        "},{"location":"hw/d5005/dev_guides/afu_dev/ug_dev_afu_d5005/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                        Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                         
                                                        OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                        "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/","title":"Intel\u00ae FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA","text":""},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#1-introduction","title":"1. Introduction","text":""},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#11-about-this-document","title":"1.1. About This Document","text":"
                                                        This document serves as a design guide for FPGA developers, system architects and hardware developers using OFS as a starting point for the creating the FPGA Interface Manager (FIM) for a custom FPGA acceleration board or Platform with Intel FPGAs.
                                                         
                                                        This document uses the Intel\u00ae FPGA PAC D5005 as an example platform to illustrate key points and demonstrate how to extend the capabilities provided in OFS (Open FPGA Stack) to custom platforms. The demonstration steps serves as a tutorial for the development of your OFS knowledge.
                                                         
                                                        This document covers OFS architecture lightly.  For more details on the OFS architecture, please see [Open FPGA Stack Technical Reference Manual].
                                                         
                                                        You are encouraged to read [OFS AFU Development Guide] to fully understand how AFU Developers will use your newly developed FIM.
                                                         Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#12-introduction","title":"1.2. Introduction","text":"
                                                        Open FPGA Stack (OFS) addresses the scalability for FPGA acceleration boards and workloads by providing a powerful and systematic methodology for the rapid development of FPGA-based Acceleration systems.  This methodology addresses the key challenges of hardware, software and workload developers by providing a complete FPGA project consisting of RTL and simulation code, build scripts and software. The FPGA project released in OFS can be rapidly customized to meet new market requirements by adding new features, custom IPs and Intel interface subsystems IPs. 
                                                         
                                                        A high-level overview of the OFS Intel\u00ae Stratix 10\u00ae FPGA hardware architecture on the Intel\u00ae Stratix 10\u00ae FPGA reference platform, Intel\u00ae FPGA PAC D5005 is shown in the below figure. The provided FPGA architecture is divided into two main components 
                                                         
                                                        - The outer area in white, the FPGA Interface manager (or FIM)    - The inner area in green, the Acceleration Function Unit or AFU Region. 
                                                         
                                                        The outer area, the FIM, provides the core infrastructure and interfaces within the FPGA. The AFU region is where a user\u2019s custom logic would reside for their specific workload. 
                                                         
                                                        * FPGA external interfaces and IP cores (e.g. Ethernet, DDR-4, PCIe, etc)   * PLLs/resets   * FPGA - Board management infrastructure   * Interface to Acceleration Function Unit (AFU)
                                                         
                                                        The AFU region has both static and dynamic partial reconfiguration regions enabling a lot of customization. 
                                                         
                                                        * Uses the FIM interfaces to perform useful work inside the FPGA   * Contains logic supporting partial reconfiguration   * Remote Signal Tap core for remote debugging of workload
                                                         
                                                        Outside of the FPGA is the Board Management Controller which provides board management, root of trust, board monitoring, and remote system updates. 
                                                         
                                                        The overall architecture is built to be very composable and modular in blocks that can be modified while leaving the rest of the infrastructure intact so you may only need to modify a few of these blocks. 
                                                         
                                                        "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#12-release-capabilities","title":"1.2. Release Capabilities","text":"
                                                        This release of OFS FIM supports the following key features:
                                                         
                                                           
                                                        • 1 - Host channel interface via PCIe Gen 3 x 16 SRIOV (1PF, 3 VF, AXI-S TLP packets)
                                                          •  
                                                        • DDR4 SDRAM External memory interface (AXI-M)
                                                          •  
                                                        • 1 - 10G Ethernet interfaces (1x10G)
                                                          •  
                                                        • MSI-X Interrupts (PF, VF)
                                                          •  
                                                        • 1 - AFU
                                                          •  
                                                        • Exercisers demonstrating PCIe, external memory and Ethernet interfaces
                                                          •  
                                                        • Port, FME CSR
                                                          •  
                                                        • Remote Signal Tap
                                                          •  
                                                         
                                                        OFS is extensible to meet the needs of a broad set of customer applications, however not all use cases are easily served.  The general uses cases listed below are examples where the OFS base design can be easily re-used to build a custom FIM: 1. Use OFS reference design as-is   - Porting the code to another platform that is identical to the OFS reference platform only changing target FPGA device and pinout   - Change I/O assignments without changing design 2. Update the configuration of peripheral IP in OFS reference design, not affecting FIM architecture   - External memory settings   - HSSI analog settings 3. Remove/update peripheral feature in OFS reference design, not affecting FIM architecture   - External memory speed/width change   - Change 10G Ethernet to 25 or 100G Ethernet IP    - Change number of VFs supported 4. Add new features as an extension to OFS reference design, not affecting FIM architecture   - Add/remove external memory interface to the design   - Add/remove user clocks for AFU   - Add/remove IP to the design with connection to AFU
                                                         
                                                        More advanced use cases requiring changes or additions to the host PCIe channel are not easily supported with this release of the OFS FIM.
                                                         
                                                        Reuse of the provided host management FPGA logic and software is the fastest and most simple approach to FIM customization.
                                                        "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#13-prerequisites","title":"1.3. Prerequisites","text":""},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#131-base-knowledge-and-skills-prerequisites","title":"1.3.1. Base Knowledge and Skills Prerequisites","text":"
                                                        OFS is an advanced application of FPGA technology. This guide assumes you have the following FPGA logic design-related knowledge and skills:
                                                         
                                                           
                                                        • FPGA compilation flows using Intel\u00ae Quartus\u00ae Prime Pro Edition design flow.
                                                          •  
                                                        • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
                                                          •  
                                                        • RTL and coding practices for FPGA implementation.
                                                          •  
                                                        • RTL simulation tools.
                                                          •  
                                                        • Intel\u00ae Quartus\u00ae Prime Pro Edition Signal Tap Logic Analyzer tool software.
                                                          •  
                                                        "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#132-development-environment","title":"1.3.2. Development Environment","text":"
                                                        To run the tutorial steps in this guide requires this development environment:
                                                         Item Version Intel Quartus Prime Pro Intel Quartus Prime Pro 23.3 (with license patch) Target D5005 Sever Operating System RHEL 8.6 OPAE SDK 2.10.0-1  Linux DFL ofs-2023.3-6.1-2 Python 3.6.8 cmake 3.15 GCC 7.4.0 perl 5.8.8 
                                                        The following server and Intel PAC card are required to run the examples in this guide:
                                                         
                                                           
                                                        1. Qualified Intel Xeon \u00ae server see Qualified Servers.
                                                          2.  
                                                        2. Intel\u00ae FPGA PAC D5005 with root entry hash erased (Please contact Intel for root entry hash erase instructions).  The standard Intel\u00ae FPGA PAC D5005 card is programmed to only allow the FIM binary files signed by Intel to be loaded.  The root entry hash erase process will allow newly created, unsigned FIM binary files to be loaded.
                                                          3.  
                                                        3. Intel\u00ae FPGA PAC D5005 installed in the qualified server following instructions in [OFS Getting Started User Guide].
                                                          4.  
                                                         
                                                        The steps included in this guide have been verified in the Dell R740 and HPE ProLiant DL380 Gen10 servers.
                                                        "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#2-high-level-description","title":"2. High Level Description","text":"
                                                        The FIM targets operation in the Intel\u00ae FPGA PAC D5005 card.  The block diagram of the D5005 is shown below:
                                                         
                                                         
                                                        The key D5005 FPGA interfaces are:
                                                         
                                                           
                                                        • Host interface      - PCIe Gen3 x 16
                                                          •  
                                                        • Network interface
                                                             
                                                          • 2 - QSFP28 cages
                                                            •  
                                                          • Current FIM supports 1 x 10 GbE, other interfaces can be created  
                                                            •  
                                                           
                                                          •  
                                                        • External Memory
                                                             
                                                          • 2 or 4 channels of DDR4-2400 to RDIMM modules
                                                            •  
                                                          • RDIMM modules =  8GB organized as 1 Gb X 72
                                                            •  
                                                           
                                                          •  
                                                        • Board Management
                                                             
                                                          • SPI interface
                                                            •  
                                                          • FPGA configuration
                                                            •  
                                                           
                                                          •  
                                                        "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#21-fpga-interface-manager-overview","title":"2.1. FPGA Interface Manager Overview","text":"
                                                        The FPGA Interface Manager architecture is shown in the below diagram:
                                                         
                                                         
                                                        The FIM consists of the following components    - PCIe Subsystem    - Memory Subsystem    - HSSI Subsystem    - Platform Management Component Intercommunications (PMCI)     - Board Peripheral Fabric (BPF)     - AFU Peripheral Fabric (APF)    - Port Gasket    - AXI-S PF/VF Demux/Mux    - Host Exerciser Modules - HE-MEM, HE-LB, HE-HSSI    - FPGA Management Engine (FME)
                                                        "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#22-fim-fpga-resource-usage","title":"2.2. FIM FPGA Resource Usage","text":"
                                                        The FIM uses a small portion of the available FPGA resources.  The table below shows resource usage for a base FIM built with 2 channels of external memory, a small AFU instantiated that has host CSR read/write, external memory test and Ethernet test functionality.
                                                         
                                                         Entity ALMs Used % ALMS Used M20Ks % M20Ks used DSP Blocks Pins IOPLLs OFS_top 125009.4 13.0% 661 5.4% 0 630 15 afu_top 70522.7 7.0% 228 2.4% 0 0 1 auto_fab_0 1305.7 0.0% 9 0.1% 0 0 0 bpf_rsv_5_slv 0.6 0.0% 0 0.0% 0 0 0 bpf_rsv_6_slv 0.6 0.0% 0 0.0% 0 0 0 bpf_rsv_7_slv 0.4 0.0% 0 0.0% 0 0 0 bpf 241.9 0.0% 0 0.0% 0 0 0 emif_top_inst 10508.6 1.0% 0 0.0% 0 0 12 eth_ac_wrapper 6024.8 0.5% 9 0.1% 0 0 0 fme_top 615.5 0.2% 7 0.1% 0 0 0 pcie_wrapper 35424.7 3.5% 348 2.9% 0 0 1 pmci_top 318.5 0.1% 0 0.0% 0 0 0 rst_ctrl 40.2 0.0% 0 0.0% 0 0 0 sys_pll 0.5 0.0% 0 0.0% 0 0 1 Total ALMS 933,120 Total M20Ks 11,721 Summary FPGA Resource Utilization Logic utilization (in ALMs) 124,092 / 933,120 ( 13 % ) Total dedicated logic registers 282822 Total pins 630 / 912 ( 69 % ) Total block memory bits 3,425,120 / 240,046,080 ( 1 % ) Total RAM Blocks 661 / 11,721 ( 6 % ) Total DSP Blocks 0 / 5,760 ( 0 % ) Total eSRAMs 0 / 75 ( 0 % ) Total HSSI P-Tiles 17 / 48 ( 35 % ) Total HSSI E-Tile Channels 17 / 48 ( 35 % ) Total HSSI HPS 0 / 1 ( 0 % ) Total HSSI EHIPs 0 / 2 ( 0 % ) Total PLLs 36 / 104 ( 35 % )"},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#23-ofs-directory-structure","title":"2.3. OFS Directory Structure","text":"
                                                        The OFS Git OFS repository ofs-d5005 directory structure is shown below:
                                                         
                                                         
                                                        \u251c\u2500\u2500 eval_script\n|   \u251c\u2500\u2500 ofs_d5005_eval.sh\n|   \u2514\u2500\u2500 README_ofs_d5005_eval.txt\n\u251c\u2500\u2500 ipss\n\u2502   \u251c\u2500\u2500 hssi\n|   \u251c\u2500\u2500 mem\n|   \u251c\u2500\u2500 pcie\n|   \u251c\u2500\u2500 pmci\n|   \u251c\u2500\u2500 spi\n|   \u2514\u2500\u2500 README.md\n\u251c\u2500\u2500 license\n\u2502   \u2514\u2500\u2500 quartus-0.0-0.01Intel OFS-linux.run\n\u251c\u2500\u2500 ofs-common\n|   \u251c\u2500\u2500 scripts\n|   \u251c\u2500\u2500 src\n|   \u251c\u2500\u2500 verification\n|   \u251c\u2500\u2500 LICENSE.txt\n|   \u2514\u2500\u2500 README.md\n\u251c\u2500\u2500 sim\n|   \u251c\u2500\u2500 bfm\n|   \u251c\u2500\u2500 rp_bfm\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 scripts\n|   \u251c\u2500\u2500 unit_test \n\u2502\u00a0\u00a0 \u2514\u2500\u2500 readme.txt\n\u251c\u2500\u2500 src\n\u2502   \u251c\u2500\u2500 afu_top\n\u2502   \u251c\u2500\u2500 includes\n\u2502   \u251c\u2500\u2500 pd_qsys\n\u2502   \u251c\u2500\u2500 README.md\n\u2502   \u2514\u2500\u2500 top\n\u251c\u2500\u2500 syn\n\u2502   \u251c\u2500\u2500 scripts\n\u2502   \u251c\u2500\u2500 setup\n\u2502   \u251c\u2500\u2500 syn_top\n\u2502   \u251c\u2500\u2500 readme.txt\n\u2502   \u2514\u2500\u2500 README\n\u251c\u2500\u2500 LICENSE.txt\n\u2514\u2500\u2500 README.md\n
                                                         
                                                        The contents of each directory are described below:
                                                         
                                                        Eval Script - Contains scripts for evaluation of OFS for D5005 including compiling FIM/AFU from source, unit level test. Also includes resources to report and setup D5005 development environment
                                                         
                                                        ipss - Contains the code and supporting files that define or set up the IP subsystems (HSSI, PCIe, memory, PMCI, SPI, etc...) contained in the D5005 FPGA Interface Manager (FIM).   
                                                         
                                                        license - License file for the Low Latency 10Gbps Ethernet MAC (6AF7 0119) IP core.
                                                         
                                                        ofs-common - This directory contains resources that may be used across the board-specific repositories. This directory is referenced via a link within each of the FPGA-specific repositories.
                                                         
                                                        sim - Contains the testbenches and supporting code for all the unit test simulations.     - Bus Functional Model code is contained here.    - Scripts are included for automating a myriad of tasks.    - All of the individual unit tests and their supporting code is also located here.
                                                         
                                                        src - SystemVerilog source and script files    - Contains all of the structural and behavioral code for the FIM.    - Scripts for generating the AXI buses for module interconnect.    - Top-level RTL for synthesis is located in this directory.    - Accelerated Functional Unit (AFU) infrastructure code is contained in this directory.
                                                         
                                                        syn - This directory contains all of the scripts, settings, and setup files for running synthesis on the FIM.
                                                        "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#3-description-of-sub-systems","title":"3. Description of Sub-Systems","text":""},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#31-host-control-and-data-flow","title":"3.1. Host Control and Data Flow","text":"
                                                        The host control and data flow are shown in the diagram below:
                                                         
                                                         
                                                        The control and data flow is composed of the following:
                                                         
                                                           
                                                        • Host Interface Adapter (PCIe)
                                                          •  
                                                        • Low Performance Peripherals
                                                             
                                                          • Slow speed peripherals (I2C, Smbus, etc)
                                                            •  
                                                          • Management peripherals (FME)
                                                            •  
                                                           
                                                          •  
                                                         
                                                           
                                                        • High Performance Peripherals
                                                             
                                                          • Memory peripherals
                                                            •  
                                                          • Acceleration Function peripherals
                                                            •  
                                                          • HPS Peripheral
                                                            •  
                                                           
                                                          •  
                                                         
                                                           
                                                        • Fabrics
                                                             
                                                          • Peripheral Fabric (multi drop)
                                                            •  
                                                          • AFU Streaming fabric (point to point)
                                                            •  
                                                           
                                                          •  
                                                         
                                                        Peripherals are connected to one another using AXI:
                                                         
                                                           
                                                        • Via the peripheral fabric (AXI4-Lite, multi drop)
                                                          •  
                                                        • Via the AFU streaming fabric (AXI-S, point to point)
                                                          •  
                                                         
                                                        Peripherals are presented to software as:
                                                         
                                                           
                                                        • OFS managed peripherals that implement DFH CSR structure.  
                                                          •  
                                                        • Native driver managed peripherals (i.e. Exposed via an independent PF, VF)
                                                          •  
                                                         
                                                        The peripherals connected to the peripheral fabric are primarily Intel OPAE managed resources, whereas the peripherals connected to the AFU are \u201cprimarily\u201d managed by native OS drivers. The word \u201cprimarily\u201d is used since the AFU is not mandated to expose all its peripherals to Intel OPAE. It can be connected to the peripheral fabric, but can choose to expose only a subset of its capability to Intel OPAE.
                                                         
                                                        OFS uses a defined set of CSRs to expose the functionality of the FPGA to the host software.  These registers are described in Open FPGA Stack Reference Manual - MMIO Regions section.
                                                         
                                                        If you make changes to the FIM that affect the software operation, then OFS provides a mechanism to communicate that information to the proper software driver.  The Device Feature Header (DFH) structure provides a mechanism to maintain compatibility with OPAE software.  Please see FPGA Device Feature List (DFL) Framework Overview for an excellent description of DFL operation from the driver perspective.
                                                         
                                                        When you are planning your address space for your FIM updates, please be aware that the OFS FIM targeting Intel\u00ae FPGA PAC D5005, 256KB of MMIO region is allocated for external FME features and 128kB of MMIO region is allocated for external port features. Each external feature must implement a feature DFH, and the DFH needs to be placed at 4KB boundary. The last feature in the external feature list must have the EOL bit in its DFH set to 1 to mark the end of external feature list.  Since the FPGA address space is limited, consider using an indirect addressing scheme to conserve address space.
                                                        "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#4-fim-development-flow","title":"4. FIM Development Flow","text":"
                                                        OFS provides a framework of FPGA synthesizable code, simulation environment, and synthesis/simulation scripts.  FIM designers can take the provided code and scripts and modify existing code or add new code to meet their specific product requirements.
                                                         
                                                        FIM development for a new acceleration card consists of the following steps:
                                                         
                                                           
                                                        1. Installation of OFS and familiarization with scripts and source code
                                                          2.  
                                                        2. Development of high-level block diagram with your specific functionality
                                                             
                                                          1. Determination of requirements and key performance metrics
                                                            2.  
                                                          2. Selection of IP cores
                                                            3.  
                                                          3. Selection of FPGA device
                                                            4.  
                                                          4. Software memory map
                                                            5.  
                                                           
                                                          3.  
                                                        3. Selection and implementation of FIM Physical interfaces including:
                                                             
                                                          1. External clock sources and creation of internal PLL clocks
                                                            2.  
                                                          2. General I/O
                                                            3.  
                                                          3. Transceivers
                                                            4.  
                                                          4. External memories
                                                            5.  
                                                          5. FPGA programming methodology
                                                            6.  
                                                           
                                                          4.  
                                                        4. Device physical implementation
                                                             
                                                          1. FPGA device pin assignment
                                                            2.  
                                                          2. Inclusion of logic lock regions
                                                            3.  
                                                          3. Creation of timing constraints
                                                            4.  
                                                          4. Create Quartus FIM test project and validate:
                                                               
                                                            1. Placement
                                                              2.  
                                                            2. Timing constraints
                                                              3.  
                                                            3. Build script process
                                                              4.  
                                                            4. Review test FIM FPGA resource usage
                                                              5.  
                                                             
                                                            5.  
                                                           
                                                          5.  
                                                        5. Select FIM to AFU interfaces and development of PIM
                                                          6.  
                                                        6. FIM design implementation
                                                             
                                                          1. RTL coding
                                                            2.  
                                                          2. IP instantiation
                                                            3.  
                                                          3. Development of test AFU to validate FIM
                                                            4.  
                                                          4. Unit and device level simulation
                                                            5.  
                                                          5. Timing constraints and build scripts
                                                            6.  
                                                          6. Timing closure and build validation
                                                            7.  
                                                           
                                                          7.  
                                                        7. Creation of FIM documentation to support AFU development and synthesis
                                                          8.  
                                                        8. Software Device Feature discovery
                                                          9.  
                                                        9. Hardware/software integration, validation and debugging
                                                          10.  
                                                        10. High volume production preparation
                                                          11.  
                                                         
                                                        The FIM developer works closely with the hardware design of the target board, software development and system validation.
                                                         
                                                        Understanding how the AFU developer utilizes the FIM is important for FIM development success.  Please read [OFS AFU Development Guide] for a detailed description of AFU development.
                                                        "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#41-installation-of-ofs","title":"4.1. Installation of OFS","text":"
                                                        In this section you set up a development machine for compiling the OFS FIM. These steps are separate from the setup for a deployment machine where the FPGA acceleration card is installed.  Typically, FPGA development and deployment work is performed on separate machines, however, both development and deployment can be performed on the same server if desired.  Please see [OFS Getting Started User Guide] for instructions on installing software for deployment of your FPGA FIM, AFU and software application on a server.  
                                                         
                                                        Building the OFS FIM requires the development machine to have at least 64 GB of RAM.
                                                         
                                                        The following is a summary of the steps to set up for FIM development:
                                                         
                                                           
                                                        1. Install Quartus Prime Pro 23.3 Linux and setup environment
                                                          2.  
                                                        2. Clone the github ofs-d5005 repository
                                                          3.  
                                                        3. Test installation by building the provided FIM
                                                          4.  
                                                         
                                                        Intel Quartus Prime Pro version 23.3 is the currently verified version of Quartus used for building the FIM and AFU images for this release.  Porting to newer versions of Quartus may be performed by developers.  Download Quartus Prime Pro Linux version 23.3 from Intel\u00ae Quartus\u00ae Prime Pro Edition Linux.
                                                         
                                                        Intel Quartus Prime Pro Version 23.3 is verified to work with the latest OFS release ofs-2023.3.  However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.
                                                         
                                                        Use RedHatEnterprise Linux\u00ae (RHEL) for compatibility with your development flow and also testing your FIM design in your platform. 
                                                         
                                                        Prior to installing Quartus:
                                                         
                                                           
                                                        1.  
                                                          Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.
                                                           
                                                             
                                                          • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                                                            •  
                                                          • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.
                                                            •  
                                                           
                                                          2.  
                                                        2.  
                                                          Perform the following steps to satisfy the required dependencies.
                                                           
                                                          $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                                                           
                                                          Apply the following configurations.
                                                           
                                                          $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n
                                                           
                                                          3.  
                                                        3.  
                                                          Create the default installation path: /intelFPGA_pro/, where  is the default path of the Linux workstation, or as set by the system administrator and  is your Quartus version number. 
                                                          The installation path must satisfy the following requirements:
                                                           
                                                             
                                                          • Contain only alphanumeric characters
                                                            •  
                                                          • No special characters or symbols, such as !$%@^&*<>,
                                                            •  
                                                          • Only English characters
                                                            •  
                                                          • No spaces
                                                            •  
                                                           
                                                        4.  
                                                          Download your required Quartus Prime Pro Linux version here.
                                                           
                                                          5.  
                                                        5.  
                                                          Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.23.
                                                           
                                                          6.  
                                                        6.  
                                                          After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                                                           
                                                          export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                                                           
                                                          For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:
                                                           
                                                          export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n
                                                           
                                                          7.  
                                                        7.  
                                                          Verify, Quartus is discoverable by opening a new shell:
                                                           
                                                          $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                                                           
                                                          8.  
                                                          After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                                                           
                                                          export PATH=$PATH:<Quartus install directory>/quartus/bin\n
                                                           
                                                          For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:
                                                           
                                                          export PATH=$PATH:/home/intelFPGA_pro/23.3/quartus/bin\n
                                                           
                                                          Verify, Quartus is discoverable by opening a new shell:
                                                           
                                                          which quartus\n## Output\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                                                           Note, for some Linux distributions such as RHEL 8.6, Quartus requires installation of the following libraries: 
                                                          sudo dnf install libnsl\nsudo dnf install ncurses-compat-libs\nsudo ln -s /usr/bin/python3 /usr/bin/python\n
                                                           
                                                          You will need to obtain a license for Intel Quartus Prime Pro version 23.3 to compile the design.  This license is obtained from Intel.  Additionally, OFS for Intel\u00ae Stratix 10\u00ae FPGA requires a license for the Low Latency 10Gbps Ethernet MAC (6AF7 0119) IP core.  This license is required to generate a programming file using the provided OFS source code.  The Low Latency 10Gbps Ethernet MAC (6AF7 0119) IP core license patch installer is provided in the ofs-d5005 git repository in the /license directory.  After cloning the OFS release in step 4 below, you can install this IP license.  
                                                           
                                                             
                                                          1. Install git and install git lfs to extract large files within the repository that are compressed with git lfs.  Please note, for proper operation of files retrieved from OFS repository, you will require git lfs. 
                                                            2.  
                                                           
                                                          sudo dnf install git\n
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#install-git-lfs","title":"Install git lfs:","text":"
                                                          curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\nsudo dnf install git-lfs\ngit lfs install\n
                                                           
                                                             
                                                          1. Retrieve OFS repositories:
                                                            2.  
                                                           
                                                          \u200b   The OFS FIM source code is included in the GitHub repository. Create a new directory to use as a clean starting point to store the retrieved files.  
                                                           
                                                             
                                                          1. Navigate to location for storage of OFS source, create the top-level source directory and clone OFS repositories.
                                                            2.  
                                                           
                                                          mkdir OFS_fim_build_root\ncd OFS_fim_build_root\nexport OFS_BUILD_ROOT=$PWD\ngit clone --recurse-submodules  https://github.com/OFS/ofs-d5005.git\ncd ofs-d5005\ngit checkout tags/ofs-2023.3-1\n
                                                           Verify proper tag is selected:
                                                           
                                                          git describe --tags\nofs-2023.3-1\n
                                                           2. Install the Low Latency 10Gbps Ethernet MAC (6AF7 0119) IP license by running provided license installer.
                                                           
                                                          cd license\nchmod +x quartus-0.0-0.01Intel OFS-linux.run\nsudo ./quartus-0.0-0.01Intel OFS-linux.run\n
                                                           
                                                             
                                                          1. Verify patch installed 
                                                            quartus_sh --version\n##Output\nQuartus Prime Shell\nVersion 23.3 Pro Edition\n
                                                            2.  
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#42-compiling-ofs-fim","title":"4.2. Compiling OFS FIM","text":"
                                                          OFS provides a build script with the following FPGA image creation options:
                                                           
                                                             
                                                          • Flat compile which combines the FIM and AFU into one FPGA image that is loaded into the entire FPGA device
                                                            •  
                                                           
                                                             
                                                          • A PR compile which creates a FPGA image consisting of the FIM that is loaded into the static region of the FPGA and a default AFU that is loaded into dynamic region. Additional AFUs maybe loaded into the dynamic region using partial reconfiguration.
                                                            •  
                                                           
                                                          The build scripts included with OFS are verified to run in a bash shell. Other shells have not been tested. Each build script step will take several hours to completes, Please note, building directly in Quartus GUI is not supported - you must build with the provided scripts.
                                                           
                                                          The following sections describe how to set up the environment and build the provided FIM and AFU. Follow these steps as a tutorial to learn the build flow. You will use this environment and build scripts for the creation of your specialized FIM.
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#421-setting-up-required-environment-variables","title":"4.2.1. Setting Up Required Environment Variables","text":"
                                                          Set required environment variables as shown below. These environment variables must be set prior to simulation or compilation tasks so creating a simple script to set these variables saves time.
                                                           
                                                          cd $OFS_BUILD_ROOT/ofs-d5005\nexport OFS_ROOTDIR=$PWD\n\n##   Note, OFS_ROOTDIR is the directory where you cloned the repo, e.g. /home/MyProject/ofs-d5005 *\n\nexport WORKDIR=$OFS_ROOTDIR\nexport VERDIR=$OFS_ROOTDIR/verification\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\n\n##   Note, QUARTUS_ROOTDIR is your Quartus installation directory, e.g. $QUARTUS_ROOTDIR/bin contains Quartus executuable*\n\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport OPAE_SDK_REPO_BRANCH=release/2.10.0\n
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#422-compiling","title":"4.2.2. Compiling","text":"
                                                          The usage of the compile build script is shown below: 
                                                          ofs-common/scripts/common/syn/build_top.sh [-p] target_configuration work_dir \nUsage: ofs-common/scripts/common/syn/build_top.sh [-k] [-p] <build target> [<work dir name>]\n\n  Build a FIM instance specified by <build target>. The target names an FPGA architecture, board and configuration.\n\n  The FIM is built in <work dir name>. If not specified, the target is ${OFS_ROOTDIR}/work.\n\n  The -k option preserves and rebuilds within an existing work tree instead of overwriting it.\n\n  When -p is set, if the FIM is able then a partial reconfiguration template tree is generated at the end of the FIM build. The PR template tree is located in the top of the work directory but is relocatable\n  and uses only relative paths. See ofs-common/scripts/common/syn/generate_pr_release.sh for details.\n\n  The -e option runs only Quartus analysis and elaboration.\n\n      * target_configuration - Specifies the project  \n         For example: d5005\n\n      * work_dir - Work Directory for this build in the form a directory name. It is created in the <local repo directory>/ofs-d5005/<work_dir> \n          - NOTE: The directory name must start with \"work\".  If the work directory exists, then the script stops and asks if you want to overwrite the directory.\n            - e.g.\n                - ofs-common/scripts/common/syn/build_top.sh d5005 work_d5005\n\n                work directory as a name will be created in <local repo directory>/ofs-d5005/work_d5005\n\n\n                The obmission of <work_dir> results in a default work directory (<local repo  directory>/ofs-d5005/work)\n\n        - compile reports and artifacts (.rpt, .sof, etc) are stored in <work_dir>/syn/syn_top/output_files\n\n        - There is a log file created in ofs-d5005 directory.  \n        - [-p]  Optional switch for creation of a relocatable PR build tree supporting the creation of a PR-able AFU workload.   \n        The \"-p\" switch invokes generate_pr_release.sh at the end of the FIM build and writes the PR build tree to the top of the work directory.  More information on this option is provided below. \n
                                                           In the next example, you will build the provided example design using a flat, non-PR build flow.
                                                           
                                                          Build the provided base example design:
                                                           
                                                          ```bash cd $OFS_BUILD_ROOT/ofs-d5005
                                                           
                                                          ofs-common/scripts/common/syn/build_top.sh d5005 work_d5005  ```
                                                           
                                                          ```bash     ... build takes ~5 hours to complete
                                                           
                                                          Compile work directory:     <$OFS_BUILD_ROOT>/work_d5005/syn/syn_top Compile artifact directory: <$OFS_BUILD_ROOT>/work_d5005/syn/syn_top/output_files
                                                           
                                                          ***        OFS_PROJECT: d5005 ***        Q_PROJECT:  d5005 ***        Q_REVISION: d5005 ***        SEED: 03 ***        Build Complete ***        Timing Passed!
                                                           
                                                          The build script copies the ipss, sim, src and syn directories to the specified work directory and then these copied files are used in the Quartus compilation process.  Do not edit the files in the work directory, these files are copies of source files.\n\nSome of the key files are described below:\n\n<work_dir>/syn/syn_top == \n```bash\n\u251c\u2500\u2500 syn_top                    // D5005 Quartus build area with Quartus files used this build\n\u2502\u00a0\u00a0\u251c\u2500\u2500 d5005.ipregen.rpt       // IP regeneration report states the output of IP upgrade\n\u2502\u00a0\u00a0\u251c\u2500\u2500 d5005.qpf               // Quartus Project File (qpf) mentions about Quartus version and project revision\n\u2502  \u251c\u2500\u2500 d5005.qsf               // Quartus Settings File (qsf) lists current project settings and entity level assignments\n\u2502\u00a0\u00a0\u251c\u2500\u2500 d5005.stp               // Signal Tap file included in the d5005.qsf. This file can be modified as required if you need to add an Signal Tap instance\n\u2502\u00a0\u00a0\u251c\u2500\u2500 fme_id.mif              // the fme id hex value is stored in a mif file format\n\u2502  \u251c\u2500\u2500 Intel OFS_pr_afu.json        // PR JSON file\n\u2502\u00a0\u00a0\u251c\u2500\u2500 Intel OFS_pr_afu.qsf                // PR AFU qsf file\n\u2502\u00a0\u00a0\u251c\u2500\u2500 Intel OFS_pr_afu_sources.tcl        // AFU source file list\n\u2502\u00a0\u00a0\u251c\u2500\u2500 ip_upgrade_port_diff_reports   // IP upgrade report files for reference\n
                                                           /syn/syn_top/output_files == Directory with build reports and FPGA programming files.  
                                                          The programming files consist of the Quartus generated d5005.sof and d5005.pof.  The D5005 board hardware provides a 2 Gb flash device to store the FPGA programming files and a MAX10 BMC that reads this flash and programs the D5005 Intel\u00ae Stratix 10\u00ae FPGA FPGA. The syn/build_top.sh script runs script file syn/syn_top/build_flash/build_flash.s which takes the Quartus generated d5005.sof and creates binary files in the proper format to be loaded into the 2 Gb flash device.  You can also run build_flash.sh by yourself if needed.  The build_flash  script runs PACSign (if installed) to create an unsigned FPGA programming file that can be stored in the D5005 FPGA flash. Please note, if the D5005 has the root entry hash key loaded, then PACsign must be run with d5005_page1.bin as the input with the proper key to create an authenticated FPGA binary file.  Please see [Security User Guide: Intel\u00ae Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA] for details on the security aspects of Intel\u00ae Open FPGA Stack.
                                                           
                                                          The following table provides further detail on the generated bin files.
                                                           File Description d5005.sof This is the Quartus generated programming file created by Quartus synthesis and place and route.  This file can be used to programming the FPGA using a JTAG programmer.  This file is used as the source file for the binary files used to program the FPGA flash. d5005.bin This is an intermediate raw binary image of the FPGA d5005_page1.bin This is the binary file created from input file, d5005.sof.  This file is used as the input file to the PACSign utility to generate d5005_page1_unsigned.bin binary image file. d5005_page1_unsigned.bin This is the unsigned PACSign output which can be programmed into the FPGA flash of an unsigned D5005 usign the OPAE SDK utility fpgasupdate mfg_d5005_reversed.bin A special programming file for a third party programming device used in board manufacturing.  This file is typically not used. 
                                                          build/output_files/timing_report == Directory containing clocks report, failing paths and passing margin reports
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#423-relocatable-pr-directory-tree","title":"4.2.3. Relocatable PR Directory Tree","text":"
                                                          If you are developing a FIM to be used by another team developing the AFU workload, scripts are provided that create a relocatable PR directory tree. ODM and board developers will make use of this capability to enable a broad set of AFUs to be loaded on a board using PR.  The relocatable PR directory contains the Quartus *.qdb file that goes the FIM.
                                                           
                                                          The creation of the relocatable PR directory tree requires a clone of the Intel Basic Building Blocks (BBB) repository. The OFS_PLATFORM_AFU_BBB environment variable must point to the repository, for example.
                                                           
                                                          cd $OFS_BUILD_ROOT\ngit clone https://github.com/OPAE/ofs-platform-afu-bbb\ncd ofs-platform-afu-bbb\nexport OFS_PLATFORM_AFU_BBB=$PWD\ncd $OFS_ROOTDIR\n
                                                           
                                                          You can create this relocatable PR directory tree by either:
                                                           
                                                             
                                                          • Build FIM and AFU using /syn/build_top.sh followed by running./ofs-common/scripts/common/syn/generate_pr_release.sh
                                                            •  
                                                          • Build FIM and AFU using /syn/build_top.sh with optional -p switch included
                                                            •  
                                                           
                                                          The generate_pr_release.sh has the following command structure:
                                                           
                                                          ./ofs-common/scripts/common/syn/generate_pr_release.sh -t <path to generated release tree> *Board Build Target* <work dir from build_top.sh>\n\nWhere:\n\n-t <path to generated release tree> = location for your relocatable PR directory tree\n*Board Build Target* is the name of the board target/FIM e.g. d5005\n<work dir from build_top.sh> \n
                                                           Here is an example of running the generate_pr_release.sh script:
                                                           
                                                          ofs-common/scripts/common/syn/generate_pr_release.sh -t work_d5005/build_tree d5005  work_d5005\n
                                                           
                                                          **********************************\n********* ENV SETUP **************\n\nFIM Project:\n  OFS_PROJECT = d5005\n  OFS_FIM     = .\n  OFS_BOARD   = .\n  Q_PROJECT   = d5005\n  Q_REVISION  = d5005\n  Fitter SEED = 03\nFME id\n  BITSTREAM_ID = 04010002c7cab852\n  BITSTREAM_MD = 0000000002204283\n\n...\n...\n
                                                           The resulting relocatable build tree has the following structure: 
                                                          .\n\u251c\u2500\u2500 bin\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 afu_synth\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 build_env_config\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 run.sh -> afu_synth\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 update_pim\n\u251c\u2500\u2500 hw\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 blue_bits\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u251c\u2500\u2500 d5005_page1_unsigned.bin\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u2514\u2500\u2500 d5005.sof -> ../lib/build/syn/syn_top/output_files/d5005.sof\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 lib\n\u2502\u00a0\u00a0     \u251c\u2500\u2500 build\n\u2502\u00a0\u00a0     \u251c\u2500\u2500 fme-ifc-id.txt\n\u2502\u00a0\u00a0     \u251c\u2500\u2500 fme-platform-class.txt\n\u2502\u00a0\u00a0     \u2514\u2500\u2500 platform\n
                                                           
                                                          This build tree can be moved to a different location and used for AFU development of a PR capable AFU to be used with this board.
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#424-unit-level-simulation","title":"4.2.4. Unit Level Simulation","text":"
                                                          Unit level simulation of key components is provided. These simulations provide verification of the following areas:
                                                           
                                                             
                                                          • HSSI
                                                            •  
                                                          • PCIe
                                                            •  
                                                          • External Memory
                                                            •  
                                                          • FIM management
                                                            •  
                                                           
                                                          These simulations use the Synopsys VCS simulator. Each simulation contains a readme file explaining how to run the simulation. Refer to [Simulation User Guide: Open FPGA Stack for Intel Intel\u00ae Stratix 10\u00ae FPGA]  for details of simulation examples. Your simulation shell requires Python, Quartus, and VCS to run.  To run a simulation of the dfh_walker that simulates host access to the internal DFH registers, perform the following steps:
                                                           
                                                          Before running unit simulation, you must set environment variables as described below:
                                                           
                                                          cd $OFS_BUILD_ROOT/ofs-d5005\nexport OFS_ROOTDIR=$PWD\n\n##   *Note, OFS_ROOTDIR is the directory where you cloned the repo, e.g. /home/MyProject/ofs-d5005 *\n\nexport WORKDIR=$OFS_ROOTDIR\nexport VERDIR=$OFS_ROOTDIR/verification\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\n\n##   *Note, QUARTUS_ROOTDIR is your Quartus installation directory, e.g. $QUARTUS_ROOTDIR/bin contains Quartus executuable*\n\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport OPAE_SDK_REPO_BRANCH=release/2.10.0\n
                                                           
                                                          To compile all IPs:
                                                           
                                                          To Generate Simulation Files & compile all IPs, run the following command:
                                                           
                                                          cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\nsh gen_sim_files.sh d5005\n
                                                           The RTL file list for unit_test is located here: $OFS_ROOTDIR/sim/scripts/rtl_comb.f
                                                           
                                                          The IPs are generated here: 
                                                           
                                                          $OFS_ROOTDIR/sim/scripts/qip_gen\n
                                                           The IP simulation filelist is generated here: 
                                                           
                                                          $OFS_ROOTDIR/sim/scripts/ip_flist.f\n
                                                           Once the IPs are generated, they can be used for any unit test.
                                                           
                                                          To run the simulation, run the following command:
                                                           
                                                          cd $OFS_ROOTDIR/sim/unit_test/<Unit Test Name>/scripts\nsh run_sim.sh VCS=1\n
                                                           Simulation files are located in the sim/unit_test//sim directory. 
                                                          To view simulation waveform:
                                                           
                                                          cd $OFS_ROOTDIR/sim/unit_test/<test_name>/script/sim/unit_test/<test_name>/scripts/sim_vcs\ndve -full64 -vpd vcdplus.vpd &\n
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#4241-dfh-walking-unit-simulation-output","title":"4.2.4.1. DFH Walking Unit Simulation Output","text":"
                                                          ********************************************\n Running TEST(0) : test_fme_dfh_walking\n********************************************\nREAD64: address=0x00000000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x4000000010000000\n\nFME_DFH\n   Address   (0x0)\n   DFH value (0x4000000010000000)\nREAD64: address=0x00001000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000020000001\n\nTHERM_MNGM_DFH\n   Address   (0x1000)\n   DFH value (0x3000000020000001)\nREAD64: address=0x00003000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000010000007\n\nGLBL_PERF_DFH\n   Address   (0x3000)\n   DFH value (0x3000000010000007)\nREAD64: address=0x00004000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x30000000c0001004\n\nGLBL_ERROR_DFH\n   Address   (0x4000)\n   DFH value (0x30000000c0001004)\nREAD64: address=0x00010000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x300000010000000e\n\nSPI_DFH\n   Address   (0x10000)\n   DFH value (0x300000010000000e)\nREAD64: address=0x00020000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000100000020\n\nPCIE_DFH\n   Address   (0x20000)\n   DFH value (0x3000000100000020)\nREAD64: address=0x00030000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x300000010000100f\n\nHSSI_DFH\n   Address   (0x30000)\n   DFH value (0x300000010000100f)\nREAD64: address=0x00040000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000500000009\n\nEMIF_DFH\n   Address   (0x40000)\n   DFH value (0x3000000500000009)\nREAD64: address=0x00090000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000010001005\n\nFME_PR_DFH\n   Address   (0x90000)\n   DFH value (0x3000000010001005)\nREAD64: address=0x00091000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x4000000010001001\n\nPORT_DFH\n   Address   (0x91000)\n   DFH value (0x4000000010001001)\nREAD64: address=0x00092000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000010000014\n\nUSER_CLOCK_DFH\n   Address   (0x92000)\n   DFH value (0x3000000010000014)\nREAD64: address=0x00093000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x30000000d0002013\n\nPORT_STP_DFH\n   Address   (0x93000)\n   DFH value (0x30000000d0002013)\nREAD64: address=0x000a0000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000010000002010\n\nAFU_INTF_DFH\n   Address   (0xa0000)\n   DFH value (0x3000010000002010)\nMMIO error count matches: x\n\nTest status: OK\n\n********************\n  Test summary\n********************\n   test_fme_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n
                                                           
                                                          The simulation transcript is displayed while the simulation runs.  The transcript is saved to the file transcript.out for review after the simulation completes.  The simulation waveform database is saved as vcdplus.vpd for post simulation review. You are encouraged to run the additional simulation examples to learn about each key area of the OFS shell.
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#43-compiling-the-ofs-fim-using-eval-script","title":"4.3. Compiling the OFS FIM using Eval Script","text":"
                                                          The Evaluation Script provides resources to setup and report D5005 development environment. You can use the evaluation script to compile and simulate the FIM. Refer to README_ofs_d5005_eval.txt for details of using the evaluation script.
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#44-debugging","title":"4.4. Debugging","text":"
                                                          For debugging issues within the FIM, Signal Tap can be used to gain internal visibility into your design.  This section describes the process of adding a Signal Tap instance to your FIM design
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#441-signal-tap-prerequisites","title":"4.4.1. Signal Tap Prerequisites","text":"
                                                          To use Signal Tap with OFS, you will need the following:
                                                           
                                                             
                                                          • Understanding of Signal Tap fundamentals - please review Quartus Prime Pro Edition User Guide: Debug Tools. section 2. Design Debugging with the Signal Tap Logic Analyzer.
                                                            •  
                                                           
                                                             
                                                          • The Intel\u00ae FPGA PAC D5005 has a built in Intel FPGA Download Cable II allowing JTAG access to the S10 FPGA. You can access the D5005 built in Intel FPGA Download Cable II by connecting your server to the Micro USB connector as shown below:
                                                            •  
                                                           
                                                           
                                                             
                                                          • If you are using a custom board without a built-in Intel FPGA Download Cable then an external Intel FPGA Download Cable II (see Download Cables for more information) can be used for Signal Tap access.  The custom board must have JTAG access to the target FPGA for the Intel FPGA Download Cable II.
                                                            •  
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#442-adding-signal-tap","title":"4.4.2. Adding Signal Tap","text":"
                                                          The following steps guide you through the process of adding a Signal Tap instance to your design.  The added Signal Tap instance provides hardware to capture the desired internal signals and connect the stored trace information via JTAG.  Please be aware, the added Signal Tap hardware will consume FPGA resources and may require additional floorplanning steps to accommodate these resources.  Some areas of the FIM use logic lock regions and these regions may need to be re-sized. These steps assume the use of the Intel\u00ae FPGA PAC D5005.
                                                           
                                                             
                                                          1. Perform a full compile using the script build_top.sh.
                                                            2.  
                                                          2. Once the compile completes open the Quartus GUI using the FIM project.  The Quartus project is named d5005 and is located in the work directory syn/syn_top/d5005.qpf.  Once the project is loaded, go to Tools > Signal Tap Logic Analyzer to bring up the Signal Tap GUI.
                                                            3.  
                                                           
                                                           
                                                             
                                                          1. Accept the \"Default\" selection and click \"Create\".
                                                            2.  
                                                           
                                                           
                                                             
                                                          1. This brings up Signal Tap Logic Analyzer window as shown below:
                                                            2.  
                                                           
                                                           
                                                             
                                                          1. Set up clock for STP instance.  In this example, the EMIF CSR module is being instrumented.  If unfamiliar with code, it is helpful to use the Quartus Project Navigator to find the specific block of interest and open the design instance for review.  For example, see snip below using Project Navigator to open emif_csr block:
                                                            2.  
                                                           
                                                           
                                                           
                                                             
                                                          1. After reviewing code, assign clock for sampling Signal Tap instrumented signals of interest.  Note, the clock selected and the signals you want to view should be the same for best trace fidelity.  Different clocks can be used however there maybe issues with trace inaccuracy due sampling time differences.  In the middle right of the Signal Tap window under Signal Configuration, Clock:  select \"\u2026\" as shown below: 
                                                            2.  
                                                           
                                                           
                                                             
                                                          1. After reviewing code, assign clock for sampling Signal Tap instrumented signals of interest.  In the middle right of the Signal Tap window under Signal Configuration, Clock:  select \"\u2026\" as shown below:   This brings up the Node Finder tool.  Input \"emif_csr\" into Named and select \"Search\".  This brings up all nodes from the pre-synthesis view.  Expand, \"mem\" and \"emif_csr\" and scroll through this list to become familiar with nodes, and then select csr_if.clk and click \">\"  to select this clock as shown below and click \"OK\": 
                                                            2.  
                                                           
                                                           
                                                             
                                                          1. Update the sample depth and other Signal Tap settings as needed for your debugging criteria.
                                                            2.  
                                                           
                                                           
                                                             
                                                          1. In the Signal Tap GUI add nodes to be instrumented by double clicking on \"Double-click to add nodes\".
                                                            2.  
                                                           
                                                           
                                                             
                                                          1. This brings up the Node Finder.  Add signals to be traced by the Signal Tap instance.  Click \"Insert\" to add the signals.
                                                            2.  
                                                          2. To provide a unique name for your Signal Tap instance, select \"auto signaltap_0\", right click and select rename instance and provide a descriptive name for your instance.
                                                            3.  
                                                          3. Save the newly created Signal Tap file and click \"Yes\" to add the new Signal Tap file to the project.
                                                            4.  
                                                          4. Compile the project with the Signal Tap file added to the project.
                                                            5.  
                                                          5. Once the compile successfully completes with proper timing, you can load the generated d5005.sof using the Intel FPGA Downloader cable.
                                                            6.  
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#443-signal-tap-trace-acquisition","title":"4.4.3. Signal Tap trace acquisition","text":"
                                                          To acquire signals using SignalTap, first load the Signal Tap instrumented SOF file into your target board, open the STP file in the Signal Tap GUI and start the signal acquisition. 
                                                           
                                                          Avoid system hang during programming the sof file, mask AER regsiter using below steps 
                                                           
                                                          Find Root complex - End Point mapping using the below command
                                                           
                                                          lspci -vt\n
                                                           
                                                          +-[0000:3a]-+-00.0-[3b-3c]----00.0  Intel Corporation Device bcce\n |           +-05.0  Intel Corporation Sky Lake-E VT-d\n |           +-05.2  Intel Corporation Sky Lake-E RAS Configuration Registers\n |           +-05.4  Intel Corporation Sky Lake-E IOxAPIC Configuration Registers\n |           +-08.0  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-09.0  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-0a.0  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-0a.1  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-0a.2  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-0a.3  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-0a.4  Intel Corporation Sky Lake-E Integrated Memory Controller\n |           +-0a.5  Intel Corporation Sky Lake-E LM Channel 1\n
                                                           
                                                          Use the bus information from the lspci logs to mask the AER (Advanced Error Reporting) register
                                                           
                                                          sudo su\n\nsetpci -s 0000:3b:00.0 ECAP_AER+0x08.L=0xFFFFFFFF \nsetpci -s 0000:3b:00.0 ECAP_AER+0x14.L=0xFFFFFFFF\nsetpci -s 0000:3a:00.0 ECAP_AER+0x08.L=0xFFFFFFFF\nsetpci -s 0000:3a:00.0 ECAP_AER+0x14.L=0xFFFFFFFF\necho \"1\" > /sys/bus/pci/devices/0000:3b:00.0/remove\n\nexit\n
                                                           
                                                             
                                                          1. The SOF file is located in the work directory work_d5005/syn/syn_top/output_files/d5005.sof.  If the target FPGA is on a different server, then transfer d5005.sof and STP files to the server with the target FPGA. Load the SOF using the Intel\u00ae FPGA PAC D5005 built-in Intel FPGA Download Cable II. 
                                                            2.  
                                                           
                                                          sudo su\necho \"1\" > /sys/bus/pci/rescan\n
                                                           
                                                             
                                                          1. Make sure D5005 is present by checking expected bitstream ID using command:
                                                            2.  
                                                           
                                                          sudo fpgainfo fme\nIntel FPGA Programmable Acceleration Card D5005\nBoard Management Controller, MAX10 NIOS FW version: 2.0.13 \nBoard Management Controller, MAX10 Build version: 2.0.8 \n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x40100022C164DB1\nBitstream Version                : 4.0.1\nPr Interface Id                  : 210a4631-18bb-57d1-879f-2c3d59b26e37\nBoot Page                        : user\n
                                                           
                                                             
                                                          1. Once the SOF file is loaded, start the Quartus Signal Tap GUI.
                                                            2.  
                                                           
                                                          quartus_stpw\n
                                                           The Signal Tap GUI comes up.
                                                           
                                                             
                                                          1. In the Signal Tap GUI, Hardware: selection box select the cable \"Stratix10 Darby Creek [ JTAG cable number ]\" as shown below:
                                                            2.  
                                                           
                                                           
                                                             
                                                          1. In File open your STP file.  Your STP file settings will load.  If not already set, you can create the trigger conditions, then start analysis with F5.
                                                            2.  
                                                           
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#5-fim-modification-example","title":"5. FIM Modification Example","text":"
                                                          An example of FIM modification is provided in this section.  This example can be used in your specific application as a starting point.  This example shows the basic flow and listing of files that are to be changed.
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#51-hello-fim-example","title":"5.1. Hello FIM example","text":"
                                                          If you intend to add a new module to the FIM area, then you will need to inform the host software of the new module.  The FIM exposes its functionalities to host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO).  This set of CSR registers and their operation is described in FIM MMIO Regions.
                                                           
                                                          See FPGA Device Feature List (DFL) Framework Overview for a description of the software process to read and process the linked list of Device Feature Header (DFH) CSRs within a FPGA.
                                                           
                                                          This example adds a simple DFH register set to the FIM. You can use this example as the basis for adding a new feature to your FIM.  
                                                           
                                                          The steps to add this simple DFH register are described below.
                                                           
                                                             
                                                          1. Review current design documentation: OFS Tech Ref MMIO Regions
                                                            2.  
                                                          2. Understand FME and Port regions, DFH walking, DFH register structure 
                                                            3.  
                                                          3. Run unit level simulations and review output:             i.  sim/unit_test/dfh_walker
                                                            4.  
                                                          4. Note DFH link list order, see DFH Walker Unit Level Simulation Output
                                                            5.  
                                                          5. Make code changes to top level FIM file to instantiate new DFH register
                                                            6.  
                                                          6. The DFH registers follow a link list.  This example inserts the hello_fim DFH register after the EMIF DFH register, so the emif_csr.sv parameters are updated to insert the hello_fim DFH register as next register in the link list.
                                                            7.  
                                                          7. Create the new hello_fim SystemVerilog files.
                                                            8.  
                                                          8. Update and run the dfh_walker unit simulation files
                                                            9.  
                                                          9. Update synthesis files to include the new hello_fim source files
                                                            10.  
                                                          10. Build and test the new FIM
                                                            11.  
                                                           
                                                          The following sections describe changes to add the hello_fim DFH example to the Intel provided FPGA design.
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#511-srctopiofs_topsv","title":"5.1.1. src/top/iofs_top.sv","text":"
                                                             
                                                          1. Edit top level design module: src/top/iofs_top.sv
                                                               
                                                            1. Instantiate new hello_fim module in OFS_top.sv at line 294
                                                              2.  
                                                             
                                                            2.  
                                                           
                                                          //*******************************\n// FME\n//*******************************\n\n  fme_top \n  fme_top(\n          .clk               (clk_1x                    ),\n          .rst_n             (rst_n_d_1x       ),\n          .pwr_good_n        (ninit_done                ),\n          .i_pcie_error      ('0                        ),\n\n          .axi_lite_m_if     (bpf_fme_mst_if            ),\n          .axi_lite_s_if     (bpf_fme_slv_if            )\n         );\n\n\n`ifdef INCLUDE_HELLO_FIM\n\n        hello_fim_top \n           hello_fim_top (\n        .clk   (clk_1x),\n\n       .rst_n                      (rst_n_d_1x),\n       .csr_lite_if             (bpf_rsv_5_slv_if)\n\n\n        );\n`endif\n\n//*******************************\n// AFU\n//*******************************\n
                                                           
                                                          You will connect the Hello_FIM DFH register to the existing BPF reserved link 5.  The provided OFS reference design includes 3 reserved BPF interfaces available for custom usage such as new OPAE controlled modules.  The address map of BPF is shown below:
                                                           Address Size (Byte) Feature Master 0x00000 \u2013 0x0FFFF 64K FME (FME, Error, etc) Yes 0x10000 \u2013 0x1FFFF 64K PMCI Proxy (SPI Controller) Yes 0x20000 \u2013 0x2FFFF 64K PCIe CSR 0x30000 \u2013 0x3FFFF 64K HSSI CSR 0x40000 \u2013 0x4FFFF 64K EMIF CSR 0x50000 \u2013 0x5FFFF 64K Reserved 0x60000 \u2013 0x6FFFF 64K Reserved 0x70000 \u2013 0x7FFFF 64K Reserved 
                                                          The BPF reserved link 5 is connected to a dummy connection to prevent this link from being optimized out the design during synthesis.  You will add a compiler `define that will cause this dummy connection to be removed when the variable INCLUDE_HELLO_FIM is defined by editing line 575 if iofs_top.sv as shown below:
                                                           
                                                              // Reserved address response\n   `ifndef INCLUDE_HELLO_FIM\n    bpf_dummy_slv\n    bpf_rsv_5_slv (\n        .clk            (clk_1x),\n        .dummy_slv_if   (bpf_rsv_5_slv_if)\n    );\n    `endif\n
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#512-ipssmememif_csrsv","title":"5.1.2. ipss/mem/emif_csr.sv","text":"
                                                          The Hello_FIM DFH is inserted in the DFH link list after the EMIF CSR DFH and before the FME_PR DFH.  The file ipss/d5005/emif/emif_csr.sv contains a parameter defining the next address for the next DFH in in the link list chain.  You will change the next address offset to be 0x10000 so the reveserved BPF AXI lite link connected to the Hello_FIM DFH register is next in the DFH link list.
                                                           
                                                          module emif_csr #(\n  parameter NUM_LOCAL_MEM_BANKS = 1,\n  parameter END_OF_LIST         = 1'b0,\n  `ifndef INCLUDE_HELLO_FIM\n  parameter NEXT_DFH_OFFSET     = 24'h05_0000\n  `else\n  parameter NEXT_DFH_OFFSET     = 24'h01_0000//New for Hello_FIM, next offset now at 0x50000\n  `endif\n)\n
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#513-srchello_fimhello_fim_topsv","title":"5.1.3. src/hello_fim/hello_fim_top.sv","text":"
                                                          Create hello_fim_top.sv, and store it in src/hello_fim directory. The main purpose of this RTL is to convert AXI4-Lite interface to a simple interface to interface with the registers in hello_fim_com.sv.  This register sets the DFH feature ID to 0xfff which is undefined.  Since this for test purposes, using an undefined feature ID will result in no driver being used.  Normally, a defined feature ID will be used to associate a specific driver with the FPGA module.
                                                           
                                                          // ***************************************************************************\n//                               INTEL CONFIDENTIAL\n//\n//        Copyright (C) 2021 Intel Corporation All Rights Reserved.\n//\n// The source code contained or described herein and all  documents related to\n// the  source  code  (\"Material\")  are  owned  by  Intel  Corporation  or its\n// suppliers  or  licensors.    Title  to  the  Material  remains  with  Intel\n// Corporation or  its suppliers  and licensors.  The Material  contains trade\n// secrets  and  proprietary  and  confidential  information  of  Intel or its\n// suppliers and licensors.  The Material is protected  by worldwide copyright\n// and trade secret laws and treaty provisions. No part of the Material may be\n// used,   copied,   reproduced,   modified,   published,   uploaded,  posted,\n// transmitted,  distributed,  or  disclosed  in any way without Intel's prior\n// express written permission.\n//\n// No license under any patent,  copyright, trade secret or other intellectual\n// property  right  is  granted  to  or  conferred  upon  you by disclosure or\n// delivery  of  the  Materials, either expressly, by implication, inducement,\n// estoppel or otherwise.  Any license under such intellectual property rights\n// must be express and approved by Intel in writing.\n//\n// You will not, and will not allow any third party to modify, adapt, enhance, \n// disassemble, decompile, reverse engineer, change or create derivative works \n// from the Software except and only to the extent as specifically required by \n// mandatory applicable laws or any applicable third party license terms \n// accompanying the Software.\n//\n// -----------------------------------------------------------------------------\n// Engineer     : \n// Create Date  : Nov 2021\n// Module Name  : hello_fim_top.sv\n// Project      : OFS\n// -----------------------------------------------------------------------------\n//\n// Description: \n// This is a simple module that implements DFH registers and \n// AVMM address decoding logic.\n\n\nmodule hello_fim_top  #(\n   parameter ADDR_WIDTH  = 12, \n   parameter DATA_WIDTH = 64, \n   parameter bit [11:0] FEAT_ID = 12'hfff,\n   parameter bit [3:0]  FEAT_VER = 4'h0,\n   parameter bit [23:0] NEXT_DFH_OFFSET = 24'h04_0000,\n   parameter bit END_OF_LIST = 1'b0\n)(\n   input  logic    clk,\n   input  logic    rst_n,\n// -----------------------------------------------------------\n//  AXI4LITE Interface\n// -----------------------------------------------------------\n   ofs_fim_axi_lite_if.slave   csr_lite_if\n);\n\nimport ofs_fim_cfg_pkg::*;\nimport ofs_csr_pkg::*;\n\n//-------------------------------------\n// Signals\n//-------------------------------------\n   logic [ADDR_WIDTH-1:0]              csr_waddr;\n   logic [DATA_WIDTH-1:0]              csr_wdata;\n   logic [DATA_WIDTH/8-1:0]            csr_wstrb;\n   logic                               csr_write;\n   logic                               csr_slv_wready;\n   csr_access_type_t                   csr_write_type;\n\n   logic [ADDR_WIDTH-1:0]              csr_raddr;\n   logic                               csr_read;\n   logic                               csr_read_32b;\n   logic [DATA_WIDTH-1:0]              csr_readdata;\n   logic                               csr_readdata_valid;\n   logic [ADDR_WIDTH-1:0]              csr_addr;\n\n   logic [63:0]                        com_csr_writedata;\n   logic                               com_csr_read;\n   logic                               com_csr_write;\n   logic [63:0]                        com_csr_readdata;\n   logic                               com_csr_readdatavalid;\n   logic [5:0]                         com_csr_address;\n\n// AXI-M CSR interfaces\nofs_fim_axi_mmio_if #(\n   .AWID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .AWADDR_WIDTH (ADDR_WIDTH),\n   .WDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH),\n   .ARID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .ARADDR_WIDTH (ADDR_WIDTH),\n   .RDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH)\n) csr_if();\n\n// AXI4-lite to AXI-M adapter\naxi_lite2mmio axi_lite2mmio (\n   .clk       (clk),\n   .rst_n     (rst_n),\n   .lite_if   (csr_lite_if),\n   .mmio_if   (csr_if)\n);\n\n//---------------------------------\n// Map AXI write/read request to CSR write/read,\n// and send the write/read response back\n//---------------------------------\nofs_fim_axi_csr_slave #(\n   .ADDR_WIDTH (ADDR_WIDTH),\n   .USE_SLV_READY (1'b1)\n\n   ) csr_slave (\n   .csr_if             (csr_if),\n\n   .csr_write          (csr_write),\n   .csr_waddr          (csr_waddr),\n   .csr_write_type     (csr_write_type),\n   .csr_wdata          (csr_wdata),\n   .csr_wstrb          (csr_wstrb),\n   .csr_slv_wready     (csr_slv_wready),\n   .csr_read           (csr_read),\n   .csr_raddr          (csr_raddr),\n   .csr_read_32b       (csr_read_32b),\n   .csr_readdata       (csr_readdata),\n   .csr_readdata_valid (csr_readdata_valid)\n);\n\n// Address mapping\nassign csr_addr             = csr_write ? csr_waddr : csr_raddr;\nassign com_csr_address      = csr_addr[5:0];  // byte address\nassign csr_slv_wready       = 1'b1 ;\n// Write data mapping\nassign com_csr_writedata    = csr_wdata;\n\n// Read-Write mapping\nalways_comb\nbegin\n   com_csr_read             = 1'b0;\n   com_csr_write            = 1'b0;\n   casez (csr_addr[11:6])\n      6'h00 : begin // Common CSR\n         com_csr_read       = csr_read;\n         com_csr_write      = csr_write;\n      end   \n      default: begin\n         com_csr_read       = 1'b0;\n         com_csr_write      = 1'b0;\n      end\n   endcase\nend\n\n// Read data mapping\n   always_comb begin\n      if (com_csr_readdatavalid) begin\n         csr_readdata       = com_csr_readdata;\n         csr_readdata_valid = 1'b1;\n      end\n      else begin\n         csr_readdata       = '0;\n         csr_readdata_valid = 1'b0;\n      end\n   end\n\n   hello_fim_com  #(\n   .FEAT_ID          (FEAT_ID),\n   .FEAT_VER         (FEAT_VER),\n   .NEXT_DFH_OFFSET  (NEXT_DFH_OFFSET),\n   .END_OF_LIST      (END_OF_LIST)\n) hello_fim_com_inst (\n   .clk                   (clk                     ),\n   .rst_n                 (rst_n                  ),\n   .writedata             (com_csr_writedata       ),\n   .read                  (com_csr_read            ),\n   .write                 (com_csr_write           ),\n   .byteenable            (4'hF                    ),\n   .readdata              (com_csr_readdata        ),\n   .readdatavalid         (com_csr_readdatavalid   ),\n   .address               (com_csr_address         )\n   );\n\n\nendmodule\n
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#514-srchello_fimhello_fim_comsv","title":"5.1.4. src/hello_fim/hello_fim_com.sv","text":"
                                                          Create hello_fim_com.sv, and store it in src/hello_fim directory. This is the simple RTL to implement the Hello FIM registers. You may use this set of registers as the basis for your custom implementation.
                                                           
                                                          // ***************************************************************************\n//                               INTEL CONFIDENTIAL\n//\n//        Copyright (C) 2021 Intel Corporation All Rights Reserved.\n//\n// The source code contained or described herein and all  documents related to\n// the  source  code  (\"Material\")  are  owned  by  Intel  Corporation  or its\n// suppliers  or  licensors.    Title  to  the  Material  remains  with  Intel\n// Corporation or  its suppliers  and licensors.  The Material  contains trade\n// secrets  and  proprietary  and  confidential  information  of  Intel or its\n// suppliers and licensors.  The Material is protected  by worldwide copyright\n// and trade secret laws and treaty provisions. No part of the Material may be\n// used,   copied,   reproduced,   modified,   published,   uploaded,  posted,\n// transmitted,  distributed,  or  disclosed  in any way without Intel's prior\n// express written permission.\n//\n// No license under any patent,  copyright, trade secret or other intellectual\n// property  right  is  granted  to  or  conferred  upon  you by disclosure or\n// delivery  of  the  Materials, either expressly, by implication, inducement,\n// estoppel or otherwise.  Any license under such intellectual property rights\n// must be express and approved by Intel in writing.\n//\n// You will not, and will not allow any third party to modify, adapt, enhance, \n// disassemble, decompile, reverse engineer, change or create derivative works \n// from the Software except and only to the extent as specifically required by \n// mandatory applicable laws or any applicable third party license terms \n// accompanying the Software.\n//\n// -----------------------------------------------------------------------------\n// Engineer     : \n// Create Date  : Nov 2021\n// Module Name  : hello_fim_com.sv\n// Project      : IOFS\n// -----------------------------------------------------------------------------\n//\n// Description: \n// This is a simple module that implements DFH registers and \n// AVMM address decoding logic.\n\n\nmodule hello_fim_com #(\n    parameter bit [11:0] FEAT_ID = 12'h001,\n    parameter bit [3:0]  FEAT_VER = 4'h1,\n    parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n    parameter bit END_OF_LIST = 1'b0\n )(\n input clk,\n input rst_n,\n input [63:0] writedata,\n input read,\ninput write,\ninput [3:0] byteenable,\noutput reg [63:0] readdata,\noutput reg readdatavalid,\ninput [5:0] address\n);\n\n\nreg [63:0] rdata_comb;\nreg [63:0] scratch_reg;\n\nalways @(posedge clk)  \n   if (!rst_n) readdata[63:0] <= 64'h0; else readdata[63:0] <= rdata_comb[63:0];\n\nalways @(posedge clk)\n   if (!rst_n) readdatavalid <= 1'b0; else readdatavalid <= read;\n\nwire wr = write;\nwire re = read;\nwire [5:0] addr = address[5:0];\nwire [63:0] din  = writedata [63:0];\nwire wr_scratch_reg = wr & (addr[5:0]  == 6'h30)? byteenable[0]:1'b0;\n\n// 64 bit scratch register\nalways @( posedge clk)\n   if (!rst_n)  begin\n      scratch_reg <= 64'h0;\n   end\n   else begin\n   if (wr_scratch_reg) begin \n      scratch_reg <=  din;  \n   end\nend\n\nalways @ (*)\nbegin\nrdata_comb = 64'h0000000000000000;\n   if(re) begin\n      case (addr)  \n    6'h00 : begin\n        rdata_comb [11:0]   = FEAT_ID ;  // dfh_feature_id  is reserved or a constant value, a read access gives the reset value\n        rdata_comb [15:12]  = FEAT_VER ;  // dfh_feature_rev    is reserved or a constant value, a read access gives the reset value\n        rdata_comb [39:16]  = NEXT_DFH_OFFSET ;  // dfh_dfh_ofst is reserved or a constant value, a read access gives the reset value\n        rdata_comb [40]     = END_OF_LIST ;        //dfh_end_of_list\n        rdata_comb [59:40]  = 20'h00000 ;  // dfh_rsvd1     is reserved or a constant value, a read access gives the reset value\n        rdata_comb [63:60]  = 4'h3 ;  // dfh_feat_type  is reserved or a constant value, a read access gives the reset value\n    end\n    6'h30 : begin\n        rdata_comb [63:0]   = scratch_reg; \n    end\n        6'h38 : begin\n                rdata_comb [63:0]       = 64'h6626_0701_5000_0034;\n        end\n    default : begin\n        rdata_comb = 64'h0000000000000000;\n    end\n      endcase\n   end\nend\n\nendmodule\n
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#515-unit-level-simulations","title":"5.1.5. Unit Level Simulations","text":"
                                                          To run a unit level simulation test for the updated RTL files, make modifications to your cloned /my_ofs_project/ofs-d5005/sim/d5005/unit_test/dfh_walker files.  The following simulation files are updated to test the new hello_fim.
                                                           
                                                             
                                                          1.  
                                                            Edit sim/unit_test/dfh_walker/testbench/test_csr_defs.sv
                                                             
                                                               
                                                            1. Update enum line 38
                                                              2.  
                                                             
                                                            2.  
                                                           
                                                                 typedef enum {\n      FME_DFH_IDX,\n      THERM_MNGM_DFH_IDX,\n      GLBL_PERF_DFH_IDX,\n      GLBL_ERROR_DFH_IDX,\n      SPI_DFH_IDX,\n      PCIE_DFH_IDX,\n      HSSI_DFH_IDX,\n      EMIF_DFH_IDX,\n      HELLO_FIM_DFH_IDX,//New for HELLO_FIM\n      FME_PR_DFH_IDX,\n      PORT_DFH_IDX,\n      USER_CLOCK_DFH_IDX,\n      PORT_STP_DFH_IDX,\n      AFU_INTF_DFH_IDX,\n      MAX_FME_DFH_IDX\n   } t_fme_dfh_idx;\n
                                                           
                                                          1. Edit function dfh_name line 78
                                                           
                                                          function automatic dfh_name[MAX_FME_DFH_IDX-1:0] get_fme_dfh_names();\n      dfh_name[MAX_FME_DFH_IDX-1:0] fme_dfh_names;\n\n      fme_dfh_names[FME_DFH_IDX]         = \"FME_DFH\";\n      fme_dfh_names[THERM_MNGM_DFH_IDX]  = \"THERM_MNGM_DFH\";\n      fme_dfh_names[GLBL_PERF_DFH_IDX]   = \"GLBL_PERF_DFH\";\n      fme_dfh_names[GLBL_ERROR_DFH_IDX]  = \"GLBL_ERROR_DFH\";\n      fme_dfh_names[SPI_DFH_IDX]         = \"SPI_DFH\";\n      fme_dfh_names[PCIE_DFH_IDX]        = \"PCIE_DFH\";\n      fme_dfh_names[HSSI_DFH_IDX]        = \"HSSI_DFH\";\n      fme_dfh_names[EMIF_DFH_IDX]        = \"EMIF_DFH\";\n      fme_dfh_names[HELLO_FIM_DFH_IDX]        = \"HELLO_FIM_DFH\";//New for HELLO_FIM\n      fme_dfh_names[FME_PR_DFH_IDX]      = \"FME_PR_DFH\";\n      fme_dfh_names[PORT_DFH_IDX]        = \"PORT_DFH\";\n      fme_dfh_names[USER_CLOCK_DFH_IDX]  = \"USER_CLOCK_DFH\";\n      fme_dfh_names[PORT_STP_DFH_IDX]    = \"PORT_STP_DFH\";\n      fme_dfh_names[AFU_INTF_DFH_IDX]    = \"AFU_INTF_DFH\";\n\n      return fme_dfh_names;\n   endfunction\n
                                                           
                                                          1. Update get_fme_dfh_values
                                                           
                                                            function automatic [MAX_FME_DFH_IDX-1:0][63:0] get_fme_dfh_values();\n      logic[MAX_FME_DFH_IDX-1:0][63:0] fme_dfh_values;\n\n      fme_dfh_values[FME_DFH_IDX]        = 64'h4000_0000_1000_0000;\n      fme_dfh_values[THERM_MNGM_DFH_IDX] = 64'h3_00000_002000_0001;\n      fme_dfh_values[GLBL_PERF_DFH_IDX]  = 64'h3_00000_001000_0007;\n      fme_dfh_values[GLBL_ERROR_DFH_IDX] = 64'h3_00000_00C000_1004;  \n      fme_dfh_values[SPI_DFH_IDX]        = 64'h3_00000_010000_000e;  \n      fme_dfh_values[PCIE_DFH_IDX]       = 64'h3_00000_010000_0020;  \n      fme_dfh_values[HSSI_DFH_IDX]       = 64'h3_00000_010000_100f;  \n      fme_dfh_values[EMIF_DFH_IDX]       = 64'h3_00000_010000_0009; //Update to link to Hello_FIM \n      fme_dfh_values[HELLO_FIM_DFH_IDX]  = 64'h3_00000_040000_0FFF;  //New for Hello_FIM\n      fme_dfh_values[FME_PR_DFH_IDX]     = 64'h3_00000_001000_1005;  \n      fme_dfh_values[PORT_DFH_IDX]       = 64'h4000_0000_1000_1001;\n      fme_dfh_values[USER_CLOCK_DFH_IDX] = 64'h3_00000_001000_0014;\n      fme_dfh_values[PORT_STP_DFH_IDX]   = 64'h3_00000_00D000_2013;\n      fme_dfh_values[AFU_INTF_DFH_IDX]   = 64'h3_00001_000000_2010; \n\n      return fme_dfh_values;\n   endfunction\n
                                                           
                                                             
                                                          1. Update verification/scripts/Makefile_VCS.mk to set macro for INCLUDE_HELLO_FIM starting at line 56 to add +define+INCLUDE_HELLO_FIM 
                                                            VLOG_OPT += +define+SIM_MODE +define+VCS_S10 +define+RP_MAX_TAGS=64 +define+INCLUDE_DDR4 +define+INCLUDE_SPI_BRIDGE +define+INCLUDE_USER_CLOCK +define+INCLUDE_HSSI +define+SIM_USE_PCIE_DUMMY_CSR +define+INCLUDE_HELLO_FIM\n
                                                            2.  
                                                          2. Update sim/scripts/rtl_comb.f to add the path to your new hello_fim_top and hello_top_com SystemVerilog files.  The update is shown below as the new line - 329 below:
                                                            3.  
                                                           
                                                          $WORKDIR/src/hello_fim/hello_fim_com.sv\n$WORKDIR/src/hello_fim/hello_fim_top.sv\n
                                                           
                                                          After making these changes, run the unit level simulation using sim/unit_test/dfh_walker test.  Before running, ensure your shell has the environment variables set properly as defined in Setting Up Required Environment Variables.
                                                           
                                                          cd verification/scripts\ngmake -f Makefile_VCS.mk cmplib\ngmake -f Makefile_VCS.mk build run [DUMP=1]\n
                                                           
                                                          Expected output: 
                                                           ********************************************\n Running TEST(0) : test_fme_dfh_walking\n********************************************\nREAD64: address=0x00000000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x4000000010000000\n\nFME_DFH\n   Address   (0x0)\n   DFH value (0x4000000010000000)\nREAD64: address=0x00001000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000020000001\n\nTHERM_MNGM_DFH\n   Address   (0x1000)\n   DFH value (0x3000000020000001)\nREAD64: address=0x00003000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000010000007\n\nGLBL_PERF_DFH\n   Address   (0x3000)\n   DFH value (0x3000000010000007)\nREAD64: address=0x00004000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x30000000c0001004\n\nGLBL_ERROR_DFH\n   Address   (0x4000)\n   DFH value (0x30000000c0001004)\nREAD64: address=0x00010000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x300000010000000e\n\nSPI_DFH\n   Address   (0x10000)\n   DFH value (0x300000010000000e)\nREAD64: address=0x00020000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000100000020\n\nPCIE_DFH\n   Address   (0x20000)\n   DFH value (0x3000000100000020)\nREAD64: address=0x00030000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x300000010000100f\n\nHSSI_DFH\n   Address   (0x30000)\n   DFH value (0x300000010000100f)\nREAD64: address=0x00040000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000100000009\n\nEMIF_DFH\n   Address   (0x40000)\n   DFH value (0x3000000100000009)\nREAD64: address=0x00050000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000400000fff\n\nHELLO_FIM_DFH\n   Address   (0x50000)\n   DFH value (0x3000000400000fff)\nREAD64: address=0x00090000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000010001005\n\nFME_PR_DFH\n   Address   (0x90000)\n   DFH value (0x3000000010001005)\nREAD64: address=0x00091000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x4000000010001001\n\nPORT_DFH\n   Address   (0x91000)\n   DFH value (0x4000000010001001)\nREAD64: address=0x00092000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000000010000014\n\nUSER_CLOCK_DFH\n   Address   (0x92000)\n   DFH value (0x3000000010000014)\nREAD64: address=0x00093000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x30000000d0002013\n\nPORT_STP_DFH\n   Address   (0x93000)\n   DFH value (0x30000000d0002013)\nREAD64: address=0x000a0000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   READDATA: 0x3000010000002010\n\nAFU_INTF_DFH\n   Address   (0xa0000)\n   DFH value (0x3000010000002010)\nMMIO error count matches: x\n\nTest status: OK\n\n********************\n  Test summary\n********************\n   test_fme_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#516-synsyn_topd5005qsf","title":"5.1.6. syn/syn_top/d5005.qsf","text":"
                                                             
                                                          1.  
                                                            Edit syn/syn_top/d5005.qsf      
                                                             
                                                               
                                                            1.  
                                                              Add new macro \"INCLUDE_HELLO_FIM\" line 107 
                                                                      set_global_assignment -name VERILOG_MACRO \"INCLUDE_HELLO_FIM\"\n
                                                               
                                                              2.  
                                                            2.  
                                                              Add new line 211 to source TCL script with new hello_fim files 
                                                                      set_global_assignment -name SOURCE_TCL_SCRIPT_FILE ../../../syn/setup/hello_fim_design_files.tcl\n
                                                               
                                                              3.  
                                                             
                                                            2.  
                                                           
                                                          Create \"hello_fim_design_files.tcl\" file and store in the syn/setup directory. This tcl file is called from d5005.qsf.
                                                           
                                                          # Copyright 2021 Intel Corporation.\n#\n# THIS SOFTWARE MAY CONTAIN PREPRODUCTION CODE AND IS PROVIDED BY THE\n# COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED\n# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE\n# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR\n# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\n# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR\n# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,\n# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE\n# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,\n# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# Hello FIM Files\n#--------------------\nset_global_assignment -name SYSTEMVERILOG_FILE src/hello_fim/hello_fim_com.sv\nset_global_assignment -name SYSTEMVERILOG_FILE src/hello_fim/hello_fim_top.sv\n
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#517-synsetuphello_fim_design_filestcl","title":"5.1.7. syn/setup/hello_fim_design_files.tcl","text":""},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#518-build-hello_fim-example","title":"5.1.8. Build hello_fim example","text":"
                                                          With the preceding changes complete, build the new hello_fim example using the following steps:
                                                           
                                                          cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh d5005 work_d5005_hello_fim\n
                                                           
                                                          Verify the design successfully compiled and timing closure is achieved by checking work_d5005_hello_fim/syn/syn_top/output_files/timing_report/clocks.sta.fail.summary - this file should be empty.  If there are timing failures, then this file will list the failing clock domain(s).
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#519-test-the-hello_fim-on-a-d5005","title":"5.1.9. Test the hello_fim on a D5005","text":"
                                                          Load the built FPGA binary file using an unsigned image.  The FPGA image will be in work_d5005_hello_fim/syn/syn_top/output_files/d5005_page1_unsigned.bin
                                                           
                                                          Provide the file d5005_page1_unsigned.bin on the server with the Intel\u00ae FPGA PAC D5005.
                                                           
                                                          sudo fpgasupdate d5005_page1_unsigned.bin <D5005 PCIe B:D.F>\nsudo rsu bmcimg <D5005 PCIe B:D.F>\n
                                                           Verify FPGA image is loaded. 
                                                          sudo fpgainfo fme\n## Output\nIntel FPGA Programmable Acceleration Card D5005\nBoard Management Controller, MAX10 NIOS FW version: 2.0.13 \nBoard Management Controller, MAX10 Build version: 2.0.8 \n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x40100022C164DB1\nBitstream Version                : 4.0.1\nPr Interface Id                  : 7d91e0d0-4dcd-58c3-a93d-b9295e6e29b0\nBoot Page                        : user\n
                                                           
                                                          Use the OPAE SDK tool opae.io to check default driver binding using your card under test PCIe B:D.F.  The steps below will use 0000:12:00.0 as the card under test PCIe B:D.F.
                                                           
                                                           sudo opae.io init -d 0000:12:00.0 $USER\n ##Output\n [0000:12:00.0] (0x8086, 0xbcce) Intel D5005 ADP (Driver: dfl-pci)\n
                                                           The dfl-pci driver is used by OPAE SDK fpgainfo commands.  The next steps will bind the card under test to the vfio driver to enable access to the registers.
                                                           
                                                           sudo opae.io init -d 0000:12:00.0 $USER\n ##Output\n opae.io 0.2.3\nUnbinding (0x8086,0xbcce) at 0000:12:00.0 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:12:00.0 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:12:00.0 is 35\nAssigning /dev/vfio/35 to $USER\n
                                                           Confirm the vfio driver is bound to the card under test.
                                                           
                                                          opae.io ls\n## Output\nopae.io 0.2.3\n[0000:12:00.0] (0x8086, 0xbcce) Intel D5005 ADP (Driver: vfio-pci)\n
                                                           Run the following command to walk DFH link list.  The new hello_fim register is located at offset 0x50000.
                                                           
                                                          opae.io walk -d 0000:12:00.0\n## Output\nopae.io 0.2.3\noffset: 0x0000, value: 0x4000000010000000\n    dfh: id = 0x0, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x4\noffset: 0x1000, value: 0x3000000020000001\n    dfh: id = 0x1, rev = 0x0, next = 0x2000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x3000, value: 0x3000000010000007\n    dfh: id = 0x7, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x4000, value: 0x30000000c0001004\n    dfh: id = 0x4, rev = 0x1, next = 0xc000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x10000, value: 0x300000010000000e\n    dfh: id = 0xe, rev = 0x0, next = 0x10000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x20000, value: 0x3000000100000020\n    dfh: id = 0x20, rev = 0x0, next = 0x10000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x30000, value: 0x300000010000100f\n    dfh: id = 0xf, rev = 0x1, next = 0x10000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x40000, value: 0x3000000100000009\n    dfh: id = 0x9, rev = 0x0, next = 0x10000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x50000, value: 0x3000000400000fff\n    dfh: id = 0xfff, rev = 0x0, next = 0x40000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x90000, value: 0x3000000010001005\n    dfh: id = 0x5, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x91000, value: 0x4000000010001001\n    dfh: id = 0x1, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x4\noffset: 0x92000, value: 0x3000000010000014\n    dfh: id = 0x14, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x93000, value: 0x30000000d0002013\n    dfh: id = 0x13, rev = 0x2, next = 0xd000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0xa0000, value: 0x3000010000002010\n    dfh: id = 0x10, rev = 0x2, next = 0x0, eol = 0x1, reserved = 0x0, feature_type = 0x3\n
                                                           Read the default values from the hello_fim registers:
                                                           
                                                          $ opae.io -d 0000:12:00.0 -r 0 peek 0x50000\nopae.io 0.2.3\n0x3000000400000fff\n$ opae.io -d 0000:12:00.0 -r 0 peek 0x50030\nopae.io 0.2.3\n0x0\n$ opae.io -d 0000:12:00.0 -r 0 peek 0x50038\nopae.io 0.2.3\n0x6626070150000034\n
                                                           Write the scratchpad register at 0x50030
                                                           
                                                          $ opae.io -d 0000:12:00.0 -r 0 poke 0x50038 0x123456789abcdef\nopae.io 0.2.3\n$ opae.io -d 0000:12:00.0 -r 0 peek 0x50038\nopae.io 0.2.3\n0x6626070150000034\n$ opae.io -d 0000:12:00.0 -r 0 poke 0x50030 0x123456789abcdef\nopae.io 0.2.3\n$ opae.io -d 0000:12:00.0 -r 0 peek 0x50030\nopae.io 0.2.3\n0x123456789abcdef\n$ opae.io -d 0000:12:00.0 -r 0 poke 0x50030 0xfedcba9876543210\nopae.io 0.2.3\n$ opae.io -d 0000:12:00.0 -r 0 peek 0x50030\nopae.io 0.2.3\n0xfedcba9876543210\n$ opae.io -d 0000:12:00.0 -r 0 poke 0x50030 0x55550000aaaaffff\nopae.io 0.2.3\n$ opae.io -d 0000:12:00.0 -r 0 peek 0x50030\nopae.io 0.2.3\n0x55550000aaaaffff\n
                                                           
                                                          Release the card under test from the vfio driver to re-bind to the dfl-pci driver:
                                                           
                                                          sudo opae.io release -d 0000:12:00.0\n## Output\nopae.io 0.2.3\nReleasing (0x8086,0xbcce) at 0000:12:00.0 from vfio-pci\nRebinding (0x8086,0xbcce) at 0000:12:00.0 to dfl-pci\n$ sudo opae.io ls\nopae.io 0.2.3\n[0000:12:00.0] (0x8086, 0xbcce) Intel D5005 ADP (Driver: dfl-pci)\n
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#52-memory-subsystem-modification","title":"5.2. Memory Subsystem Modification","text":"
                                                          OFS enables modifications on the different subsystems that encompass the FIM. To customize the Memory Subsystem follow these instructions.
                                                           
                                                             
                                                          1.  
                                                            Set up the environment variables as described in section 4.2.1. Setting Up Required Environment Variables
                                                             
                                                            2.  
                                                          2.  
                                                            Modify the NUM_MEM_CH parameter in src/afu_top/mux/top_cfg_pkg.sv Change NUM_MEM_CH from 4 to 2 as shown in below code 
                                                             
                                                            3.  
                                                           
                                                          //=========================================================================================================================\n//                         OFS Configuration Parameters                                                                 \n//=========================================================================================================================\n     parameter NUM_MEM_CH     = 2                                                 ,// Number of Memory/DDR Channel         \n               NUM_HOST       = 1                                                 ,// Number of Host/Upstream Ports        \n               NUM_PORT       = 4                                                 ,// Number of Functions/Downstream Ports \n               DATA_WIDTH     = 512                                               ,// Data Width of Interface              \n               TOTAL_BAR_SIZE = 20                                                ,// Total Space for APF/BPF BARs (2^N) \n           //------------+-------------+-------------+-----------------+           //--------------------------------------\n           // VF Active  |     PF #    |     VF #    |  Mux Port Map   |           //  PF/VF Mapping Parameters            \n           //------------+-------------+-------------+-----------------+           //--------------------------------------\n             CFG_VA = 0  , CFG_PF = 0  , CFG_VF =  0 ,  CFG_PID = 3    ,           //  Configuration Register Block        \n             HLB_VA = 1  , HLB_PF = 0  , HLB_VF =  0 ,  HLB_PID = 0    ,           //  HE Loopback Engine                  \n             PRG_VA = 1  , PRG_PF = 0  , PRG_VF =  1 ,  PRG_PID = 1    ,           //  Partial Reconfiguration Gasket      \n             HSI_VA = 1  , HSI_PF = 0  , HSI_VF =  2 ,  HSI_PID = 2    ;           //  HSSI interface \n
                                                           
                                                          Compile a new FIM that incorporates the newly configured Memory Subsystem. 
                                                           
                                                          cd $OFS_BUILD_ROOT/ofs-d5005\nofs-common/scripts/common/syn/build_top.sh d5005 work_d5005_mem_2channel\n
                                                           
                                                          ***********************************\n***\n***        OFS_PROJECT: d5005\n***        Q_PROJECT:  d5005\n***        Q_REVISION: d5005\n***        SEED: 03\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                                           
                                                          Program d5005_page1_unsigned.bin file using below command
                                                           
                                                          sudo fpgasupdate d5005_page1_unsigned.bin 3b:00.0\n
                                                           
                                                          Run rsu command 
                                                          sudo rsu bmcimg 3b:00.0\n
                                                           
                                                          Check if binary was loaded correctly 
                                                          fpgainfo fme\n## Output\nIntel FPGA Programmable Acceleration Card D5005\nBoard Management Controller, MAX10 NIOS FW version: 2.0.14\nBoard Management Controller, MAX10 Build version: 2.0.8\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 288511863935352239\nBitstream Version                : 4.0.1\nPr Interface Id                  : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98\nBoot Page                        : user\n
                                                           
                                                          Run Host Excersiser to check Memory Subsystem performance
                                                           
                                                          sudo host_exerciser mem\n## Output\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5365\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.054 GB/s\n    Test mem(1): PASS\n
                                                           
                                                          Verify Memory controller placement in syn/syn_top/output_files/d5005.fit.place.rpt file. Open fitter place stage report in any text editor of your choice, find keyword emif in the file. You should see emif[0] & emif[1] for Memory channel 0 & 1 respectively.
                                                           
                                                          |emif[0].ddr4_pr_freeze_sync|                ; 0.4 (0.0)            ; 0.5 (0.0)                        ; 0.1 (0.0)                            ;\n|resync_chains[0].synchronizer_nocut|        ; 0.4 (0.4)            ; 0.5 (0.5)                        ; 0.1 (0.1)                            ;\n|emif[0].ddr4_softreset_sync|                ; 0.5 (0.0)            ; 0.7 (0.0)                        ; 0.2 (0.0)                            ;\n|resync_chains[0].synchronizer_nocut|        ; 0.5 (0.5)            ; 0.7 (0.7)                        ; 0.2 (0.2)                            ;\n|emif[0].pr_frz_afu_avmm_if|                 ; 647.5 (647.5)        ; 917.3 (917.3)                    ; 272.8 (272.8)                        ;\n|emif[1].ddr4_pr_freeze_sync|                ; 0.4 (0.0)            ; 0.8 (0.0)                        ; 0.4 (0.0)                            ;\n|resync_chains[0].synchronizer_nocut|        ; 0.4 (0.4)            ; 0.8 (0.8)                        ; 0.4 (0.4)                            ;\n|emif[1].ddr4_softreset_sync|                ; 0.4 (0.0)            ; 1.0 (0.0)                        ; 0.6 (0.0)                            ;\n|resync_chains[0].synchronizer_nocut|        ; 0.4 (0.4)            ; 1.0 (1.0)                        ; 0.6 (0.6)                            ;\n|emif[1].pr_frz_afu_avmm_if|                 ; 641.1 (641.1)        ; 914.0 (914.0)                    ; 272.9 (272.9)                        ;\n|p[0].pr_frz_fn2mx_a_port|                   ; 435.4 (0.0)          ; 476.2 (0.0)                      ; 40.8 (0.0)                           ;\n|r.axis_pl_stage[0].axis_reg_inst|           ; 435.4 (435.4)        ; 476.2 (476.2)                    ; 40.8 (40.8)                          ;\n|p[0].pr_frz_fn2mx_b_port|                   ; 434.6 (0.0)          ; 494.3 (0.0)                      ; 59.6 (0.0)                           ;\n
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#6-conclusion","title":"6. Conclusion","text":"
                                                          Using the OFS reference design and OPAE SDK enables the rapid creation of market leading FPGA based Acceleration systems. OFS facilitates customization of the FIM area for your custom board or platforms. 
                                                          "},{"location":"hw/d5005/dev_guides/fim_dev/ug_dev_fim_ofs_d5005/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                          Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                           
                                                          OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                          "},{"location":"hw/d5005/doc_modules/Glossary/","title":"Glossary","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/","title":"FPGA Interface Manager Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA","text":""},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1-overview","title":"1 Overview","text":""},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#11-about-this-document","title":"1.1 About this Document","text":"
                                                          This document describes the hardware architecture of the\u200b Open FPGA Stack (OFS) targeting the Intel\u00ae Stratix 10 FPGA.  After reviewing this document you should understand the features and functions of the components that comprise the FPGA Interface Manager (FIM), also known as the \"shell.\"
                                                           Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#12-introduction-to-the-open-fpga-stack","title":"1.2 Introduction to the Open FPGA Stack","text":"
                                                          The Open FPGA Stack (OFS) is a modular collection of hardware platform components, open source upstreamed software, and broad ecosystem support that enables an efficient path to develop a custom FPGA platform.  OFS Provides a framework of FPGA synthesizable code, simulation environment and synthesis/simulation scripts.  The key components of OFS include: -   Target development platforms such as Intel-branded Programmable Acceleration Cards (PACs), Acceleration Development Platforms (ADPs) and third-party platforms.
                                                           
                                                             
                                                          • Board Management Controller RTL and firmware that supports telemetry monitoring, remote configuration updates and most importantly a root of trust for the platform.
                                                            •  
                                                           
                                                             
                                                          • Source accessible, modular FPGA Interface  manager (FIM) RTL with unit tests that can be leveraged for your own custom FIM design
                                                            •  
                                                           
                                                             
                                                          • Basic building blocks for interconnect and PF/VF translation and arbitration; Platform Interface Manager (PIM) which provides Avalon\u00ae bus compliant interfaces.
                                                            •  
                                                           
                                                             
                                                          • AFU examples both in the git repository and workload examples provided by 3rd party vendors
                                                            •  
                                                           
                                                             
                                                          • The OneAPI shim provides a layer that is used by the OneAPI runtime to communicate with the kernel.
                                                            •  
                                                           
                                                             
                                                          • OPAE software development kit (APIs, upstreamed Linux drivers and software tools)
                                                            •  
                                                           
                                                             
                                                          • Support for other frameworks to be built on top of the OPAE such as DPDK 
                                                            •  
                                                           
                                                          The OFS hardware repository supports hardware development and simulation.  Repositories for OFS high level design support and board management controller RTL and firmware source code are also provided.  These repositories can be found in the Intel Opensource Technology GitHub location, which requires entitlement access.  To request access, please contact your local Intel sales representative.
                                                           
                                                          Table 1-2 OFS GitHub Repositories (https://github.com/OFS/)
                                                           Repository Contains ofs-fim-common Contains common modules shared by all OFS designs.  This repository is a submodule of each platform repository. ofs-d5005 Contains FIM or shell RTL design, automated compilation scripts, unit tests. 
                                                           The OPAE software GitHub site is fully opensource and contains resources for both software and workload developers.
                                                           
                                                          Table 1-3 OPAE Public Git Repositories (https://github.com/OFS)
                                                           OPAE Git Repository Folder Contains linux-dfl Contains OFS Linux drivers that are being upstreamed to the Linux kernel. linux-dfl-backport Backport versions of the linux-dfl to older kernel versions. opae-sdk Contains the files for building and installing OPAE SDK from source. opae-sim Contains an AFU/Workload simulator for software/hardware co-simulation. examples-afu Contains simple AFU tutorials. 
                                                          Providing the hardware and software source code and supporting test frameworks in a GitHub repository allows you to easily customize your own designs with the latest versions.
                                                           
                                                          Most hardware and software ingredients are available in our OFS GitHub location.  For access to the board management controller firmware and RTL or our security guide for OFS, please contact a local Intel sales representative. 
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#13-ofs-features","title":"1.3 OFS Features","text":"
                                                          The OFS architecture within the FPGA comprises two partitions:
                                                           
                                                             
                                                          • FPGA Interface Manager (FIM)
                                                            •  
                                                           
                                                             
                                                          • Accelerator Functional Unit (AFU)
                                                            •  
                                                           
                                                          The FIM or shell provides platform management functionality, clocks, resets and interface access to the host and peripheral features of the acceleration platform. The FIM architecture along with the supporting OPAE software supports features such as partial reconfiguration and virtualization. The FIM provides a standard Arm* AMBA* 4 AXI4 datapath interface. The FIM resides in the static region of the FPGA.
                                                           
                                                          The AFU partition is provided for custom acceleration workloads and may contain both static and partial reconfiguration regions.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#131-fpga-interface-manager-fim","title":"1.3.1 FPGA Interface Manager (FIM)","text":"
                                                          The updated OFS architecture for Intel\u00ae Stratix 10\u00ae FPGA devices improves upon the modularity, configurability and scalability of the first release of the OFS architecture while maintaining compatibility with the original design.  The primary components of the FPGA Interface Manager or shell of the reference design are: 
                                                           
                                                             
                                                          • PCIe Subsystem
                                                            •  
                                                          • HSSI Subsystem
                                                            •  
                                                          • Memory Subsystem
                                                            •  
                                                          • Reset Controller
                                                            •  
                                                          • FPGA Management Engine
                                                            •  
                                                          • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                                                            •  
                                                          • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                                                            •  
                                                          • SPI Interface to BMC controller
                                                            •  
                                                           
                                                          The AFU Region provides design space for custom workloads and contains both static and partial reconfiguration regions.  Partial reconfiguration allows you to update your specific logic blocks or entire workload while the rest of your static design is still in operation.
                                                           
                                                          Note that as discussed previously, the BMC RTL and firmware, the OFS OPAE software stack and support for building your own customer board support package are also provided in separate OFS repositories.
                                                           
                                                          Figure 1-2 OFS for Intel Stratix 10 Block Diagram
                                                           
                                                           
                                                          The table below details the features of the OFS release targeting the Intel\u00ae Stratix 10\u00ae FPGA .
                                                           
                                                          Table 1-4 Features
                                                           Key Feature OFS Update Comments PCIe H-tile PCIe Gen3x16 Interface Integrates PCIe TLP adapter for new data mover packet format. MSI-X vector and PBA tables are located in the PCIe subsystem. Interrupts from  FME as well as four user interrupts coming from PF0.VF1 are supported. Memory Two Avalon Memory Mapped channels provided as default with capability to compile design with four channels support. - HSSI 1 Arm* AMBA* 4 AXI4-Stream channel of 10G Ethernet, using the low latency Ethernet 10G MAC Intel FPGA IP interfacing to an E-tile PHY.  - Manageability SPI interface to Board Management Controller targeting Intel FPGA PAC D5005 - CoreFIM Flexible configuration support using Arm* AMBA* 4 AXI4-Stream Physical Function/Virtual Function (PF/VF) Demux/Mux and AFU Peripheral Fabric (APF) and Board Peripheral (BPF) Fabric Interconnects.   APF and BPF fabrics are Platform Designer generated IPs. The Arm* AMBA* 4 AXI4-Stream PF/VF Demux/Mux is a new component.  Physical Function/Virtual 1 PF/3VF configuration is provided as an example but the architecture now supports full virtualization with the ability to expand to whatever the PCIe tile supports. - Partial Reconfiguration 1 Partial Reconfiguration region supported in hardware and software - Sample test PR AFUs Host exerciser modules provided to exercise interfaces.  These modules are provided in both the flat and PR AFU examples. - OneAPI Yes Available Q1 2023 Software Support OFS software stack with support for full virtualization. -"},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#fpga-management-engine-fme","title":"FPGA Management Engine (FME)","text":"
                                                          The FIM contains only one FME, regardless of the number of host interfaces to the FIM. The FME provides management features for the platform and controls reset and loading of the AFU into the partial reconfiguration region of the FPGA.
                                                           
                                                          Each FME feature exposes its capability to host software drivers through a device feature header (DFH) register found at the beginning of its control status register (CSR) space. The FME CSR maps to physical function 0 (PF0) Base address register 0 (BAR0) so that software can access it through a single PCIe link.  For more information about DFHs, refer to the Device Feature Header (DFH) structure.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#streaming-datapath","title":"Streaming Datapath","text":"
                                                          The FIM implements an AXI4-Stream bus protocol for data transfer in the FIM. AXI4-Stream channels send data packets to and from the host channel IP without data abstraction. Memory-mapped I/O (MMIO) CSR accesses are routed to the ST2MM module which converts the AXI4-Stream to an AXI4 memory mapped protocol.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#virtualization","title":"Virtualization","text":"
                                                          This design supports virtualization by making use of the virtualization functionality in the PCIe Hard IP and mapping packets to the appropriate physical or virtual function through a PF/VF multiplexer.  This reference FIM supports 1 PF and 3 VFs as an example; however, you may extend your configuration to whatever the PCIe Hard IP can support or what your application requires.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#132-afu","title":"1.3.2 AFU","text":"
                                                          An AFU is an acceleration workload that interfaces to the FIM. The AFU boundary in this design comprises both static and partial reconfiguration (PR) regions.  You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region.  A port gasket within the design provides all the PR specific modules and logic required partial reconfiguration. Only one partial reconfiguration region is supported in this design.
                                                           
                                                          Similar to the FME, the port gasket exposes its capability to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space.  The port is part of the FPGA Interface Unit (FIU) that resides in the FIM.
                                                           
                                                          You can compile your design in one of the following ways: * Your entire AFU resides in a partial reconfiguration region of the FPGA  * The AFU is part of the static region and is compiled a flat design
                                                           
                                                          In this design, PF0.VF1 and PF0.VF2 map to host exerciser modules (HEM) that map to HE-LB and HE-HSSI respectively.  
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#133-platform-interface-manager","title":"1.3.3 Platform Interface Manager","text":"
                                                          The PIM provides a way to abstract the AXI4-Stream interface to the AFU by providing a library of shims that convert the host channel native packet into other protocols such as CCI-P, AXI4 memory-mapped, Avalon\u00ae streaming (Avalon-ST) or Avalon\u00ae memory-mapped (Avalon-MM). The FPGA or AFU developer implement these interface abstractions in the AFU region of the design.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#134-opae-sdk-fpga-platform-feature-discovery","title":"1.3.4 OPAE SDK FPGA Platform Feature Discovery","text":"
                                                          The OPAE C library in the OPAE software development kit is built on top of the OPAE Intel FPGA driver stack that abstracts the hardware and operating system specific details of the platform to the host. The FIM implements a DFH linked list to allow an FPGA platform driver running on the host to discover FME, port and AFU features. This model is similar to how PCIe enumeration occurs. You must implement a 64-bit DFH Device Feature Header register at the beginning (first 8B aligned address) of the feature CSR space for a new feature to be discovered or enumerated by a driver.
                                                           
                                                          A driver starts the traversing by reading the DFH of the first feature from the first address on PF0 BAR0. Based on the information in the DFH, a driver can determine the CSR address range of the feature and other associated details of the feature. The end of the DFH contains a \"next DFH offset\" field that points the driver to the DFH of the next feature. The software must continue traversing the linked list until it sees the EOL (End-Of-List) bit set to 1 in the \"next DFH offset\" field it is inspecting. A 1 indicates this is the last feature in the feature set. Figure below gives a simple illustration of the feature discovery by traversing the DFH registers.
                                                           
                                                          Figure 1-3 Device Feature Header Linked List Traversal 
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#135-ofs-reference-design","title":"1.3.5 OFS Reference Design","text":"
                                                          OFS provides FIM designs you can use as a starting point for your own custom design. These designs target a specific programmable acceleration card or development kit and exercise key FPGA device interfaces. The Intel Stratix\u00ae 10 code line for OFS targets the Intel FPGA PAC D5005.  FIM designs are released to OFS D5005 FIM Github Branch for evaluation and use.  
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#136-fim-simulation","title":"1.3.6 FIM Simulation","text":"
                                                          OFS provides a UVM environment for the FIM and a framework for new feature verification. UVM provides a modular, reusable, and scalable testbench structure by providing an API framework that can be deployed across multiple projects. The FIM testbench is UVM compliant and integrates third-party verification IPs from Synopsys that require license to use. Verification components include:
                                                           
                                                             
                                                          • FIM monitor to detect correct design behavior
                                                            •  
                                                           
                                                             
                                                          • FIM assertions for signal level integrity testing
                                                            •  
                                                           
                                                             
                                                          • Arm AMBA AXI4 scoreboards to check data integrity
                                                            •  
                                                           
                                                             
                                                          • FIM coverage to collect functional data
                                                            •  
                                                           
                                                          The verification infrastructure can be in the verification folder here OFS D5005 FIM Github Branch  for evaluation and use. 
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#2-ofs-high-level-architecture","title":"2 OFS High Level Architecture","text":"
                                                          OFS provides distinct datapaths that simplifies the design and integration process for add or for removing interface modules:
                                                           
                                                             
                                                          • High Bandwidth datapath for AFU-attached high performance peripherals (HSSI, Memory, HPS, workload).
                                                            •  
                                                          • Low Bandwidth datapath for OFS management and slow peripheral components (JTAG, I2C, SMBus).
                                                            •  
                                                          • AFU Peripheral Fabric (APF) to Board Peripheral Fabric (BPF) path to communicate with interface control and status registers (CSRs) and board components.
                                                            •  
                                                          • Peer-to-peer datapath between AFU components.
                                                            •  
                                                          • Peer-to-peer datapath between BPF components.
                                                            •  
                                                           
                                                          Depending on your design goals, you can present peripherals to software as:
                                                           
                                                             
                                                          • OFS managed peripherals with a device feature header that is part of a device feature list.
                                                            •  
                                                          • Native driver managed peripherals that are exposed through an independent physical function or virtual function.
                                                            •  
                                                           
                                                          Figure 2-1 OFS Datapath Structure
                                                           
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#3-pcie-interface","title":"3 PCIe Interface","text":"
                                                          The FIM's H-tile PCIe* hard IP is a Gen3x16 design.  The IP supports SR-IOV and is configured to provide one PF and three VFs.  Native PCIe TLP packets are sent through the PCIe using Arm AMBA 4 AXI-4 Stream Protocol.  Before they reach the AFU, however, the packets go through an adapter that converts any headers to a data mover format that is forward compatible with Intel Agilex FPGA devices and beyond.   
                                                           
                                                          Figure 3-1 OFS FIM RX-TX Datapath
                                                           
                                                           
                                                          Some key features of the PCIe interface are:
                                                           Feature OFS for Intel Stratix 10 Configuration Mode PCIe Gen3x16 Port Mode Native Endpoint SR-IOV 1 PF, 3 VFs MSI-X Support Yes Functional Mode Data Mover Profile Virtual+ TLP Bypass No Header Packing Scheme Simple Data Width 512-bit (64-byte) PLD Clock Frequency 250 MHz Tags Supported 128 Reordering No reordering of requests, no completion reordering Maximum Payload Size 256 Bytes Memory Requests Supported 1CL, 2CL, 4CL MMIO transaction Size 4B, 8B"},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#31-receiver-datapath","title":"3.1 Receiver Datapath","text":"
                                                          The path of data received by the FIM is as follows:
                                                           
                                                             
                                                          1.  
                                                            The PCIe Hard IP receives TLP packets from the host.  Host response types can be:
                                                             
                                                            * MMIO read or response   * Memory write request   * Interrupt response   * Memory read request or response
                                                             
                                                            2.  
                                                          2.  
                                                            The PCIe IP routes the TLP packets to the PCIe bridge interface in     the FIM, where they get buffered into the RX FIFO.
                                                             
                                                            3.  
                                                          3.  
                                                            The TLP checker in the bridge examines the packets and filters     erroneous packets including packets with unsupported requests or fields.     Errors are sent to the error logger and are logged as advance error     reporting (AER). Error status is sent to the PCIe hard IP and to the     FIM error status registers. Appropriate action is taken for the     errors as described in the Reliability, Accessibility, Serviceability (RAS) and Error Handling section. The TLP checker also     maintains RX buffer credits for TLP completions with data (CplD)     that are received from the host in response to a memory read request     (MRd request sent by the AFU). The TLP checker notifies the PCIe TX bridge when there are not enough RX buffer credits available in the PCIe RX bridge. If there are not enough credits, the PCIe TX bridge pauses MRd requests from the AFU to Host until there     is availability.
                                                             
                                                            4.  
                                                          4.  
                                                            The TLP checker forwards packets that pass TLP checking to an AXI4     adapter which moves the packets into an Arm AMBA 4 AXI4-Stream bus.
                                                             
                                                            5.  
                                                          5.  
                                                            AXI4-Stream packets are sent downstream to datamover AXI4-Stream adapter to be modified into the new datamover packet format.
                                                             
                                                            6.  
                                                          6.  
                                                            Datamover packets are sent to the PF/VF Mux in the AFU.
                                                             
                                                            7.  
                                                          7.  
                                                            The AXI4-Stream to Memory Mapped (ST2MM) module in the AFU region routes MMIO requests targeting FIM CSRs (FME, peripherals and AFU).
                                                             
                                                            8.  
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#32-transmit-datapath","title":"3.2 Transmit Datapath","text":"
                                                          The Transmit (TX) datapath refers to all TLP packet types that originate in the FPGA.  A single Arm AMBA 4 AXI4-Stream channel at the port interface boundary of the FIM carries PCIe TLP packets and interrupt requests upstream from the AFU.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#33-data-handshaking","title":"3.3 Data Handshaking","text":"
                                                          The Arm AMBA 4 AXI4 interfaces to the AFU use the VALID and READY signal for handshaking and backpressure management.  The FIM holds the DATA and VALID asserted until the receiver asserts the READY signal.  The AFU accepts the data when both the VALID and READY signals are asserted. 
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#34-arm-amba-4-axi4-stream-interface","title":"3.4 Arm AMBA 4 AXI4-Stream Interface","text":"
                                                          The table below shows the high-level signal mapping of the channels for the OFS for Intel Stratix 10 FPGA.  If you have previously used the OFS EA architecture, that mapping is provided as a comparison as well in this table.  
                                                           
                                                          Table 3-1 AXI4-Stream RX Channel
                                                           AXI4-Stream Signal Source OFS Stratix 10 Mapping OFS Early Access Mapping ACLK Clock Source PCLK = 250 MHz PCLK = 250 MHz AResetn Reset Source System Reset System Reset TVALID Master Data Valid Data Valid TREADY Slave Ready Ready TDATA Master Width=512 bits When packet includes a header (32 bytes) and data the packing scheme is: 16B data mover header: 4B prefix PF NumberVM Number VF Active Slot Number Memory Mapped Number8B Address/Metadata Data: 32B If the packet is only data then all 64 bytes comprise data [TLP_CH] [8n]  n=49 (392 bits)TLP_CH=2 (2TLP data streams)Mapping of each TLP data stream [391:136] payload (32-byte data payload) [135:8]: hdr (16 byte header) [7:3]: rsvd0 (reserved bits)[2]: end of packet (eop)  [1]: start of packet (sop)  [0]: valid (TLP packet on data stream is valid) TLAST Master Set to 1'b1 when end of packet is sent; otherwise TLAST is 1'b0 Set to 1'b1 when end of packet is sent; otherwise TLAST is 1'b0 TKEEP Byte Qualifier Signal indicates whether content of the associated byte is valid.  Invalid bytes are allowed only during TLAST cycle.  Valid bytes always start from Byte 0. Signal indicates whether content of the associated byte is valid.  Invalid bytes are allowed only during TLAST cycle.  Valid bytes always start from Byte 0. TUSER Master WIDTH = 10  Bit 0 is always equal to 1'b1 to indicate data mover header format [4:1] - Indicates header position on the TDATA bus and is always equal to 4'b0001 indicating that the header starts from Byte0.All other bits of TUSER are unused. [TLP_CH][u-1:0] u=21 Sideband of each TLP data stream[20]: ummio_rd (Unsupported MMIO request)[19:0]: destination routing ID, where:[19:17]= BAR offset[2:0][16:4]=VF number[12:0][3:1]=PF number[2:0][0]=vf_active, indicating if the virtual function feature is enabled 
                                                          Figure 3-3 AXI4-Stream RX Request Cycle Header Format
                                                           
                                                           
                                                          All Host requests sent to the AFU are memory-mapped I/O requests.  Of the fields below, the following are not supported in the design: * Prefix * Slot number (set to 0) * Local Address
                                                           
                                                          Figure 3-4 AXI4-Stream RX Completion Header Format
                                                           
                                                           
                                                          All completions in the RX direction are data completions.  Of the fields below, the following are not supported in the design: * Prefix * MM mode * Slot number (set to 0) * Meta Data
                                                           
                                                          Note that: * VF Active, VF Num and PF Num are obtained from TUSER. * Data packet responses (for memory read requests from the AFU) from the PCIe may come out of order when the size is greater than 64 bytes.
                                                           
                                                          Table 3-2 AXI4-Stream TX Channel
                                                           AXI4-Stream Signal Source OFS Stratix 10 Mapping OFS Early Access Mapping ACLK Clock Source PCLK = 250 MHz PCLK = 250 MHz AResetn Reset Source System Reset System Reset TVALID Master Data Valid Data Valid TREADY Slave Ready ReadyOnly 1 ready signal for the two channels TDATA Master Width=512 bits When packet includes a header (32 bytes) and data the packing scheme is: 16B data mover header: 4B prefix PF NumberVM Number VF Active Slot Number Memory Mapped Number8B Address/Metadata Data: 32B If the packet is only data then all 64 bytes comprise data [TLP_CH] [8n]  n=49 (392 bits)TLP_CH=2 (2TLP data streams)Mapping of each TLP data stream [391:136] payload (32 byte data payload) [135:8]: hdr (16 byte header) [7:3]: rsvd0 (reserved bits)[2]: end of packet (eop)  [1]: start of packet (sop)  [0]: valid (TLP packet on data stream is valid) TLAST Master Per protocol Set to 1'b1 TKEEP Byte Qualifier Signal indicates whether content of the associated byte is valid.  Invalid bytes are allowed only during TLAST cycle.  Valid bytes always start from Byte 0. Signal indicates whether content of the associated byte is valid.  Invalid bytes are allowed only during TLAST cycle.  Valid bytes always start from Byte 0. TUSER Master WIDTH = 10  Bit 0 \u2013 Indicates Header Format: 0 \u2013 Power user mode header format 1 \u2013 Data mover header format Bit [4:1] - Indicates header position on the TDATA bus and is always equal to 4'b0001 indicating that the header starts from Byte0.All other bits of TUSER are unused. [TLP_CH][u-1:0] u=2Sideband of each TLP data stream[0]=vf_active, indicating if the virtual function feature is enabled[0]: afu_irq (AFU interrupt) 
                                                          Table 3-3 Interrupt Response Channel
                                                           AXI4-Stream Signal Source Mapping ACLK Clock Source PCLK AResetn Reset Source System Reset TVALID Master Valid TREADY Slave Ready TDATA [8n-1:0]  n=3 (24 bits) Master [23:16]: 8-bit interrupt ID  [15:0]: Requester ID 
                                                          Figure 3-5:  AXI4-Stream TX Request Cycle Header Format
                                                           
                                                           
                                                          All requests in the TX direction are Memory Read/Write. The requester ID does not come from the AFU; the AXI-Stream adapter supplies it.  The tag must come from the AFU.  Of the fields below, the following are not used in the H-Tile PCIe subsystem design: * Prefix * MM Mode * Slot number (set to 0) * Local Address
                                                           
                                                          Note that VF Active, VF Num and PF Num are obtained from the header packet.
                                                           
                                                          Figure 3-4 AXI4-Stream TX Completion Header Format
                                                           
                                                           
                                                          All completions in the TX direction are for MMIO.  Of the fields below, the following are not supported in the design: * Prefix * MM mode * Slot number (set to 0) * Meta Data
                                                           
                                                          Note that: * VF Active, VF Num and PF Num are obtained from TUSER.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#4-platform-interface-manager","title":"4 Platform Interface Manager","text":"
                                                          The FIM interfaces to an AFU through AXI4-Stream channels.  This format allows the AFU to access the host channel's raw interface without any translation. As a FIM developer, you have the option to provide the raw data format associated with the host interface channel to the workload or AFU developer or you can provide an intermediate protocol using Platform Interface Manager Components or your own custom interface. If you expose the raw AXI4-Stream interface of the FIM, workload developers also have the option to convert to a desired protocol using the PIM resources as well.  
                                                           
                                                          Refer to https://github.com/OPAE/ofs-platform-afu-bbb for more information on options for implementing the PIM.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#5-afu-interface-handler","title":"5 AFU Interface Handler","text":"
                                                          The AFU Interface Handler resides inline between the PCIe AXI4-Stream Adapter and the AXI4-Stream PF/VF Demux/Mux logic. Its main function is to provide:  * Unique PCIe tags \u2013 Each PCIe transaction shares the 128 tags across all VFs in the AFU region * AFU error logging for all VFs in the AFU region
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#51-afu-error-handling","title":"5.1 AFU Error Handling","text":"
                                                          In this OFS design, the AFU Interface Handler handles error logging for all VFs in the AFU. Errors handled are as follows
                                                           Checker Field Description AFU protocol checker (PCIe TLP) TxReqCounterOverflow Pending memory write or memory read requests exceed the predefined limit TxFifoOverflow Tx FIFO in the port that buffers TLP packets from AFU is overflow Malformed TLP AFU PCIe TLP contains unsupported format type MaxPayloadError AFU memory write payload size exceeds max_payload_length limit MaxReadReqSizeError AFU memory read payload size exceeds max_read_request_size limit MaxTagError AFU memory read request tag value exceeds the maximum supported tag count TagOccupiedErr AFU sends out memory read request using a tag that is already used for a pending memory read request UnalignedAddrErr The address field in AFU memory write/read request TLP is not DW-aligned. UnexpMMIOResp AFU is sending a MMIO read response with no matching MMIO read request. MMIOTimedOutAFU is not responding to a MMIO read request within the pre-defined response timeout period.  MMIODataPayloadOverrunThe number of data payload sent by AFU for a MMIO response (cplD) is more than the data length specified in the response. MMIOInsufficientDataThe number of data payload sent by AFU for a MMIO response (cplD) is less than the data length specified in the response. TxMWrDataPayloadOverrun    The number of data payload sent by AFU for a memory write request is more than the data length specified in the request. TxMWrInsufficientData  The number of data payload sent by AFU for a memory write request is less than the data length specified in the request. AFU Protocol Checker (AXI4-Stream)TxValidViolationThree checkers are implemented in the FIM to catch errors and protocol violations. 
                                                          To view the CSR space for the AFU interface handle, go to the src/afu_top/AFU_INTF_CSR.xls file OFS D5005 FIM Github Branch.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#6-interconnect-fabric","title":"6 Interconnect Fabric","text":"
                                                          There are three types of interconnect fabric in the OFS FIM design:  * AXI4-Stream mux/demux fabric  * AFU Periheral Fabric (APF)  * Board Peripheral Fabric (BPF)
                                                           
                                                          Figure 6-1 Interonnect Fabric Diagram
                                                           
                                                           
                                                          TLP packets sent from upstream PCIe Subsystem on AXI4-Stream channel are demultiplexed in the AXI4-Stream PF/VF mux/demux fabric and routed to the respective PF/VF function based on the PF/VF information in the TLP header, such as vf_active or the PF/VF number. On the opposite direction, TLP packets from downstream PF/VF function are muxed in the fabric and sent to PCIe subsystem over AXI4-Stream channel.
                                                           
                                                          All host MMIO requests targeting PF0 BAR0 are routed to the ST2MM module. The ST2MM converts MMIO TLP packets into AXI-Lite memory requests and places the requests onto AFU Peripheral Fabric (APF). AFU peripherals, such as OFS managed AFU features and ST2MM) and Board Peripheral Fabric (BPF) are interconnected by APF. The BPF is the interconnect fabric one hiearchy below APF which connects all the board peripherals. Both APF and BPF allow multiple AXI4-Lite master and slave interconnect topology.
                                                           
                                                          The following table summarizes the mechanism for configuring PF/VF functions:
                                                           
                                                          Table 6-1 Interconnect Configuration Methods 
                                                           InterconnectConfiguration Mechanism APF or BPFEither:Use Platform Designer (PD) to generate the fabrics directly, or  Specify desired cfg in iofs_dfl.txt and run dfh2tcl.pl script to generate the necessary HW TCL scripts as needed by PD. This PERL script also takes care of invoking PD in script mode to generate the end result of RTL design Verilog files. This is the preferred method for generation. AXI-S PF/VF Demux/MuxUpdate parameters in these RTL files: * src/includes/top_cfg_pkg.sv * src/common/pf_vf_mux.sv Then make the corresponding update to AFU top level instantiation and connections: * src/FIMs/.../afu_top.sv 
                                                          Note:
                                                           
                                                          In all cases, you must configure and regenerate the PCIe IP to match the new PF/VF configuration if it deviates from the reference FIM provided.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#61-fabric-generation-flow","title":"6.1 Fabric Generation Flow","text":"
                                                          You have two options to generate APF/BPF IPs.
                                                           
                                                             
                                                          • Use Quartus Platform Designer.  This method requires familiarity with Quartus Platform Designer.  With this option, you manually enter each address space and define the associated master and slave interface in a table provided by the Platform Designer.  The parameters and attributes such as data width, address width, number of outstanding cycles, \u2026 etc. are also set to the desired values.  After this is completed, you then connect each master and slave interface in the table.  Platform Designer then generates the RTL files according to the table.  For more details, please refer to the Intel Quartus Prime Pro Edition User Guide.
                                                            •  
                                                           
                                                             
                                                          • Use the APF/BPF script that reads in iofs_dfl.txt and automatically generates APF/BPF IPs.  Both APF and BPF are generated from Platform Designer using hardware TCL scripts. To provide a more user friendly experience to OFS Rel 1 and AC ADP customers, a perl script dfh2tcl.pl has been developed to provide a higher level of abstraction. The below figure illustrates the high level flow:
                                                            •  
                                                           
                                                          Figure 6-2 APF/BPF Generation
                                                           
                                                           
                                                          Note that the only input required is the iofs_dfl.txt text file which allows you to specify how many ports, which fabric, type of AXI4-Lite port (master, slave, or both), port addresses and sizes.  Using iofs_dfl.txt and dfh2tcl.pl to generate the APF and BPF is the preferred method.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#62-iofs_dfltxt-format","title":"6.2   iofs_dfl.txt Format","text":"
                                                          The following table describes the format of the iofs_dfl.txt input file:
                                                           
                                                          Table 6-2 iofs_dfl.txt Format Types
                                                           ColumnDescription REGISTERName of the port on the APF/BPF fabrics. Note that each entry must be unique. FABRICAllows the user to specify which fabric this port is connected to, and also the type of AXI4-Lite port (master, slave, or both). The format is FABRIC-PORT_TYPE, where: * FABRIC = APF or PBF * PORT_TYPE = MST, SLV, or BID (Master, Slave, or Bi-Directional) BASE_ADDRSpecifies the address offset of the AXI4-Lite port. BAR_SIZE=2^N   Specifies the size of the port. For simplicity reasons, all AXI4-Lite ports are configured as 64KB. i.e. 2^16 
                                                          Additional AXI4-Lite ports can be easily created by adding more rows in the iofs_dfl.txt file. 
                                                           
                                                          Note that there are several reserved ports in both APF and BPF so it might not be necessary for you to regenerate the fabrics if the design does not exceed the number of ports as implemented in the APF/BPF in the reference FIMs.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#63-afu-peripheral-fabric-apf","title":"6.3   AFU Peripheral Fabric (APF)","text":"
                                                          The AFU Peripheral Fabric (APF) is a 64-bit AXI4-lite compliant interconnect fabric that connects AFU peripheral modules to board peripheral modules through the Board Peripheral Fabric (BPF).  The fabric is clocked by clk_csr and has a read allowance and write allowance of 1, i.e. only 1 active write/read is allowed in the fabric at any single time. 
                                                           
                                                          The address mapping for components interconnected by the APF is listed below. All components are mapped to PF0 BAR0 and implement AXI-lite slave interface. The Master column indicates if a component also implements AXI4-lite master interface which can send request to APF.
                                                           
                                                          Table 6-3 APF Address Mapping
                                                           AddressSize (Byte)FeatureMaster 0x00000 \u2013 0x7FFFF512KBoard Peripherals (See BPF address mapping)   No AFU Peripherals 0x80000 \u2013 0x8FFFF64KST2MMYes (Send MMIO request to all the peripherals) 0x90000 \u2013 0x9FFFF64KPort GasketYes 4KPR Control & Status 4KUser clock 16KRemote STP 0xA0000 \u2013 0xAFFFF64KAFU Interface HandlerNo 0xB0000 \u2013 0xBFFFF64KRSV_b_DFHAvailable for customer useMay be programmed as master if used.  By default this is a reserved base address. 0xC0000 \u2013 0xCFFFF64KRSV_c_DFHAvailable for customer useMay be programmed as master if used.  By default this is a reserved base address. 0xD0000 \u2013 0xDFFFF64KRSV_d_DFHAvailable for customer useMay be programmed as master if used.  By default this is a reserved base address. 0xE0000 \u2013 0xEFFFF64KRSV_e_DFHAvailable for customer useMay be programmed as master if used.  By default this is a reserved base address. 0xF0000 \u2013 0xFFFFF64KRSV_f_DFHAvailable for customer useMay be programmed as master if used.  By default this is a reserved base address. 
                                                          The five reserved regions have associated ports available in the APF for customer use.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#64-board-peripheral-fabric-bpf","title":"6.4   Board Peripheral Fabric (BPF)","text":"
                                                          The Board Peripheral Fabric is the 64-bit AXI4-Lite compliant interconnect fabric that connects board peripheral modules to APF. The fabric is clocked by clk_csr and has a read allowance and write allowance of 1, i.e. only 1 active write/read is allowed in the fabric at any single time.
                                                           
                                                          The address mapping for components interconnected by BPF is listed below. All components are mapped to PF0 BAR0 and implement AXI4-lite slave interface. The Master column indicates if a component also implements AXI4-lite master interface which can send request to BPF.
                                                           
                                                          Table 6-4 BPF Address Mapping
                                                           AddressSize (Byte)FeatureMaster 0x00000 \u2013 0x0FFFF64KFME (FME, Error, etc)Yes 0x10000 \u2013 0x1FFFF64KSPI ControllerYes 0x20000 \u2013 0x2FFFF64KPCIe CSR- 0x30000 \u2013 0x3FFFF64KHSSI CSR- 0x40000 \u2013 0x4FFFF64KEMIF CSR- 0x50000 \u2013 0x5FFFF64KReservedAvailable for customer use.Dependent on user programming 0x60000 \u2013 0x6FFFF64KReservedAvailable for customer use.Dependent on user programming 0x70000 \u2013 0x7FFFF64KReservedAvailable for customer use.Dependent on user programming 
                                                          The three reserved regions have associated ports available in the BPF for customer use.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#65-axi4-stream-pfvf-muxdemux","title":"6.5   AXI4-Stream PF/VF Mux/Demux","text":"
                                                          The AXI4-Stream PF/VF Mux/Demux routes the PCIe TLP packets from the PCIe subsytem AXI4-Stream RX channel to downstream PF/VF based on the pf_num and vf_num information in the PCIe TLP header.
                                                           
                                                          The AXI4-Stream PF/VF mux arbitrates PCIe TLP packets from downstream PF/VF to the PCIe SS AXI-S TX channel. The PF/VF Mux/Demux is an M x N switch that allows any M port to target any N port, and any N port to target any M port, where M is the number of host/upstream ports, and N is the numbers functions/downstream ports.  M and N values are parameterized in the RTL for adding, removing, or remapping of FPGA functional units/modules to PF/VF.  
                                                           
                                                          The fpga top package file, found in the src/includes/ofs_fim_cfg_pkg.sv file OFS D5005 FIM Github Branch  contains these parameters as well as the mapping of N port\u2019s PF/VF.
                                                           
                                                          Structurally, M x N switch is composed of M number of N:1 mux, and N number of M:1 mux.  Each mux output has an arbiter that perform round robin priority arbitration of its inputs.  At the mux output is a FIFO with depth greater than the handshake round trip delay.  The FIFO allows the switch to arbitrarily insert pipeline/register stages for timing.
                                                           
                                                          Note that M x N switch is design for AXI streaming, but it can be easily converted to AVST.  The protocol signals pass through switch intact \u2013 only ready, valid, and last (common between AVST and AXI) effect switch operation.  The data width of the switch is also parameterized in the src/includes/ofs_fim_cfg_pkg.sv file OFS D5005 FIM Github Branch.  
                                                           
                                                          The default mapping is shown below:
                                                           
                                                          Table 6-5 PF/VF Mapping
                                                           DevicePhysical Function #Virtual Function #Switch Port ID APF/BPF0x3 HE Loopback000 Port Gasket011 HSSI022 
                                                          For information on how to modify the PF/VF mapping for your own design, refer to the [Intel\u00ae FPGA Interface Manager Developer Guide: Open Stack for Intel\u00ae Stratix 10\u00ae].
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#66-unified-tag-remapping","title":"6.6 Unified Tag Remapping","text":"
                                                          When a FPGA function sends out a read cycle, it allocates a unique tag which is subsequently used to identify the read completion.  The tag is considered busy; it cannot be assigned to another read cycle until read completion.  While a tag may be unique within a unit, two different units could unknowingly send out two read cycles of the same tag.  The PCIe subsystem requires unique tags for all read cycles irrespective of their origins.  Therefore, a mechanism is needed to uniquify tag globally across different units.
                                                           
                                                          OFS contains a tag remapper (tag_remap) that intercepts the read cycle, finds a globally unique tag, and replaces the original tag value.  It also restores the original tag value when returning completion to the read requester.  tag_remap is placed between the AXI4-Stream interface of the PCIE subsystem and the PF/VF Mux/Demux.
                                                           
                                                          The logic is described as follows:
                                                           
                                                             
                                                          1. A sub-module (ofs_fim_tag_pool) maintains a pool of available tags.
                                                            2.  
                                                          2. TX read requests are held until a tag is available from the pool by setting tvalid=0 to the host, and tready=0 to the PF/VF Mux/Demux.
                                                            3.  
                                                          3. When a TX read is dispatched, the tag is marked busy in the pool.
                                                            4.  
                                                          4. The original tag is stored in tag_reg, so it can be recovered when returning a completion to the unit/function.
                                                            5.  
                                                          5. Since completion to a read request can split into multiple smaller transfer sizes, responses are monitored and the final completion is detected using PCIe TLP rules.
                                                            6.  
                                                          6. Tags are released in the pool only when all requested data are transferred.
                                                            7.  
                                                          7. When the completion returns, the original tag is restored from tag_reg.
                                                            8.  
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#67-tlp-to-axi4-lite-memory-mapped-bridge-st2mm","title":"6.7 TLP to AXI4-Lite Memory Mapped Bridge (ST2MM)","text":"
                                                          ST2MM implements the following key features: * Host MMIO bridge     * Maps MMIO TLP packets received from the PCIe Subsystem over streaming interface to AXI4-Lite memory-mapped request. The memory-mapped request is sent to AFU or Board peripherals over APF and BPF.     * Maps AXI4-lite MM response received from AFU or Board peripherals to TLP packets and send the packets over ST streaming channel to host HIA subsystem. * Sends MMIO response of all 0\u2019s for MMIO read to unused BAR region. * Interrupt     * Sends interrupt packets to the PCIe subsystem when interrupt requests are received from the peripherals. Interrupts can be requested by a peripheral through a memory write to interrupt CSR registers in the ST2MM.
                                                           
                                                          Figure 6-2 APF/BPF Generation
                                                           
                                                           
                                                          ST2MM implements both AXI4-lite master and slave interfaces that are connected to the designated slave and master port on APF. Host memory requests are sent on the ST2MM master interface to AFP where the requests are routed to the targeted peripherals. 
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#7-mmio-regions","title":"7 MMIO Regions","text":"
                                                          The FIM and AFU expose their functionalities to the host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). An MMIO region is an address space within a base address register (BAR) region to which features are memory mapped.  For example, when a feature is mapped to an MMIO region, the CSR registers of that feature are located within the address range of that region. There can be multiple MMIO regions within a BAR region. 
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#71-base-address-register-bar-layout","title":"7.1 Base Address Register (BAR) Layout","text":"
                                                          The function, BAR and external feature region starting address are put into a platform specific parameter SystemVerilog package file src/includes/ofs_fim_cfg_pkg.sv file OFS D5005 FIM Github Branch.  
                                                           
                                                          You can modify the parameterization according to your platform requirements, however you must ensure the corresponding software driver is also updated to align with the new assignment. 
                                                           
                                                          Table 7-1 BAR Layouts
                                                           PF VF Feature BAR BAR Size PF0 - OFS Managed Peripherals BAR 0 512K AFU PeripheralsBoard Peripherals 256K256K VF0 HE-LB BAR0 4K VF1 HE-MEM in PR slot BAR0 4K VF2 HE-HSSI BAR0 4K"},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#72-feature-region","title":"7.2 Feature Region","text":"
                                                          A group of related CSRs can be categorized as a feature region. For example, a DMA engine has queue management function and quality of service (QoS) function; these are two different features of the DMA engine. A feature region is contained within a single PCIe BAR and cannot span across two BAR region boundaries.   You can view the PF0 BAR0 MMIO mapping by referencing thesrc/common/fme/fme_csr_pkg.sv file OFS D5005 FIM Github Branch file.
                                                           
                                                          A Device Feature Header (DFH) register marks the start of the feature region and sub-feature region, and you must place it at the first address of the region. Each DFH starts at 4KB boundary. A DFH register contains information that OPAE software requires to enumerate the feature. It also has an offset field that points to the next DFH in a feature list. OPAE software traverses the linked list of DFHs in each BAR region to discover all the features implemented on the platform. The EOL field in a DFH marks the end of a DFH list and is only set in the DFH of the last feature in the feature list. The feature type field in the DFH is used to differentiate between the different types of feature region. Basic building blocks (BBB) and private features are always a child of an AFU or FPGA Interface Unit (FIU) and must be contained within an AFU or FIU, respectively.  
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#721-device-feature-header-dfh-structure","title":"7.2.1 Device Feature Header (DFH) Structure","text":"
                                                          All DFHs must follow the following structure to be compatible with OPAE software.
                                                           
                                                          Table 7-2: DFH Structure
                                                           Bitfield Name Range Access Description FeatureType 63:60 RO 4\u2019b0000 \u2013 Reserved 4\u2019b0001 \u2013 AFU4\u2019b0010 \u2013 BBB4\u2019b0011 \u2013 Private Feature4'b0100 \u2013 FIU/FIM Reserved 59:41 Rsvd Reserved EOL 40 RO End of DFH List1'b0=No other feature header beyond this one1'b1=This is the last feature header NextDFHByteOffset 39:16 RO Next DFH byte offsetNext DFH Address= Current DFH address + Next DFH byte offset.  You can also use this value as an indication of the maximum size of the MMIO region occupied by this feature. FeatureRev 15:12 RO For AFU Feature type= AFU major version number that is user defined.All other feature types= Feature revision number FeatureID 11:0 RO For AFU feature type= CoreFIM version numberFor BBB feature type= Intel defined ID for BBBFor private feature type= User-defined ID to identify within an AFU For FIU type=ID for FIU unit (ex. 0 for FME, 1 for Port) 
                                                          You must increment a feature revision number if a feature changes.  This change requires a corresponding change in the software to detect the new version and report mismatches between the hardware and software revision number. 
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#73-control-and-status-registers","title":"7.3 Control and Status Registers","text":"
                                                          All the Control and Status Registers (CSRs) in the FIM are 64-bit registers with the following MMIO write and MMIO read support.
                                                           
                                                          Table 7-3: CSR MMIO Read and Write Support
                                                           Request Memory Attribute Payload size Memory Ordering MMIO Write UC 4B or 8B Strongly ordered MMIO Read UC 4B or 8B Strongly ordered 
                                                          The FIM does not reorder the MMIO requests or responses. For MMIO writes, there is no reordering of requests in FIM, and UC ordering rules are followed. Similarly, for MMIO reads, there is no re-ordering of requests or responses in the FIM. An AFU may opt to re-order the MMIO read responses but the FIM does not enforce read response ordering. 
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#731-software-access-to-registers","title":"7.3.1    Software Access to Registers","text":"
                                                             
                                                          • Software accesses 64-bit registers as aligned quadwords. For example, to modify a field (bit or byte) in a 64-bit register, the entire quadword is read, the appropriate field(s) are modified, and the entire quadword is written back.
                                                            •  
                                                          • When updating registers through multiple accesses (whether in software or due to hardware disassembly), certain registers may have specific requirements on how the accesses must be ordered for proper behavior. These are documented as part of the respective register descriptions.
                                                            •  
                                                          • For compatibility with future extensions or enhancements, software must assign the last read value to all \u201cReserved and Preserved\u201d (RsvdP) fields when written. In other words, any updates to a register must be read so that the appropriate merge between the RsvdP and updated fields occurs. Also, software must assign a value of zero for \u201cReserved and Zero\u201d (RsvdZ) fields when written.
                                                            •  
                                                          • PCIe locked operations to FPGA hardware registers are not supported. Software must not issue locked operations to access FPGA hardware registers.
                                                            •  
                                                           
                                                          In the following two cases, the FIM terminates MMIO Read requests by sending a completion with the data (CplD) specified below: * MMIO Timeout:  This occurs when the AFU does not respond within a set timeout. The timeout value is currently configured to 512 pclks (clk_2x). In this case, the FIM returns all 1s.
                                                           
                                                             
                                                          • Illegal MMIO Accesses: This occurs when the read is accessing undefined registers in the FIM or if an AFU access violation.  An example of an access violation is when a PF attempts to access the AFU when it is set to VF mode, or when a VF attempts to access the AFU when it is set to PF mode. In this case, the FME will return all 0s.
                                                            •  
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#732-register-attribute-definition","title":"7.3.2    Register Attribute Definition","text":"
                                                          Table 7-4: OFS Register Attribute Definitions
                                                           Attribute Expansion Description RW Read/Write This bit can be read or written by software. RO Read Only The bit is set by hardware only. Software can only read this bit. Writes do not have any effect. RW1C Read/ Write 1 to Clear Software can read or clear this bit. The software must write 1 to clear this bit. Writing zero to RW1C bit has no effect. Note that a multi-bit RW1C field may exist. In this case, all bits in the field are cleared if a 1 is written to any of the bits. RW1S Read/ Write 1 to Set Software can read this bit. Writing a 1 to the bit sets it to 1. Writing a 0 has no effect. It is not possible for software to set this bit to 0. The 1 to 0 transition can only be performed by HW. RW1CS Read/Write 1 to Clear Sticky Software can read and clear this bit. Writing a 1 to a bit clears it, while writing a 0 to a bit has no effect. This bit is only reinitialized to its default value by a power-on reset. RWD Read/Write Sticky across Hard Reset The bit can be read or written by SW. This bit is sticky or unchanged by any reset type, including Hard Reset. The bit gets cleared only with power on. *S Sticky across Soft Reset The bit will be sticky or unchanged by soft reset. These bits are only re-initialized to their default value by a power-on reset. *D Sticky across Hard Reset The bit is sticky or unchanged by or unchanged by any reset type, including hard reset. The bit gets cleared only with power on. Rsvd Reserved Reserved for future definitions. Currently don\u2019t care bits. RsvdP Reserved and Protected Reserved for future RW implementations. The software must preserve the value of this bit by read modify write. RsvdZ Reserved and Zero Reserved for future RW1C implementations. The software must write zero to this bit."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#733-csr-offset-in-bars","title":"7.3.3 CSR Offset in BARs","text":"
                                                          The table below captures the FIM and AFU features in the supported BAR regions. The offset highlighted in red indicates the first DFH in the DFH list of a BAR region where device driver starts the DFH traversal.
                                                           
                                                          Table 3-6: PF0 BAR0 Features
                                                           Offset Feature CSR set 0x00000 FME 0x03000 Global Performance 0x04000 Global Error 0x10000 SPI Controller 0x20000 PCIe CSR Interface 0x30000 HSSI CSR Interface 0x40000 EMIF CSR Interface 0x80000 Reserved for ST2MM Bridge 0x90000 PR Control & Status (Port Gasket) 0x91000 Port CSRs (Port Gasket) 0x92000 User Clock (Port Gasket) 0x93000 Remote SignalTap (Port Gasket) 0xA000 AFU Errors (AFU Interface Handler) 
                                                          Table 3-7: PF0 BAR4 Features
                                                           Offset Feature CSR set 0x02000 MSI-X 0x03000 MSI-X PBA Tables 
                                                          Table 3-8: PF0-VF0 BAR0 Features
                                                           Offset Feature CSR set 0x00000 HE-LBK 
                                                          Table 3-9: PF0-VF1 BAR0 Features
                                                           Offset Feature CSR set 0x00000 HE-MEM 
                                                          Table 3-10: PF0-VF2 BAR0 Features
                                                           Offset Feature CSR set 0x00000 HE-HSSI"},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#8-fim-clocks","title":"8 FIM Clocks","text":"
                                                          The following table provides the clocks available in the OFS reference design that targets the Intel FPGA PAC D5005.  Clocks that the high speed serial interface (HSSI) or external memory interface provide to the FIM may be different depending on if you modify your external features with different components.
                                                           
                                                          Table 8-1: External Clock Source
                                                           Clock Frequency Description SYS_RefClk 100 MHz Reference clock to system IOPLL (sys_pll) which provides FIM system clocks. qsfp*_644_53125_clk 644.5312 5MHz HSSI reference clocks ddr4_mem*.ref_clk 150 MHz Reference clocks to DDR4 interfaces PCIE_REFCLK 100MHz PCIe reference clock 
                                                          Table 8-2: Internal Clocks
                                                           Clock Frequency Description clk_1x 250 MHz Generated by the system IOPLL (sys_pll). This clock drives CoreFIM datapath and the AFU interface. clk_div2 125 MHz Generated by the system IOPLL, synchronous to clk_1x. This clock drives IM datapath and AFU interface. clk_100 100 MHz Generated by the system IOPLL, synchronous to clk_1x.  This clock also supplies the HSSI reconfiguration clock. avl_clk 250 MHz PCIe H-tile Hard IP clock output. This clock is not synchronous to Clk_* DDR4x_USERCLK 299.76 MHz Each of the four DDR interfaces generates one of these clocks, which provides the the clock to the  DDR4* Avalon Memory Mapped interfaces.  Your memory clock output may be different depending on the memory interface you are implementing in your design. uclk_usr User defined Provides an AFU user clock running at a user specified frequency. Generated by user IOPLL. Not synchronous to clk_*. uclk_usr_div2 uclk_usr/2 Second user clock to AFU running at half the frequency of uclk_usr. Synchronous to uclk_usr. Generated by user IOPLL. hssi[*].f2a_tx_parallel_clk_x1 156.2 MHz 1x TX clock generated from fPLL in the HSSI module in FIM, used to clock the HSSI TX datapath in AFU. hssi[*].f2a_tx_parallel_clk_x2 312.5 MHz 2x TX clock generated from fPLL in the HSSI module in FIM, used to clock the HSSI TX datapath in AFU. hssi[*].f2a_rx_clkout 322.265625MHz RX parallel clock from the HSSI PHY channels, used to clock the HSSI RX data datapath in AFU."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#9-reset","title":"9 Reset","text":""},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#91-reset-signals","title":"9.1   Reset Signals","text":"
                                                          The system reset of OFS reference platform is driven by nPERST pin, pcie_reset_status signal from the PCIe hard IP, the INIT_DONE and nCONFIG pins of the FPGA, and the locked signal of the SYS IOPLL that provides system clocks to FIM.  
                                                           
                                                          Upon power-on, the reset module in the FIM holds the FIM in reset until all the reset conditions are de-activated:
                                                           
                                                             
                                                          • nPERST signal is asserted.
                                                            •  
                                                          • The INIT_DONE pin is driven high to indicate core configuration is complete.
                                                            •  
                                                          • The SYS IOPLL is locked.
                                                            •  
                                                          • The reset status from PCIe hard IP is de-asserted indicating the IP is ready for transfer. 
                                                            •  
                                                           
                                                          The reset module places the FIM back into reset if any of these conditions becomes active again. The only way to invoke a system reset to the FIM after power-up is to deassert the nPERST pin either by performing a warm reboot or through PCIe driver intervention. There are soft reset signals set aside to allow software to reset the Port, AFU and partial reconfiguration IP. 
                                                           
                                                          Table 9-1: FIM System Resets
                                                           Reset Description nPERST pin Active low PCIe reset pin that serves as the system reset pin on the platform. nCONFIG pin Active low input to the FPGA that causes the FPGA to lose its configuration data, enter a reset state, and tri-state all I/O pins. Host software must reload the FPGA FIM after nCONFIG is activated. ninit_done Active low signal derived from the INIT_DONE pin which indicates the FPGA core configuration is complete and has entered usermode. pcie_reset_status Active high reset status from PCIe hard IP. When driven high, this signal indicates that the PCIe IP core is not ready for usermode. pll_locked Active high SYS IOPLL locked signal 
                                                          Table 9-2: Soft Resets
                                                           Soft Reset Bitfield Register Description PortSoftReset PORT_CONTROL[0] Resets Port and AFU. FlrPortReset PORT_CONTROL[3] PCIe function level reset that resets Port and AFU when SR-IOV is enabled. PRReset FME_PR_CTRL[0] Resets the partial reconfiguration (PR) controller."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#92-platform-power-up-sequence","title":"9.2   Platform Power up Sequence","text":"
                                                          Upon power up, the HSSI interfaces of the FIM go through an internal reset and calibration sequence. After the nPERST is de-activated, the PCIe interface and EMIF interfaces are first released from reset, followed by SYS IOPLL when the ninit_done is de-asserted. The rest of the FIM logic is still being hold in reset. The nPOR signal to the PCIe hard IP de-activates following nPERST assertion, which releases PCIe hard IP from reset. The PCIe hard IP asserts pld_clk_inuse to indicate to the application layer that the HIP transaction layer is using the pld_clk as its clock and is ready for operation with the Application layer (pld_clk is stable). Finally, reset_status from PCIe IP is de-asserted. When SYS IOPLL is locked and the reset_status from the PCIe interface is de-asserted, the FIM is released from reset while the Port and AFU is still held in reset by the PortSoftReset register bit (PORT_CONTROL[0]) that is held high until software writes a 0 to this bit to de-activate port reset. At this point, the platform is fully released from reset and PCIe link is going through link training and PCIe enumeration. Once the platform is successfully enumerated, driver can then be loaded to start the device feature discovery process and provision the platform for AFU application usage.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#10-interrupts","title":"10 Interrupts","text":"
                                                          The OFS platform supports interrupt through MSI-X feature. The OFS reference platform supports at least 4 FME interrupts (PF only) and 4 AFU interrupts (PF and VF). 
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#101-msi-x","title":"10.1 MSI-X","text":"
                                                          In the default implementation the MSI-X feature that handles FME and AFU interrupts is inside the PCIe Subsystem.  The MSI-X vector table and Pending Bit Array (PBA) table for PF0 and PF0/VF1 are provided as an example.  FME interrupts are primarily used to notify the host of error events occurring in the FIM. 
                                                           
                                                          All interrupt requests arrive inband through the AXI4-Stream interface to the TX AXI4-stream adapter inside the PCIe Subsystem.  
                                                           
                                                          An AFU sends an interrupt to the MSI-X module on the AXI interrupt request channel. After the interrupt request is serviced, the MSI-X module sends an interrupt response back to the AFU on the AXI interrupt response channel. The AFU has the flexibility to define the use of each AFU interrupt.
                                                           
                                                          The following interrupts are supported: PF supports 7 interrupt vectors: - 0-3:  User AFU triggered  - 4:    Port error triggered - 6:    FME error triggered
                                                           
                                                          VF supports 5 interrupt vectors: - 0-3:  User AFU triggered  - 4:    Port error triggered
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#11-external-memory-interface-emif","title":"11 External Memory Interface (EMIF)","text":"
                                                          There are four DDR4 external memory interfaces on the OFS EA FIM that targets the Intel FPGA PAC D5005 FIM for each of the four DDR4 banks (DDR4a, DDR4b, DDR4c, and DDR4d). Two of the DDR4 external memory interfaces and the associated clocks are directly exposed to AFU except for two Avalon Memory Mapped pipeline bridges to facilitate timing closure across PR boundary. The Avalon Memory Mapped interfaces of each external memory interface are connected to an Avalon-MM pipeline bridge (avmm_bridge) in the FIM, which is then connected to another Avalon-MM pipeline bridge in the PR or AFU region.  An AFU should use the USER_CLK associated with a memory interface when interfacing with the memory interface for better timing performance.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#111-emif-csr","title":"11.1 EMIF CSR","text":"
                                                          The CSR for the EMIF feature is memory mapped to the FME BAR region. Following table captures the EMIF CSR registers.
                                                           
                                                          Table 9-1: EMIF CSR Registers
                                                           EMIF_DFH 0x40000 0x3000000050000009 EMIF Management DFH FIELD NAMERANGEACCESSDEFAULT DESCRIPTION FeatureType [63:60] RO 0x3 Feature Type = Private Feature Reserved40 [59:40] RsvdZ 0x0 Reserved NextDfhByteOffset [39:16] RO 0x050000 Next DFH Byte offset FeatureRev [15:12] RO 0x0 Feature Revision FeatureID [11:0] RO 0x9 Feature Id EMIF_STAT 0x40008 0x0000000000000000 EMIF Status FIELD NAMERANGEACCESSDEFAULT DESCRIPTION Reserved [63:16] RsvdZ 0x0 Reserved Reserved [15:12] RsvdZ 0x0 Reserved EmifCalFail [11:8] RO 0x0 EMIF PHY Calibration Failure (1 bit per interface) Reserved [7:4] RsvdZ 0x0 Reserved EmifCalSuccess [3:0] RO 0x0 EMIF PHY Calibration Successful (1 bit per interface) 
                                                          \u200b          \u200b               
                                                           EMIF_CTRL 0x40010 0x0000000000000000 EMIF Control FIELD NAMERANGEACCESSDEFAULT DESCRIPTION Reserved [63:0] RsvdZ 0x0 Reserved"},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#92-afu-emif-interface","title":"9.2 AFU EMIF Interface","text":"
                                                          The FIM exposes 576-bits of Avalon Memory-Mapped data to the AFU, with 512-bit data and additional 64 bits that can either be used for additional metadata, parity or ECC. The AFU has the flexibility to decide the use of the extra 64 bits of data.  The ECC soft IP is not enabled in the EMIF IP to allow for the afore-mentioned flexibility. AFU developers can implement the ECC logic in the AFU by making use of the extra 64-bit of data.  Avalon Memory-Mapped is the native interface protocol used by Intel EMIF IP. AFU developers who desire other interface protocol in their designs over Avalon Memory-Mapped, such as AXI4 Memory-Mapped, can leverage the bridge in the PIM library. 
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#12-hssi-subsystem","title":"12 HSSI Subsystem","text":""},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#121-hssi-subsystem-overview","title":"12.1 HSSI Subsystem Overview","text":"
                                                          The high speed serial interface (HSSI) subsystem architecture provides portability to different Ethernet configurations across platforms and generations and reusability of the hardware framework and software stack. This reference FIM contains the Low Latency Ethernet 10G MAC IP and provides a Linux driver that can be leveraged for customization. 
                                                           
                                                          The HSSI design is leveraged from our OFS EA release so prior customers can easily maintain compatibility with past designs.
                                                           
                                                          A host exerciser, named he-hssi, is provided in the pr_slot of the AFU partition. The Ethernet interface to the AFU has an AXI4-Stream data and sideband interface. The HSSI control and status registers in the FIM are accessible by the AXI4-Stream memory mapped interface. 
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#122-ofs-hssi-subsystem-interfaces","title":"12.2 OFS HSSI Subsystem Interfaces","text":""},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1221-hssi-subsystem-fim-interfaces","title":"12.2.1 HSSI Subsystem FIM Interfaces","text":"
                                                          There are three interfaces to the HSSI Subsystem that is part of the FIM:
                                                           
                                                             
                                                          1. AXI4 Memory Mapped to access HSSI-CSR (to FIM)
                                                            2.  
                                                          2. AXI4-Stream Ethernet Data Interface (from FIM)
                                                            3.  
                                                          3. AXI4-Stream Ethernet Sideband Interface (from FIM)
                                                            4.  
                                                           
                                                          The PCIe subystem uses AXI Memory Mapped accesses to read and write HSSI Control and Status Registers in the FIM. The Ethernet MAC interface typically has a data streaming interface which is mapped to standard AXI4-Stream.
                                                           
                                                          Figure 12-1: HSSI Subsystem
                                                           
                                                           
                                                          Additionally, the Ethernet MAC interface has an interface for status and flow control. Status and flow control signals vary across IPs and operating modes so for this design we group the signals into a AXI4-Stream sideband interface which provides a standard interface to the AFU along with platform customizations if needed. The Avalon to Arm AMBA 4 AXI4 bridge (av_axi_st_bridge) converts native Avalon interfaces to an AXI4 Stream interface.
                                                           
                                                          The following flow control are implemented in the Ethernet MAC: * IEEE 802.3 flow control: this flow control implements the IEEE 802.3 Annex 31B standard to manage congestion. When the Low Latency Ethernet 10G MAC IP experiences congestion, the core sends a pause frame to request its link partner to suspend transmission for a given period of time. This flow control is a mechanism to manage congestion at the local or remote partner. When the receiving device experiences congestion, it sends an XOFF pause frame to the emitting device to instruct the emitting device to stop sending data for a duration specified by the congested receiver. Data transmission resumes when the emitting device receives an XON pause frame (pause quanta = zero) or when the timer expires. 
                                                           
                                                          \u2022 Priority-based flow control (PFC): this flow control implements the IEEE 802.1Qbb standard. PFC manages congestion based on priority levels. It supports up to 8 priority queues. When the receiving device experiences congestion on a priority queue, it sends a PFC frame requesting the emitting device to stop transmission on the priority queue for a duration specified by the congested receiver. When the receiving device is ready to receive transmission on the priority queue again, it sends a PFC frame instructing the emitting device to resume transmission on the priority queue
                                                           
                                                          To use flow control, set the following registers:
                                                           
                                                          On the TX datapath:  1. Set tx_pfc_priority_enable[7:0] (Address :0x0046 -> 11A0) to 0 to disable the PFC. The rest of the bits are unused. 2. Set tx_pauseframe_enable[0] (Address :0x0044 -> 1142) to 1 to enable the flow control. 
                                                           
                                                          On the RX datapath:
                                                           
                                                          \u2022   Set rx_pfc_control[7:0] (Address :0x00C0 -> 818) to 1 to disable the PFC. The rest of the bits are mostly unused. \u2022   Set the IGNORE_PAUSE (Address :0x00AC -> 800) bit in the rx_frame_control register to 0 to enable the flow control.
                                                           
                                                          To use Priority-Based Flow Control Follow these steps to use the priority-based flow control (PFC):  1. Enable the Priority-based flow control (PFC) parameter and specify the number of priority levels using the Number of PFC priorities parameter. You can specify between 2 to 8 PFC priority levels.  2. Set the following registers. 
                                                           
                                                          On the TX datapath: * Set tx_pauseframe_enable (Address :0x0044 -> 1142) to 0 to disable the flow control. * Set tx_pfc_priority_enable[n] (Address :0x0046 -> 11A0) to 1 to enable the PFC for priority queue n. 
                                                           
                                                          On the RX datapath:  * Set the IGNORE_PAUSE bit in the rx_frame_control (Address :0x00AC -> 800) register to 1 to disable the flow control. * Set the rx_pfc_control[7:0] (Address :0x00C0 -> 818) register bits to 0 to enable the PFC. Most of the rest of the bits are unused.  * Set PFC Quanta bit for the appropriate queue. Eg: pfc_pause_quanta_0 (0x048 -> 1180) for queue0 and so on. *  Connect the avalon_st_tx_pfc_gen_data signal to the corresponding RX client logic and the avalon_st_rx_pfc_pause_data signal to the corresponding TX client logic.  * You have the option to configure the MAC RX to forward the PFC frame to the client by setting the rx_pfc_control[16] register to 1. By default, the MAC RX drops the PFC frame after processing it
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1222-hssi-subsystem-afu-interfaces","title":"12.2.2 HSSI Subsystem AFU Interfaces","text":"
                                                          The HSSI subsystem provides the following interfaces to the AFU region:
                                                           
                                                             
                                                          1. AXI4-Memory Mapped access to the HSSI CSR (to FIM)
                                                            2.  
                                                          2. AXI4-Stream Ethernet Data Interface (from FIM)
                                                            3.  
                                                          3. AXI4-Stream Ethernet Sideband Interface (from FIM)
                                                            4.  
                                                          4. Ethernet Clock Interface (eth_clock) (from FIM)
                                                            5.  
                                                           
                                                          The he-hssi uses the APF interface for HSSI CSR (MMIO) accesses. The AXI4-Stream Ethernet data and side band interface along with Ethernet clocks communicate directly to the he-hssi module in the AFU region through platform independent data structures provided by the PIM.  Even if you implement a different MAC you typically can leverage these data structures defined in the hssi/inc/ofs_fim_eth_avst_if.sv file here without modification.
                                                           
                                                          While the platform-independent interfaces in ofs_fim_eth_if.sv are convenient containers for passing data streams through the design hierarchy, both the MAC and AFU traffic generator require platform-specific data types. The payloads of the streams in ofs_fim_eth_if.sv are defined in platform-specific structures, with fields that are MAC-specific. In this 10GbE reference design, the payload datatypes are defined in the ipss/hssi/s10/includes/ofs_fim_eth_plat_if_pkg.sv file hhere. Implementers connecting a new MAC should generally edit only ofs_fim_eth_plat_if_pkg.sv when defining payloads.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1223-hssi-sideband-interface","title":"12.2.3 HSSI Sideband Interface","text":"
                                                          The AXI4-Stream sideband interface has been been defined to allow for time sensitive status and flow control information. It does not have the optional tready signal and assumes the slave always accepts  information.  The Ethernet sideband interface varies widely across IPs and operating modes and device generations. In OFS Stratix 10 FIM, the Ethernet sideband signals are mapped to the AXI4-Stream interface and interface variations can be accommodated by customizing the tdata signals of the sideband interface in platform specific interface packages using the PIM. 
                                                           
                                                          As an example, please refer to ofs_fim_eth_sideband_tx_axis_if interface in the ipss/hssi/inc/ofs_fim_eth_if.sv found OFS D5005 FIM Github Branch.
                                                           
                                                          The t_axis_eth_sideband_tx and t_axis_eth_sideband_rx structures are found in the ipss/hssi/s10/includes/ofs_fim_eth_plat_if_pkg.sv file OFS D5005 FIM Github Branch.
                                                           
                                                          Platform specific details for the 10GbE example are from the ipss/hssi/s10/includes/ofs_fim_eth_plat_if_pkg.sv file OFS D5005 FIM Github Branch.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1224-reconfiguration-interfaces","title":"12.2.4 Reconfiguration Interfaces","text":"
                                                          The reconfiguration interface in the OFS EA design consists of abstracted and consolidated memory-mapped transceiver reconfiguration interfaces that are exposed to the HSSI CSRs. The reconfiguration interface directly exposes the address space of the MAC and PHY IPs in order to allow a software driver to perform dynamic reconfiguration of those IPs (i.e. read/write access to the Native PHY CSRs). Therefore, to use this interface you must be familiar with the CSR memory maps of the corresponding Intel IP cores. 
                                                           
                                                          The table below summarizes all the ports associated with Reconfiguration Interfaces. 
                                                           Name Width Domain Description i_xcvr_reconfig_cmd 2 i_reconfig_clk Command port used to specify a read or write access operation to a MAC/PHY CSRs on a selected CSR_interface i_xcvr_reconfig_addr 20 i_reconfig_clkCSR access address of  MAC/PHY IP. Note that only the lower 16 bits are used, i_xcvr_reconfig_addr[15:0]. The remaining upper bits, i_xcvr_reconfig_addr[20:16], should be set to zero.  i_xcvr_reconfig_writedata  32 i_reconfig_clk CSR data to be written on a write command.  o_xcvr_reconfig_readdata 32 i_reconfig_clk CSR data that was read on a read command. o_xcvr_reconfig_ack 1 i_reconfig_clk Reconfiguration acknowledgement bit. Asserted when Reconfiguration Controller is finished performing an Avalon-MM read or write request (i.e. waitrequest is low). Deasserted when i_xcvr_reconfig_cmd is all-zero (i.e. command is invalidated). Note that the controller assumes a valid command when i_xcvr_reconfig_cmd is non-zero."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#12241-reconfiguration-sequence","title":"12.2.4.1 Reconfiguration Sequence","text":"
                                                          The diagram below explains the sequence of operation and handshaking between software and a memory-mapped dynamic reconfiguration interface.
                                                           
                                                          Figure 12-2: Sequence of Operation and Handshaking between Software and a Memory-Mapped Dynamic Reconfiguration Interface.
                                                           
                                                           
                                                          (0)  Idle state \u2013 command and address buses are cleared (all-zero). 1.  Software sets a desired non-zero command and address to initiate reconfiguration.
                                                           
                                                          a.  Memory-Mapped Reconfiguration Controller (MM CTRL) converts the command and address to a single Avalon Memory Mapped read or write request and handles Avalon Memory Mapped protocol details.\n\nb. MM CTRL completes the Avalon Memory Mapped transaction when the `waitrequest` signal of a given Avalon Memory Mapped interface is deasserted.\n\nc. MM CTRL sets the reconfiguration acknowledgment bit and `readdata` (in case of a read command) back to the FME and waits for command and address ports to be cleared by software. \n1.\n
                                                           
                                                             
                                                          1.  
                                                            Meanwhile, software continuously polls the reconfiguration acknowledgment bit and waits for it to get asserted. Assertion of the acknowledgment bit confirms that the MM CTRL has completed the current Avalon Memory Mapped read/write request. 
                                                             
                                                            2.  
                                                          2.  
                                                            Software reads readdata from the  HSSI_RCFG_DATA CSR that was returned by the MM CTRL (in case of a read command). 
                                                             
                                                            3.  
                                                          3.  
                                                            Software clears the command and address buses to communicate back to the MM CTRL that the operation is finished from the CPU\u2019s perspective. 
                                                             
                                                            a.  MM CTRL gets cleared (all-zero) command and address signals from the HSSI_CSR.    b.  MM CTRL clears (deasserts) the reconfiguration acknowledgment bit back to the HSSI_CSR and is finished / back to idle state.  5.  Meanwhile, software continuously polls the reconfiguration acknowledgment bit and waits for it to get deasserted. Deassertion of the acknowledgment bit confirms that the MM CTRL has completed its handshake and is now back to idle state. 
                                                             
                                                            4.  
                                                           
                                                          NOTE
                                                           
                                                          Reads and writes cannot be performed at the same time. Remember that when multiple CSRs are at the same address, a Read-Modify-Write operation may be required to change the desired CSR without changing the CSRs in the same address.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1225-hssi-control-and-status-register-csr-map","title":"12.2.5   HSSI Control and Status Register (CSR) Map","text":"
                                                          The HSSI CSR Map structure is designed to scale according to IP capabilities.
                                                           
                                                             
                                                          • HSSI_DFH allows for identifying HSSI as an external FME feature.
                                                            •  
                                                          • HSSI_CAPABILITY register exposes design capabilities and provides direction to SW/Driver. The fields num_channels, Num_channels_CSR interface, Num_CSR_interface indicate to the software how many CSR interfaces are exposed by the design and how to use mailbox registers (HSSI_RCFG_CMD and HSSI_RCFG_DATA). The number of mailbox registers(HSSI_RCFG_CMD and HSSI_RCFG_DATA) must scale to the number of CSR interfaces exposed by the design. This implementation facilitates flexibility in the design and reuse of the software stack. If you are modifying the HSSI interface, you must update these CSR fields according to HW configuration .  
                                                            •  
                                                           
                                                          Example: If Num_CSR_interface=2 & Num_channels_CSR_interface=2 then channel(0,1) are behind CSR interface 0 handled by HSSI_RCFG_CMD0/DATA0 , channel (2,3) are behind CSR interface 1 handled by HSSI_RCFG_CMD1/DATA1 HSSI_CTRL, HSSI_STATUS0, HSSI_STATUS1 provide control and status information and can scale upto 8 channels.  HSSI_RCFG_CMD0, HSSI_RCFG_DATA0 registers are for the reconfiguration interface,  and additional mailbox registers could be added depending on the number of CSR interfaces the design exposes. 
                                                           
                                                          The HSSI CSR Map can be found in the ipss/hssi/s10/hssi_ss_csr.xls file OFS D5005 FIM Github Branch.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1226-hssi-host-exercisier-he-hssi","title":"12.2.6 HSSI Host Exercisier (HE-HSSI)","text":"
                                                          HE-HSSI is an Ethernet AFU that handles client side ethernet traffic. The reference HE-HSSI has following features:
                                                           
                                                             
                                                          • HE-HSSI wraps the 10G Ethernet AFU that was provided in the OFS EA FIM with a wrapper that provides an E-tile compatible interface with OFS for Intel Stratix 10 and Intel Agilex FPGAs.
                                                            •  
                                                          • Includes 10GbE traffic generator and checker (monitor)
                                                            •  
                                                          • Provides pause signals to the HSSI subsystem for XON and XOFF generation
                                                            •  
                                                          • It can generate traffic or incoming traffic that can be looped back into transmit path by enabling loopback mode, which will bypass traffic generator
                                                            •  
                                                          • At the HE-HSSI interface boundary the Ethernet data interface is AXI4-Stream with 64-bit data at eth_clk clock
                                                            •  
                                                          • An AXI4-Stream to Avalon-ST bridge converts Avalon-ST Ethernet traffic from the HE-HSSI traffic generator to AXI4-Stream traffic and AXI4-Stream RX data from the FIM to Avalon-ST for the HE-HSSI traffic checker. The data width for all the interfaces in this bridge is 64 bits at eth_clk clock.
                                                            •  
                                                          • The Traffic generator and checker modules have a 64-bit data interface at eth_clk clock.
                                                            •  
                                                          • The traffic generator supports the following modes:     * Fixed length or Random Length     * Incremental pattern or Random pattern
                                                            •  
                                                          • The traffic checker does a 32-bit CRC check
                                                            •  
                                                          • The CSR of this AFU is accessible through AXI4-Stream PCIe TLP interfac
                                                            •  
                                                          • The PCIe TLP to CSR Interface Conversion module converts PCIe TLPs into simple CSR interface
                                                            •  
                                                          • The CSR space of the traffic generator and checker modules are accessed in an indirect way using mailbox registers
                                                            •  
                                                          • Though the default configuration for this reference HE-HSSI is 1x10GbE, it can be scaled up to eight 10G ethernet traffic generators and checkers in one HE-HSSI
                                                            •  
                                                          • If used for more than one channel, each channel has a separate traffic generator and traffic checker with separate CSR space.
                                                            •  
                                                          • Reads and Writes to individual traffic controller CSR spaces can be done by selecting that particular channel using channel select register.
                                                            •  
                                                           
                                                          The HE-HSSI Ethernet block diagram is below. 
                                                           
                                                          Figure 12-6: HE-HSSI Block Diagram Block Diagram
                                                           
                                                           
                                                          Figure 12-7: 10G Ethernet AFU Clock Domains
                                                           
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#12261-he-hssi-csr-map","title":"12.2.6.1 HE-HSSI CSR Map","text":"
                                                          The reference HSSI AFU contains the following registers and a similar arrangement of register space can be implemented for other usecase specific HSSI AFUs. * AFU DFH Register: Device feature header for the AFU (AFU_DFH) * AFU ID Registers: 128-bit UUID for the AFU which occupies two 64-bit registers (AFU_ID_L, AFU_ID_H) * Mailbox Registers: Command and Data register for traffic controller register access. It follows the standard access method defined for OFS. Access method and implementation is same as Reconfiguration Interface defined for the HSSI FIM. (TRAFFIC_CTRL_CMD, TRAFFIC_CTRL_DATA) * Channel Select Register: Channel select register for traffic controller mailbox access. It is used in cases where more than one channel is in the AFU, else it defaults to zero, meaning channel-0 is selected. (TRAFFIC_CTRL_PORT_SEL) * Scratchpad Register: Scratchpad register for CSR access checking. (AFU_SCRATCHPAD) 
                                                           
                                                          The CSR excel for the 10G HSSI reference AFU can be found ipss/hssi/s10/hssi_ss_csr.xls OFS D5005 FIM Github Branch.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#123-hssi-software","title":"12.3 HSSI Software","text":"
                                                          There are two pieces of software related to running the HSSI Subsystem and the HE-HSSI host exerciser: The Linux* dfl network driver and a user space application.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1231-hssi-linux-driver","title":"12.3.1 HSSI Linux Driver","text":"
                                                          The HSSI subystem is exposed as a feature in the PCIe PF BAR0 region.  It has a Device Feature Header (DFH) indicating the HSSI interface. The feature ID in the DFH causes the following driver to be instantiated for the HSSI interface: drivers/net/ethernet/intel/s10hssi.c Kernel Driver Branch
                                                           
                                                          The primary functionality of the driver is to interact with the ethernet MAC and PHY through an indirect register access mailbox implemented by the HSSI_RCFG_CMD0, HSSI_RCFG_DATA0 registers described above. To aid in RTL bringup, the driver provides a debugfs interface directly to the indirect register access mailbox. For each HSSI interface in the system there would be a directory with the following form containing two files, regaddr and regval: /sys/kernel/debug/dfl-fme.X.Y
                                                           
                                                          To read a register offset in the MAC/PHY write the offset to the regaddr file as a C hex string (e.g. 0x04) and then read the value as string out of regval file. To write a register offset in the MAC/PHY write the offset to the regaddr file as a C hex string (e.g. 0x04) and then write the value as a C hex string to regval file.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1232-hssi-user-space-tool","title":"12.3.2   HSSI User Space Tool","text":"
                                                          The HSSI user space application exports a control interface to the HSSI AFU's packet generator logic. Context-sensitive help is given by the --help option, doc/src/fpga_tools/hssi/hssi.md,  OPAE SDK Branch.
                                                           
                                                          $ hssi --help\n\n
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#124-user-guidelines","title":"12.4 User Guidelines","text":"
                                                          You can either leverage Ethernet example designs from platform designer or use your own custom IP\u2019s. However below recommendations would help leverage the infrastructure of the OFS stack:\n* Follow the Ethernet-GBS interface standard, customize platform specific sideband and clock intefaces.\n* Follow the reconfiguration interface example and reuse the hssi_csr block by modifying the memory map (change address decoder map accordingly as well.)
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#13-partial-reconfiguration","title":"13 Partial Reconfiguration","text":"
                                                          Partial Reconfiguration (PR) is an Intel FPGA technology that allows user to reconfigure parts of the FPGA device dynamically, while the remainder of the device continues to operate. In a non-partial reconfiguration flow, any change to the design requires full reprogramming of the entire configuration RAM (CRAM) arrays in the device. With partial reconfiguration, you can dynamically reprogram one or more CRAM frames. A partial reconfiguration design has a static region, and one or more PR regions, which can be modified to implement new logic. The portion of the CRAM on the chip to be reconfigured is contained within a PR region.\nFor the PR flow, the design should be partitioned into static region and reconfigurable region. The static region is the area of your FPGA that is not reconfigured without reprogramming the entire FPGA. An area of the chip that you plan to partially reconfigure is a PR region. 
                                                          \n
                                                          The Port Gasket contains all the PR specific modules and logic, such as PR slot reset/freeze control, user clock, remote STP etc. For this reference example only one PR slot is supported.\nThe following figure depicts the high level view of the Port Gasket:
                                                          \n
                                                          Figure 13-1 Partial Reconfiguration Gasket\n
                                                          \n
                                                          \n
                                                          The isolation logic is provided on the output signals of the PR region to ensure they don\u2019t glitch and affect the interfacing logic in the Static Region (SR). The isolation logic is controlled by the PR Freeze logic during PR operation.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#14-reliability-accessibility-serviceability-ras-and-error-handling","title":"14 Reliability, Accessibility, Serviceability (RAS) and Error Handling","text":"
                                                            \n
                                                          1. Downstream AFU checker: Identifies AFU violations.  For example, this checker flags violations of the interface specification. 
                                                            2. \n
                                                          2. Upstream software or PCIe link checker: Identifies invalid traffic from PCIe that violates either FIM specifications or PCIe specifications. For example, this checker flags an application sending traffic if it violates the FIM specification or creates a PCIe link issue by causing completion timeout or malformed TLP. 
                                                            3. \n
                                                          3. FIM - Checks for bugs in the FIM fabric.
                                                            4. \n
                                                          \n
                                                          Errors reported by the checker are logged in either the FME error registers or Port error registers, or both, as shown in the table below.  For more details on each of the registers, please refer to src/common/protocol_checker/protocol_checker_csr.xml file OFS FIM_COMMON Github Branch or the SystemVerilog file src/common/fme/xls/d5005/FME_CSR.xls found OFS FIM_COMMON Github Branch .
                                                          \n
                                                          Table 14-1: Error Registers
                                                          \n\n\n\nMMIO Region\nArea\nRegister\nDescription\n\n\n\n\nFME\nCoreFIM\nFME_ERROR\nFME Error Status Register 0.  Registers parity errors, underflow or overflow errors and access mismatches.\n\n\nFME\nCoreFIM\nFME_ERROR0_MASK\nFME Error Mask Register 0.  Write a 0 to mask errors in the FME Error Status Register 0.\n\n\nFME\nExternal\nPCIE0_ERROR\nPCIe0 Error Status Register.\n\n\nFME\nExternal\nPCIE0_ERROR_MASK\nPCIe0 Error Mask Register 0.  Write a 0 to mask errors in the PCIe0 Error Status Register 0.\n\n\nFME\nCoreFIM\nFME_FIRST_ERROR\nFirst FME Error Register.\n\n\nFME\nCoreFIM\nFME_NEXT_ERROR\nFME Next Error Register.\n\n\nFME\nCoreFIM\nRAS_NOFAT_ERR_STAT\nReliability/Accessibility/Serviceability (RAS) Non-Fatal Error Status Register.\n\n\nFME\nCoreFIM\nRAS_NOFAT_ERR_MASK\nRAS Non-Fatal Error Mask Register. Write 0 to mask error fields in RAS_NOFAT_ERR_STAT Register.\n\n\nFME\nCoreFIM\nRAS_CATFAT_ERR_STAT\nRAS Catastrophic and Fatal Errors Status Register.\n\n\nFME\nCoreFIM\nRAS_CATFAT_ERR_MASK\nRAS Catastrophic and Fatal Errors Mask Register. Write 0 to mask error fields in the RAS_CATFAT_ERR_STAT Register.\n\n\nFME\nCoreFIM\nRAS_ERROR_INJ\nRAS error Injection Register.\n\n\nPORT\nCoreFIM\nPORT_ERROR\nPort Error Status Register.\n\n\nPORT\nCoreFIM\nPORT_FIRST_ERROR\nPort First Error Register .\n\n\nPORT\nCoreFIM\nPORT_MALFORMED_REQ0\nPort Malformed Request Register 0.  Provides malformed request header LSBs.\n\n\nPORT\nCoreFIM\nPORT_MALFORMED_REQ1\nPort Malformed Request Register 1.  Provides malformed request header MSBs."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#141-fme-errors","title":"14.1 FME Errors","text":""},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1411-fme_error0","title":"14.1.1   FME_ERROR0","text":"
                                                          The FME_ERROR0 register flags CoreFIM FME errors in the Global Error (GLBL_ERROR) private feature. The error bits in this register are sticky bits. You can only clear these bits through software or through a system reset. Writing a 1 to the error field in the register clears the corresponding error bit. Writing a 1 to the corresponding bit in FME_ERROR0_MASK register masks the error.
                                                          \n
                                                          Table 14-2: FME Error Types
                                                          \n\n\n\nError Type\nDescription\n\n\n\n\nFabric errors\nFIFO underflow/overflow condition in CoreFIM. These errors only occur if you have introduced bugs into the FIM or during very rare single event upset (SEU) or SEU-like events.\n\n\nInvalid port access\nA port can either be mapped to a PF or a VF, and not both. The checker reads the AfuAccessCtrl field in the FME CSR PORT0_OFFSET register to determine the access type of the Port. If it finds a PF is trying to access a port that is mapped to a VF or vice-versa, an error will be reported.\n\n\nInvalid AFU access\nAn AFU can either be mapped to a PF or a VF, and not both. The checker reads the AfuAccessCtrl field in the FME CSR PORT0_OFFSET register to determine the access type of the AFU associated with the Port. If it finds a PF is trying to access an AFU that is mapped to a VF or vice-versa, an error is reported and a fake response is sent back to the requester to avoid a completion timeout on the host."},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1412-pcie0_error","title":"14.1.2 PCIE0_ERROR","text":"
                                                          The PCIe Avalon-ST to AXI4-Stream bridge monitors the PCIe link for errors and logs any such errors in the PCIE0_ERROR register (in PCIE0 feature region) and PCIE0_ERROR register in the GLBL_ERR private feature.    The error bits in the PCIE0_ERROR register are sticky bits that you can only clear through software or through a system reset. Writing a 1 to the error field in the register clears the corresponding error bit. Writing a 1 to the corresponding bit in PCIE0_ERROR0_MASK masks the error.  
                                                          \n
                                                          If you have other external FME features, you can add similar _ERROR registers to this space. Please refer to the following spreadsheet, bbs/csr/stratix10/pac_d5005/fme_csr_pcie.xls,  OFS D5005 FIM Github Branch or the SystemVerilog file the SystemVerilog file src/common/fme/xls/d5005/FME_CSR.xls found OFS FIM_COMMON Github Branch for more details on this register. \n\n
                                                          NOTE
                                                          \n
                                                          The PCIE0_ERROR register is located in both the Global Error external feature memory space and a separate PCIe external feature memory space.  OPAE software supports the PCIe external feature memory space beginning at offset 0x40000 for OFS EA and going forward.  PCIe registers beginning at 0x4000 in the Global Error external feature memory space is there for backward compatibility to the Intel FPGA PAC D5005 v2.0.1 Acceleration Stack.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1413-fme_first_error-fme_next_error","title":"14.1.3   FME_FIRST_ERROR, FME_NEXT_ERROR","text":"
                                                          The FME_FIRST_ERROR register flags which of the FME error reporting registers, such as FME_ERROR0, PCIE0_ERROR0, has reported the first error occurrence. The error fields of the first error register are then continuously logged into the FME_FIRST_ERROR register until a system reset or software clears all the errors in that first error register.\nLikewise, the FME_NEXT_ERROR indicates which of the FME error reporting registers (except the first error register) has reported the next occurrence of error after the first error register. The error fields of the next error register are continuously logged into the FME_NEXT_ERROR register until a system reset or software clears all the errors in the second error register.
                                                          \n
                                                          Please refer tobbs/csr/stratix10/pac_d5005/s10_iofs_csr_map_update_fme_ral_.xls, OFS D5005 FIM Github Branch for individual register field descriptions or the SystemVerilog file: src/fme/fme_csr.sv, OFS D5005 FIM Github Branch
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1414-reliability-accessibility-serviceability-ras-error-status","title":"14.1.4   Reliability, Accessibility, Serviceability (RAS) Error Status","text":"
                                                          The RAS feature in CoreFIM labels errors as non-fatal, fatal or catastrophic based on their impact to the system. \n* A non-fatal error usually originates from software or an AFU.  With a non-fatal error, the user application may still be able to recover from the error by performing a soft reset on the AFU, fixing the user application software if necessary, and clearing the error. On the other hand, a fatal or catastrophic error is non-recoverable and requires the platform to be reset.\n* Non-fatal errors are logged in the RAS_NOFAT_ERR_STAT register and fatal or catastrophic errors are logged in the RAS_CATFAT_ERR_STAT register.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#14141-non-fatal-errors","title":"14.1.4.1    Non-Fatal Errors","text":"
                                                          The RAS_NOFAT_ERR_STAT is a read-only status register that is specifically added for the RAS feature. It logs the high-level status of non-fatal errors in the hardware.  Unlike the error bits in the PCIE0_ERROR and FME_ERROR0 registers which are RW1C (software can write a 1 to clear the error), the error bits in this register are read-only and can only be cleared by system reset. Software has an option to mask the error using RAS_NOFAT_ERR_MASK.\nPlease refer to bbs/csr/stratix10/pac_d5005/s10_iofs_csr_map_update_fme_ral_.xls, OFS D5005 FIM Github Branch for individual register field descriptions or the SystemVerilog file: src/fme/fme_csr.sv, OFS D5005 FIM Github Branch.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#14142-catastrophic-fatal-errors","title":"14.1.4.2    Catastrophic & Fatal Errors","text":"
                                                          The RAS_CATFAT_ERR_STAT is a read-only status register that is specifically added for the RAS feature. It captures the high-level status of errors that can only be recovered with a system reset. Therefore, the error bits in the RAS_CATFAT_ERR_STAT register are read-only and can only be cleared by system reset or masked through RAS_CATFAT_ERR_MASK.\nPlease refer to bbs/csr/stratix10/pac_d5005/s10_iofs_csr_map_update_fme_ral_.xls, OFS D5005 FIM Github Branch for individual register field descriptions or the SystemVerilog file: src/fme/fme_csr.sv OFS D5005 FIM Github Branch.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1415-ras-error-injection","title":"14.1.5   RAS Error Injection","text":"
                                                          For software testing purposes, you can inject non-fatal, fatal and catastrophic errors into the platform through the RAS_ERROR_INJ register.  These errors are reported in the RAS_CATFAT_ERR_STAT and RAS_NOFAT_ERR_STAT registers.\nPlease refer to bbs/csr/stratix10/pac_d5005/s10_iofs_csr_map_update_fme_ral_.xls OFS D5005 FIM Github Branch for individual register field descriptions or the SystemVerilog file: src/fme/fme_csr.sv OFS D5005 FIM Github Branch.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1416-fme-error-interrupts","title":"14.1.6   FME Error Interrupts","text":"
                                                          In an event of an FME error, the MSI-X module in the FIM generates an interrupt so the host can decide on the next course of action. The FIM does not stall upstream and downstream traffic after the FME error. However, a port error triggered by invalid request from a user AFU stalls all the traffic going from AFU to PCIe.\nThe interrupt capability is discoverable by querying the NumbSuppInterrupt field of the PORT_CAPABILITY register in the Port private feature.  The MSI-X vector number is recorded in the InterruptVectorNumber field of the GLBL_ERROR_CAPABILITY register of the Global Error external feature.
                                                          \n
                                                          An FME error interrupt is generated in response to the FME_ERROR0, PCIE0_ERROR0, RAS_NOFAT_ERR_STAT or RAS_CATFAT_ERR_STAT registers recording a new, unmasked, error per the rules defined by CoreFIM interrupt feature.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#14161-msi-x-masking-pending-bit-array-pba-clearing","title":"14.1.6.1    MSI-X Masking & Pending Bit Array (PBA) Clearing","text":"
                                                          If the MSI-X vector corresponding to the FME error interrupt is masked, events are recorded in the PBA array.  Clearing the FME error status registers clears the corresponding MSI-X PBA entries.  If only some events are cleared, the normal interrupt triggering rules apply and a new pending interrupt is registered.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1417-fme-error-handling","title":"14.1.7   FME Error Handling","text":"
                                                          When the host receives an FME error interrupt, it must take the recommended actions described below to bring the system back to its normal operation.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#14171-catastrophicfatal-error","title":"14.1.7.1    Catastrophic/Fatal Error","text":"
                                                          A system reset is mandatory for any catastrophic or fatal error.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#14172-non-fatal-error","title":"14.1.7.2    Non-Fatal Error","text":"
                                                          When software receives a non-fatal error interrupt which does not require a system reset, it can take the following steps to clear the error after software handles the error:\n1.  Set the *_ERROR_MASK register to all 1\u2019s to mask all errors\n2. Clear the *_FIRST_ERROR register\n3. Clear the *_ERROR register\n4. Set *_ERROR_MASK register to all 0\u2019s to enable all errors
                                                          \n
                                                            \n
                                                          • Result: The *_ERROR & *_FIRST_ERROR registers begin capturing new errors.
                                                            • \n
                                                          \n\n
                                                          NOTE
                                                          \n
                                                          A system reset can only clear RAS Error status registers.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#142-mmio-requests","title":"14.2   MMIO Requests","text":"
                                                          The FIM is designed to gracefully handle MMIO request scenarios.  
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1421-unsupported-functions-and-bars","title":"14.2.1 Unsupported Functions and BARs","text":"
                                                          The OFS FIM EA has only one PCIe link and all MMIO requests from the host are sent through this link. The PCIe hard IP in the FIM guarantees that only TLP packets for the functions and BARs supported by the FIM (as configured in PCIe HIP IP instantiation) are sent to the FIM on the PCIe Avalon Streaming interface.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1422-mmio-request-decoding","title":"14.2.2   MMIO Request Decoding","text":"
                                                          The packet router and memory decoder in the FIM ensure that only legal MMIO requests are forwarded to the targeted MMIO region. Full address and BAR decoding is done both in the packet router and the memory decoder to ensure the requests are forwarded to the designated CSR region as defined in the MMIO Regions chapter.  Any unsolicited/illegal MMIO request is dropped, and an error is reported back to host through the FME error register.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1423-unused-fmeport-csr-regions","title":"14.2.3   Unused FME/Port CSR Regions","text":"
                                                          All the CSR slaves in FIM which are mapped to the FME or Port CSR regions must always respond to MMIO read requests targeting its associated CSR region. A CSR slave must return all 0s for MMIO reads to its unused CSR region such as a reserved space or a region where no CSR register is implemented for the address.\nThe FIM ensures MMIO reads to FME or Port CSR regions that are not mapped to any CSR slave always gets a response of all 0s. The memory decoder module and fake responder module in the FIM provide this guaranteed response.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1424-unsupported-mmio-request","title":"14.2.4   Unsupported MMIO Request","text":"
                                                          Any MMIO request targeting FME or Port CSR regions with a length or address alignment that are not  supported by the FIM is dropped, and an error is logged in PCIE0_ERROR register. The MMIO checker module in the FIM guarantees this response. When an unsupported MMIO read request to the FIM CSR region is detected, the FIM sends back a CPL (completion without data) with error status (UR) back to host.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1425-afu-access-violation","title":"14.2.5   AFU Access Violation","text":"
                                                          AFU access violations refer to the scenarios where a PF is attempting to access the MMIO region of an AFU bound to a VF (virtualization enabled), or when a VF is trying to access the MMIO region of an AFU bound to a PF (virtualization disabled). When such a violation is detected, the FIM drops the request and logs the error in the FME_ERROR0 register. If the request is an MMIO read request, the FIM returns a fake response to the host.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#1426-afu-mmio-response-timeout","title":"14.2.6   AFU MMIO Response Timeout","text":"
                                                          An AFU MMIO Response timeout functions in the same manner described in the MMIO Response Timeout section.
                                                          "},{"location":"hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_d5005/#15-design-guidance","title":"15 Design Guidance","text":"
                                                          The OFS FIM is designed with configurability and scalability in mind.  At a high level, these are the necessary steps for a user to customize the design.  Please refer to the FPGA Interface Manager Developer Guide: Open FPGA Stack for Intel Stratix 10
                                                          \n
                                                          Table 15-1 Features
                                                          \n\n    \n        \n            Step\n            Description\n            Comments  \n        \n    \n    \n        \n            1\n            Re-configure PCIe HIP for additional VFs (if necessary)\n            * PF0 is mandatory for running OPAE software* Only modification of the VFs (added or subtracted) by the user is recommended.\n\u2022   Default configuration supports 1 PF and 3 VFs.\n        \n        \n        \n            2\n            Update AXI4-Stram PF/VF MUX-DEMUX configuration (if necessary)\n\n            * The PF/VF MUX-DEMUX is parameterized for flexibility.  You can change, add or delete PF or VF functions by updating the top_cfg_pkg.sv file.\n* You also have the option of keeping the default configuration and tying off the unused VFs if needed.\n\n        \n \n            3\n            Update top level and AFU level as necessary\n\n     * If you integrating additional external interfaces, make the edits at the top level (iofs_top.sv) and propagate the interface down to the AFU level (afu_top.sv)\n \n            4\n            Add user implemented function(s) in AFU\n            * All of your implemented functions must have the required AXI4-Stream interface for both the data path and the MMIO control path to CSRs. * All CSRs in the user implemented function must have the required DFH layout. * See host exerciser CSRs for reference.\n   \n        \n\n\nFor more information on modifying the FIM, refer to the [Open FPGA Stack Technical Reference Manual].\n\n\n\n## Notices & Disclaimers\n\nIntel\u00ae technologies may require enabled hardware, software or service activation.\nNo product or component can be absolutely secure. \nPerformance varies by use, configuration and other factors.\nYour costs and results may vary. \nYou may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein.\nNo license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document.\nThe products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request.\nIntel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade.\nYou are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards. \n\u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. \n\nOpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. \n\n\n\n\n\n\n\n*[AFU]: Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region\n*[BBB]: Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID.\n*[BKC]: Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against.\n*[BMC]: Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors.\n*[DFL]: Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\n*[FIM]: FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring.\n*[FME]: FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform.\n*[HEM]:  Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc.\n*[JTAG]:  Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology.\n*[OFS]: Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs.\n*[OPAE SDK]: Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE.\n*[PIM]: Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols.\n*[PR]: Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page.\n*[RSU]: Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration.\n*[UVM]: Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework.\n*[TB]: Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output.\n*[Intel VT-d]: Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization.\n*[SR-IOV]: Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance.\n*[MMIO]: Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators.\n*[VFIO]: Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace.\n*[IOCTL]: Input/Output Control, System calls used to manipulate underlying device parameters of special files.\n*[AER]: Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting.\n*[PAC]: Programmable Acceleration Card: FPGA based Accelerator card"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/","title":"Platform Evaluation Script: Open FPGA Stack for Intel Stratix 10 FPGA","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#1-overview","title":"1 Overview","text":""},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#11-about-this-document","title":"1.1 About this Document","text":"
                                                          This document serves as a set-up and user guide for the checkout and evaluation of an Intel\u00ae FPGA PAC D5005 development platform using Open FPGA Stack (OFS). After reviewing the document, you will be able to:
                                                           
                                                             
                                                          • Set-up and modify the script to your environment
                                                            •  
                                                           
                                                             
                                                          • Compile and simulate an OFS reference design
                                                            •  
                                                           
                                                             
                                                          • Run hardware and software tests to evaluate the complete OFS flow
                                                            •  
                                                          "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#12-table-software-version-summary","title":"1.2 Table : Software Version Summary","text":"Component Version Description FPGA Platform Intel\u00ae FPGA PAC D5005 Intel platform you can use for your custom board development OFS FIM Source Code Branch: ofs-d5005, Tag: release/ofs-2023.3 OFS Shell RTL for Intel Stratix 10 FPGA (targeting Intel\u00ae FPGA PAC D5005) OFS FIM Common Branch: ofs-2023.3-2, Tag: ofs-2023.3-2 Common RTL across all OFS-based platforms AFU Examples Branch: examples-afu , Tag: ofs-examples-ofs-2023.3-1 Tutorials and simple examples for the Accelerator Functional Unit region (workload region) OPAE SDK Branch: 2.10.0-1, Tag: 2.10.0-1 Open Programmable Acceleration Engine Software Development Kit Kernel Drivers Branch: ofs-2023.3-6.1-2, Tag: ofs-2023.3-6.1-2 OFS specific kernel drivers OPAE Simulation Branch: opae-sim, Tag: 2.10.0-1 Accelerator Simulation Environment for hardware/software co-simulation of your AFU (workload) Intel Quartus Prime Pro Edition Design Software 23.3 Intel\u00ae Quartus\u00ae Prime Pro Edition Linux Software tool for Intel FPGA Development Operating System RHEL 8.6 Operating system on which this script has been tested 
                                                          A download page containing the release and already-compiled FIM binary artifacts that you can use for immediate evaluation on the Intel\u00ae FPGA PAC D5005 can be found on the OFS 2023.3 official release drop on GitHub.
                                                           
                                                           Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#2-introduction-to-ofs-evaluation-script","title":"2 Introduction to OFS Evaluation Script","text":"
                                                          By following the setup steps and using the OFS evaluation script you can quickly evaluate many features that the OFS framework provides and also leverage this script for your own development.  
                                                          "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#21-pre-requisites","title":"2.1 Pre-Requisites","text":"
                                                          This script uses the following set of software tools which should be installed using the directory structure below. Tool versions can vary.
                                                           
                                                             
                                                          • Intel Quartus\u00ae Prime Pro Software
                                                            •  
                                                          • Synopsys\u00ae VCS Simulator
                                                            •  
                                                          • Siemens\u00ae Questa\u00ae Simulator
                                                            •  
                                                           
                                                          Figure 2-1 Folder Hierarchy for Software Tools
                                                           
                                                           
                                                             
                                                          1.  
                                                            You must create a directory named \"ofs-X.X.X\" where the X represents the current release number, for example ofs-2023.3-2. 
                                                             
                                                            2.  
                                                          2.  
                                                            You must clone the required OFS repositories as per Figure 2-2 . Please refer to the BKC table for locations., Please go to [OFS Getting Started User Guide] for the instructions for the BKC installation.
                                                             
                                                            3.  
                                                          3.  
                                                            Once the repositories are cloned, copy the evaluation script (ofs_d5005_eval.sh) which is located at [eval_scripts] beneath the ofs-2023.3-2 directory location as shown in the example below:
                                                             
                                                            4.  
                                                           
                                                          Figure 2-2 Directory Structure for OFS Project
                                                           
                                                          ## ofs-2023.3-2\n##  -> examples-afu\n##  -> linux-dfl\n##  -> ofs-d5005\n##  -> oneapi-asp\n##  -> oneAPI-samples\n##  -> opae-sdk\n##  -> opae-sim\n##  -> ofs_d5005_eval.sh\n
                                                           
                                                             
                                                          1. Open the README file named (README_ofs_D5005_eval.txt) which is located at [eval_scripts] which informs the user which sections to modify in the script prior to building the FIM and running hardware, software and simulation tests. 
                                                            2.  
                                                          "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#22-intel-fpga-pac-d5005-evaluation-script-modification","title":"2.2 Intel\u00ae FPGA PAC D5005 Evaluation Script modification","text":"
                                                          To adapt this script to the user environment please follow the instructions below which explains which line numbers to change in the ofs_d5005_eval.sh script.
                                                          "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#user-directory-creation","title":"User Directory Creation","text":"
                                                          The user must create the top-level source directory and then clone the OFS repositories
                                                           
                                                          mkdir ofs-2023.3-2\n
                                                           
                                                          In the example above we have used ofs-2023.3-2 as the directory name
                                                          "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#set-up-proxy-server-lines-65-67","title":"Set-Up Proxy Server (lines 65-67)","text":"
                                                          Please enter the location of your proxy server to allow access to external internet to build software packages.
                                                           
                                                          Note: Failing to add proxy server will prevent cloning of repositories and the user will be unable to build the OFS framework.
                                                           
                                                          export http_proxy=<user_proxy>\nexport https_proxy=<user_proxy>\nexport no_proxy=<user_proxy>\n
                                                          "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#license-files-lines-70-72","title":"License Files (lines 70-72)","text":"
                                                          Please enter the the license file locations for the following tool variables
                                                           
                                                          export LM_LICENSE_FILE=<user_license>\nexport DW_LICENSE_FILE=<user_license>\nexport SNPSLMD_LICENSE_FILE=<user_license>\n
                                                          "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#tools-location-line-85-86-87-88","title":"Tools Location (line 85, 86, 87, 88)","text":"
                                                          Set Location of Quartus, Synopsys, Questasim and oneAPI Tools
                                                           
                                                          export QUARTUS_TOOLS_LOCATION=/home\nexport SYNOPSYS_TOOLS_LOCATION=/home\nexport QUESTASIM_TOOLS_LOCATION=/home\nexport ONEAPI_TOOLS_LOCATION=/opt\n
                                                          "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#quartus-tools-version-line-93","title":"Quartus Tools Version (line 93)","text":"
                                                          Set version of Quartus
                                                           
                                                          export QUARTUS_VERSION=23.3\n
                                                           
                                                          In the example above \"23.3\" is used as the Quartus tools version
                                                          "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#opae-tools-line-106","title":"OPAE Tools (line 106)","text":"
                                                          change OPAE SDK VERSION
                                                           
                                                          export OPAE_SDK_VERSION=2.10.0-1\n
                                                           
                                                          In the example above \"2.10.0-1\" is used as the OPAE SDK tools version
                                                          "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#pcie-bus-number-lines-231-and-238","title":"PCIe (Bus Number) (lines 231 and 238)","text":"
                                                          The Bus number must be entered by the user after installing the hardware in the chosen server, in the example below \"b1\" is the Bus Number for a single card as defined in the evaluation script.
                                                           
                                                          export ADP_CARD0_BUS_NUMBER=b1\n
                                                           
                                                          The evaluation script uses the bus number as an identifier to interrogate the card. The command below will identify the accelerator card plugged into a server. 
                                                           
                                                          lspci | grep acc\n\n86:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                                                           
                                                          The result identifies the card as being assigned \"86\" as the bus number so the entry in the script changes to
                                                           
                                                          export ADP_CARD0_BUS_NUMBER=86\n
                                                           
                                                          The user can also run the following command on the ofs_d5005_eval.sh script to automatically change the bus number to 86 in the ofs_d5005_eval.sh script.
                                                           
                                                          grep -rli '86' * | xargs -i@ sed -i '86' @
                                                           
                                                          if the bus number is 85 for example 
                                                           
                                                          85:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)
                                                           
                                                          the command to change to 85 in the evaluation script would be
                                                           
                                                          grep -rli '86' * | xargs -i@ sed -i '85' @
                                                           
                                                          The ofs_d5005_eval.sh script has now been modified to the server set-up and the user can proceed to build, compile and simulate the OFS stack
                                                           
                                                          "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#3-using-the-evaluation-script","title":"3 Using the Evaluation Script","text":""},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#31-overview","title":"3.1 Overview","text":"
                                                          The evaluation script focuses on different evaluation areas. Each of these menu options is described in the next section.  
                                                           
                                                          The figure below shows a snapshot of the full evaluation script menu showing all 57 options and each one of 10 sub-menus which focus on different areas of evaluation. Each of these menu options is described in the next section.
                                                           
                                                          Figure 3-1 ofs_d5005_eval.sh Evaluation Menu
                                                           
                                                          "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#311-tools-menu","title":"3.1.1 TOOLS MENU","text":"
                                                          By selecting \"List of Documentation for ADP Intel\u00ae FPGA PAC D5005 Project,\" a list of links to the latest OFS documentation appears. Note that these links will take you to documentation for the most recent release which may not correspond to the release version you are evaluating. To find the documentation specific to your release, ensure you clone the intel-ofs-docs tag that corresponds to your OFS version.
                                                           
                                                          By selecting \"Check Versions of Operating System and Quartus Premier Design Suite\", the tool verifies correct Operating System, Quartus version, kernel parameters, license files and paths to installed software tools.
                                                           
                                                           Menu Option Example Output 1 - List of Documentation for ADP D5005 Project Open FPGA Stack Overview Guides you through the setup and build steps to evaluate the OFS solution https://ofs.github.io 2 - Check versions of Operating System and Quartus Premier Design Suite (QPDS) Checking Linux release Linux version 6.1.41-dfl Checking RedHat release Red Hat Enterprise Linux release RHEL 8.6  Checking Ubuntu release cat: /etc/lsb-release: No such file or directory  Checking Kernel parameters BOOT_IMAGE=(hd0,msdos1)/vmlinuz-6.1.41-dfl-2023.3-1 root=/dev/mapper/rhel-root ro crashkernel=auto resume=/dev/mapper/rhel-swap rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200  Checking Licenses LM_LICENSE_FILE is set to port@socket number:port@socket number DW_LICENSE_FILE is set to port@socket number:port@socket number SNPSLMD_LICENSE_FILE is set to port@socket number:port@socket number  Checking Tool versions QUARTUS_HOME is set to /home/intelFPGA_pro/23.3/quartus QUARTUS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus IMPORT_IP_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../ip QSYS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../qsys/bin  Checking QPDS Patches Quartus Prime Shell Version 23.3"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#312-hardware-menu","title":"3.1.2 HARDWARE MENU","text":"
                                                          Identifies card by PCIe number, checks power, temperature and current firmware configuration.
                                                           
                                                           Menu Option Example Output 3 - Identify Acceleration Development Platform (ADP) D5005 Hardware via PCIe PCIe card detected as 86:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01) Host Server is connected to SINGLE card configuration 4 - Identify the Board Management Controller (BMC) Version and check BMC sensors Intel FPGA Programmable Acceleration Card D5005Board Management Controller, MAX10 NIOS FW version: 2.0.14Board Management Controller, MAX10 Build version: 2.0.8//****** BMC SENSORS ******//Object Id : 0xF000000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00Ports Num : 01Bitstream Id : 288511863935352239Bitstream Version : 4.0.1Pr Interface Id : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98 5 - Identify the FPGA Management Engine (FME) Version Intel FPGA Programmable Acceleration Card D5005Board Management Controller, MAX10 NIOS FW version: 2.0.14Board Management Controller, MAX10 Build version: 2.0.8//****** FME ******//Object Id : 0xF000000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00Ports Num : 01Bitstream Id : 288511863935352239Bitstream Version : 4.0.1Pr Interface Id : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98Boot Page : user 6 - Check Board Power and Temperature Intel FPGA Programmable Acceleration Card D5005Board Management Controller, MAX10 NIOS FW version: 2.0.14Board Management Controller, MAX10 Build version: 2.0.8//****** POWER ******//Object Id : 0xF000000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00Ports Num : 01Bitstream Id : 288511863935352239Bitstream Version : 4.0.1Pr Interface Id : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98( 1) VCCERAM Voltage : 0.90 Voltsetc ......................Intel FPGA Programmable Acceleration Card D5005Board Management Controller, MAX10 NIOS FW version: 2.0.14Board Management Controller, MAX10 Build version: 2.0.8//****** TEMP ******//Object Id : 0xF000000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00Ports Num : 01Bitstream Id : 288511863935352239Bitstream Version : 4.0.1Pr Interface Id : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98( 1) VCCT Temperature : 57.00 Celsiusetc ...................... 7 - Check Accelerator Port status //****** PORT ******// Object Id : 0xEF00000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00 8 - Check MAC and PHY status Intel FPGA Programmable Acceleration Card D5005Board Management Controller, MAX10 NIOS FW version: 2.0.14Board Management Controller, MAX10 Build version: 2.0.8//****** MAC ******//Object Id : 0xF000000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00Ports Num : 01Bitstream Id : 288511863935352239Bitstream Version : 4.0.1Pr Interface Id : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98MAC address : 64:4c:36:f:44:1fIntel FPGA Programmable Acceleration Card D5005Board Management Controller, MAX10 NIOS FW version: 2.0.14Board Management Controller, MAX10 Build version: 2.0.8//****** PHY ******//Object Id : 0xF000000PCIe s:b:d.f : 0000:86:00.0Vendor Id : 0x8086Device Id : 0xBCCESubVendor Id : 0x8086SubDevice Id : 0x138DSocket Id : 0x00Ports Num : 01Bitstream Id : 288511863935352239Bitstream Version : 4.0.1Pr Interface Id : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#313-fimpr-build-menu","title":"3.1.3 FIM/PR BUILD MENU","text":"
                                                          Builds FIM, Partial Reconfiguration Region and Remote Signal Tap
                                                           
                                                           Menu Option Description 9 - Check ADP software versions for ADP Intel\u00ae FPGA PAC D5005 Project OFS_ROOTDIR is set to /home/user_area/ofs-2023.3-2/ofs-d5005OPAE_SDK_REPO_BRANCH is set to release/2.10.0-1 OPAE_SDK_ROOT is set to /home/user_area/ofs-2023.3-2/ofs-d5005/../opae-sdk LD_LIBRARY_PATH is set to /home/user_area/ofs-2023.3-2/ofs-d5005/../opae-sdk/lib64: 10 - Build FIM for Intel\u00ae FPGA PAC D5005 Hardware This option builds the FIM based on the setting for the $ADP_PLATFORM, $FIM_SHELL environment variable. Check these variables in the following file ofs_d5005_eval.sh 11 - Check FIM Identification of FIM for Intel\u00ae FPGA PAC D5005 Hardware The FIM is identified by the following file fme-ifc-id.txt located at $OFS_ROOTDIR/$FIM_WORKDIR/syn/syn_top/ 12 - Build Partial Reconfiguration Tree for Intel\u00ae FPGA PAC D5005 Hardware This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development and also for the OneAPI build flow 13 - Build Base FIM Identification(ID) into PR Build Tree template This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU and OneAPI workloads 14 - Build Partial Reconfiguration Tree for Intel\u00ae FPGA PAC D5005 Hardware with Remote Signal Tap This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development and also for the OneAPI build flow and for the Remote Signal Tap flow 15 - Build Base FIM Identification(ID) into PR Build Tree template with Remote Signal Tap This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree for Remote Signal Tap to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU and OneAPI workloads"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#314-hardware-programmingdiagnostic-menu","title":"3.1.4 HARDWARE PROGRAMMING/DIAGNOSTIC MENU","text":"
                                                          The following submenu allows you to: * Program and check flash  * Perform a remote system update (RSU) of the FPGA image into the FPGA * Bind virtual functions to VFIO PCIe driver  * Run host exerciser (HE) commands such as loopback to test interfaces VFIO PCI driver binding * Read the control and status registers (CSRs) for bound modules that are part of the OFS reference design.
                                                           
                                                           Menu Option Description 16 - Program BMC Image into Intel\u00ae FPGA PAC D5005 Hardware The user must place a new BMC flash file in the following directory $OFS_ROOTDIR/bmc_flash_files. Once the user executes this option a new BMC image will be programmed. A remote system upgrade command is initiated to store the new BMC image 17 - Check Boot Area Flash Image from Intel\u00ae FPGA PAC D5005 Hardware This option checks which location area in FLASH the image will boot from, the default is user1  Boot Page : user1 18 - Program FIM Image into user1 area for Intel\u00ae FPGA PAC D5005 Hardware This option programs the FIM image \"D5005_page1_unsigned.bin\" into user1 area in flash 19 - Initiate Remote System Upgrade (RSU) from user1 Flash Image into Intel\u00ae FPGA PAC D5005 Hardware This option initiates a Remote System Upgrade and soft reboots the server and re-scans the PCIe bus for the new image to be loaded  2022-12-13 07:31:33,244 - [[pci_address(0000:86:00.0), pci_id(0x8086, 0xbcce, 0x8086, 0x138d)]] performing RSU operation 2022-12-13 07:31:33,249 - [[pci_address(0000:85:00.0), pci_id(0x8086, 0x2030, 0x1590, 0x00ea)]] removing device from PCIe bus 2022-12-13 07:31:34,333 - waiting 10.0 seconds for boot 2022-12-13 07:31:44,344 - rescanning PCIe bus: /sys/devices/pci0000:85/pci_bus/0000:85 2022-12-13 07:31:44,377 - RSU operation complete 20 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 21 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 22 - Create Virtual Functions (VF) and bind driver to vfio-pci Intel\u00ae FPGA PAC D5005 Hardware This option creates vfio-pci driver binding for the PF's and VF's Once the VF's have been bound to the driver the user can select menu option 20 to check that the new drivers are bound 23 - Run HE-LB Test This option runs 5 tests  1) checks and generates traffic with the intention of exercising the path from the AFU to the Host at full bandwidth 2) run a loopback throughput test using one cacheline per request 3) run a loopback read test using four cachelines per request 4) run a loopback write test using four cachelines per request 5) run a loopback throughput test using four cachelines per request 24 - Run HE-MEM Test This option runs 2 tests  1) Checking and generating traffic with the intention of exercising the path from FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host 2) run a loopback throughput test using one cacheline per request 25 - Run HE-HSSI Test This option runs 1 test  HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs, and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic  1) Send traffic through the 10G AFU 26 - Read from CSR (Command and Status Registers) for Intel\u00ae FPGA PAC D5005 Hardware This option reads from the following CSR's HE-LB Command and Status Register Default Definitions HE-MEM Command and Status Register Default Definitions HE-HSSI Command and Status Register Default Definitions"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#315-hardware-afu-testing-menu","title":"3.1.5  HARDWARE AFU TESTING MENU","text":"
                                                          This submenu tests partial reconfiguration by building and loading an memory-mapped I/O example AFU/workload, executes software from host, and tests remote signal tap.
                                                           
                                                           Menu Option Description 27 - Build and Compile host_chan_mmio example This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB_EXTERNAL/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS (Green Bit Stream) binary file ready for hardware programming 28 - Execute host_chan_mmio example This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test 29 - Modify host_chan_mmio example to insert Remote Signal Tap This option inserts a pre-defined host_chan_mmio.stp Signal Tap file into the OFS code to allow a user to debug the host_chan_mmio AFU example 30 - Build and Compile host_chan_mmio example with Remote Signal Tap This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB_EXTERNAL/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS (Green Bit Stream) binary file ready for hardware programming with Remote Signal tap enabled 31 - Execute host_chan_mmio example with Remote Signal Tap This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test. The user must open the Signal Tap window when running the host code to see the transactions in the Signal Tap window"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#316-hardware-afu-bbb-testing-menu","title":"3.1.6 HARDWARE AFU BBB TESTING MENU","text":"
                                                          This submenu tests partial reconfiguration using a hello_world example AFU/workload, executes sw from the host
                                                           
                                                           Menu Option Description 32 - Build and Compile hello_world example This option builds the hello_ world example from the following repo $FPGA_BBB_CCI_SRC/tutorial/afu_types/01_pim_ifc/$AFU_BBB_TEST_NAME, where AFU_BBB_NAME=hello_world. This produces a GBS (Green Bit Stream) file ready for hardware programming 33 - Execute hello_world example This option builds the host code for hello_world example and programs the GBS file and then executes the test"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#317-adp-oneapi-project-menu","title":"3.1.7 ADP ONEAPI PROJECT MENU","text":"
                                                          Builds OneAPI kernel, executes the software from host and runs diagnostic tests
                                                           
                                                           Menu Option Result 34 - Check oneAPI software versions for Intel\u00ae FPGA PAC D5005 Project This option checks the setup of the oneAPI software and adds the relevant oneAPI environment variables to the terminal. This option also informs the user to match the oneAPI software version to the oneAPI-samples version 35 - Build and clone shim libraries required by oneAPI host This option builds the oneAPI directory structure 36 - Install OneAPI Host Driver This option Installs the oneAPI Host driver at the following location /opt/Intel/OpenCLFPGA/oneAPI/Boards/, and requires sudo permission 37 - Uninstall One API Host Driver This option Uninstall's the oneAPI Host driver, and requires sudo permissions 38 - Diagnose oneAPI Hardware This option Checks ICD (Intel Client Driver) and FCD (FPGA Client Driver), oneAPI library locations and detects whether oneAPI BSP is loaded into the FPGA 39 - Build oneAPI BSP ofs-d5005 Default Kernel (hello_world) This option Builds the oneAPI BSP using hello_world kernel 40 - Build oneAPI MakeFile Environment This option Builds the oneAPI environment using a Makefile for kernel insertion 41 - Compile oneAPI Sample Application (board_test) for Emulation This option compiles the board_test kernel for Emulation 42 - Run oneAPI Sample Application (board_test) for Emulation This option executes the board_test kernel for Emulation 43 - Generate oneAPI Optimization report for (board_test) This option generates an optimization report for the board_test kernel 44 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 45 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 46 - Create Virtual Function (VF) and bind driver to vfio-pci Intel\u00ae FPGA PAC D5005 Hardware This option creates vfio-pci driver binding for the PF's and VF's Once the VF's have been bound to the driver the user can select menu option 45 to check that the new drivers are bound 47 - Program oneAPI BSP ofs-d5005 Default Kernel (hello_world) This option programs the FPGA with a aocx file based on the hello_world kernel 48 - Compile oneAPI Sample Application (board_test) for Hardware This option compiles the board_test kernel for Hardware 49 - Run oneAPI Sample Application (board_test) for Hardware This option builds the host code for board_test kernel and executes the program running through kernel and host bandwidth tests"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#318-unit-test-project-menu","title":"3.1.8 UNIT TEST PROJECT MENU","text":"
                                                          Builds, compiles and runs standalone simulation block tests. More unit test examples are found at the following location ofs-d5005/sim/unit_test 
                                                           
                                                           Menu Option Result 50 - Generate Simulation files for Unit Test This option builds the simulation file set for running a unit test simulation 51 - Simulate Unit Test dfh_walker and log waveform This option runs the dfh_walker based on the environment variable \"UNIT_TEST_NAME=dfh_walker\" in the evaluation script. A user can change the test being run by modifying this variable"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#319-adp-uvm-project-menu","title":"3.1.9 ADP UVM PROJECT MENU","text":"
                                                          Builds, compiles and runs full chip simulation tests. The user should execute the options sequentially ie 52, 53, 54 and 55
                                                           
                                                           Menu Option Description 52 - Check UVM software versions for Intel\u00ae FPGA PAC D5005 Project DESIGNWARE_HOME is set to /home/synopsys/vip_common/vip_Q-2020.03A UVM_HOME is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm VCS_HOME is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel VERDIR is set to /home/user_area/ofs-2023.3-2/ofs-d5005/verification VIPDIR is set to /home/user_area/ofs-2023.3-2/ofs-d5005/verification 53 - Compile UVM IP This option compiles the UVM IP 54 - Compile UVM RTL and Testbench This option compiles the UVM RTL and Testbench 55 - Simulate UVM ofs_mmio_test and log waveform This option runs the dfh_walking test based on the environment variable \"UVM_TEST_NAME=dfh_walking_test\" in the evaluation script. A user can change the test being run by modifying this variable 56 - Simulate all UVM test cases (Regression Mode) This option runs the Intel\u00ae FPGA PAC D5005 regression mode, cycling through all UVM tests defined in /ofs-d5005/verification/tests/test_pkg.svh file"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#3110-adp-build-all-project-menu","title":"3.1.10 ADP BUILD ALL PROJECT MENU","text":"
                                                          Builds the complete OFS flow, good for regression testing and overnight builds
                                                           
                                                          For this menu, a user can run a sequence of tests (compilation, build and simulation) and executes them sequentially. After the script is successfully executed, a set of binary files is produced which a you can use to evaluate your hardware. Log files are also produced which checks whether the tests passed.
                                                           
                                                          A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 57 from the main menu the script will execute 23 tests ie (main menu options 2, 9, 10, 11, 12, 13, 14, 15, 27, 29, 30, 32, 34, 35, 39, 40, 48, 50, 51, 52, 53, 54 and 55. These 23 menu options are chosen to build the complete OFS flow covering build, compile and simulation.
                                                           
                                                           Menu Option Result 57 - Build and Simulate Complete Intel\u00ae FPGA PAC D5005  Project Generating Log File with date and timestamp Log file written to /home/user_area/ofs-2023.3-2/log_files/D5005_log_2022_11_10-093649/ofs-d5005_eval.log 
                                                          Definition of Multi-Test Set-up
                                                           
                                                          Menu Option 57 above in the evaluation script can be refined to tailor the number of tests the users runs. The set-up is principally defined by the variable below
                                                           
                                                          MULTI_TEST[A,B]=C
                                                           
                                                          where
                                                           
                                                          A= Total Number of menu options in script B= Can be changed to a number to select the test order C= Menu Option in Script
                                                           
                                                          Example 1 MULTI_TEST[57,0]=2
                                                           
                                                          A= 57 is the total number of options in the script B= 0 indicates that this is the first test to be run in the script C= Menu option in Script ie 2- List of Documentation for ADP Intel\u00ae FPGA PAC D5005 Project
                                                           
                                                          Example 2 MULTI_TEST[57,0]=2 MULTI_TEST[57,1]=9
                                                           
                                                          In the example above two tests are run in order ie 0, and 1 and the following menu options are executed ie 2- List of Documentation for ADP Intel\u00ae FPGA PAC D5005 Project and 9 - Check ADP software versions for ADP Intel\u00ae FPGA PAC D5005 Project
                                                           
                                                          The user can also modify the build time by de-selecting options they do not wish to use, see below for a couple of use-case scenarios.
                                                           
                                                          Default User Case
                                                           
                                                          A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 57 from the main menu the script will execute 23 tests ie (main menu options 2, 9, 10, 11, 12, 13, 14, 15, 27, 29, 30, 32, 34, 35, 39, 40, 48, 50, 51, 52, 53, 54 and 55. All other tests with an \"X\" indicates do not run that test
                                                           
                                                           
                                                          User Case for ADP FIM/PR BUILD MENU
                                                           
                                                          In the example below when the user selects option 57 from the main menu the script will only run options from the ADP FIM/PR BUILD MENU (7 options, main menu options 9, 10, 11, 12, 13, 14 and 15). All other tests with an \"X\" indicates do not run that test.
                                                           
                                                           
                                                          "},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#4-common-test-scenarios","title":"4 Common Test Scenarios","text":"
                                                          This section will describe the most common compile build scenarios if a user wanted to evaluate an acceleration card on their server. The Pre-requisite column indicates the menu commands that must be run before executing the test eg To run Test 5 then a user needs to have run option 10, 12 and 13 before running options 20, 21, 22, 27 and 28.
                                                           Test Test Scenario Pre-Requisite Menu Option Menu Option Test 1 FIM Build - 10 Test 2 Partial Reconfiguration Build 10 12 Test 3 Program FIM and perform Remote System Upgrade 10 18, 19 Test 4 Bind PF and VF to vfio-pci drivers - 20, 21, 22 Test 5 Build, compile and test AFU on hardware 10, 12, 13 20, 21, 22, 27, 28 Test 6 Build, compile and test AFU Basic Building Blocks on hardware 10, 12, 13 20, 21, 22, 32, 33 Test 7 Build, compile and test oneAPI on hardware 10, 12, 13 34, 35, 36, 39, 40, 44, 45, 46, 47, 48, 49 Test 8 Build and Simulate Unit Tests - 50, 51 Test 9 Build and Simulate UVM Tests - 52, 53, 54, 55"},{"location":"hw/d5005/user_guides/ug_eval_ofs_d5005/ug_eval_script_ofs_d5005/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                          Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                           
                                                          OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/","title":"Getting Started Guide: Open FPGA Stack for Intel Stratix 10","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#10-introduction","title":"1.0 Introduction","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#11-about-this-document","title":"1.1 About This Document","text":"
                                                          This document helps users get started in evaluating Open FPGA Stack (OFS) for Intel\u00ae Stratix 10\u00ae FPGA targeting the Intel\u00ae FPGA PAC D5005. After reviewing the document a user shall be able to:
                                                           
                                                             
                                                          • Set up a development environment with all OFS ingredients
                                                            •  
                                                          • Build and install the OFS Linux Kernel drivers
                                                            •  
                                                          • Build and install the Open Programmable Acceleration Engine Software Development Kit (OPAE SDK) software on top of the OFS Linux kernel drivers
                                                            •  
                                                          • Flash an OFS FIM binary onto the Intel\u00ae FPGA PAC D5005
                                                            •  
                                                          • Verify the functionality of OFS on an Intel\u00ae FPGA PAC D5005 board
                                                            •  
                                                          • Know where to find additional information on all OFS ingredients
                                                            •  
                                                           
                                                          The following flow charts show a high level overview of the initial bringup process, split into three sequential diagrams.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#diagram-1-installing-the-opae-sdk","title":"Diagram 1: Installing the OPAE SDK","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#diagram-2-installing-the-linux-dfl-drivers","title":"Diagram 2: Installing the Linux DFL Drivers","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#diagram-3-bringing-up-the-intel-d5005","title":"Diagram 3: Bringing up the Intel D5005","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#12-terminology","title":"1.2 Terminology","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#13-introduction-to-ofs","title":"1.3 Introduction to OFS","text":"
                                                          Each OFS reference FIM targets a specific platform, but the modular hardware, software, simulation and test infrastructure allows you to modify each part of the design and test environment for your own custom acceleration platform card. The current OFS reference FIM for Stratix 10 FPGA targets the Intel\u00ae FPGA PAC D5005 board. This document focuses exclusively on the OFS release targeting the Intel\u00ae FPGA PAC D5005 board.
                                                           
                                                          The OFS repositories (in OFS ) on GitHub provide the following components targeting an Intel\u00ae FPGA PAC D5005:
                                                           
                                                             
                                                          • opae-sdk: Contains the Open Programmable Acceleration Software Development Kit source code and build scripts.
                                                            •  
                                                          • linux-dfl: Contains Linux kernel-level driver source code and build scripts.
                                                            •  
                                                          • intel-ofs-fim: Contains the source code, build scripts and verification suite for FPGA RTL source code
                                                            •  
                                                          • ofs-hld-shim: Contains the necessary files to generate Shim/BSP for OFS Cards, using OPAE SDK Interfaces.
                                                            •  
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#14-intended-audience","title":"1.4 Intended Audience","text":"
                                                          The information in this document is intended for customers evaluating the Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA on the Intel PAC D5005. This document will cover key topics related to initial setup and development, with links for deeper dives on the topics discussed therein.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#15-reference-documents","title":"1.5 Reference Documents","text":"
                                                          Please refer to the README on the OFS GitHub for an updated list of collateral on the OFS GitHub page.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#16-component-version-summary","title":"1.6 Component Version Summary","text":"
                                                          The OFS 2023.3 Release targeting the Intel\u00ae Stratix 10\u00ae FPGA is built upon tightly coupled software and firmware versions. Use this section as a general reference for the versions which comprise this release.
                                                           
                                                          The following table highlights the hardware which makes up the Best Known Configuration (BKC) for the OFS 2023.3 release.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#table-1-2-hardware-bkc","title":"Table 1-2: Hardware BKC","text":"Component 1 x Intel\u00ae FPGA PAC D5005 1 x Supported Server Model 1 x Intel FPGA Download Cable II (Optional, only required if loading images via JTAG) 
                                                          The following table highlights the versions of the software which comprise the OFS stack. The installation of the user-space OPAE SDK on top of the kernel-space linux-dfl drivers is discussed in subsequent sections of this document.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#table-1-3-software-version-summary","title":"Table 1-3: Software Version Summary","text":"Component Version FPGA Platform Intel\u00ae FPGA PAC D5005 OPAE SDK Tag: 2.10.0-1 Kernel Drivers Tag: ofs-2023.3-6.1-2 OFS FIM Source Code Branch: release/ofs-2023.3 Intel Quartus Prime Pro Edition Design Software 23.3 Intel\u00ae Quartus\u00ae Prime Pro Edition Linux Operating System RHEL 8.6 
                                                          A download page containing the release and already-compiled FIM binary artifacts that you can use for immediate evaluation on the Intel\u00ae FPGA PAC D5005 can be found on the OFS 2023.3 official release drop on GitHub.
                                                           
                                                          Note: If you wish to freeze your Red Hat operating system version on the RHEL 8.6, refer to the following solution provided in the Red Hat customer portal.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#20-ofs-stack-architecture-overview-for-reference-platform","title":"2.0 OFS Stack Architecture Overview for Reference Platform","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#21-hardware-components","title":"2.1 Hardware Components","text":"
                                                          The OFS hardware architecture decomposes all designs into a standard set of modules, interfaces, and capabilities. Although the OFS infrastructure provides a standard set of functionality and capability, the user is responsible for making the customizations to their specific design in compliance with the specifications outlined in the FPGA Interface Manager Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA.
                                                           
                                                          OFS is a blanket term which can be used to collectively refer to all ingredients of the OFS reference design, which includes the core hardware components discussed below and software.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#211-fpga-interface-manager","title":"2.1.1 FPGA Interface Manager","text":"
                                                          The FPGA Interface Manager (FIM) or 'shell' provides platform management functionality, clocks, resets, and interface access to the host and peripheral features on the acceleration platform. The FIM is implemented in a static region of the FPGA device.
                                                           
                                                          The primary components of the FIM reference design are:
                                                           
                                                             
                                                          • PCIe Subsystem
                                                            •  
                                                          • Transceiver Subsystem
                                                            •  
                                                          • Memory Subsystem
                                                            •  
                                                          • FPGA Management Engine
                                                            •  
                                                          • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                                                            •  
                                                          • Board Peripheral Fabric for master to slave CSR accesses from host or AFU
                                                            •  
                                                          • Interface to Board Management Controller (BMC)
                                                            •  
                                                           
                                                          The FPGA Management Engine (FME) provides management features for the platform and the loading/unloading of accelerators through partial reconfiguration.
                                                           
                                                          For more information on the FIM and its external connections, please refer to the FPGA Interface Manager Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA, and the Intel FPGA Programmable Acceleration Card D5005 Data Sheet. Below is a high-level block diagram of the FIM.
                                                           
                                                          Figure 2-1 FIM Overview
                                                           
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#212-afu","title":"2.1.2 AFU","text":"
                                                          An AFU is an acceleration workload that interfaces to the FIM. The AFU boundary in this reference design comprises both static and partial reconfiguration (PR) regions. You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region. A port gasket within the design provides all the PR specific modules and logic required for partial reconfiguration. Only one partial reconfiguration region is supported in this design.
                                                           
                                                          Similar to the FME, the port gasket exposes its capabilities to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space.
                                                           
                                                          You can compile your design in one of the following ways:
                                                           
                                                             
                                                          • Your AFU resides in a partial reconfiguration (PR) region of the FPGA.
                                                            •  
                                                          • Your AFU is a part of the static region (SR) and is a compiled flat design.
                                                            •  
                                                          • Your AFU contains both static and PR regions.
                                                            •  
                                                           
                                                          The AFU provided in this release is comprised of the following functions:
                                                           
                                                             
                                                          • AFU interface handler to verify transactions coming from the AFU region.
                                                            •  
                                                          • PV/VF Mux to route transactions to and from corresponding AFU components, including the ST2MM module, PCIe loopback host exerciser (HE-LB), HSSI host exerciser (HE-HSSI), and Memory Host Exerciser (HE-MEM).
                                                            •  
                                                          • AXI4 Streaming to Memory Map (ST2MM) Module that routes MMIO CSR accesses to FME and board peripherals.
                                                            •  
                                                          • Host exercisers to test PCIe, memory and HSSI interfaces (these can be removed from the AFU region after your FIM design is complete to provide more resource area for workloads).
                                                            •  
                                                          • Port gasket and partial reconfiguration support.
                                                            •  
                                                           
                                                          For more information on the Platform Interface Manager (PIM) and AFU development and testing, please refer to the [OFS AFU Development Guide].
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#22-ofs-software-overview","title":"2.2 OFS Software Overview","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#221-kernel-drivers-for-ofs","title":"2.2.1 Kernel Drivers for OFS","text":"
                                                          OFS DFL driver software provides the bottom-most API to FPGA platforms. Libraries such as OPAE and frameworks like DPDK are consumers of the APIs provided by OFS. Applications may be built on top of these frameworks and libraries. The OFS software does not cover any out-of-band management interfaces. OFS driver software is designed to be extendable, flexible, and provide for bare-metal and virtualized functionality. An in depth look at the various aspects of the driver architecture such as the API, an explanation of the DFL framework, and instructions on how to port DFL driver patches to other kernel distributions can be found on the DFL Wiki page.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#30-intel-fpga-pac-d5005-card-and-server-requirements","title":"3.0 Intel FPGA PAC D5005 Card and Server Requirements","text":"
                                                          Currently OFS for Intel\u00ae Stratix 10\u00ae FPGA targets the Intel\u00ae FPGA PAC D5005. Because the Intel\u00ae FPGA PAC D5005 is a production card, you must prepare the card in order to receive a new non-production bitstream. For these instructions, please contact an Intel representative.
                                                           
                                                          In addition, refer to sections 2.1-2.3 of the Intel Acceleration Stack Quick Start Guide: Intel FPGA Programmable Acceleration Card D5005 for a complete overview of the physical installation process and ESD precautions for the D5005 platform.
                                                           
                                                          Note: Ensure that the system meets all the following requirements   before proceeding to install the Intel\u00ae FPGA PAC D5005 into a server.
                                                           
                                                          Table 3-1 Server Requirements for Intel D5005
                                                           Component Description Server Qualified Servers Main Board PCI Express 3.0 compliant motherboard with at least one dual-width x16 PCIe slot available for card installation Board Power Supply* Auxiliary Power (12V) 
                                                          * For more information on the required auxiliary power supply, refer to section 2.2.2 of the Intel FPGA Programmable Acceleration Card D5005 Data Sheet.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#31-supported-processors-for-intel-d5005","title":"3.1 Supported Processors for Intel D5005","text":"
                                                          OFS requires that the deployment machine's Xeon processor must support the following technologies. These options must also be enabled in the BIOS and as kernel parameters. The process to enable these parameters will be discussed in the section on driver installation:
                                                           
                                                             
                                                          • Intel VT-d (Intel Virtualization Technology for IA-32 and Intel 64 Processors)
                                                            •  
                                                          • Intel VT-x (Intel Virtualization Technology for Directed I/O)
                                                            •  
                                                          • Intel IOMMU
                                                            •  
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#32-cooling-requirements-for-the-intel-fpga-pac-d5005","title":"3.2 Cooling Requirements for the Intel FPGA PAC D5005","text":"
                                                          Please refer to sections 8.1 and 8.2 of the Intel FPGA Programmable Acceleration Card D5005 Data Sheet for guidance on cooling specifications that must be met when using the D5005 card. Failure to adhere to these guidelines may result in thermal runaway and/or performance degradation.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#40-ofs-dfl-kernel-drivers","title":"4.0 OFS DFL Kernel Drivers","text":""},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#41-ofs-dfl-kernel-driver-environment-setup","title":"4.1 OFS DFL Kernel Driver Environment Setup","text":"
                                                          All OFS DFL kernel driver code resides in the Linux DFL GitHub repository. This repository is open source and does not require any permissions to access. It includes a snapshot of the latest best-known configuration (BKC) Linux kernel with the OFS driver included in the drivers/fpga/* directory. Downloading, configuration, and compilation will be discussed in this section. Please refer to Table 1-3 for the latest supported OS.
                                                           
                                                          The DFL driver suite can be automatically installed using a supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3 Release Page.
                                                           
                                                          It is recommended you boot into your operating system's native 4.18.x kernel before attempting to upgrade to the dfl enabled 6.1.41 You may experience issues when moving between two dfl enabled 6.1.41 kernels.
                                                           
                                                          This installation process assumes the user has access to an internet connection in order to pull specific GitHub repositories, and to satisfy package dependencies.
                                                           
                                                          1. You must make the following changes in order to install all dependencies on the latest BKC Operating System. These are required to both build and install the drivers from source, and to install them from pre-built packages:
                                                           
                                                           
                                                          $ subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms\n$ sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n
                                                           
                                                          2. You must satisfy the following package dependencies if building and installing the drivers from source. Double check that all packages have been found and installed:
                                                           
                                                           
                                                          $ sudo dnf install -y python3 python3-pip python3-devel \\\ngdb vim git gcc gcc-c++ make cmake libuuid-devel rpm-build systemd-devel sudo nmap \\\npython3-jsonschema json-c-devel tbb-devel rpmdevtools libcap-devel \\\nspdlog-devel cli11-devel python3-pyyaml hwloc-devel libedit-devel openssl-devel\n\n$ python3 -m pip install --user jsonschema virtualenv pudb pyyaml\n\n$ sudo pip3 uninstall setuptools\n\n$ sudo pip3 install --upgrade setuptools --prefix=/usr\n\n# To Install pybind11 following are the steps\n\n$ curl 'ftp://ftp.pbone.net/mirror/archive.fedoraproject.org/epel/8.2.2020-11-04/Everything/x86_64/Packages/p/python3-pybind11-2.4.3-2.el8.x86_64.rpm' --output ./python3-pybind11-2.4.3-2.el8.x86_64.rpm\n\n$ curl 'ftp://ftp.pbone.net/mirror/archive.fedoraproject.org/epel/8.2.2020-11-04/Everything/x86_64/Packages/p/pybind11-devel-2.4.3-2.el8.x86_64.rpm' --output ./pybind11-devel-2.4.3-2.el8.x86_64.rpm\n\n$ sudo dnf localinstall ./python3-pybind11-2.4.3-2.el8.x86_64.rpm ./pybind11-devel-2.4.3-2.el8.x86_64.rpm\n
                                                           
                                                          It is recommended you create an empty top-level directory for their OFS related repositories to keep the working environment clean. All steps in this installation will use a generic top-level directory at /home/user/OFS/. If you have created a different top-level directory, replace this path with your custom path.
                                                           
                                                          3. Initialize an empty git repository and clone the LTS tagged DFL driver source code: 
                                                           
                                                          $ cd /home/user/OFS/\n$ git init\n$ git clone https://github.com/OPAE/linux-dfl\n$ cd /home/user/OFS/linux-dfl\n$ git checkout tags/ofs-2023.3-6.1-2 -b fpga-ofs-dev-6.1.41\n
                                                           
                                                          4. Verify that the correct tag has been checkout out.
                                                           
                                                          $ git describe \n$ ofs-2023.3-6.1-2\n
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#42-building-and-installing-the-ofs-dfl-kernel-drivers-from-source","title":"4.2 Building and Installing the OFS DFL Kernel Drivers from Source","text":"
                                                          1. The following set of instructions walk you through copying an existing kernel configuration file on your machine and changing the minimal required configuration settings:
                                                           
                                                          $ cd /home/user/OFS/linux-dfl\n$ cp /boot/config-`uname -r` .config\n$ cat configs/dfl-config >> .config\n$ echo 'CONFIG_LOCALVERSION=\"-dfl\"' >> .config\n$ echo 'CONFIG_LOCALVERSION_AUTO=y' >> .config\n$ sed -i -r 's/CONFIG_SYSTEM_TRUSTED_KEYS=.*/CONFIG_SYSTEM_TRUSTED_KEYS=\"\"/' .config\n$ sed -i '/^CONFIG_DEBUG_INFO_BTF/ s/./#&/' .config\n$ echo 'CONFIG_DEBUG_ATOMIC_SLEEP=y' >> .config\n$ export LOCALVERSION=\n$ make olddefconfig\n
                                                           
                                                          (Optional) To use the built-in GUI menu for editing kernel configuration parameters, you can opt to run make menuconfig.
                                                           
                                                          2. Linux kernel builds take advantage of multiple processors to parallelize the build process. Display how many processors are available with the nproc command, and then specify how many make threads to utilize with the -j option. Note that number of threads can exceed the number of processors. In this case, the number of threads are set to the number of processors in the system.
                                                           
                                                          $ cd /home/user/OFS/linux-dfl\n$ make -j `nproc`\n$ make -j `nproc` modules\n
                                                           3. The following options are available to o locally build a set of packages:
                                                           
                                                             
                                                          • rpm-pkg: Build both source and binary RPM kernel packages
                                                            •  
                                                          • binrpm-pkg: Build only the binary kernel RPM package
                                                            •  
                                                          • deb-pkg: Build both source and binary deb kernel packages
                                                            •  
                                                          • bindeb-pkg: Build only the binary kernel deb package
                                                            •  
                                                           
                                                          If you are concerned about the size of the resulting package and binaries, you can significantly reduce the size of the package and object files by using the make variable INSTALL_MOD_STRIP. If this is not a concern, feel free to skip this step. The below instructions will build a set of binary RPM packages:
                                                           
                                                          $ cd /home/user/OFS/linux-dfl\n$ make INSTALL_MOD_STRIP=1 binrpm-pkg\n
                                                           
                                                          4. By default a directory is created in your home directory called rpmbuild. This directory will house all of the kernel packages which have been built. You need to navigate to the newly built kernel packages and install them. The following files were generated using the build command executed in the previous step:
                                                           
                                                          $ cd ~/rpmbuild/RPMS/x86_64\n$ ls\n$ kernel-6.1.41_dfl-1.x86_64.rpm  kernel-headers-6.1.41_dfl-1.x86_64.rpm\n$ sudo dnf localinstall kernel*.rpm\n
                                                           
                                                          5. The system will need to be rebooted for changes to take effect. After a reboot, select the newly built kernel as the boot target. This can be done pre-boot using the command grub2-reboot, which removes the requirement for user intervention. After boot, verify that the currently running kernel matches expectation.
                                                           
                                                          $ uname -r\n$ 6.1.41-dfl\n
                                                           
                                                          6. Verify the DFL drivers have been successfully installed. If an Intel\u00ae FPGA PAC D5005 card with the appropriate FIM is on the local system, the kernel driver modules will have been loaded. In the lsmod output the second column corresponds to the size of the kernel module in bytes, the third column displays the number of devices registered to that driver, and the fourth column displays the names of the devices using it. Verify their versions against the below.
                                                           
                                                          $ lsmod | grep -e fpga -e dfl\n\nuio_dfl                20480  0\nspi_altera_dfl         20480  0\nuio                    20480  1 uio_dfl\ndfl_emif               16384  0\nspi_altera_core        16384  1 spi_altera_dfl\ndfl_fme_region         20480  0\ndfl_fme_br             16384  0\ndfl_fme_mgr            20480  0\ndfl_afu                36864  0\ndfl_fme                49152  0\ndfl_pci                20480  0\ndfl                    40960  8 dfl_pci,s10hssi,uio_dfl,dfl_fme,dfl_fme_br,dfl_afu,spi_altera_dfl,dfl_emif\nfpga_region            20480  3 dfl_fme_region,dfl_fme,dfl\nfpga_bridge            20480  4 dfl_fme_region,fpga_region,dfl_fme,dfl_fme_br\nfpga_mgr               24576  4 dfl_fme_region,fpga_region,dfl_fme_mgr,dfl_fme\n
                                                           
                                                          If an Intel\u00ae FPGA PAC D5005 card is not installed in the system and/or does not have the appropriate FIM configured, the user may read version information of the DFL drivers directly from /lib/modules:
                                                           
                                                          $ cd /usr/lib/modules/`uname -r`/kernel/drivers/fpga\n$ modinfo dfl* fpga* | grep ^name\n\nname:           dfl_afu\nname:           dfl_fme_br\nname:           dfl_fme\nname:           dfl_fme_mgr\nname:           dfl_fme_region\nname:           dfl_hssi\nname:           dfl\nname:           dfl_n3000_nios\nname:           dfl_pci\nname:           fpga_bridge\nname:           fpga_mgr\nname:           fpga_regions\n
                                                           
                                                          7. Four kernel parameters must be added to the boot command-line for the newly installed kernel. First, open the file grub:
                                                           
                                                          $ sudo vim /etc/default/grub\n
                                                           
                                                          8. In the variable GRUB_CMDLINE_LINUX add the parameters shown after quiet:
                                                           
                                                          GRUB_CMDLINE_LINUX=\"crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\"\n
                                                           
                                                          Note: If you wish to instead set hugepages on a per session basis, you can perform the following step. These settings will be lost on reboot.
                                                           
                                                          $ mkdir -p /mnt/huge \n$ mount -t hugetlbfs nodev /mnt/huge \n$ echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages \n$ echo 2048 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages \n
                                                           
                                                          9. Save your edits, then apply them to the GRUB2 configuration file.
                                                           
                                                          $ sudo grub2-mkconfig  -o /boot/efi/EFI/redhat/grub.cfg\n
                                                           
                                                          10. Warm reboot. Your kernel parameter changes should have taken affect.
                                                           
                                                          $ cat /proc/cmdline\nBOOT_IMAGE=(hd0,gpt2)/vmlinuz-6.1.41-dfl root=/dev/mapper/rhel_bapvedell028-root ro crashkernel=auto resume=/dev/mapper/rhel_bapvedell028-swap rd.lvm.lv=rhel_bapvedell028/root rd.lvm.lv=rhel_bapvedell028/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\n
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#50-opae-software-development-kit","title":"5.0 OPAE Software Development Kit","text":"
                                                          The OPAE SDK software stack sits in user space on top of the OFS kernel drivers. It is a common software infrastructure layer that simplifies and streamlines integration of programmable accelerators such as FPGAs into software applications and environments. OPAE consists of a set of drivers, user-space libraries, and tools to discover, enumerate, share, query, access, manipulate, and reconfigure programmable accelerators. OPAE is designed to support a layered, common programming model across different platforms and devices.
                                                           
                                                          The OPAE SDK source code is contained within a single GitHub repository hosted at the OPAE GitHub. This repository is open source.
                                                           
                                                          You may choose to use the supplied Python 3 installation script to handle OPAE SDK installation. This script ships with a README detailing execution instructions on the OFS 2023.3.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#51-opae-sdk-build-environment-setup","title":"5.1 OPAE SDK Build Environment Setup","text":"
                                                          Ensure the local environment matches the supported Operating System discussed in section Table 1-3: Software Version Summary. This installation process assumes you have access to an internet connection in order to pull specific GitHub repositories, and to satisfy package dependencies.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#511-installing-the-opae-sdk-with-pre-built-packages","title":"5.1.1 Installing the OPAE SDK with Pre-Built Packages","text":"
                                                          You can skip the entire build process and use a set of pre-built binaries supplied by Intel. Visit the OFS 2023.3 and navigate to the bottom of the page, under the Assets tab you will see a file named opae-2.10.0-1.x86_64-<>_<>.tar.gz. Download this package and extract its contents: 
                                                          $ dnf install --enablerepo=codeready-builder-for-rhel-8-x86_64-rpms -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel\n\n$ pip3 install --upgrade --prefix=/usr pip setuptools pybind11\n\n$ tar xf opae-2.10.0-1.x86_64-<<date>>_<<build>>.tar.gz\n
                                                           
                                                          For a fast installation you can delete the source RPM as it isn't necessary, and install all remaining OPAE RPMs:
                                                           
                                                          $ rm opae-*.src.rpm\n$ sudo dnf localinstall opae*.rpm\n
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#512-building-and-installing-the-opae-sdk-from-source","title":"5.1.2 Building and Installing the OPAE SDK from Source","text":"
                                                          1. Before OPAE SDK installation the user must remove any prior OPAE frameworks.  To remove these packages:
                                                           
                                                           
                                                          $ sudo dnf remove opae*\n
                                                           
                                                          2. It is recommended you create an empty top-level directory for their OFS related repositories to keep the working environment clean. All steps in this installation will use a generic top-level directory at /home/user/OFS/. If you have created a different top-level directory, replace this path with your custom path.
                                                           
                                                          3. Initialize an empty git repository and clone the tagged OPAE SDK source code:
                                                           
                                                           
                                                          $ cd /home/user/OFS/\n$ git init\n$ git clone https://github.com/OPAE/opae-sdk.git\n$ cd opae-sdk\n$ git checkout tags/2.10.0-1 -b release/2.10.0\n
                                                           
                                                          4. Verify that the correct tag has been checkout out:
                                                           
                                                          $ git describe --tags\n2.10.0-1\n
                                                           
                                                          5. Build the OPAE SDK source code, and pack it into several local RPM packages. Building the code into packages allows for easier installation and removal.
                                                           
                                                          $ cd /home/user/OFS\n\n$ podman pull registry.access.redhat.com/ubi8:8.6\n$ podman run -ti -v \"$PWD\":/src:Z -w /src registry.access.redhat.com/ubi8:8.6\n\n# Everything after runs within container:\n\n$ dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n\n$ dnf install --enablerepo=codeready-builder-for-rhel-8-x86_64-rpms -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel\n\n$ pip3 install --upgrade --prefix=/usr pip setuptools pybind11\n\n$ ./opae-sdk/packaging/opae/rpm/create unrestricted\n\n$ exit\n
                                                           
                                                          6. After a successful compile there should be 8 packages present:
                                                           
                                                          $ ls | grep rpm\n\nopae-2.10.0-1.el8.src.rpm\nopae-2.10.0-1.el8.x86_64.rpm\nopae-debuginfo-2.10.0-1.el8.x86_64.rpm\nopae-debugsource-2.10.0-1.el8.x86_64.rpm\nopae-devel-2.10.0-1.el8.x86_64.rpm\nopae-devel-debuginfo-2.10.0-1.el8.x86_64.rpm\nopae-extra-tools-2.10.0-1.el8.x86_64.rpm\nopae-extra-tools-debuginfo-2.10.0-1.el8.x86_64.rpm\n
                                                           Remove the opae-2.10.0-1.el8.src.rpm file as it is not used. 
                                                          $ rm opae-2.10.0-1.el8.src.rpm\n
                                                           
                                                          7. Install the user-built OPAE SDK packages:
                                                           
                                                          $ sudo dnf clean all\n\n$ sudo dnf localinstall -y opae*.rpm\n
                                                           
                                                          8. Check that all packages have been installed:
                                                           
                                                          $ rpm -qa | grep opae\n\nopae-extra-tools-2.10.0-1.el8.x86_64\nopae-debugsource-2.10.0-1.el8.x86_64\nopae-2.10.0-1.el8.x86_64\nopae-extra-tools-debuginfo-2.10.0-1.el8.x86_64\nopae-debuginfo-2.10.0-1.el8.x86_64\nopae-devel-2.10.0-1.el8.x86_64\nopae-devel-debuginfo-2.10.0-1.el8.x86_64\n
                                                           
                                                          You can query information about each installed package using rpm -qi <package__name>.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#513-fpga-device-access-permissions","title":"5.1.3 FPGA Device Access Permissions","text":"
                                                          Access to FPGA accelerators and devices is controlled using file access permissions on the Intel\u00ae FPGA device files, /dev/dfl-fme.* and /dev/dfl-port.*, as well as to the files reachable through /sys/class/fpga_region/.
                                                           
                                                          In order to allow regular (non-root) users to access accelerators, you need to grant them read and write permissions on /dev/dfl-port.* (with * denoting the respective socket, i.e. 0 or 1). E.g.:
                                                           
                                                          sudo chmod a+rw /dev/dfl-port.0\n
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#514-memlock-limit","title":"5.1.4 Memlock limit","text":"
                                                          Depending on the requirements of your application, you may also want to increase the maximum amount of memory a user process is allowed to lock. The exact way to do this depends on your Linux distribution.
                                                           
                                                          You can check the current memlock limit using
                                                           
                                                          ulimit -l\n
                                                           
                                                          A way to permanently remove the limit for locked memory for a regular user is to add the following lines to your /etc/security/limits.conf:
                                                           
                                                          user1    hard   memlock           unlimited\nuser1    soft   memlock           unlimited\n
                                                           
                                                          This removes the limit on locked memory for user user1. To remove it for all users, you can replace user1 with *:
                                                           
                                                          *    hard   memlock           unlimited\n*    soft   memlock           unlimited\n
                                                           
                                                          Note that settings in the /etc/security/limits.conf file don't apply to services. To increase the locked memory limit for a service you need to modify the application's systemd service file and add the line:
                                                           
                                                          [Service]\nLimitMEMLOCK=infinity\n
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#52-opae-tools-overview","title":"5.2 OPAE Tools Overview","text":"
                                                          The OPAE SDK user-space tools sit upon the kernel-space DFL drivers. In order to use OPAE SDK functionality the user needs to complete the steps outlined in the previous section 4.1 OFS DFL Kernel Driver Environment Setup before attempting to run any OPAE commands or flows. You must have at least one D5005 card with the appropriate FIM present in your system. The steps to read and load a new FIM version are discussed in section 6.1 Programming the OFS FIM). After both the DFL kernel-space drivers have been installed and the FIM has been upgraded, you may proceed to test the OPAE commands discussed below.
                                                           
                                                          This section covers basic functionality of the commonly used OPAE tools and their expected results. These steps may also be used to verify that all OFS software installation has been completed successfully. A complete overview of the OPAE tools can be found on the OPAE GitHub and in your cloned GitHub repo at <your path>/opae-sdk/doc/src/fpga_tools. More commands are listed than are defined in the list below - most of these are called by other tools and do not need to be called directly themselves.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#521-fpgasupdate","title":"5.2.1 fpgasupdate","text":"
                                                          The fpgasupdate tool updates the Intel Max10 BMC image and firmware, root entry hash, and FPGA Static Region (SR) and user image (PR). The fpgasupdate will only accept images that have been formatted using PACsign. If a root entry hash has been programmed onto the board, then the image will also need to be signed using the correct keys. Please refer to the [Security User Guide: Intel Open FPGA Stack] for information on created signed images and on programming and managing the root entry hash.
                                                           
                                                          The Intel FPGA PAC ships with a factory and user programmed image for both the FIM and BMC FW and RTL on all cards.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#table-5-1-fpgasupdate-overview","title":"Table 5-1: fpgasupdate Overview","text":"
                                                          Synopsis:
                                                           
                                                          fpgasupdate [--log-level=<level>] file [bdf]\n
                                                           
                                                          Description: The fpgasupdate command implements a secure firmware update.
                                                           Command args (optional) Description --log-level  Specifies the log-level which is the level of information output to your command tool. The following seven levels  are available: state, ioctl, debug, info, warning, error, critical. Setting --log-level=state provides the most verbose output. Setting --log-level=ioctl provides the second most information, and so on. The default level is info. file Specifies the secure update firmware file to be programmed. This file may be to program a static region (SR), programmable region (PR), root entry hash, key cancellation, or other device-specific firmware. bdf The PCIe address of the PAC to program. bdf is of the form [ssss:]bb:dd:f, corresponding to PCIe segment, bus, device, function. The segment is optional. If you do not specify a segment, the segment defaults to 0000. If the system has only one PAC you can omit the bdf and let fpgasupdate  determine the address automatically."},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#522-fpgainfo","title":"5.2.2 fpgainfo","text":"
                                                          Synopsis:
                                                           
                                                             fpgainfo [-h] [-S <segment>] [-B <bus>] [-D <device>] [-F <function>] [PCI_ADDR]\n            {errors,power,temp,fme,port,bmc,mac,phy,security}\n
                                                           
                                                          Description:  Displays FPGA information derived from sysfs files. The command argument is one of the following: errors, power, temp, port, fme, bmc, phy or mac, security. Some commands may also have other arguments or options that control their behavior.
                                                           
                                                          For systems with multiple FPGA devices, you can specify the BDF to limit the output to the FPGA resource with the corresponding PCIe configuration. If not specified, information displays for all resources for the given command.
                                                           Command args (optional) Description --help, -h Prints help information and exits. --version, -v Prints version information and exits. -S, --segment PCIe segment number of resource. -B, --bus PCIe bus number of resource. -D, --device PCIe device number of resource. -F, --function PCIe function number of resource. errors {fme, port, all} --clear, -c First agument to the errors command specifies the resource type to display in human readable format. The second optional argument clears errors for the given FPGA resource. power Provides total power in watts that the FPGA hardware consumes temp Provides FPGA temperature values in degrees Celsius port Provides information about the port fme Provides information about the FME bmc Provides BMC sensors information mac Provides information about MAC ROM connected to FPGA security Provides information about the security keys, hashes, and flash count, if available. 
                                                          Note: Your Bitstream ID and PR Interface Id may not match the below examples.
                                                           
                                                          The following examples walk through sample outputs generated by fpgainfo.
                                                           
                                                          $ sudo fpgainfo fme\n\nOpen FPGA Stack Platform\nBoard Management Controller, MAX10 NIOS FW version: 2.0.14\nBoard Management Controller, MAX10 Build version: 2.0.8\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 288511863935352239\nBitstream Version                : 4.0.1\nPr Interface Id                  : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98\nBoot Page                        : user\n
                                                           
                                                          $ sudo fpgainfo bmc\n\nOpen FPGA Stack Platform\nBoard Management Controller, MAX10 NIOS FW version: 2.0.14\nBoard Management Controller, MAX10 Build version: 2.0.8\n//****** BMC SENSORS ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 288511863935352239\nBitstream Version                : 4.0.1\nPr Interface Id                  : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98\n( 1) VCCERAM Voltage                                    : 0.90 Volts\n( 2) VCCT Temperature                                   : 29.00 Celsius\n( 3) 12v Backplane Voltage                              : 12.17 Volts\n( 4) VCCERAM Current                                    : 0.18 Amps\n( 5) FPGA Transceiver Temperature                       : 36.50 Celsius\n( 6) QSFP1 Supply Voltage                               : 0.00 Volts\n( 7) 3.3v Temperature                                   : 29.00 Celsius\n( 8) 12v Backplane Current                              : 2.28 Amps\n( 9) RDIMM3 Temperature                                 : 25.50 Celsius\n(10) VCCR Voltage                                       : 1.12 Volts\n(11) Board Inlet Air Temperature                        : 24.50 Celsius\n(12) 1.8v Temperature                                   : 27.50 Celsius\n(13) 12v AUX Voltage                                    : 12.14 Volts\n(14) VCCR Current                                       : 0.55 Amps\n(15) RDIMM0 Temperature                                 : 24.50 Celsius\n(16) FPGA Core Voltage                                  : 0.88 Volts\n(17) VCCERAM Temperature                                : 27.50 Celsius\n(18) 12v AUX Current                                    : 1.19 Amps\n(19) QSFP0 Temperature                                  : N/A\n(20) VCCT Voltage                                       : 1.12 Volts\n(21) FPGA Core Current                                  : 11.60 Amps\n(22) FPGA Core Temperature                              : 42.50 Celsius\n(23) 12v Backplane Temperature                          : 24.00 Celsius\n(24) VCCT Current                                       : 0.14 Amps\n(25) RDIMM1 Temperature                                 : 24.00 Celsius\n(26) 3.3v Voltage                                       : 3.30 Volts\n(27) VCCR Temperature                                   : 33.50 Celsius\n(28) 1.8v Voltage                                       : 1.80 Volts\n(29) 3.3v Current                                       : 0.32 Amps\n(30) Board Exhaust Air Temperature                      : 26.00 Celsius\n(31) 12v AUX Temperature                                : 25.00 Celsius\n(32) QSFP0 Supply Voltage                               : 0.00 Volts\n(33) QSFP1 Temperature                                  : N/A\n(34) 1.8v Current                                       : 0.54 Amps\n(35) RDIMM2 Temperature                                 : 26.00 Celsius\n
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#523-rsu","title":"5.2.3 rsu","text":"
                                                          The rsu performs a Remote System Update operation on a device, given its PCIe address. A rsu operation sends an instruction to the device to trigger a power cycle of the card only. This will force reconfiguration from flash for either the BMC or FPGA.
                                                           
                                                          The Intel FPGA PAC contains a region of flash the user may store their FIM image. After an image has been programmed with fpgasupdate the user may choose to perform rsu to update the image on the device.
                                                           
                                                          Note: The D5005 platform only supports storing and configuring a single user image from flash for the FPGA. It does not include support for the user1/user2 partitions as shown in other OFS related acceleration boards.
                                                           
                                                          rsu Overview
                                                           
                                                          Synopsis
                                                           
                                                          rsu [-h] [-d] {bmc,bmcimg,retimer,sdm,fpgadefault} [PCIE_ADDR]\n
                                                           
                                                          rsu bmc --page=(user) [PCIE_ADDR]\nrsu retimer [PCIE_ADDR]\nrsu sdm [PCIE_ADDR]\n
                                                           
                                                          Perform RSU (remote system update) operation on PAC device given its PCIe address. An RSU operation sends an instruction to the device to trigger a power cycle of the card only. This will force reconfiguration from flash for either BMC, Retimer, SDM, (on devices that support these) or the FPGA.
                                                           
                                                          Note: As a result of using the rsu command, the host rescans the PCI bus and may assign a different Bus/Device/Function (B/D/F) value than the originally assigned value.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#525-bitstreaminfo","title":"5.2.5 bitstreaminfo","text":"
                                                          Displays authentication information contained with each provided file on the command line. This includes any JSON header strings, authentication header block information, and a small portion  of the payload. The binary is installed by default at /usr/bin/bitstreaminfo.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#526-hssi","title":"5.2.6 hssi","text":"
                                                          The hssi application provides a means of interacting with the 10G and with the 100G HSSI AFUs. In both 10G and 100G operating modes, the application initializes the AFU, completes the desired transfer as described by the mode-specific options. Only the hssi_10g MODE is currently supported. An example of this command's output can be found in section 5.2.9 Running the Host Exerciser Modules. The binary is installed by default at /usr/bin/hssi.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#527-opaeio","title":"5.2.7 opae.io","text":"
                                                          Opae.io is a interactive Python environment packaged on top of libopaevfio.so, which provides user space access to PCIe devices via the vfio-pci driver. The main feature of opae.io is its built-in Python command interpreter, along with some Python bindings that provide a means to access Configuration and Status Registers (CSRs) that reside on the PCIe device. opae.io has two operating modes: command line mode and interactive mode. An example of this command's output can be found in section 5.2.9 Running the Host Exerciser Modules. The binary is installed by default at /usr/bin/opae.io.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#528-host_exerciser","title":"5.2.8 host_exerciser","text":"
                                                          The host exerciser is used to exercise and characterize the various host-FPGA interactions eg. MMIO, Data transfer from host to FPGA , PR, host to FPGA memory etc. An example of this command's output can be found in section 5.2.9 Running the Host Exerciser Modules. The binary is installed by default at /usr/bin/host_exerciser. For more information refer to - Host Exerciser
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#529-running-the-host-exerciser-modules","title":"5.2.9 Running the Host Exerciser Modules","text":"
                                                          The reference FIM and unchanged compilations contain Host Exerciser Modules (HEMs). These are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. 
                                                           
                                                          Note: Before continuing, if huge pages are not set refer to section 4.2, step 7.
                                                           
                                                          There are three HEMs present in the OFS FIM - HE-LPBK, HE-HSSI, and HE-MEM. These exercisers are tied to three different VFs that must be enabled before they can be used. The user should enable the VF for each HEM using the below steps:
                                                           
                                                          1. Determine the BDF of the Intel\u00ae FPGA PAC D5005 card.
                                                           
                                                          The PCIe BDF address is initially determined when the server powers on. The user can determine the addresses of all Intel\u00ae FPGA PAC D5005 boards using lspci:
                                                           
                                                          # lspci -d :bcce\n\n3b:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                                                           
                                                          Note: Before continuing, if you updated your OFS installation, please also update your PAC FIM to run HEMs.
                                                           
                                                          2. Enable three VFs.
                                                           
                                                          In this example, the BDF address is 0000:3b:00.0. With this information the user can now enable three VFs with the following:
                                                           
                                                          # sudo pci_device 0000:3b:00.0 vf 3\n
                                                           
                                                          3. Verify that all three VFs have been created.
                                                           
                                                          # lspci -s 3b:00\n\n3b:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n3b:00.1 Processing accelerators: Intel Corporation Device bccf (rev 01)\n3b:00.2 Processing accelerators: Intel Corporation Device bccf (rev 01)\n3b:00.3 Processing accelerators: Intel Corporation Device bccf (rev 01)\n
                                                           
                                                          4. Bind the 3 VFs to the vfio-pci driver.
                                                           
                                                          $ sudo opae.io init -d 0000:3b:00.1 $USER\n\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:3b:00.1 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:3b:00.1 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:3b:00.1 is 142\nAssigning /dev/vfio/142 to $USER:$USER\nChanging permissions for /dev/vfio/142 to rw-rw----\n\n\n$ sudo opae.io init -d 0000:3b:00.2 $USER\n\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:3b:00.2 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:3b:00.2 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:3b:00.2 is 143\nAssigning /dev/vfio/143 to $USER:$USER\nChanging permissions for /dev/vfio/143 to rw-rw----\n\n\n$ sudo opae.io init -d 0000:3b:00.3 $USER\n\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:3b:00.3 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:3b:00.3 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:3b:00.3 is 144\nAssigning /dev/vfio/144 to $USER:$USER\nChanging permissions for /dev/vfio/144 to rw-rw----\n
                                                           
                                                          5. Check that the accelerators are present using fpgainfo. Note your port configuration may differ from the below.
                                                           
                                                          $ sudo fpgainfo port\n\n//****** PORT ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0x603B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : 823c334c-98bf-11ea-bb37-0242ac130002\n//****** PORT ******//\nObject Id                        : 0x403B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n//****** PORT ******//\nObject Id                        : 0x203B000000000000\nPCIe s:b:d.f                     : 0000:3B:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#table-5-5-vf-to-hem-mappings","title":"Table 5-5 VF to HEM Mappings","text":"VF BDF HEM BBBB:DD.1 HE-LB BBBB:DD.2 HE-MEM BBBB:DD.3 He-HSSI 
                                                          HE-MEM / HE-LB
                                                           
                                                          HE-LB is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth. HE-MEM is used to exercise the DDR interface; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host. HE-MEM uses external DDR memory (i.e. EMIF) to store data. It has a customized version of the AVMM interface to communicate with the EMIF memory controller. Both exercisers rely on the user-space tool host_exerciser. The following commands are supported by the HE-LB/HE-MEM OPAE driver program. They may need to be run using sudo privileges, depending on your server configuration.
                                                           
                                                          Basic operations:
                                                           
                                                          $ sudo host_exerciser lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5342\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.067 GB/s\n    Test lpbk(1): PASS\n\n$ sudo host_exerciser --mode lpbk lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5358\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.058 GB/s\n    Test lpbk(1): PASS\n\n$ sudo host_exerciser --mode write lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 0\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 2592\n    Total number of Reads sent: 0\n    Total number of Writes sent: 1024\n    Bandwidth: 6.321 GB/s\n    Test lpbk(1): PASS\n\n$ sudo host_exerciser --mode trput lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 512\n    Host Exerciser numWrites: 513\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 3384\n    Total number of Reads sent: 512\n    Total number of Writes sent: 512\n    Bandwidth: 4.842 GB/s\n    Test lpbk(1): PASS\n
                                                           
                                                          Number of cachelines per request 1, 2, and 4. The user may replace --mode lpbk with read, write, trput. The target lpbk can be replaced with mem:
                                                           
                                                          $ sudo host_exerciser --mode lpbk --cls cl_1 lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5475\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 2.993 GB/s\n    Test lpbk(1): PASS\n\n\n$ sudo host_exerciser --mode lpbk --cls cl_2 lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5356\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.059 GB/s\n    Test lpbk(1): PASS\n\n\n$ sudo host_exerciser --mode lpbk --cls cl_4 lpbk\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 4481\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.656 GB/s\n    Test lpbk(1): PASS\n
                                                           
                                                          Interrupt tests (only valid for mode mem):
                                                           
                                                          $ sudo host_exerciser --interrupt 0 mem\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\nUsing Interrupts\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1026\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5140\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.188 GB/s\n    Test mem(1): PASS\n\n$ sudo host_exerciser --interrupt 1 mem\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\nUsing Interrupts\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1026\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5079\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.226 GB/s\n    Test mem(1): PASS\n\n\n$ sudo host_exerciser --interrupt 2 mem\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\nUsing Interrupts\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1026\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 5525\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.439 GB/s\n    Test mem(1): PASS\n\n\n$ sudo host_exerciser --interrupt 3 mem\n\n    starting test run, count of 1\nAPI version: 1\nAFU clock: 250 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\nUsing Interrupts\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1026\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 4735\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.460 GB/s\n    Test mem(1): PASS\n
                                                           
                                                          HE-HSSI
                                                           
                                                          HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G ethernet AFU and includes a 10G traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic. Context sensitive information is given by the hssi --help command. Help for the 10G specific test is given by hssi hssi_10g --help Example useage:
                                                           
                                                          $ sudo hssi --pci-address 3b:00.3 hssi_10g --eth-ifc s10hssi0 --eth-loopback on --he-loopback=off  --num-packets 100\n\n10G loopback test\n  port: 0\n  eth_loopback: on\n  he_loopback: off\n  num_packets: 100\n  packet_length: 64\n  src_address: 11:22:33:44:55:66\n    (bits):  0x665544332211\n  dest_address: 77:88:99:aa:bb:cc\n    (bits): 0xccbbaa998877\n  random_length: fixed\n  random_payload: incremental\n  rnd_seed0: 5eed0000\n  rnd_seed1: 5eed0001\n  rnd_seed2: 25eed\n  eth: s10hssi0\n
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#60-compiling-ofs-fim","title":"6.0 Compiling OFS FIM","text":"
                                                          Pre-Compiled FIM binaries are at OFS 2023.3 release page and to compile the OFS FIM for Intel\u00ae FPGA PAC D5005 follow the below steps :
                                                           
                                                          1) Compile OFS FIM manually - Steps are provided in the developer guide to compile FIM and generate binaries. Refer to FPGA Interface Manager Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA.
                                                           
                                                          2) Compile OFS FIM using evaluation script - The script guides you to the steps required for compilation via selecting options from the menu. Refer to Platform Evaluation Script: Open FPGA Stack for Intel Stratix 10 FPGA.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#70-programming-the-ofs-fim-and-bmc","title":"7.0 Programming the OFS FIM and BMC","text":"
                                                          Instructions surrounding the compilation and simulation of the OFS FIM have fully moved into the FPGA Interface Manager Technical Reference Manual: Open FPGA Stack for Intel\u00ae Stratix 10\u00ae FPGA.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#71-programming-the-ofs-fim","title":"7.1 Programming the OFS FIM","text":"
                                                          In order to program the OFS FIM, both the OPAE SDK and DFL drivers need to be installed on the host system. Please complete the steps in sections 4.0 OFS DFL Kernel Drivers and 5.0 OPAE Software Development Kit. The OFS FIM version can be identified using the OPAE tool fpgainfo. A sample output of this command is included below.
                                                           
                                                          $ sudo fpgainfo fme\n\nIntel FPGA Programmable Acceleration Card D5005\nBoard Management Controller, MAX10 NIOS FW version: 2.0.14\nBoard Management Controller, MAX10 Build version: 2.0.8\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:3B:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x138D\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 288511863935352239\nBitstream Version                : 4.0.1\nPr Interface Id                  : b2d7971b-dd7e-53c4-a4d0-34e6c9391a98\nBoot Page                        : user\n
                                                           
                                                          Use the value under PR Interface ID to identify that FIM that has been loaded. Refer to the table below for a list of previous FIM releases:
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#table-7-1-previous-fim-releases","title":"Table 7-1 Previous FIM Releases","text":"PR Release PR Interface ID 2023.2 edad864c-99d6-5831-ab67-62bfd81ec654 2022.2 (tag 1.3.0) bf531bcf-a896-5171-ab31-601a4ab754b6 2022.1 Beta (tag: 1.2.0-beta) 2fae83fc-8568-53aa-9157-8f75e9c0ba92 OFS 2.1 Beta (tag: 1.1.0-beta) 99160d37e42a 3f8b586f-c275-594c-92e2-d9f2c23e94d1 OFS 1.0 (tag: ofs-1.0.0) b5f6a71e-daec-59c3-a43a-85567b51fd3f Intel Acceleration Stack for Intel\u00ae FPGA PAC D5005 2.0.1 9346116d-a52d-5ca8-b06a-a9a389ef7c8d 
                                                          If the user's card does not report a PR Interface ID which matches the above table, then a new FIM will need to be programmed.
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#711-programming-the-fim","title":"7.1.1 Programming the FIM","text":"
                                                          1. Download the file d5005_page1_unsigned.bin from OFS 2023.3 release page.
                                                           
                                                          2. Run PACSign to create an unsigned image with added header for use by fpgasupdate
                                                           
                                                          $ PACSign SR -y -v -t UPDATE -s 0 -H openssl_manager -i d5005_page1_unsigned.bin -o d5005_PACsigned_unsigned.bin\n
                                                           
                                                          3. Run fpgasupdate to load the image into the user location of the Intel\u00ae FPGA PAC D5005 FPGA flash, NOTE: use \"sudo fpgainfo fme\" command to find the PCIe address for your card.
                                                           
                                                          $ sudo fpgasupdate d5005_PACsigned_unsigned.bin 3B:00.0\n
                                                           
                                                          4. Run RSU command.
                                                           
                                                          $ sudo rsu bmcimg 0000:3B:00.0\n
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#72-programming-the-bmc","title":"7.2 Programming the BMC","text":"
                                                          1. Download intel-fpga-bmc images(To download OFS Stratix 10 BMC binaries contact Intel Technical Sales Representative)
                                                           
                                                          2. The file unsigned_bmc_fw.bin has the newly binary format. This bitstream is programmed with remote system update (RSU) and the bitstream must be signed with PACSign tool to generate.
                                                           
                                                          3. Run PACSign to create an unsigned image with added header for use by fpgasupdate
                                                           
                                                          $ PACSign BMC -y -v -t UPDATE -s 0 -H openssl_manager -i unsigned_bmc_fw.bin -o PACsigned_unsigned_bmc_fw.bin\n\n2022-04-22 03:07:05,626 - PACSign.log - INFO - OpenSSL version \"OpenSSL 1.1.1k  FIPS 25 Mar 2021\" matches \"1.1.1\"\n2022-04-22 03:07:05,648 - PACSign.log - INFO - Bitstream not previously signed\n2022-04-22 03:07:05,648 - PACSign.log - INFO - platform value is '688128'\n2022-04-22 03:07:05,745 - PACSign.log - INFO - Starting Block 0 creation\n2022-04-22 03:07:05,745 - PACSign.log - INFO - Calculating SHA256\n2022-04-22 03:07:05,747 - PACSign.log - INFO - Calculating SHA384\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Done with Block 0\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Starting Root Entry creation\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Calculating Root Entry SHA\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Starting Code Signing Key Entry creation\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Calculating Code Signing Key Entry SHA\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Code Signing Key Entry done\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Starting Block 0 Entry creation\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Calculating Block 0 Entry SHA\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Block 0 Entry done\n2022-04-22 03:07:05,749 - PACSign.log - INFO - Starting Block 1 creation\n2022-04-22 03:07:05,750 - PACSign.log - INFO - Block 1 done\n2022-04-22 03:07:05,757 - PACSign.log - INFO - Writing blocks to file\n2022-04-22 03:07:05,758 - PACSign.log - INFO - Processing of file 'PACsigned_unsigned_bmc_fw.bin' complete\n
                                                           
                                                          4. Run fpgasupdate to perform an upgrade of the BMC.
                                                           
                                                          $ sudo fpgasupdate PACsigned_unsigned_bmc_fw.bin 3B:00.0\n\n[2022-04-22 03:08:34.15] [WARNING ] Update starting. Please do not interrupt.\n[2022-04-22 03:08:34.15] [INFO    ] updating from file pacsign_unsigned_bmc_fw.bin with size 819968\n[2022-04-22 03:08:34.15] [INFO    ] waiting for idle\n[2022-04-22 03:08:34.15] [INFO    ] preparing image file\n[2022-04-22 03:09:02.18] [INFO    ] writing image file\n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [819968/819968 bytes][Elapsed Time: 0:00:13.01]\n[2022-04-22 03:09:15.20] [INFO    ] programming image file\n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588][Elapsed Time: 0:00:29.03]\n[2022-04-22 03:09:44.24] [INFO    ] update of 0000:3B:00.0 complete\n[2022-04-22 03:09:44.24] [INFO    ] Secure update OK\n[2022-04-22 03:09:44.24] [INFO    ] Total time: 0:01:10.08\n
                                                          "},{"location":"hw/d5005/user_guides/ug_qs_ofs_d5005/ug_qs_ofs_d5005/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                          Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                           
                                                          OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/","title":"Simulation User Guide: Open FPGA Stack for Intel Intel\u00ae Stratix 10\u00ae FPGA","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel FPGA PAC D5005 Intel FPGA Programmable Acceleration Card D5005, A high performance PCI Express (PCIe)-based FPGA acceleration card for data centers. This card is the target platform for the initial OFS release. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to userspace."},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#1-overview","title":"1 Overview","text":""},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#11-about-this-document","title":"1.1 About this Document","text":"
                                                          This document serves as a set-up and user guide for the UVM simulation tool using OFS. After reviewing the document, you will be able to:
                                                           
                                                             
                                                          • Set-up the UVM verification tool suite
                                                            •  
                                                          • Run pre-existing UVM unit tests and also create new UVM tests for your design
                                                            •  
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#2-introduction-to-uvm","title":"2 Introduction to UVM","text":"
                                                          OFS (Open FPGA Stack) provides a UVM (Universal Verification Methodology) environment for the FIM (FPGA Interface Manager) with a modular, reusable, and scalable testbench structure via an API framework.
                                                           
                                                          The framework consists of a FIM Testbench which is UVM compliant and integrates third party VIPs from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this Testbench. UVM RAL (Register Abstaction Layer) is used for CSR (Command and Status Registers) verification.
                                                           
                                                          The qualified verification IPs will help to detect incorrect protocol behavior, help to focus on FIM features and accelerate the verification process.
                                                           
                                                          Verification components include:
                                                           
                                                             
                                                          • FIM monitor to detect correct design behavior
                                                            •  
                                                          • FIM assertions for signal level integrity testing
                                                            •  
                                                          • Arm AMBA Arm\u00ae AMBA\u00ae 4 AXI4 scoreboards to check data integrity
                                                            •  
                                                          • FIM coverage to collect functional data
                                                            •  
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#3-testbench-architecture","title":"3 Testbench Architecture","text":"
                                                          The testbench connects to the full chip that includes major RTL blocks depicted in Figure 1.
                                                           
                                                           
                                                          Figure 1 Testbench Diagram
                                                           
                                                          The major interface is between the Xeon and FPGA where PCIe Verification IP is connected to PCIe Subsystem. Therefore, as a full chip simulation environment, PCIe host VIP is the sole VIP/BFM used. PCIe host VIP connects to PCIe device which resides in FPGA in serial mode. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#4-testbench-infrastructure","title":"4 Testbench Infrastructure","text":""},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#41-traffic-flow","title":"4.1 Traffic Flow","text":"
                                                          PCIe Host, as the master of FPGA, initiates MMIO read/write requests to FPGA to program registers. The PCIe host also passively receives memory requests from FPGA to read from or write to host memory. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#42-link-up-and-enumeration","title":"4.2 Link Up and Enumeration","text":"
                                                          With serial mode connection between PCIe host and device, link training and enumeration has to be done before the regular traffic starts. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#421-link-up","title":"4.2.1 Link Up","text":"
                                                          Linkup sequence(pcie_device_bring_up_link_sequence) is part of configure sequence(ofs_config_seq), which is started in UVM configure phase. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#422-enumeration","title":"4.2.2 Enumeration","text":"
                                                          PCIe host driver needs to retrieve information from the device hard IP and program necessary configuration space registers, such as PF/VF BAR values. This is done in enumerate_seq, which follows link up sequence in configure sequence(ofs_config_seq). 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#423-pfvf-bar","title":"4.2.3 PF/VF BAR","text":"
                                                          PF0 BAR0 is set in the base sequence and can be randomized. During enumeration, PF0 BAR0, along with PCIe device hard IP configuration, derives other PF and VF BAR values. These BAR values are stored into base sequence variables and can be used throughout any test sequences that extend the base sequence. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#43-mmio-apis","title":"4.3 MMIO APIs","text":"
                                                          The base sequence provides APIs for 32-bit and 64-bit MMIO read/write accesses, as well as blocking or non-blocking for MMIO read as described in Table 1. The users can use MMIO APIs without knowing the underlining PCIe sequence items.
                                                           Name API 32-bit MMIO Write task mmio_write32(input bit [63:0] addr_, input bit [31:0] data_); 64-bit MMIO Write task mmio_write64(input bit [63:0] addr_, input bit [63:0] data_); 32-bit MMIO Read task mmio_read32(input bit [63:0] addr_, output bit [31:0] data_, input blocking_ = 1); 64-bit MMIO Read task mmio_read64(input bit [63:0] addr_, output bit [63:0] data_, input blocking_ = 1); 
                                                          Table 1 MMIO APIs
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#44-ral","title":"4.4 RAL","text":"
                                                          UVM RAL is integrated in the testbench providing alternative ways of accessing CSRs in test sequences. RAL is generated from an excel format CSR specification where register name, field, offset, bitmap, attribute, and description are specified.
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#45-vip-dut-connection","title":"4.5 VIP DUT Connection","text":"
                                                          PCIe host verification IP and DUT connection is achieved by connecting 16 bits lanes. The module for connection from VIP is svt_pcie_device_agent_serdes_x16_x8g_hdl.  
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#5-test-plan","title":"5 Test Plan","text":"
                                                          The test plan consists of four major categories: MMIO path, HE-LB, HE-MEM, HE-HSSI and interrupt tests. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#51-mmio-path","title":"5.1 MMIO Path","text":"
                                                          The tests under this category exercise MMIO path including all destination functions or blocks as well as PF/VF mux and different fabrics. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#52-he-lb","title":"5.2 HE-LB","text":"
                                                          The tests under this category target HE-LB function only. Software which test sequences needs to configure HE-LB CSRs before starting it. These CSRs include SRC_ADDR, DST_ADDR, DSM_ADDR, NUM_LINES, CFG etc.   
                                                           
                                                          If HE-LB is configured to have memory read transactions, PCIe host memory has to be initialized before HE-LB is started. This is done by svt_pcie_mem_target_service sequence. In other words, PCIe host VIP programs its internal memory model entries in backdoor way. The same process applies to DSM memory entry. 
                                                           
                                                          Once HE-LB is started, HE-LB will function based on what it is programmed to do. When HE-LB is done with all necessary memory transactions, it will perform a final memory write to DSM memory entry. Since the software does not know when hardware is done, software polls DSM memory entry periodically until the DSM status bit is asserted. 
                                                           
                                                          For loopback mode, data is compared between source buffer and destination buffer in host memory. 
                                                           
                                                          RTL statistic counters are also compared against the corresponding variables inside the test sequence at the end of the simulation. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#53-he-mem","title":"5.3 HE-MEM","text":"
                                                          HE-MEM tests are duplicates from HE-LB with MMIO to CSRs targeting HE-MEM instead of HE-LB. 
                                                           
                                                          The DDR simulation model is inside memory controller IP when being generated.  
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#54-he-hssi","title":"5.4 HE-HSSI","text":"
                                                          HE-HSSI has indirect registers that are associated with HSSI subsystem, MMIO for indirect registers is different from other functions. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#541-indirect-registers","title":"5.4.1 Indirect Registers","text":"
                                                          To obtain access to indirect registers, either reading or writing, a MMIO write must be performed.
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#542-tx-loopback","title":"5.4.2 TX Loopback","text":"
                                                          In TX loopback, HE-HSSI initiates ethernet packets to HSSI subsystem and the packets are looped back to HE-HSSI. The loopback is achieved by hard-wiring HSSI TX and RX lanes. This is done inside RTL and for simulation purposes only. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#55-interrupt-test","title":"5.5 Interrupt Test","text":"
                                                          The test plan covers the basic interrupt flow for FME error, PORT error and user AFU interrupts. The MSI-X table must be programmed in PF0 BAR4. Corresponding PBA bit is expected to be asserted. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#56-performance-test","title":"5.6 Performance Test","text":"
                                                          Performance tests are derived from HE-LB tests and they are directed tests. At the end of the simulation, performance number is calculated and printed to terminal and a log file. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#57-csr-test","title":"5.7 CSR Test","text":"
                                                          CSR consists of two parts. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#571-reset-value-check","title":"5.7.1 Reset Value Check","text":"
                                                          Front-door MMIO read data is compared against RAL register reset value out of reset. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#572-rw-attribute-csr","title":"5.7.2 RW Attribute CSR","text":"
                                                          MMIO write-read-compare is performed after reset value check for RW attribute CSRs. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#6-checking-mechanism","title":"6 Checking Mechanism","text":"
                                                          Since there is only PCIe host verification component in testbench, data checking is done by a self-checking mechanism in the test sequence.  
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#61-protocol-violation","title":"6.1 Protocol Violation","text":"
                                                          PCIe host VIP has built-in protocol checking on TLP received from FPGA. Abnormal responses are also flagged by the VIP. 
                                                           
                                                          Internal AXI Streaming interface has integrated RTL assertion to check AXI Streaming protocol violations. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#62-data-checking","title":"6.2 Data Checking","text":"
                                                          Data checking is done by self-checking inside a test sequence. MMIO write/read/compare to read-writable CSRs is done inside a sequence. 
                                                           
                                                          For memory transactions initiated by functions, backdoor reads from host memory on source buffer and destination buffer is done inside a sequence. Data is compared in case of loopback mode. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#63-counter-checking","title":"6.3 Counter Checking","text":"
                                                          RTL statistic counters records the number of transactional information that can be read at the end of the simulation and compared against the test expected number. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#64-afu-error-csr","title":"6.4 AFU Error CSR","text":"
                                                          AFU interface handler provides an error log for illegal transactions that can be read at the end of the simulation. 
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#7-uvm-set-up","title":"7 UVM set-up","text":"
                                                          To run the tutorial steps in this guide requires the following development environment:
                                                           Item Version Intel Quartus Prime Pro Intel Quartus Prime Pro 23.3 Simulator (VCS) Synopsys VCS P-2019.06-SP2-5 or newer for UVM simulation of top level FIM Simulator (Questasim) Questasim 2023.2 or newer for UVM simulation of top level FIM"},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#71-uvm-prerequisite","title":"7.1 UVM Prerequisite","text":"
                                                          Retrieve OFS repositories.
                                                           
                                                          The OFS FIM source code is included in the GitHub repository. Create a new directory to use as a clean starting point to store the retrieved files.  The following is a short description of each repository, followed by the git commands for cloning.  The instructions section uses the HTTPS git method for cloning repositories.  Cloning the repo using the HTTPS method requires a personal access token.  Please see this blog post for information about obtaining a personal access token Token authentication requirements for Git operations.
                                                           
                                                          Navigate to the location for storage of OFS source, create the top-level source directory and clone OFS repositories.
                                                           
                                                          $ mkdir ofs-2023.3-2\n$ cd ofs-2023.3-2\n$ export OFS_BUILD_ROOT=$PWD\n$ git clone --recurse-submodules https://github.com/OFS/ofs-d5005.git\n\nCloning into 'ofs-d5005' ...\nUsername for 'https://github.com': <<Enter your git hub username>>\nPassword for 'https://<<Your username>>': <<Enter your personal access token>>\nremote: Enumerating objects:  ....\n...\n...\nResolving deltas  ..., done.\n\n$ cd ofs-d5005\n$ git checkout tags/release/ofs-2023.3\n
                                                           
                                                          Verify that the correct tag/branch have been checked out        
                                                           
                                                          $ git describe --tags\n\n$ release/ofs-2023.3\n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#72-license-requirements","title":"7.2 License Requirements","text":"
                                                          The FIM Testbench is UVM compliant and integrates third party VIPs from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this TB. UVM RAL (Register Abstraction Layer) is used for CSR Verification.
                                                           
                                                          The Qualified Verification IPs will help to detect incorrect protocol behavior easily, help to focus on BBS features and accelerate the verification process.
                                                           
                                                             
                                                          • VCS & DVE
                                                            •  
                                                          • SNPS-Assertions
                                                            •  
                                                          • Verdi
                                                            •  
                                                          • VerdiCoverage
                                                            •  
                                                          • VerdiSimDB
                                                            •  
                                                          • VerdiTransactionDebugUltra
                                                            •  
                                                          • VIP-AMBA-AXI-SVT
                                                            •  
                                                          • VIP-AMBA-STREAM-SVT
                                                            •  
                                                          • VIP-PCIE-SVT
                                                            •  
                                                          • VIP-PCIE-TS-SVT
                                                            •  
                                                          • VIP-PCIE-G3-OPT-SVT
                                                            •  
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#73-software-tools-requirements","title":"7.3 Software Tools Requirements","text":"
                                                          The following tools are required for successful UVM set-up
                                                           
                                                             
                                                          • Python 3.6.8
                                                            •  
                                                          • Synopsys PCIE and AMBA AXI UVM VIP Q-2020.03A License
                                                            •  
                                                          • Synopsys Verdi R-2020.12-SP2 License         Note: Makefile can be modified to use DVE instead of Verdi
                                                            •  
                                                           
                                                             
                                                          • VCS R-2020.12-SP2 License 
                                                            •  
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#74-creating-a-software-tools-script","title":"7.4 Creating a Software Tools Script","text":"
                                                          The UVM tool set-up is best done by creating a simple set-up script so all applicable tools are sourced before running the tests.
                                                           
                                                          The following environment variables can be pasted into a script and used prior to running the UVM verification environment
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#license-files","title":"License Files","text":"
                                                          export LM_LICENSE_FILE=\nexport SNPSLMD_LICENSE_FILE=\n
                                                           
                                                          The license environment variables LM_LICENSE_FILE and SNPSLMD_LICENSE_FILE can point to a server license on your system.
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#general-environment-variables","title":"General Environment Variables","text":"
                                                          export OFS_BUILD_ROOT=$PWD\nexport OFS_ROOTDIR=<user_path>/ofs-d5005\nexport WORKDIR=$OFS_ROOTDIR\n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#quartus-tools","title":"Quartus Tools","text":"
                                                          export QUARTUS_HOME=<user_path>/intelFPGA_pro/23.3/quartus\nexport QUARTUS_ROOTDIR=$QUARTUS_HOME\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$PATH\n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-verification-tools","title":"Synopsys Verification Tools","text":"
                                                          export DESIGNWARE_HOME=<user_path>/synopsys/vip_common/vip_Q-2020.03A\nexport PATH=$DESIGNWARE_HOME/bin:$PATH\nexport UVM_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport PATH=$VCS_HOME/bin:$PATH\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim-verification-tools","title":"QuestaSIM Verification Tools","text":"
                                                          export MTI_HOME=<user_path>/mentor/questasim/2023.2/linux64\nexport PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\nexport QUESTA_HOME=$MTI_HOME\n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#8-running-a-uvm-simulation-test-and-analysing-results","title":"8 Running a UVM Simulation Test and Analysing Results","text":""},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#81-simulation","title":"8.1 Simulation","text":"
                                                          The default simulator used in this document is Synopsys VCS-MX but there will be references to Questasim. Users can refer to the options and adopt the options for other simulators. 
                                                           
                                                          The script is a makefile that calls vlogan, vcs and simv for compilation, elaboration and simulation, respectively
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#82-file-structure","title":"8.2 File Structure","text":"
                                                          After cloning the repo, the verification and ofs-common directories contain all UVM verification related files. The directory structure is shown in Figure 2 below.
                                                           
                                                           
                                                          Figure 2 UVM Verification Directory File Structure
                                                           
                                                          ofs-d5005/testbench has a testbench, uvm env, virtual sequencer, RAL etc. 
                                                           
                                                          ofs-d5005/verification/tests contains all uvm tests and sequences. 
                                                           
                                                          Users can run the simulation under \"ofs-d5005/verification/scripts\" directory and the simulation result is outputted to a \"sim\" directory for Synopsys VCS or \"sim_msim\" for Questasim.
                                                           
                                                          The simulation result folder is named after the test name with increasing suffix number. If user runs the same test multiple times, the suffix is incremented by 1 each time.
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#83-uvm-test-suite","title":"8.3 UVM Test Suite","text":"
                                                          The UVM environment contains a variety of tests that have been developed to test out the FIM portion of OFS.
                                                           
                                                          The table below lists out the \"Test Name\" which will be used on the command line to execute the test, the \"Test Scenario\" and the \"Checking Criteria\".
                                                           
                                                          Tests are located at ofs-d5005/verification/tests
                                                           Test Name DUT Scope Test Scenario Checking Criteria dfh_walking_test DFH DHF walking offset checking, eol checking flr_reset_test FLR Reset FLR reset to all PFs Reset checking flr_vf0_reset_test FLR Reset FLR reset to VF0 Reset checking flr_vf1_reset_test FLR Reset FLR reset to VF1 Reset checking flr_vf2_reset_test FLR Reset FLR reset to VF2 Reset checking fme_csr_test FME CSR CSR accesses data checking fme_hemem_intr_test Interrupt FME and HE MEM interrupt Interrupts assertion, PBA bits check fme_intr_test Interrupt FME error interrupt Interrupts assertion, PBA bits check he_hssi_csr_test HE-HSSI CSR accesses for HSSI data checking he_hssi_err_test HE-HSSI Error Cases counter checking he_hssi_rx_lpbk_test HE-HSSI RX loopback data checking he_hssi_tx_lpbk_test HE-HSSI TX loopback counter checking he_lpbk_cont_test HE-LPBK Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking he_lpbk_long_rst_test HE-MEM Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_lpbk_long_test HE-MEM Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_lpbk_port_rst_test HE-LPBK Loopback mode. Randomize num_lines, addresses, req_len with port rst data checking he_lpbk_rd_cont_test HE-LPBK Read only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_lpbk_rd_test HE-LPBK Read only mode. Randomize num_lines, addresses, req_len counter checking he_lpbk_reqlen1_test HE-LPBK Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_lpbk_reqlen2_test HE-LPBK Loopback mode. 128 CLs, req_len = 2CL, random addresses. data checking, counter checking he_lpbk_reqlen4_test HE-LPBK Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_lpbk_reqlen8_test HE-LPBK Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_lpbk_test HE-LPBK Loopback mode. Randomize num_lines, addresses, req_len data checking, counter checking he_lpbk_thruput_contmode_test HE-LPBK Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses req_len data checking, counter checking he_lpbk_thruput_test HE-LPBK Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_lpbk_wr_cont_test HE-LPBK Write only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_lpbk_wr_test HE-LPBK Write only mode. Randomize num_lines, addresses, req_len counter checking he_mem_cont_test HE-MEM Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking he_mem_lpbk_long_rst_test HE-LPBK Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_mem_lpbk_long_test HE-LPBK Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_mem_lpbk_reqlen1_test HE-MEM Loopback mode. 128 CLs, req_len = 1CL, random addresses. data checking, counter checking he_mem_lpbk_reqlen2_test HE-MEM Loopback mode. 128 CLs, req_len = 2CL, random addresses. data checking, counter checking he_mem_lpbk_reqlen4_test HE-MEM Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_mem_lpbk_test HE-MEM Loopback mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_rd_cont_test HE-MEM Read only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_rd_test HE-MEM Read only mode. Randomize num_lines, addresses, req_len counter checking he_mem_thruput_contmode_test HE-MEM Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_thruput_test HE-MEM Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_mem_wr_cont_test HE-MEM Write only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_wr_test HE-MEM Write only mode. Randomize num_lines, addresses, req_len counter checking he_random_long_test All HE's Enable all HEs and randomize modes for multiple iterations data checking if in lpbk mode, counter checking he_random_test All HEs Enable all HEs and randomize modes data checking if in lpbk mode, counter checking hehssi_csr_test HE-HSSI CSR accesses for Traffic Control Mail box registers data checking helb_csr_test HE-LPBK CSR accesses data checking helb_rd_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Read only mode data checking, counter checking helb_rd_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Read only mode data checking, counter checking helb_rd_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Read only mode data checking, counter checking helb_thruput_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Thruput mode data checking, counter checking helb_thruput_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Thruput mode data checking, counter checking helb_thruput_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Thruput mode data checking, counter checking helb_wr_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Write only mode data checking, counter checking helb_wr_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Write only mode data checking, counter checking helb_wr_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Write only mode data checking, counter checking hemem_csr_test HE-MEM CSR accesses data checking hemem_intr_test Interrupt HE MEMN Interrupt Interrupts assertion, PBA bits check malformedtlp_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing. maxpayloaderror_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing. MaxTagError_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3.Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing. mini_smoke_test All HEs shorter simpler version of random test for turn-in sanity check data checking if in lpbk mode, counter checking mmio_64b_bar_test PCIe MMIO Path 64-bit bar addess for MMIO data checking mmio_stress_nonblocking_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux with non-blocking MMIO reads data checking mmio_stress_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux data checking mmio_test PCIe MMIO Path MMIO targeting PF0(ST2MM, FME, PMCI, HSSI SS), PF1, PF1.VF1, PF1.VF2 data checking mmio_unimp_test PCIe MMIO Path MMIO acccess to unimplemented addresses MMIO checking MMIODataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing. MMIOInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing. MMIOTimedout_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing. msix_csr_test MSIX CSR CSR accesses data checking pmci_csr_test PMCI CSR CSR accesses data checking port_gasket_csr_test PORT GASKET Port Gasket CSR test port csr checking TxMWrDataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing TxMWrInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing UnexpMMIORspErr_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors. 1. Apply the error 2. Wait 5us 1us 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be retuened on resds. Write a 0x5 to set and a 0x4 to clear) 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.) 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4 6. Read the AFU_INTF_ERROR register, Be sure only the expected error(s) are set 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector. After clearing the error register,check if normal transcation are completing 
                                                          The next section describes how to compile and build the UVM environment prior to running each UVM test and analyzing the results in the log files
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#84-ip-compile","title":"8.4 IP Compile","text":""},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-vcs","title":"Synopsys VCS","text":"
                                                          To compile all IPs for the Synopsys VCS simulater:
                                                           
                                                              cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk cmplib\n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim","title":"Questasim","text":"
                                                          To compile all IPs for the Questasim simulater:
                                                           
                                                              cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk cmplib \n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#85-rtl-test-bench-compile","title":"8.5 RTL & Test Bench Compile","text":"
                                                          The RTL file list for compilation is located here: verification/scripts/rtl_comb.f
                                                           
                                                          The TB file list for compilation is located here: verification/scripts/ver_list.f
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-vcs_1","title":"Synopsys VCS","text":"
                                                          To compile RTL and Testbench for the Synopsys VCS simulater
                                                           
                                                              cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build DUMP=1\n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim_1","title":"Questasim","text":"
                                                          To compile RTL and Testbench for the Questasim simulater
                                                           
                                                              cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build DUMP=1\n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#86-ip-and-rtl-test-bench-compile","title":"8.6 IP and RTL & Test Bench Compile","text":""},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-vcs_2","title":"Synopsys VCS","text":"
                                                          If the user wants to compile all IPs and RTL Testbench in one command for Synopsys VCS then follow the procedure below
                                                           
                                                              cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_all DUMP=1\n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim_2","title":"Questasim","text":"
                                                          If the user wants to compile all IPs and RTL Testbench in one command for Questasim then follow the procedure below
                                                           
                                                              cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_all DUMP=1\n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-vcs_3","title":"Synopsys VCS","text":"
                                                          To run a simulation for Synopsys VCS:
                                                           
                                                              cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk run TESTNAME=mmio_test DUMP=1\n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim_3","title":"Questasim","text":"
                                                          To run a simulation for Questasim:
                                                           
                                                              cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=mmio_test DUMP=1 \n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-vcs_4","title":"Synopsys VCS","text":"
                                                          To dump the waveform, \"DUMP=1\" must be added into the command line for Synopsys VCS build and simulation. 
                                                           
                                                              ofs-d5005/verification/scripts  gmake -f Makefile_VCS.mk build DUMP=1\n\n    ofs-d5005/verification/scripts  gmake -f Makefile_VCS.mk run TESTNAME=<test_case_name> DUMP=1\n
                                                           Or
                                                           
                                                              ofs-d5005/verification/scripts  gmake -f Makefile_VCS.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim_4","title":"Questasim","text":"
                                                          To dump the waveform, \"DUMP=1\" must be added into the command line for Questasim build and simulation. 
                                                           
                                                              ofs-d5005/verification/scripts  gmake -f Makefile_MSIM.mk build DUMP=1\n\n    ofs-d5005/verification/scripts  gmake -f Makefile_MSIM.mk run TESTNAME=<test_case_name> DUMP=1\n
                                                           Or
                                                           
                                                              ofs-d5005/verification/scripts  gmake -f Makefile_MSIM.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                                                           
                                                          There are some optimizations in the Table below for convenience if you want to bypass some commands for both Synopsys VCS and Questasim:
                                                           Command (Synopsys VCS) Command (Questasim) Details gmake -f Makefile_VCS.mk build_all DUMP=1 gmake -f Makefile_MSIM.mk build_all DUMP=1 compile IP + compile RTL gmake -f Makefile_VCS.mk build_run TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk build_run TESTNAME= DUMP=1 compile RTL + run test gmake -f Makefile_VCS.mk do_it_all TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk do_it_all TESTNAME= DUMP=1 compile IP, RTL and run test gmake -f Makefile_VCS.mk rundb TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk rundb TESTNAME= DUMP=1 run test in sim dir + over-writes content"},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#87-uvm-regression-test","title":"8.7 UVM Regression Test","text":"
                                                          cd $VERDIR/scripts\n\nFor Regression in VCS with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -s vcs -c\n\nResults are created in a sim directory ($VERDIR/sim) with individual testcase log dir\n\nFor Regression in MSIM with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -s msim -c\n
                                                           
                                                          Results are created in a sim directory ($VERDIR/sim_msim) with individual testcase log dir
                                                          "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#88-uvm-waveform-and-transcript-analysis","title":"8.8 UVM Waveform and Transcript Analysis","text":""},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#synopsys-vcs_5","title":"Synopsys VCS","text":"
                                                          Running Synopsys VCS UVM tests will generate a ofs-d5005/verification/sim directory
                                                           
                                                             
                                                          • All build time logs are at ofs-d5005/verification/sim
                                                            •  
                                                           
                                                             
                                                          • Each testcase will have separate directory inside sim ofs-d5005/verification/sim/ 
                                                            There are two tracker or log files that are available: runsim.log and trans.log. 
                                                             
                                                            runsim.log is the simulation log file generated from Synopsys VCS. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 3
                                                             
                                                             
                                                            Figure 3 runsim.log
                                                             
                                                            trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 4
                                                             
                                                             
                                                            Figure 4 trans.log
                                                             
                                                            The waveform generated is named as \"inter.vpd\". To open the waveform, go to simulation result directory and run 
                                                             
                                                            dve -full64 -vpd inter.vpd &\n
                                                            "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#questasim_5","title":"Questasim","text":"
                                                            Running Questasim UVM tests will generate a ofs-d5005/verification/sim_msim directory
                                                             
                                                               
                                                            • All build time logs are at ofs-d5005/verification/sim_msim
                                                              •  
                                                             
                                                               
                                                            • Each testcase will have separate directory inside sim ofs-d5005/verification/sim_msim/ 
                                                              There are two tracker or log files that are available: runsim.log and trans.log. 
                                                               
                                                              runsim.log is the simulation log file generated from Questasim. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 4
                                                               
                                                               
                                                              Figure 4 runsim.log
                                                               
                                                              trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 5
                                                               
                                                               
                                                              Figure 5 trans.log
                                                               
                                                              The waveform generated is named as \"vsim.wlf\". To open the waveform, go to simulation result directory and run 
                                                               
                                                                  vsim -view vsim.wlf &    \n
                                                              "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#9-modifying-uvm-testbench","title":"9 Modifying UVM Testbench","text":"
                                                              The next section describe what needs to be considered when modifying the UVM, targeting a different device, adding a new interface to the testbench and creating a new UVM test for a customized OFS Accelerator platform.
                                                              "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#91-modifying-uvm-environment-when-targeting-different-device","title":"9.1 Modifying UVM environment when targeting different device","text":"
                                                              A new device may have different design feature or flow. The base address must be allocated for the new device. The MMIO targeting the new device must be based on the base address. If it is a new PF or VF, PCIe HIP must be regenerated and enumeration sequence must be updated accordingly. 
                                                              "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#92-modifying-uvm-environment-when-adding-a-new-interface","title":"9.2 Modifying UVM environment when adding a new interface","text":"
                                                              Adding a new interface requires signal connections in the testbench. An additional BFM or verification IP is needed to drive the new interface. The main testbench file tb_top.sv is found at the following location
                                                               
                                                                  $OFS_ROOTDIR/verification/testbench\n
                                                              "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#93-adding-a-new-uvm-test","title":"9.3 Adding a new UVM test","text":"
                                                              In the following example we will modify an existing test \"he_lpbk\" and name it \"he_lpbk_new\", and rebuild the test to check it. Please follow the steps below 
                                                               
                                                                 
                                                              1.  
                                                                Create a new test sequence file under ofs-d5005/verification/tests/sequences
                                                                 
                                                                he_lpbk_seq_new.svh\n
                                                                 
                                                                2.  
                                                              2.  
                                                                Modify ifndef, define and endif statements in new test sequence case i.e he_lpbk_seq_new.svh file
                                                                 
                                                                `ifndef HE_LPBK_SEQ_NEW_SVH \n`define HE_LPBK_SEQ_NEW_SVH\n`endif // HE_LPBK_SEQ_NEW_SVH\n
                                                                 
                                                                also replace all occurences of he_lpbk_seq with he_lpbk_seq_new in the he_lpbk_seq_new.svh file
                                                                 
                                                                3.  
                                                              3.  
                                                                Append the new sequence name into ofs-d5005/verification/tests/sequences/seq_lib.svh file
                                                                 
                                                                `include \"he_lpbk_seq_new.svh\"\n
                                                                 
                                                                4.  
                                                              4.  
                                                                Create a new test under ofs-d5005/verification/tests
                                                                 
                                                                he_lpbk_test_new.svh\n
                                                                 
                                                                5.  
                                                              5.  
                                                                Modify ifndef, define and endif statements in new test case i.e he_lpbk_test_new.svh file
                                                                 
                                                                `ifndef HE_LPBK_TEST_NEW_SVH \n`define HE_LPBK_TEST_NEW_SVH\n`endif // HE_LPBK_TEST_NEW_SVH\n
                                                                 
                                                                also replace all occurences of he_lpbk_test with he_lpbk_test_new in the he_lpbk_test_new.svh file
                                                                 
                                                                6.  
                                                              6.  
                                                                Append the new test name into ofs-d5005/verification/tests/test_pkg.svh file
                                                                 
                                                                `include \"he_lpbk_test_new.svh\"\n
                                                                 
                                                                7.  
                                                              7.  
                                                                Rebuild UVM test suite for either Synopsys VCS or Questasim simulater
                                                                 
                                                                cd $VERDIR/scripts\ngmake -f Makefile_VCS.mk build_all\n
                                                                 
                                                                or
                                                                 
                                                                cd $VERDIR/scripts\ngmake -f Makefile_MSIM.mk build_all\n
                                                                 
                                                                8.  
                                                              8.  
                                                                Execute new test for either Synopsys VCS or Questasim simulater
                                                                 
                                                                cd $VERDIR/scripts\ngmake -f Makefile_VCS.mk run TESTNAME=he_lpbk_test_new\n
                                                                 
                                                                or
                                                                 
                                                                cd $VERDIR/scripts\ngmake -f Makefile_MSIM.mk run TESTNAME=he_lpbk_test_new\n
                                                                 
                                                                9.  
                                                               
                                                              9) Check new test and log files         cd ofs-d5005/verification/sim/he_lpbk_test_new
                                                               
                                                              ```sh\nopen runsim.log\n```\n
                                                              "},{"location":"hw/d5005/user_guides/ug_sim_ofs_d5005/ug_sim_ofs_d5005/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                              Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                               
                                                              OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                              "},{"location":"hw/doc_modules/Glossary/","title":"Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/doc_modules/Notices_%26_Disclaimers/","title":"Notices & Disclaimers","text":""},{"location":"hw/doc_modules/Notices_%26_Disclaimers/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                              Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                               
                                                              OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                              "},{"location":"hw/doc_modules/links/","title":"AFU Dev","text":""},{"location":"hw/doc_modules/quartus_installation/","title":"Quartus installation","text":"
                                                              Intel ${{ env.QUARTUS_PP_VER_L }} is verified to work with the latest OFS release ${{ env.RELEASE }}.  However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.
                                                               
                                                              Use ${{ env.HOST_OS_L }} for compatibility with your development flow and also testing your FIM design in your platform. 
                                                               
                                                              Prior to installing Quartus:
                                                               
                                                                 
                                                              1.  
                                                                Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.
                                                                 
                                                                   
                                                                • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                                                                  •  
                                                                • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.
                                                                  •  
                                                                 
                                                                2.  
                                                              2.  
                                                                Perform the following steps to satisfy the required dependencies.
                                                                 
                                                                $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                                                                 
                                                                Apply the following configurations.
                                                                 
                                                                $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n
                                                                 
                                                                3.  
                                                              3.  
                                                                Create the default installation path: /intelFPGA_pro/, where  is the default path of the Linux workstation, or as set by the system administrator and  is your Quartus version number. 
                                                                The installation path must satisfy the following requirements:
                                                                 
                                                                   
                                                                • Contain only alphanumeric characters
                                                                  •  
                                                                • No special characters or symbols, such as !$%@^&*<>,
                                                                  •  
                                                                • Only English characters
                                                                  •  
                                                                • No spaces
                                                                  •  
                                                                 
                                                              4.  
                                                                Download your required Quartus Prime Pro Linux version here.
                                                                 
                                                                5.  
                                                              5.  
                                                                Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are ${{ env.QUARTUS_PATCHES }}.
                                                                 
                                                                6.  
                                                              6.  
                                                                After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                                                                 
                                                                export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                                                                 
                                                                For example, if the Quartus install directory is /home/intelFPGA_pro/${{ env.QUARTUS_PP_VER_S } then the new line is:
                                                                 
                                                                export PATH=/home/intelFPGA_pro/${{env.QUARTUS_PP_VER_S }}/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/${ env.QUARTUS_PP_VER_S }}/qsys/bin:$PATH\n
                                                                 
                                                                7.  
                                                              7.  
                                                                Verify, Quartus is discoverable by opening a new shell:
                                                                 
                                                                $ which quartus\n/home/intelFPGA_pro/${{ env.QUARTUS_PP_VER_S }/quartus/bin/quartus\n
                                                                 
                                                                8. "},{"location":"hw/doc_modules/wt_clone_fim_repo/","title":"Wt clone fim repo","text":"
                                                                Perform the following steps to clone the OFS Agilex PCIe Attach FIM Repository:
                                                                 
                                                                   
                                                                1. Create a new directory to use as a clean starting point to store the retrieved files.     
                                                                  mkdir OFS_BUILD_ROOT\ncd OFS_BUILD_ROOT\nexport OFS_BUILD_ROOT=$PWD\n
                                                                  2.  
                                                                2. Clone GitHub repository using the HTTPS git method     
                                                                  git clone --recurse-submodules ${{ env.REPO }}.git\n
                                                                  3.  
                                                                3. Check out the correct tag of the repository     
                                                                  cd ${{ env.FIM_REPO }}\ngit checkout --recurse-submodules tags/${{ env.TAG }}\n
                                                                  4.  
                                                                "},{"location":"hw/doc_modules/wt_compile_fim_in_prep_for_afu/","title":"Wt compile fim in prep for afu","text":"
                                                                Perform the following steps to compile a FIM for where the exercisers are removed and replaced with an he_null module while keeping the PF/VF multiplexor connections.
                                                                 
                                                                Pre-requisites:
                                                                 
                                                                   
                                                                • This walkthrough requires a development environment. Refer to the [Walkthrough: Set Up Development Environment] Section for instructions on setting up a development environment.
                                                                  •  
                                                                 
                                                                Steps:
                                                                 
                                                                   
                                                                1.  
                                                                  Clone the FIM repository (or use an existing cloned repository). Refer to the [Walkthrough: Clone FIM Repository] section for step-by-step instructions.
                                                                   
                                                                  2.  
                                                                2.  
                                                                  Set development environment variables. Refer to the [Walkthrough: Set Development Environment Variables] section for step-by-step instructions.
                                                                   
                                                                  3.  
                                                                3.  
                                                                  Compile the FIM with the HE_NULL compile options
                                                                   
                                                                  cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/${{ env.MODEL }}.ofss ${{ env.MODEL }}:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_${{ env.MODEL }}\n
                                                                   
                                                                  4.  
                                                                "},{"location":"hw/doc_modules/wt_install_git_lfs_rhel/","title":"Wt install git lfs rhel","text":"
                                                                To install the Git Large File Storage (LFS) extension, execute the following commands:
                                                                 
                                                                   
                                                                1. Obtain Git LFS package     
                                                                  curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\n
                                                                  2.  
                                                                2. Install Git LFS package     
                                                                  sudo dnf install git-lfs\n
                                                                  3.  
                                                                3. Install Git LFS     
                                                                  git lfs install\n
                                                                  4.  
                                                                "},{"location":"hw/doc_modules/wt_run_individual_unit_level_sim/","title":"Wt run individual unit level sim","text":"
                                                                Perform the following steps to run an individual unit test.
                                                                 
                                                                Pre-requisites:
                                                                 
                                                                   
                                                                • This walkthrough requires a development environment. Refer to the [Walkthrough: Set Up Development Environment] Section for instructions on setting up a development environment.
                                                                  •  
                                                                 
                                                                Steps:
                                                                 
                                                                   
                                                                1.  
                                                                  Clone the FIM repository (or use an existing cloned repository). Refer to the [Walkthrough: Clone FIM Repository] section for step-by-step instructions.
                                                                   
                                                                  2.  
                                                                2.  
                                                                  Set development environment variables. Refer to the [Walkthrough: Set Development Environment Variables] section for step-by-step instructions.
                                                                   
                                                                  3.  
                                                                3.  
                                                                  Generate the simulation files for the ${{ env.MODEL }}
                                                                   
                                                                  cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/${{ env.MODEL }}.ofss ${{ env.MODEL }}\n
                                                                   
                                                                  4.  
                                                                4.  
                                                                  Navigate to the common simulation directory   
                                                                  cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n
                                                                   
                                                                  5.  
                                                                5.  
                                                                  Run the desired unit test using your desired simulator
                                                                   
                                                                     
                                                                  •  
                                                                    Using VCS
                                                                     
                                                                    sh run_sim.sh TEST=<test_name>\n
                                                                     
                                                                    •  
                                                                   
                                                                     
                                                                  •  
                                                                    Using VCSMX
                                                                     
                                                                    sh run_sim.sh TEST=<test_name> VCSMX=1\n
                                                                     
                                                                    •  
                                                                   
                                                                     
                                                                  •  
                                                                    Using QuestaSim
                                                                     
                                                                    sh run_sim.sh TEST=<test_name> MSIM=1\n
                                                                     
                                                                    •  
                                                                   
                                                                     
                                                                  •  
                                                                    For example, to run the DFH walker test using VCSMX:
                                                                     
                                                                    sh run_sim.sh TEST=dfh_walker VCSMX=1\n
                                                                     
                                                                    •  
                                                                   
                                                                  6.  
                                                                6.  
                                                                  Once the test is complete, check the output for the simulation results. Review the log for detailed test results.
                                                                   
                                                                  Test status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/${{ env.FIM_REPO }}/sim/unit_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time         356233750000\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 356233750000 fs\nCPU Time:     57.730 seconds;       Data structure size:  47.2Mb\nTue Sep  5 09:44:19 2023\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                                                                   
                                                                  7.  
                                                                "},{"location":"hw/doc_modules/wt_set_fim_dev_env_vars/","title":"Wt set fim dev env vars","text":"
                                                                Perform the following steps to set the required environment variables. These environment variables must be set prior to simulation or compilation tasks so it is recommended that you create a script to set these variables.
                                                                 
                                                                   
                                                                1.  
                                                                  Navigate to the top level directory of the cloned OFS FIM repository.
                                                                   
                                                                  cd ${{ env.FIM_REPO }}\n
                                                                   
                                                                  2.  
                                                                2.  
                                                                  Set project variables   
                                                                  # Set OFS Root Directory - e.g. this is the top level directory of the cloned OFS FIM repository\nexport OFS_ROOTDIR=$PWD\n
                                                                   
                                                                  3.  
                                                                3.  
                                                                  Set variables based on your development environment   
                                                                  # Set proxies if required for your server\nexport http_proxy=<YOUR_HTTP_PROXY>\nexport https_proxy=<YOUR_HTTPS_PROXY>\nexport ftp_proxy=<YOUR_FTP_PROXY>\nexport socks_proxy=<YOUR_SOCKS_PROXY>\nexport no_proxy=<YOUR_NO_PROXY>\n\n# Set Quartus license path\nexport LM_LICENSE_FILE=<YOUR_LM_LICENSE_FILE>\n\n# Set Synopsys License path (if using Synopsys for simulation)\nexport DW_LICENSE_FILE=<YOUR_DW_LICENSE_FILE>\nexport SNPSLMD_LICENSE_FILE=<YOUR_SNPSLMD_LICENSE_FILE>\n\n# Set Quartus Installation Directory - e.g. $QUARTUS_ROOTDIR/bin contains Quartus executables\nexport QUARTUS_ROOTDIR=<YOUR_QUARTUS_INSTALLATION_DIRECTORY>\n\n# Set the Tools Directory - e.g. $TOOLS_LOCATION contains the 'synopsys' directory if you are using Synopsys. Refer to the $VCS_HOME variable for an example.\nexport TOOLS_LOCATION=<YOUR_TOOLS_LOCATION>\n
                                                                   
                                                                  4.  
                                                                4.  
                                                                  Set generic environment variables
                                                                   
                                                                  # Set Work directory \nexport WORKDIR=$OFS_ROOTDIR\n\n# Set Quartus Tools variables\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport QUARTUS_VER_AC=$QUARTUS_ROOTDIR\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport IMPORT_IP_ROOTDIR=$IP_ROOTDIR\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\n\n# Set Verification Tools variables (if running simulations)\nexport DESIGNWARE_HOME=$TOOLS_LOCATION/synopsys/vip_common/vip_Q-2020.03A\nexport UVM_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport MTI_HOME=$QUARTUS_ROOTDIR/../questa_fse\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n\n# Set PATH to include compilation and simulation tools\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/../qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/bin:$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$DESIGNWARE_HOME/bin:$VCS_HOME/bin:$PATH\n
                                                                   
                                                                  5.  
                                                                "},{"location":"hw/doc_modules/wt_set_up_development_environment/","title":"Wt set up development environment","text":"
                                                                This walkthrough guides you through the process of setting up your development environment in preparation for FIM development. This flow only needs to be done once on your development machine.
                                                                 
                                                                   
                                                                1.  
                                                                  Ensure that ${{ env.QUARTUS_PP_VER_L }} for Linux with Intel Agilex FPGA device support is installed on your development machine. Refer to the [Walkthrough: Install Quartus Prime Pro Software] section for step-by-step installation instructions.
                                                                   
                                                                     
                                                                  1. Verify version number
                                                                    quartus_sh --version\n
                                                                     
                                                                    Example Output:
                                                                     
                                                                    Quartus Prime Shell\nVersion ${{ env.QUARTUS_PP_VER_S }} Build 94 06/14/2023 SC Pro Edition\nCopyright (C) 2023  Intel Corporation. All rights reserved.\n
                                                                     
                                                                    2.  
                                                                   
                                                                  2.  
                                                                2.  
                                                                  Ensure that all support tools are installed on your development machine, and that they meet the version requirements.
                                                                   
                                                                     
                                                                  1.  
                                                                    Python ${{ env.PYTHON_VER }} or later
                                                                     
                                                                       
                                                                    1.  
                                                                      Verify version number
                                                                       
                                                                      python --version\n
                                                                       
                                                                      Example Output:
                                                                       
                                                                      Python ${{ env.PYTHON_VER }}\n
                                                                       
                                                                      2.  
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    GCC ${{ env.GCC_VER }} or later
                                                                     
                                                                       
                                                                    1.  
                                                                      Verify version number
                                                                       
                                                                      gcc --version\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      gcc (GCC) ${{ env.GCC_VER }}\n
                                                                       
                                                                      2.  
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    cmake ${{ env.CMAKE_VER }} or later
                                                                     
                                                                       
                                                                    1.  
                                                                      Verify version number
                                                                       
                                                                      cmake --version\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      cmake version ${{ env.CMAKE_VER }}\n
                                                                       
                                                                      2.  
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    git with git-lfs ${{ env.GIT_VER }} or later. Refer to the [Walkthrough: Install Git Large File Storage Extension] section for step-by-step instructions on installing the Git Large File Storage (LFS) extension.
                                                                     
                                                                       
                                                                    1.  
                                                                      Verify version number
                                                                       
                                                                      git --version\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      git version ${{ env.GIT_VER }}\n
                                                                       
                                                                      2.  
                                                                     
                                                                    5.  
                                                                   
                                                                  3.  
                                                                3.  
                                                                  Clone the ${{ env.FIM_REPO }} repository. Refer to the [Walkthrough: Clone FIM Repository] section for step-by-step instructions.
                                                                   
                                                                  4.  
                                                                4.  
                                                                  Install UART IP license patch .02.
                                                                   
                                                                     
                                                                  1.  
                                                                    Navigate to the license directory
                                                                     
                                                                    cd $IOFS_BUILD_ROOT/license\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Install Patch 0.02
                                                                     
                                                                    sudo ./quartus-0.0-0.02iofs-linux.run\n
                                                                     
                                                                    3.  
                                                                   
                                                                  5.  
                                                                5.  
                                                                  Install Quartus Patches ${{ env.QUARTUS_PATCHES }}. All required patches are provided in the Assets of the OFS FIM Release: ${{ env.RELEASE_PATH }}
                                                                   
                                                                  6.  
                                                                6.  
                                                                  Verify that patches have been installed correctly. They should be listed in the output of the following command.
                                                                   
                                                                  quartus_sh --version\n
                                                                   
                                                                  7.  
                                                                7.  
                                                                  Set required environment variables. Refer to the [Walkthrough: Set Environment Variables] section for step-by-step instructions.
                                                                   
                                                                  8.  
                                                                 
                                                                This concludes the walkthrough for setting up your development environment. At this point you are ready to begin FIM development.
                                                                "},{"location":"hw/f2000x/","title":"Index","text":"
                                                                Location for SoC Attach Collateral for OFS.
                                                                "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/","title":"AFU Development Guide: OFS for Intel\u00ae Intel\u00ae Agilex\u00ae 7 FPGA SoC Attach FPGAs","text":"
                                                                Last updated: February 03, 2024 
                                                                "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#1-introduction","title":"1. Introduction","text":"
                                                                This document is a design guide for the creation of an Accelerator Functional Unit (AFU) using Open FPGA Stack (OFS) for Intel\u00ae Agilex\u00ae 7 FPGAs SoC Attach. The AFU concept consists of separating out the FPGA design development process into two parts, the construction of the foundational FPGA Interface Manager (FIM), and the development of the Acceleration Function Unit (AFU), as shown in the diagram below.
                                                                 
                                                                 
                                                                This diagram shows the separation of FPGA board interface development from the internal FPGA workload creation.  This separation starts with the FPGA Interface Manager (FIM) which consists of the external interfaces and board management functions.  The FIM is the base system layer and is typically provided by board vendors. The FIM interface is specific to a particular physical platform.  The AFU makes use of the external interfaces with user defined logic to perform a specific application.  By separating out the lengthy and complicated process of developing and integrating external interfaces for an FPGA into a board allows the AFU developer to focus on the needs of their workload.  OFS for Intel\u00ae Agilex\u00ae 7 FPGAs SoC Attach provides the following tools for rapid AFU development:
                                                                 
                                                                   
                                                                • Scripts for both compilation and simulation setup
                                                                  •  
                                                                • Optional Platform Interface Manager (PIM) which is a set of SystemVerilog shims and scripts for flexible FIM to AFU interfacing
                                                                  •  
                                                                • Acceleration Simulation Environment (ASE) which is a hardware/software co-simulation environment scripts for compilation and Acceleration
                                                                  •  
                                                                • Integration with Open Programmable Acceleration Engine (OPAE) SDK for rapid software development for your AFU application
                                                                  •  
                                                                 
                                                                Please notice in the above block diagram that the AFU region consists of static and partial reconfiguration (PR) regions where the PR region can be dynamically reconfigured while the remaining FPGA design continues to function.  Creating AFU logic for the static region is described in FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae SoC Attach FPGAs. This guide covers logic in the AFU Main region.
                                                                "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#11-document-organization","title":"1.1. Document Organization","text":"
                                                                This document is organized as follows:
                                                                 
                                                                   
                                                                • Description of design flow
                                                                  •  
                                                                • Interfaces and functionality provided in the Intel\u00ae Agilex\u00ae 7 FPGAs SoC Attach FIM
                                                                  •  
                                                                • Setup of the AFU Development environment
                                                                  •  
                                                                • Synthesize the AFU example
                                                                  •  
                                                                • Testing the AFU example on the Intel IPU Platform F2000X-PL card
                                                                  •  
                                                                • Hardware/Software co-simulation using ASE
                                                                  •  
                                                                • Debugging an AFU with Remote Signal Tap
                                                                  •  
                                                                 
                                                                This guide provides theory followed by tutorial steps to solidify your AFU development knowledge.
                                                                 
                                                                NOTE: 
                                                                 
                                                                This guide uses the Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL as the platform for all tutorial steps. Additionally, this guide and the tutorial steps can be used with other Intel\u00ae Agilex\u00ae 7 FPGAs SoC Attach platforms..
                                                                 
                                                                Some of the document links in this guide are specific to the Intel IPU Platform F2000X-PL .   If using a different platform, please use the associated documentation for your platform as there could be differences in building the FIM and downloading FIM images. 
                                                                 
                                                                If you have worked with previous Intel Programmable Acceleration products, you will find out that OFS for Intel\u00ae Agilex\u00ae 7 FPGAs SoC Attach is similar. However, there are differences and you are advised to carefully read and follow the tutorial steps to fully understand the design tools and flow.
                                                                "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#12-prerequisite","title":"1.2. Prerequisite","text":"
                                                                This guide assumes you have the following FPGA logic design-related knowledge and skills:
                                                                 
                                                                   
                                                                • FPGA compilation flows including the Intel\u00ae Quartus\u00ae Prime Pro Edition design flow
                                                                  •  
                                                                • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition software, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
                                                                  •  
                                                                • RTL and coding practices to create synthesizable logic.
                                                                  •  
                                                                • Understanding of AXI and Avalon memory mapped and streaming interfaces.
                                                                  •  
                                                                • Simulation of complex RTL using industry standard simulators (Synopsys\u00ae VCS\u00ae or Siemens\u00ae QuestaSim\u00ae).
                                                                  •  
                                                                • Signal Tap Logic Analyzer tool in the Intel\u00ae Quartus\u00ae Prime Pro Edition software.
                                                                  •  
                                                                 
                                                                You are strongly encouraged to review the FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae SoC Attach FPGAs
                                                                "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#13-acceleration-functional-unit-afu-development-flow","title":"1.3. Acceleration Functional Unit (AFU) Development Flow","text":"
                                                                The AFU development flow is shown below: 
                                                                "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#131-understanding-platform-capabilities","title":"1.3.1. Understanding Platform Capabilities","text":"
                                                                The block diagram of the F2000x Board is shown below:
                                                                 
                                                                 
                                                                The F2000x FIM provided with this release is shown below:
                                                                 
                                                                 
                                                                This release F2000x FIM provides the following features:
                                                                 
                                                                   
                                                                • Host interface
                                                                     
                                                                  • PCIe Gen4 x 16
                                                                    •  
                                                                  • 2 - PFs 
                                                                    •  
                                                                  • MSI-X interrupts
                                                                    •  
                                                                  • Logic to demonstrate simple PCIe loopback
                                                                    •  
                                                                   
                                                                  •  
                                                                • Network interface
                                                                     
                                                                  • 2 - QSFP28/56 cages
                                                                    •  
                                                                  • 8 X 25 GbE with exerciser logic demonstrating traffic generation/monitoring
                                                                    •  
                                                                   
                                                                  •  
                                                                • External Memory - DDR4 - 2400
                                                                     
                                                                  • 4 Banks - 4 GB organized as 1 Gb x 32 with 1 Gb x 8 ECC
                                                                    •  
                                                                  • Memory exerciser logic demonstrating external memory operation
                                                                    •  
                                                                   
                                                                  •  
                                                                • Board Management
                                                                     
                                                                  • SPI interface
                                                                    •  
                                                                  • FPGA management and configuration
                                                                    •  
                                                                  • Example logic showing DFH operation
                                                                    •  
                                                                   
                                                                  •  
                                                                • Partial reconfiguration control logic
                                                                  •  
                                                                • SoC - Xeon Icelake-D subsystem with embedded Linux
                                                                     
                                                                  • PCIe Gen4 x 16 interface to FPGA
                                                                    •  
                                                                  • 1 - PF, 3 - VF, AXI-S TLP packets
                                                                    •  
                                                                  • DDR Memory
                                                                       
                                                                    • 2 Banks - 8 GB organized as 1 Gb x 64 with 1 Gb x 8 ECC
                                                                      •  
                                                                    • NVMe SSD - 64 GB
                                                                      •  
                                                                     
                                                                    •  
                                                                   
                                                                  •  
                                                                "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#132-high-level-data-flow","title":"1.3.2. High Level Data Flow","text":"
                                                                The OFS high level data flow is shown below:
                                                                 
                                                                "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#133-considerations-for-pim-usage","title":"1.3.3. Considerations for PIM Usage","text":"
                                                                When creating an AFU, a designer needs to decide of what type of interfaces the AFU will provide to the platform (FIM).  The AFU can use the native interfaces (i.e. PCIe TLP commands) provided by the FIM or standard memory mapped interfaces (i.e. AXI-MM or AVMM) by using the PIM.  The PIM is an abstraction layer consisting of a collection of SystemVerilog interfaces and shims to enable partial AFU portability across hardware despite variations in hardware topology and native interfaces. The PIM adds a level of logic between the AFU and the FIM converting the native interfaces from the FIM to match the interfaces provided by the AFU.
                                                                 
                                                                 
                                                                The following resources are available to assist in creating an AFU:
                                                                 
                                                                PIM Core Concepts provides details on using the PIM and its capabilities. 
                                                                 
                                                                Connecting an AFU to a Platform using PIM guides you through the steps needed to connect a PIM Based AFU to the FIM. 
                                                                 
                                                                The AFU Tutorial provides several AFU examples.  These examples can be run with the current OFS FIM package.  There are three AFU types of examples provided (PIM based, hybrid and native).  Each example provides the following:
                                                                 
                                                                   
                                                                • RTL, which includes the following interfaces: 
                                                                     
                                                                  • Host Channel: 
                                                                       
                                                                    • Host memory, providing a DMA interface.
                                                                      •  
                                                                    • MMIO, providing a CSR interface.  
                                                                      •  
                                                                     
                                                                    •  
                                                                  • Local Memory
                                                                    •  
                                                                   
                                                                  •  
                                                                • Host software example interfacing to the CSR interface and host memory interface, using the OPAE C API.
                                                                  •  
                                                                • Accelerator Description File .json file
                                                                  •  
                                                                • Source file list
                                                                  •  
                                                                "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#134-afu-interfaces-included-with-intel-ipu-platform-f2000x-pl","title":"1.3.4. AFU Interfaces Included with Intel IPU Platform F2000X-PL","text":"
                                                                The figure below shows the interfaces available to the AFU in this architecture. It also shows the design hierarchy with module names from the fim (top.sv) to the PR region AFU (afu_main.sv). One of the main differences from the Stratix 10 PAC OFS architecture to this one is the presence of the static port gasket region (port_gasket.sv) that has components to facilitate the AFU and also consists of the PR region (afu_main.sv) via the PR slot. The Port Gasket contains all the PR specific modules and logic, e.g., PR slot reset/freeze control, user clock, remote STP etc. Architecturally, a Port Gasket can have multiple PR slots where user workload can be programmed into. However, only one PR slot is supported for OFS Release for Intel Agilex. Everything in the Port Gasket until the PR slot should be provided by the FIM developer. The task of the AFU developer is to add their desired application in the afu_main.sv module by stripping out unwanted logic and instantiating the target accelerator. As shown in the figure below, here are the interfaces connected to the AFU (highlighted in green) via the SoC Attach FIM:
                                                                 
                                                                   
                                                                1. AXI Streaming (AXI-S) interface to the Host via PCIe Gen4x16
                                                                  2.  
                                                                2. AXI Memory Mapped Channels (4) to the DDR4 EMIF interface
                                                                  3.  
                                                                3. AXI Streaming (AXI-S) interface to the HSSI 25 Gb Ethernet
                                                                  4.  
                                                                 
                                                                "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#2-set-up-afu-development-environment","title":"2. Set Up AFU Development Environment","text":"
                                                                This section covers the setup of the AFU development environment.
                                                                "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#21-prepare-afu-development-environment","title":"2.1. Prepare AFU development environment","text":"
                                                                A typical development and hardware test environment consists of a development server or workstation with FPGA development tools installed and a separate server with the target OFS compatible FPGA PCIe card installed.  The typical usage and flow of data between these two servers is shown below:
                                                                 
                                                                 
                                                                Note: both development and hardware testing can be performed on the same server if desired.
                                                                 
                                                                This guide uses Intel IPU Platform F2000X-PL as the target OFS compatible FPGA PCIe card for demonstration steps.  The Intel IPU Platform F2000X-PL must be fully installed following the Getting Started User Guide: OFS for Intel\u00ae Agilex\u00ae SoC Attach FPGAs. If using a different OFS FPGA PCIe card, contact your supplier for instructions on how to install and operate user developed AFUs.
                                                                 
                                                                The following is a summary of the steps to set up for AFU development:
                                                                 
                                                                   
                                                                1. Install Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 for Linux with Agilex device support and required Quartus patches.
                                                                  2.  
                                                                2. Make sure support tools are installed and meet version requirements.
                                                                  3.  
                                                                3. Install OPAE SDK.
                                                                  4.  
                                                                4. Download the Basic Building Blocks repository.
                                                                  5.  
                                                                5. Build or download the relocatable AFU PR-able build tree based on your Intel\u00ae Agilex FPGA PCIe Attach FIM.
                                                                  6.  
                                                                6. Download FIM to the Intel\u00ae Agilex FPGA PCIe Attach platform.
                                                                  7.  
                                                                 
                                                                Building AFUs with OFS for Agilex requires the build machine to have at least 64 GB of RAM. 
                                                                "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#22-installation-of-quartus-and-required-patches","title":"2.2. Installation of Quartus and required patches","text":"
                                                                Intel Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 is verified to work with the latest OFS release release/ofs-2023.3.  However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.
                                                                 
                                                                Use Ubuntu 22.04 for compatibility with your development flow and also testing your FIM design in your platform. 
                                                                 
                                                                Prior to installing Quartus:
                                                                 
                                                                   
                                                                1.  
                                                                  Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.
                                                                   
                                                                     
                                                                  • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                                                                    •  
                                                                  • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.
                                                                    •  
                                                                   
                                                                  2.  
                                                                2.  
                                                                  Perform the following steps to satisfy the required dependencies.
                                                                   
                                                                  $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                                                                   
                                                                  Apply the following configurations.
                                                                   
                                                                  $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n
                                                                   
                                                                  3.  
                                                                3.  
                                                                  Create the default installation path: /intelFPGA_pro/, where  is the default path of the Linux workstation, or as set by the system administrator and  is your Quartus version number. 
                                                                  The installation path must satisfy the following requirements:
                                                                   
                                                                     
                                                                  • Contain only alphanumeric characters
                                                                    •  
                                                                  • No special characters or symbols, such as !$%@^&*<>,
                                                                    •  
                                                                  • Only English characters
                                                                    •  
                                                                  • No spaces
                                                                    •  
                                                                   
                                                                4.  
                                                                  Download your required Quartus Prime Pro Linux version here.
                                                                   
                                                                  5.  
                                                                5.  
                                                                  Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.13 patch (Generic Serial Flash Interface IP), 0.21 (PCIe Subsystem).
                                                                   
                                                                  6.  
                                                                6.  
                                                                  After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                                                                   
                                                                  export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                                                                   
                                                                  For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:
                                                                   
                                                                  export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n
                                                                   
                                                                  7.  
                                                                7.  
                                                                  Verify, Quartus is discoverable by opening a new shell:
                                                                   
                                                                  $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                                                                   
                                                                  8. "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#23-installation-of-support-tools","title":"2.3. Installation of Support Tools","text":"
                                                                  Make sure support tools are installed and meet version requirements.
                                                                   
                                                                  The OFS provided Quartus build scripts require the following tools. Verify these are installed in your development environment.
                                                                   Item Version Python 3.6.8 GCC 7.2.0 cmake 3.15 git 1.8.3.1 perl 5.8.8"},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#24-installation-of-opae-sdk","title":"2.4. Installation of OPAE SDK","text":"
                                                                  Follow the instructions in the Getting Started Guide: Open FPGA Stack for Intel IPU Platform F2000X-PL, section 6.2 Installing the OPAE SDK On the Host to build and install the required OPAE SDK for the Intel IPU Platform F2000X-PL.
                                                                   
                                                                  Working with the Intel\u00ae Intel IPU Platform F2000X-PL card requires opae-2.10.0-1. Follow the instructions in the Getting Started Guide: Intel\u00ae Open FPGA Stack for Intel IPU Platform F2000X-PL section 6.2 Installing the OPAE SDK On the Host. Make sure to check out the cloned repository to tag 2.10.0-1 and branch release/2.10.0.
                                                                   
                                                                  $ git checkout tags/2.10.0-1 -b release/2.10.0\n
                                                                   
                                                                  Note: The tutorial steps provided in the next sections assume the OPAE SDK is installed in default system locations, under the directory, /usr. In most system configurations, this will allow the OS and tools to automatically locate the OPAE binaries, scripts, libraries and include files required for the compilation and simulation of the FIM and AFUs.
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#25-download-the-basic-building-blocks-repositories","title":"2.5. Download the Basic Building Blocks repositories","text":"
                                                                  The ofs-platform-afu-bbb repository contains the PIM files as well as example PIM-based AFUs that can be used for testing and demonstration purposes. This guide will use the host_chan_mmio AFU example in the ofs-platform-afu-bbb repository and the hello_world sample in the examples AFU repository to demonstrate how to synthesize, load, simulate, and test a PIM-based AFU using the Intel IPU Platform F2000X-PL card with the SoC Attach FIM.
                                                                   
                                                                  Execute the next commands to clone the BBB repositories.
                                                                   
                                                                    # Create top level directory for AFU development\n$ mkdir OFS_BUILD_ROOT\n$ cd OFS_BUILD_ROOT\n$ export OFS_BUILD_ROOT=$PWD\n\n  # Clone the ofs-platform-afu-bbb repository.\n$ cd $OFS_BUILD_ROOT\n$ git clone https://github.com/OFS/ofs-platform-afu-bbb.git\n\n  # Verify retrieval\n$ cd $OFS_BUILD_ROOT/ofs-platform-afu-bbb\n$ ls\nLICENSE  plat_if_develop  plat_if_release  plat_if_tests  README.md\n
                                                                   
                                                                  The documentation in the ofs-platform-afu-bbb repository further addresses   - The PIM concept.   - The structure of the PIM-based AFU examples.   - How to generate a release and configure the PIM.   - How to connect an AFU to an FIM.
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#26-build-or-download-the-relocatable-pr-build-tree","title":"2.6. Build or download the relocatable PR build tree","text":"
                                                                  A relocatable PR build tree is needed to build the AFU partial reconfiguration area for the intended FIM. The tree is relocatable and may be copied to a new location. It does not depend on files in the original FIM build.
                                                                   
                                                                  You can use the Intel IPU Platform F2000X-PL release package and download the PR build tree and FIM images, to develop your AFU.  These are located at OFS-F2000X-PL release 
                                                                   
                                                                  Or you can build your own FIM and generate the PR build tree during the process.
                                                                   
                                                                  To download and untar the pr_build_template:
                                                                   
                                                                  $ wget https://github.com/OFS/ofs-f2000x-pl/releases/download/ofs-2023.3-1/f2000x-images_ofs-2023-3-2.tar.gz\n$ tar -zxvf f2000x-images_ofs-2023-3-2.tar.gz\n$ cd f2000x-images_ofs-2023-3-2/\n$ mkdir pr_build_template\n$ tar -zxvf pr_template-f2000x.tar.gz -C ./pr_build_template\n$ cd pr_build_template\n$ export OPAE_PLATFORM_ROOT=$PWD\n
                                                                   
                                                                  To build your own FIM and generate the PR build tree for the Intel IPU Platform F2000X-PL platform, refer the FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae SoC Attach FPGAs and follow the Out-of-Tree PR FIM build flow.  If you are using a different platform, refer to the FPGA Interface Manager Developer Guide for your platform and follow the Out-of-Tree PR FIM build flow.
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#27-download-fim-to-fpga","title":"2.7. Download FIM to FPGA","text":"
                                                                  The AFU requires that the FIM from which the AFU is derived be loaded onto the FPGA.   
                                                                   
                                                                  If you are using the Intel IPU Platform F2000X-PL release package downloaded in the previous section, copy the associated FIM files to the SoC: 
                                                                  # On Development Host\n$ cd $OFS_BUILD_ROOT/f2000x-images_ofs-2023-3-2/\n$ scp ofs_top_page1_unsigned_user1.bin <user>@<SoC IP address>:</remote/directory>\n$ scp ofs_top_page2_unsigned_user2.bin <user>@<SoC IP address>:</remote/directory>\n
                                                                   
                                                                  If you are generating your own FIM, use the unsigned FPGA binary images from your FIM build and copy over to the SoC. 
                                                                   
                                                                  To downlaod the FIM to the Intel IPU Platform F2000X-PL platform: 
                                                                  # On SoC\n$ sudo fpgasupdate ofs_top_page1_unsigned_user1.bin \n$ sudo fpgasupdate ofs_top_page2_unsigned_user2.bin \n$ sudo rsu fpga --page=user1 \n
                                                                   
                                                                  If you are using a different platform, refer to the documentation for your platform to download the FIM images onto your Intel\u00ae Agilex\u00ae SoC Attach Platform.
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#28-set-up-required-environment-variables","title":"2.8. Set up required Environment Variables","text":"
                                                                  Set the required environment variables as shown below. These environment variables must be set prior to simulation or compilation tasks. You can create a simple script to set these variables and save time going forward.
                                                                   
                                                                  # If not already done, export OFS_BUILD_ROOT to the top level directory for AFU development\n$ export OFS_BUILD_ROOT=<path to ofs build directory>\n\n# If not already done, export OPAE_PLATFORM_ROOT to the PR build tree directory\n$ export OPAE_PLATFORM_ROOT=<path to pr build tree>\n\n# Quartus Tools\n# Note, QUARTUS_HOME is your Quartus installation directory, e.g. $QUARTUS_HOME/bin contains Quartus executable.\n$ export QUARTUS_HOME=<user_path>/intelFPGA_pro/23.3/quartus\n$ export QUARTUS_ROOTDIR=$QUARTUS_HOME\n$ export QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\n$ export QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\n$ export IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\n$ export IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\n$ export QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys\n$ export PATH=$QUARTUS_HOME/bin:$QSYS_ROOTDIR/bin:$QUARTUS_HOME/sopc_builder/bin/:$PATH\n\n# OPAE SDK release\n$ export OPAE_SDK_REPO_BRANCH=release/2.10.0\n\n# The following environment variables are required for compiling the AFU examples. \n\n# Location to clone the ofs-platform-afu-bbb repository which contains PIM files and AFU examples.\n$ export OFS_PLATFORM_AFU_BBB=$OFS_BUILD_ROOT/ofs-platform-afu-bbb \n\n# OPAE and MPF libraries must either be on the default linker search paths or on both LIBRARY_PATH and LD_LIBRARY_PATH.  \n$ export OPAE_LOC=/usr\n$ export LIBRARY_PATH=$OPAE_LOC/lib:$LIBRARY_PATH\n$ export LD_LIBRARY_PATH=$OPAE_LOC/lib64:$LD_LIBRARY_PATH\n
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#3-compiling-an-afu","title":"3. Compiling an AFU","text":"
                                                                  In this section, you will use the relocatable PR build tree created in the previous steps from the FIM to compile an example PIM-based AFU. This section will be developed around the host_chan_mmio and hello_world AFU examples to showcase the synthesis of a PIM-based AFU.
                                                                   
                                                                  The build steps presented below demonstrate the ease in building and running an actual AFU on the board. To successfully execute the instructions in this section, you must have set up your development environment and have a relocateable PR Build tree as instructed in section 2 of this document.
                                                                   
                                                                  The build steps presented below demonstrate the ease in building and running an actual AFU on the board. To successfully execute the instructions in this section, you must have set up your development environment and have a relocateable PR Build tree as instructed in section 2 of this document.
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#31-creating-the-afu-synthesis-environment","title":"3.1. Creating the AFU Synthesis Environment","text":"
                                                                  The PIM flow provides the script afu_synth_setup to create the synthesis environment to build the AFU examples. See how to use it below.
                                                                   
                                                                  usage: afu_synth_setup [-h] -s SOURCES [-p PLATFORM] [-l LIB] [-f] dst\n\nGenerate a Quartus build environment for an AFU. A build environment is\ninstantiated from a release and then configured for the specified AFU. AFU\nsource files are specified in a text file that is parsed by rtl_src_config,\nwhich is part of the OPAE base environment.\n\npositional arguments:\n  dst                   Target directory path (directory must not exist).\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -s SOURCES, --sources SOURCES\n                        AFU source specification file that will be passed to\n                        rtl_src_config. See \"rtl_src_config --help\" for the\n                        file's syntax. rtl_src_config translates the source\n                        list into either Quartus or RTL simulator syntax.\n  -p PLATFORM, --platform PLATFORM\n                        FPGA platform name.\n  -l LIB, --lib LIB     FPGA platform release hw/lib directory. If not\n                        specified, the environment variables OPAE_FPGA_HW_LIB\n                        and then BBS_LIB_PATH are checked.\n  -f, --force           Overwrite target directory if it exists.\n
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#32-building-and-running-host_chan_mmio-example-afu","title":"3.2. Building and Running host_chan_mmio example AFU","text":"
                                                                  The $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio is a simple example demonstrating both hardware and software access to an AFU. The host_chan_mmio example AFU consists of the following files.  The hw directory contains the RTL to implement the hardware functionality using Avalon and AXI interfaces. However, this guide will use the AXI version of the host_chan_mmio AFU to go through the compilation steps. The sw directory of the AFU contains the source code of the host application that communicates with the actual AFU hardware.
                                                                   
                                                                  host_chan_mmio\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_avalon.sv\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_axi.sv\n\u2502       \u251c\u2500\u2500 host_chan_mmio.json\n\u2502       \u251c\u2500\u2500 test_mmio_avalon0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon1.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon2_512rw.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi1.txt\n\u2502       \u2514\u2500\u2500 test_mmio_axi2_512rw.txt\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 main.c\n    \u251c\u2500\u2500 Makefile\n
                                                                   
                                                                  Execute afu_synth_setup as follows to create the synthesis environment for a host_chan_mmio AFU that fits the SoC Attach FIM previously constructed.
                                                                   
                                                                  $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio\n$ afu_synth_setup -s ./hw/rtl/test_mmio_axi1.txt afu_dev\n
                                                                   
                                                                  Now, move into the synthesis environment afu_dev directory just created. From there, execute the afu_synth command. The successful completion of the command will produce the host_chan_mmio.gbs file under the synthesis environment directory, $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_dev.
                                                                   
                                                                  $ cd afu_dev\n$ $OPAE_PLATFORM_ROOT/bin/afu_synth\nCompiling ofs_top ofs_pr_afu\nGenerating host_chan_mmio.gbs\n==================================\n...\n...\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n Design meets timing\n===========================================================================\n
                                                                   
                                                                  The previous output indicates the successful compilation of the AFU and the compliance with the timing requirements. Analyze the reports generated in case the design does not meet timing. The timing reports are stored in the directory, $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_dev/build/syn/syn_top/output_files/timing_report.
                                                                   
                                                                  Once the compilation finishes successfully, load the new host_chan_mmio.gbs bitstream file into the partial reconfiguration region of the target Intel IPU Platform F2000X-PL board. Keep in mind, that the loaded image is dynamic - this image is not stored in flash and if the card is power cycled, then the PR region is re-loaded with the default AFU.
                                                                   
                                                                  To load the image, perform the following steps:
                                                                   
                                                                  # On Development Host\n$ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_dev\n# Copy FIM files to SoC\n$ scp host_chan_mmio.gbs <user>@<SoC IP address>:</remote/directory>\n
                                                                   
                                                                  # On SoC\n$ cd </remote/directory>\n$ fpgasupdate host_chan_mmio.gbs \n[2022-04-15 20:22:18.85] [WARNING ] Update starting. Please do not interrupt.\n[2022-04-15 20:22:19.75] [INFO    ] \nPartial Reconfiguration OK\n[2022-04-15 20:22:19.75] [INFO    ] Total time: 0:00:00.90\n
                                                                   
                                                                  Set up your board to work with the newly loaded AFU.
                                                                   
                                                                  # On SoC\n\n# Verify PCIe b.d.f\n# For the following example, the F2000x SKU2 PCIe b:d.f is 15:00.0,\n# however this may be different in your system\n$  fpgainfo fme\nIntel IPU Platform F2000X-PL\nBoard Management Controller NIOS FW version: 1.2.4\nBoard Management Controller Build version: 1.2.4\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\n...\n\n# Create the Virtual Functions (VFs) provided by the FIM, the default FIM has 3 VFs\n$ pci_device 15:00.0 vf 3\n\n# Verify the VFs have been added (device id: bccf)\n$ lspci -s 15:00\n15:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n15:00.1 Processing accelerators: Intel Corporation Device bccf\n15:00.2 Processing accelerators: Intel Corporation Device bccf\n15:00.3 Processing accelerators: Intel Corporation Device bccf\n\n\n# Bind VFs to VFIO driver. \n\n$  opae.io init -d 0000:15:00.1 \nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:15:00.1 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:15:00.1 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:15:00.1 is 52\n\n$ opae.io init -d 0000:15:00.2\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:15:00.2 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:15:00.2 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:15:00.2 is 53\n\n$  opae.io init -d 0000:15:00.3\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:15:00.3 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:15:00.3 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:15:00.3 is 54\n\n# Verify the new AFU is loaded.  The host_chan_mmio AFU GUID is \"76d7ae9c-f66b-461f-816a-5428bcebdbc5\".\n\n$ fpgainfo port\n//****** PORT ******//\nObject Id                        : 0xF100000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0x6015000000000000\nPCIe s:b:d.f                     : 0000:15:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nAccelerator GUID                 : d15ab1ed-0000-0000-0210-000000000000\n//****** PORT ******//\nObject Id                        : 0x4015000000000000\nPCIe s:b:d.f                     : 0000:15:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nAccelerator GUID                 : d15ab1ed-0000-0000-0110-000000000000\n//****** PORT ******//\nObject Id                        : 0x2015000000000000\nPCIe s:b:d.f                     : 0000:15:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nAccelerator GUID                 : 76d7ae9c-f66b-461f-816a-5428bcebdbc5\n
                                                                   
                                                                  Now, navigate to the directory of the host_chan_mmio AFU containing the host application's source code, $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw. Once there, compile the host_chan_mmio host application and execute it on the host server to excercise the functionality of the AFU.
                                                                   
                                                                  Note: If OPAE SDK libraries were not installed in the default systems directories under /usr, you need to set the OPAE_LOC, LIBRARY_PATH, and LD_LIBRARY_PATH environment variables to the custom locations where the OPAE SDK libraries were installed.
                                                                   
                                                                  # On Development Host, move to the sw directory of the the host_chan_mmio AFU. This directory holds the source for the host application.\n$ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw\n$ export OPAE_LOC=/usr\n$ export LIBRARY_PATH=$OPAE_LOC/lib:$LIBRARY_PATH\n$ export LD_LIBRARY_PATH=$OPAE_LOC/lib64:$LD_LIBRARY_PATH\n$ make\n# Copy application to SoC\n$ scp host_chan_mmio <user>@<SoC IP address>:</remote/directory>\n
                                                                   
                                                                  # On SoC, Run the application\n$ cd </remote/directory>\n$  ./host_chan_mmio\nAFU ID:  76d7ae9cf66b461f 816a5428bcebdbc5\nAFU MMIO interface: AXI Lite\nAFU MMIO read bus width: 64 bits\n512 bit MMIO write supported: yes\nAFU pClk frequency: 470 MHz\n\nTesting 32 bit MMIO reads:\n  PASS - 4 tests\n\nTesting 32 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 64 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 512 bit MMIO writes:\n  PASS\n
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#33-building-and-running-the-hello_world-example-afu","title":"3.3. Building and running the hello_world example AFU","text":"
                                                                  The platform-independent examples AFU repository also provides some interesting example AFUs. In this section, you will compile and execute the PIM based hello_world AFU. The RTL of the hello_world AFU receives from the host application an address via memory mapped I/O (MMIO) write and generates a DMA write to the memory line at that address. The content written to memory is the string \"Hello world!\". The host application spins, waiting for the memory line to be updated. Once available, the software prints out the string.
                                                                   
                                                                  The hello_world example AFU consists of the following files. The hw directory contains the RTL to implement the hardware functionality using CCIP, Avalon, and AXI interfaces. However, this guide will use the AXI version of the AFU to go through the compilation steps. The sw directory of the AFU contains the source code of the host application that communicates with the AFU hardware.
                                                                   
                                                                  hello_world\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u251c\u2500\u2500 ccip\n\u2502       \u2502   \u251c\u2500\u2500 hello_world_ccip.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu.sv\n\u2502       \u2502   \u2514\u2500\u2500 sources.txt\n\u2502       \u2514\u2500\u2500 hello_world.json\n\u251c\u2500\u2500 README.md\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 hello_world\n    \u251c\u2500\u2500 hello_world.c\n    \u251c\u2500\u2500 Makefile\n    \u2514\u2500\u2500 obj\n        \u251c\u2500\u2500 afu_json_info.h\n        \u2514\u2500\u2500 hello_world.o\n
                                                                   
                                                                  The following instructions can be used to compile other AFU samples accompanying this repository.
                                                                   
                                                                     
                                                                  1. If not done already, download and clone the examples AFU repository.
                                                                    2.  
                                                                   
                                                                  $ cd $OFS_BUILD_ROOT \n$ git clone https://github.com/OFS/examples-afu.git\n
                                                                   
                                                                     
                                                                  1.  
                                                                    Compile the hello_word sample AFU.  
                                                                    $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world\n$ afu_synth_setup --source hw/rtl/axi/sources.txt afu_dev\n$ cd afu_dev\n$ ${OPAE_PLATFORM_ROOT}/bin/afu_synth\nCompiling ofs_top ofs_pr_afu\nGenerating hello_world.gbs\n==================================\n.\n.\n.\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'hello_world.gbs'\n Design meets timing\n===========================================================================\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    To test the AFU in actual hardware, load the hello_world.gbs to the Intel IPU Platform F2000X-PL card. For this step to be successful, the SoC Attach FIM must have already been loaded to the Intel IPU Platform F2000X-PL card following the steps described in Section 2 of this document.
                                                                     
                                                                    3.  
                                                                   
                                                                  $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/afu_dev/\n# Copy FIM files to SoC\n$ scp hello_world.gbs <user>@<SoC IP address>:</remote/directory>\n
                                                                   
                                                                  # On SoC\n$ cd </remote/directory>\n$ fpgasupdate hello_world.gbs \n[2022-04-15 20:22:18.85] [WARNING ] Update starting. Please do not interrupt.\n[2022-04-15 20:22:19.75] [INFO    ] \nPartial Reconfiguration OK\n[2022-04-15 20:22:19.75] [INFO    ] Total time: 0:00:00.90\n
                                                                   
                                                                  Set up your Intel IPU Platform F2000X-PL board to work with the newly loaded hello_world.gbs file.
                                                                   
                                                                  # On SoC\n\n# Verify PCIe b.d.f\n# For the following example, the F2000x SKU2 PCIe b:d.f is 15:00.0,\n# however this may be different in your system\n$  fpgainfo fme\nIntel IPU Platform F2000X-PL\nBoard Management Controller NIOS FW version: 1.2.4\nBoard Management Controller Build version: 1.2.4\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\n...\n\n# Create the Virtual Functions (VFs) provided by the FIM, the default FIM has 3 VFs\n$ pci_device 15:00.0 vf 3\n\n# Verify the VFs have been added (device id: bccf)\n$ lspci -s 15:00\n15:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n15:00.1 Processing accelerators: Intel Corporation Device bccf\n15:00.2 Processing accelerators: Intel Corporation Device bccf\n15:00.3 Processing accelerators: Intel Corporation Device bccf\n\n\n# Bind VFs to VFIO driver.\n\n$ opae.io init -d 0000:15:00.1\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:15:00.1 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:15:00.1 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:15:00.1 is 52\n\n$ opae.io init -d 0000:15:00.2\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:15:00.2 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:15:00.2 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:15:00.2 is 53\n\n$ opae.io init -d 0000:15:00.3\nopae.io 0.2.5\nUnbinding (0x8086,0xbccf) at 0000:15:00.3 from dfl-pci\nBinding (0x8086,0xbccf) at 0000:15:00.3 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:15:00.3 is 54\n\n# Verify the new AFU is loaded.  The hello_world AFU GUID is \"c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\".\n\n$ fpgainfo port\n//****** PORT ******//\nObject Id                        : 0xF100000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0x6015000000000000\nPCIe s:b:d.f                     : 0000:15:00.3\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nAccelerator GUID                 : d15ab1ed-0000-0000-0210-000000000000\n//****** PORT ******//\nObject Id                        : 0x4015000000000000\nPCIe s:b:d.f                     : 0000:15:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nAccelerator GUID                 : d15ab1ed-0000-0000-0110-000000000000\n//****** PORT ******//\nObject Id                        : 0x2015000000000000\nPCIe s:b:d.f                     : 0000:15:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nAccelerator GUID                 : c6aa954a-9b91-4a37-abc1-1d9f0709dcc3\n
                                                                   
                                                                     
                                                                  1. Compile and execute the host application of the hello_world AFU. You should see the application outputs the \"Hello world!\" message in the terminal.
                                                                    2.  
                                                                   
                                                                  # On Development Host, move to the sw directory of the hello_world AFU and build application\n$ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/sw\n$ make\n# Copy application to SoC\n$ scp hello_world <user>@<SoC IP address>:</remote/directory>\n
                                                                   
                                                                  # On SoC, Run the application\n$ cd </remote/directory>\n$ ./hello_world\nHello world!\n
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#34-modify-the-afu-user-clocks-frequency","title":"3.4. Modify the AFU user clocks frequency","text":"
                                                                  An OPAE compliant AFU specifies the frequency of the uclk_usr and uclk_usr_div2 clocks through the JSON file for AFU configuration located under the <afu_example>/hw/rtl directory of an AFU design. For instance, the AFU configuration file of the host_chan_mmio example is $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/host_chan_mmio.json.
                                                                   
                                                                  The AFU specifies the frequency for uClk_usr in its platform configuration file using the following key:value pairs:
                                                                   
                                                                    \"clock-frequency-high\": [<float-value>|\u201dauto\u201d|\u201dauto-<float-value>\u201d]\n  \"clock-frequency-low\": [<float-value>|\u201dauto\u201d|\u201dauto-<float-value>\u201d]\n
                                                                   
                                                                  These key:value tuples are used to configure the PLL of the target platform that provides the user clocks through the AFU clocks interface. In addition, the specified frequency affects the timing closure process on the user clocks during AFU compilation. 
                                                                   
                                                                  Setting the value field to a float number (e.g., 315.0 to specify 315 MHz) drives the AFU generation process to close timing within the bounds set by the low and high values and sets the AFU's JSON metadata to specify the user clock PLL  frequency values.
                                                                   
                                                                  The following example shows the JSON file of the host_chan_mmio to set the AFU uClk to 500 MHz and uClk_div2 to 250 MHz.
                                                                   
                                                                  {\n   \"version\": 1,\n   \"afu-image\": {\n      \"power\": 0,\n      \"clock-frequency-high\": 500,\n      \"clock-frequency-low\": 250,\n      \"afu-top-interface\":\n         {\n            \"class\": \"ofs_plat_afu\"\n         },\n      \"accelerator-clusters\":\n         [\n            {\n               \"name\": \"host_chan_mmio\",\n               \"total-contexts\": 1,\n               \"accelerator-type-uuid\": \"76d7ae9c-f66b-461f-816a-5428bcebdbc5\"\n            }\n         ]\n   }\n}\n
                                                                   
                                                                  Save the changes to host_chan_mmio.json file, then execute the afu_synth_setup script to create a new copy of the AFU files with the modified user clock settigns.
                                                                   
                                                                  $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/\n$ afu_synth_setup -s $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/hw/rtl/test_mmio_axi1.txt afu_clks\n\nCopying build from /home/<user_area>/ofs-f2000x-pl/work_pr/pr_build_template/hw/lib/build...\nConfiguring Quartus build directory: build_F2000x_afu_clks/build\nLoading platform database: /home/<user_area>/ofs-f2000x-pl/work_pr/pr_build_template/hw/lib/platform/platform_db/ofs_agilex_adp.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting platform/platform_afu_top_config.vh\nWriting platform/platform_if_addenda.qsf\nWriting ../hw/afu_json_info.vh\n
                                                                   Compile the host_chan_mmio AFU with the new frequency values.
                                                                   
                                                                  $ cd afu_clks\n$ $OPAE_PLATFORM_ROOT/bin/afu_synth\n
                                                                   
                                                                  During the compilation phase, you will observe the Timing Analyzer uses the specified user clock frequency values as the target to close timing.
                                                                   
                                                                   
                                                                  AFU developers must ensure the AFU hardware design meets timing. The compilation of an AFU that fails timing shows a message similar to the following.
                                                                   
                                                                  .\n.\n.\n\nWrote host_chan_mmio.gbs\n\n===========================================================================\n PR AFU compilation complete\n AFU gbs file is 'host_chan_mmio.gbs'\n\n  *** Design does not meet timing\n  *** See build/syn/syn_top/output_files/timing_report\n\n===========================================================================\n
                                                                   
                                                                  The previous output indicates the location of the timing reports for the AFU designer to identify the failing paths and perform the necessary design changes. Next, is a listing of the timing report files from a host_chan_mmio AFU that fails to meet timing after modifying the user clock frequency values.
                                                                   
                                                                  $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_clks\n$ ls build/syn/syn_top/output_files/timing_report\n\nclocks.rpt  clocks.sta.fail.summary  clocks.sta.pass.summary\n
                                                                   
                                                                  Warning: AFU developers must inform software developers of the maximum operating frequency (Fmax) of the user clocks to avoid any unexpected behavior of the accelerator and potentially of the overall system.
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#4-simulating-an-afu-using-ase","title":"4. Simulating an AFU using ASE","text":"
                                                                  The Application Simulation Environment (ASE) is a hardware/software co-simulation environment for your AFU. See diagram below illustrating ASE operation:
                                                                   
                                                                   
                                                                  ASE uses the simulator Direct Programming Interface (DPI) to provide HW/SW connectivity.  The PCIe connection to the AFU under testing is emulated with a transactional model.
                                                                   
                                                                  The following list describes ASE operation:
                                                                   
                                                                     
                                                                  • Attempts to replicate the transactions that will be seen in real system.
                                                                    •  
                                                                  • Provides a memory model to AFU, so illegal memory accesses can be identified early.
                                                                    •  
                                                                  • Not a cache simulator.
                                                                    •  
                                                                  • Does not guarantee synthesizability or timing closure.
                                                                    •  
                                                                  • Does not model system latency.
                                                                    •  
                                                                  • No administrator privileges are needed to run ASE.  All code is user level.
                                                                    •  
                                                                   
                                                                  The remainder of this section is a tutorial providing the steps on how to run ASE with either Synopsys\u00ae VCS\u00ae or Siemens\u00ae QuestaSim\u00ae using an example AFU and the AFU build tree previously created in this guide.
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#41-set-up-steps-to-run-ase","title":"4.1. Set Up Steps to Run ASE","text":"
                                                                  In this section you will set up your server to support ASE by independently downloading and installing OPAE SDK and ASE. Then, set up the required environment variables.
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#411-install-opae-sdk","title":"4.1.1. Install OPAE SDK","text":"
                                                                  Follow the instructions documented in the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 FPGAs Targeting the Intel IPU Platform F2000X-PL, section 6.2 Installing the OPAE SDK On the Host to build and install the required OPAE SDK for the Intel IPU Platform F2000X-PL card.
                                                                   
                                                                  The F2000x SKU2 card requires 2.10.0-1. Follow the instructions provided in the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 FPGAs Targeting the Intel IPU Platform F2000X-PL, section 6.2 Installing the OPAE SDK On the Host. However, just make sure to check out the cloned repository to tag 2.10.0-1 and branch release/2.10.0.
                                                                   
                                                                  $ git checkout tags/2.10.0-1 -b release/2.10.0\n
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#412-install-ase-tools","title":"4.1.2 Install ASE Tools","text":"
                                                                  ASE is an RTL simulator for OPAE-based AFUs. The simulator emulates both the OPAE SDK software user space API and the AFU RTL interface. The majority of the FIM as well as devices such as PCIe and local memory are emulated with simple functional models.
                                                                   
                                                                  ASE must be installed separatedly from the OPAE SDK. However, the recommendation is to install it in the same target directory as OPAE SDK.
                                                                   
                                                                     
                                                                  1.  
                                                                    If not done already, set the environment variables as described in section, Set Up AFU Development Environment.
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Clone the opae-sim repository. 
                                                                    $ cd $OFS_BUILD_ROOT\n$ git clone https://github.com/OFS/opae-sim.git\n$ cd opae-sim\n$ git checkout tags/2.10.0-1 -b release/2.10.0 \n
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Create a build directory and build ASE to be installed under the default system directories along with OPAE SDK. 
                                                                    $ mkdir build\n$ cd build\n$ cmake  -DCMAKE_INSTALL_PREFIX=/usr ..\n$ make\n
                                                                     
                                                                    4.  
                                                                   
                                                                  Optionally, if the desire is to install ASE binaries in a different location to the system's default, provide the path to CMAKE through the CMAKE_INSTALL_PREFIX switch, as follows. 
                                                                  $ cmake -DCMAKE_INSTALL_PREFIX=<</some/arbitrary/path>> ..  \n
                                                                   
                                                                     
                                                                  1. Install ASE binaries and libraries under the system directory /usr. 
                                                                    $ sudo make install  \n
                                                                    2.  
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#413-setup-required-ase-environment-variables","title":"4.1.3. Setup Required ASE Environment Variables","text":"
                                                                  The values set to the following environment variables assume the OPAE SDK and ASE were installed in the default system directories below /usr. Setup these variables in the shell where ASE will be executed. You may wish to add these variables to the script you created to facilitate configuring your environment.
                                                                   
                                                                  $ export OPAE_PLATFORM_ROOT=<path to PR build tree>\n$ cd /usr/bin\n$ export PATH=$PWD:$PATH\n$ cd /usr/lib/python*/site-packages\n$ export PYTHONPATH=$PWD\n$ cd /usr/lib\n$ export LIBRARY_PATH=$PWD\n$ cd /usr/lib64\n$ export LD_LIBRARY_PATH=$PWD\n$ cd $OFS_BUILD_ROOT/ofs-platform-afu-bbb\n$ export OFS_PLATFORM_AFU_BBB=$PWD\n\n  ## For VCS, set the following:\n$ export VCS_HOME=<Set the path to VCS installation directory>\n$ export PATH=$VCS_HOME/bin:$PATH\n\n  ## For QuestaSIM, set the following:\n$ export MTI_HOME=<path to Modelsim installation directory>\n$ export PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\n
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#42-simulating-the-host_chan_mmio-afu","title":"4.2. Simulating the host_chan_mmio AFU","text":"
                                                                  The $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio is a simple example demonstrating both hardware and software access to an AFU. The host_chan_mmio example AFU consists of the following files:
                                                                   
                                                                  host_chan_mmio\n\u251c\u2500\u2500 hw\n\u2502   \u2514\u2500\u2500 rtl\n\u2502       \u251c\u2500\u2500 avalon\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_avalon.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_avalon_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_avalon.sv\n\u2502       \u251c\u2500\u2500 axi\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 afu_axi.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi512.sv\n\u2502       \u2502   \u251c\u2500\u2500 ofs_plat_afu_axi_from_ccip.sv\n\u2502       \u2502   \u2514\u2500\u2500 ofs_plat_afu_axi.sv\n\u2502       \u251c\u2500\u2500 host_chan_mmio.json\n\u2502       \u251c\u2500\u2500 test_mmio_avalon0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon1.txt\n\u2502       \u251c\u2500\u2500 test_mmio_avalon2_512rw.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi0_from_ccip.txt\n\u2502       \u251c\u2500\u2500 test_mmio_axi1.txt\n\u2502       \u2514\u2500\u2500 test_mmio_axi2_512rw.txt\n\u2514\u2500\u2500 sw\n    \u251c\u2500\u2500 main.c\n    \u251c\u2500\u2500 Makefile\n
                                                                   
                                                                  This example AFU contains examples using both Avalon and AXI interface buses. This guide will use the AXI version of the host_chan_mmio AFU.
                                                                   
                                                                  ASE uses client-server application architecture to deliver hardware/software co-simulation.  You require one shell for the hardware based simulation and another shell where the software application is running. The hardware is started first with a simulation compilation and simulator startup script, once the simulator has loaded the design, it will wait until the software process starts. Once the software process starts, the simulator proceeds.  Transaction logging and waveform capture is performed.
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#421-set-up-and-run-the-hw-simulation-process","title":"4.2.1 Set Up and Run the HW Simulation Process","text":"
                                                                  You will run the afu_sim_setup script to create the scripts for running the ASE environment.  The afu_sim_setup script has the following usage:
                                                                   
                                                                  usage: afu_sim_setup [-h] -s SOURCES [-p PLATFORM] [-t {VCS,QUESTA,MODELSIM}]\n                     [-f] [--ase-mode ASE_MODE] [--ase-verbose]\n                     dst\n\nGenerate an ASE simulation environment for an AFU. An ASE environment is\ninstantiated from the OPAE installation and then configured for the specified\nAFU. AFU source files are specified in a text file that is parsed by\nrtl_src_config, which is also part of the OPAE base environment.\n\npositional arguments:\n  dst                   Target directory path (directory must not exist).\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -s SOURCES, --sources SOURCES\n                        AFU source specification file that will be passed to\n                        rtl_src_config. See \"rtl_src_config --help\" for the\n                        file's syntax. rtl_src_config translates the source\n                        list into either Quartus or RTL simulator syntax.\n  -p PLATFORM, --platform PLATFORM\n                        FPGA Platform to simulate.\n  -t {VCS,QUESTA,MODELSIM}, --tool {VCS,QUESTA,MODELSIM}\n                        Default simulator.\n  -f, --force           Overwrite target directory if it exists.\n  --ase-mode ASE_MODE   ASE execution mode (default, mode 3, exits on\n                        completion). See ase.cfg in the target directory.\n  --ase-verbose         When set, ASE prints each CCI-P transaction to the\n                        command line. Transactions are always logged to\n                        work/ccip_transactions.tsv, even when not set. This\n                        switch sets ENABLE_CL_VIEW in ase.cfg.\n
                                                                   
                                                                  Run afu_sim_setup to create the ASE simulation environment for the host_chan_mmio example AFU. The '-t VCS' option indicates to prepare the ASE simulation environment for Synopsys\u00ae VCS\u00ae.
                                                                   
                                                                  $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio\n$ afu_sim_setup -s ./hw/rtl/test_mmio_axi1.txt -t VCS afu_sim\n\nCopying ASE from /opae-sdk/install-opae-sdk/share/opae/ase...\n#################################################################\n#                                                               #\n#             OPAE Intel(R) Xeon(R) + FPGA Library              #\n#               AFU Simulation Environment (ASE)                #\n#                                                               #\n#################################################################\n\nTool Brand: VCS\nLoading platform database: /ofs-f2000x-pl/work_pr/build_tree/hw/lib/platform/platform_db/ofs_agilex_adp.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting rtl/platform_afu_top_config.vh\nWriting rtl/platform_if_addenda.txt\nWriting rtl/platform_if_includes.txt\nWriting rtl/ase_platform_name.txt\nWriting rtl/ase_platform_config.mk and rtl/ase_platform_config.cmake\nASE Platform: discrete (FPGA_PLATFORM_DISCRETE)\n
                                                                   
                                                                  The afu_sim_setup creates the ASE scripts in the directory host_chan_mmio_sim where the afu_sim_setup script was run.  Start the simulator as shown below:
                                                                   
                                                                  $ cd afu_sim\n$ make\n$ make sim\n
                                                                   
                                                                  This process launches the AFU hardware simulator. Before moving to the next section, pay attention to the simulator output highlighted in the image below.
                                                                   
                                                                   
                                                                  The simulation artifacts are stored in host_chan_mmio/work and consist of:
                                                                   
                                                                  log_ase_events.tsv\nlog_ofs_plat_host_chan.tsv \nlog_ofs_plat_local_mem.tsv \nlog_pf_vf_mux_A.tsv \nlog_pf_vf_mux_B.tsv \n
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#422-set-up-and-run-the-sw-process","title":"4.2.2 Set Up and Run the SW Process","text":"
                                                                  Open an additional shell to build and run the host application that communicates with the actual AFU hardware. Set up the same environment variable you have set up in the shell you have been working on until this point. 
                                                                   
                                                                  Additionally, as indicated by the hardware simulator output that is currently executing in the \"simulator shell\", copy and paste the line \"export ASE_WORKDIR=...\", into the new \"software shell\". See the last image of the previous section.
                                                                   
                                                                  $ export ASE_WORKDIR= $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/afu_sim/work\n
                                                                   Then, go to the sw directory of the host_chan_mmio AFU example to compile the host application.
                                                                   
                                                                  $ cd $OFS_PLATFORM_AFU_BBB/plat_if_tests/host_chan_mmio/sw  \n$ make\n\nafu_json_mgr json-info --afu-json=../hw/rtl/host_chan_mmio.json --c-hdr=obj/afu_json_info.h\nWriting obj/afu_json_info.h\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c main.c -o obj/main.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c test_host_chan_mmio.c -o obj/test_host_chan_mmio.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/connect.c -o obj/connect.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/csr_mgr.c -o obj/csr_mgr.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/hash32.c -o obj/hash32.o\ncc -g -O2 -std=gnu99 -fstack-protector -fPIE -fPIC -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -I../../common/sw -I./obj -c ../../common/sw/test_data.c -o obj/test_data.o\ncc -o host_chan_mmio obj/main.o obj/test_host_chan_mmio.o obj/connect.o obj/csr_mgr.o obj/hash32.o obj/test_data.o  -z noexecstack -z relro -z now -pie -luuid -lopae-c-ase\n
                                                                   
                                                                  Now, launch the host application to exercise the AFU hardware running on the simulator shell. The next image shows the AFU hardware simulation process on the left side shell. The right hand shell shows the host application's output of a successful simulation.
                                                                   
                                                                  $ with_ase ./host_chan_mmio\n  [APP]  Initializing simulation session ...\nRunning in ASE mode\nAFU ID:  76d7ae9cf66b461f 816a5428bcebdbc5\nAFU MMIO interface: AXI Lite\nAFU MMIO read bus width: 64 bits\n512 bit MMIO write supported: yes\nAFU pClk frequency: 470 MHz\n\nTesting 32 bit MMIO reads:\n  PASS - 4 tests\n\nTesting 32 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 64 bit MMIO writes:\n  PASS - 5 tests\n\nTesting 512 bit MMIO writes:\n  PASS\n  [APP]  Deinitializing simulation session\n  [APP]         Took 1,003,771,568 nsec\n  [APP]  Session ended\n
                                                                   
                                                                   
                                                                   
                                                                  Finally, on the hardware simulation shell, you can view the wave forms by invoking the following command.
                                                                   
                                                                  $ make wave\n
                                                                   
                                                                  This brings up the Synopsys\u00ae VCS\u00ae simulator GUI and loads the simulation waveform files. Use the Hierarchy window to navigate to the afu instance located under, ase_top | ase_top_plat | ase_afu_main_pcie_ss | ase_afu_main_emul | afu_main | port_afu_instances | ofs_plat_afu | afu , as shown below.
                                                                   
                                                                   
                                                                  Right click on the afu (afu) entry to display the drop-down menu. Then, click on Add to Waves | New Wave View to display the following waveforms window.
                                                                   
                                                                   
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#43-simulating-the-hello_world-afu","title":"4.3 Simulating the hello_world AFU","text":"
                                                                  In this section you will quickly simulate the PIM-based hello_world sample AFU accompanying the examples-afu repository.
                                                                   
                                                                     
                                                                  1.  
                                                                    Set the environment variables as described in section 4.1. Set Up Steps to Run ASE.
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Prepare an RTL simulation environment for the AXI version of the hello_world AFU.
                                                                     
                                                                    3.  
                                                                   
                                                                  Simulation with ASE requires two software processes, one to simulate the AFU RTL and the other to run the host software that excercises the AFU. To construct an RTL simulation environment under the directory simulation, execute the following.
                                                                   
                                                                  $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world\n$ afu_sim_setup -s ./hw/rtl/axi/sources.txt -t VCS afu_sim\n\n\nCopying ASE from /usr/local/share/opae/ase...\n#################################################################\n#                                                               #\n#             OPAE Intel(R) Xeon(R) + FPGA Library              #\n#               AFU Simulation Environment (ASE)                #\n#                                                               #\n#################################################################\n\nTool Brand: VCS\nLoading platform database: /home/<user_area>/ofs-f2000x-pl/work_pr/pr_build_template/hw/lib/platform/platform_db/ofs_agilex_adp.json\nLoading platform-params database: /usr/share/opae/platform/platform_db/platform_defaults.json\nLoading AFU database: /usr/share/opae/platform/afu_top_ifc_db/ofs_plat_afu.json\nWriting rtl/platform_afu_top_config.vh\nWriting rtl/platform_if_addenda.txt\nWriting rtl/platform_if_includes.txt\nWriting rtl/ase_platform_name.txt\nWriting rtl/ase_platform_config.mk and rtl/ase_platform_config.cmake\nASE Platform: discrete (FPGA_PLATFORM_DISCRETE)\n
                                                                   
                                                                  The afu_sim_setup script constructs an ASE environment in the hello_world_sim subdirectory. If the command fails, confirm that the path to the afu_sim_setup is on your PATH environment variable (in the OPAE SDK bin directory) and that your Python version is at least 2.7.
                                                                   
                                                                     
                                                                  1. Build and execute the AFU RTL simulator.
                                                                    2.  
                                                                   
                                                                  $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/afu_sim\n$ make\n$ make sim  \n
                                                                   
                                                                  The previous commands will build and run the Synopsys\u00ae VCS\u00ae RTL simulator, which prints a message saying it is ready for simulation. The simulation process also prints a message instructing you to set the ASE_WORKDIR environment variable in a second shell.
                                                                   
                                                                     
                                                                  1.  
                                                                    Open a second shell where you will build and execute the host software. In this new \"software shell\", set up the environment variables you have set up so far in the \"hardware simulation\" shell.
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Also, set the ASE_WORKDIR environment variable following the instructions given in the \"hardware simulation\" shell. 
                                                                    $ export ASE_WORKDIR=$OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/afu_sim/work\n
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Then, move to the sw directory of the hello_world AFU sample to build the host software. 
                                                                    $ cd $OFS_BUILD_ROOT/examples-afu/tutorial/afu_types/01_pim_ifc/hello_world/sw\n$ make      \n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Run the hello_world host application to resume the work of the RTL simulation. The host software process and the RTL simulation execute in lockstep. If successful, you should see the Hello world! output.
                                                                     
                                                                    5.  
                                                                   
                                                                  $ with_ase ./hello_world\n\n  [APP]  Initializing simulation session ...\nHello world!\n  [APP]  Deinitializing simulation session\n  [APP]         Took 43,978,424 nsec\n  [APP]  Session ended\n
                                                                   
                                                                  The image below shows the simulation of the AFU hardware and the execution of the host application side-by-side.
                                                                   
                                                                   
                                                                   
                                                                   
                                                                     
                                                                  1. Finally, on the hardware simulation shell, you can view the wave forms by invoking the following command. 
                                                                    make wave\n
                                                                    2.  
                                                                   
                                                                  This brings up the DVE GUI and loads the simulation waveform files. Use the Hierarchy window to navigate to the afu instance located under, ase_top | ase_top_plat | ase_afu_main_pcie_ss | ase_afu_main_emul | afu_main | port_afu_instances | ofs_plat_afu | hello_afu, as shown below.
                                                                   
                                                                   
                                                                  Right click on the hello_afu entry to display the drop-down menu. Then, click on Add to Waves | New Wave View to display the following waveforms window.
                                                                   
                                                                   
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#5-adding-remote-signal-tap-logic-analyzer-to-debug-the-afu","title":"5. Adding Remote Signal Tap Logic Analyzer to debug the AFU","text":"
                                                                  Remote Signal Tap is currently not supported in F2000x base FIM configuration.
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#6-how-to-modify-the-pfvf-mux-configuration","title":"6. How to modify the PF/VF MUX configuration","text":"
                                                                  For information on how to modify the PF/VF mapping for your own design, refer to the FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae SoC Attach FPGAs.
                                                                  "},{"location":"hw/f2000x/dev_guides/afu_dev/ug_dev_afu_ofs_f2000x/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                  Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                   
                                                                  OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/","title":"Intel\u00ae FPGA Interface Manager Developer Guide: Intel Agilex\u00ae 7 SoC Attach: Open FPGA Stack","text":"
                                                                  Last updated: February 03, 2024 
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#1-introduction","title":"1 Introduction","text":""},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#11-about-this-document","title":"1.1 About This Document","text":"
                                                                  This document serves as a design guide for FPGA developers, system architects, and hardware developers using Open FPGA Stack (OFS) as a starting point for the creation of an FPGA Interface Manager (FIM) for a custom FPGA acceleration board.   
                                                                   
                                                                  OFS addresses the demand for FPGA acceleration boards and workloads by providing a powerful methodology for the rapid development of FPGA Acceleration systems.  This methodology addresses the challenges and responsibilities of the board, platform, and workload developers by providing a complete FPGA project consisting of RTL and simulation code, build scripts, and software.  This provided FPGA project can be rapidly customized to meet new market requirements.
                                                                   
                                                                  OFS separates the FPGA design into two areas: FPGA Interface Manager (FIM) and workload (or Acceleration Function Unit) as shown in the figure below:
                                                                   
                                                                   
                                                                  As can be seen in this diagram, the OFS FPGA structure has a natural separation into two distinct areas: 
                                                                   
                                                                     
                                                                  • FPGA Interface Manager (FIM or sometimes called the \"the shell\") containing:
                                                                       
                                                                    • FPGA external interfaces and IP cores (e.g. Ethernet, DDR-4, PCIe, etc)
                                                                      •  
                                                                    • PLLs/resets
                                                                      •  
                                                                    • FPGA - Board management infrastructure
                                                                      •  
                                                                    • Interface to Acceleration Function Unit (AFU)
                                                                      •  
                                                                     
                                                                    •  
                                                                  • Acceleration Function Unit (\"the workload\")
                                                                       
                                                                    • Uses the FIM interfaces to perform useful work inside the FPGA
                                                                      •  
                                                                    • Contains logic supporting partial reconfiguration
                                                                      •  
                                                                    • Remote Signal Tap core for remote debugging of SoC AFU PR region
                                                                      •  
                                                                     
                                                                    •  
                                                                   
                                                                  This guide is organized as follows: 
                                                                   
                                                                     
                                                                  • Introduction
                                                                    •  
                                                                  • Top Level Block Diagram description
                                                                       
                                                                    • Control and data flow
                                                                      •  
                                                                     
                                                                    •  
                                                                  • Description of Sub-systems
                                                                       
                                                                    • Command/status registers (CSR) and software interface
                                                                      •  
                                                                    • Clocking, resets, and interfaces
                                                                      •  
                                                                    • High-Speed Serial Interface Sub-System (HSSI-SS) - also known as the Ethernet Sub-System
                                                                      •  
                                                                    • External Memory Interface Sub-System (MEM-SS)
                                                                      •  
                                                                     
                                                                    •  
                                                                  • High-level development flow description
                                                                       
                                                                    • Installation of OFS RTL and development packages
                                                                      •  
                                                                    • Compiling FIM
                                                                      •  
                                                                    • Simulation  
                                                                      •  
                                                                     
                                                                    •  
                                                                  • Design customization walkthroughs
                                                                    •  
                                                                   
                                                                  This document uses the  Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL as the platform to illustrate key points and demonstrate how to extend the capabilities provided in OFS.  The demonstration steps serve as a tutorial for the development of your OFS knowledge.  
                                                                   
                                                                  This document covers OFS architecture lightly. For more details on the OFS architecture, please see Open FPGA Stack Technical Reference Manual.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#111-glossary","title":"1.1.1 Glossary","text":"
                                                                  The following table describes several terms that are used in this document.
                                                                   Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#12-release-capabilities","title":"1.2 Release Capabilities","text":"
                                                                  Intel Agilex 7 SoC Attach OFS supports the following features.
                                                                   FIM BASE Intel IPU Platform F2000X-PL f2000x PCIe Configuration Host: PCIe Gen4x16SoC: PCIe Gen4x16 SR-IOV support Host: 2 PFs, No VFsSoC:  1 PFs, 3 VFs AXI ST datapath 512b @ 470MHz Transceiver Subsystem Configuration 2x4x25G 
                                                                  The FIM also integrates:
                                                                   
                                                                     
                                                                  • SoC AFU and Host AFU
                                                                    •  
                                                                  • Exercisers demonstrating PCIe, external memory, and Ethernet interfaces
                                                                    •  
                                                                  • FME CSR
                                                                    •  
                                                                  • Remote Signal Tap
                                                                    •  
                                                                  • Partial Reconfiguration of the SoC AFU
                                                                    •  
                                                                   
                                                                  The Host exercisers are provided for the quick evaluation of the FIM and can be leveraged for the verification of the platform's functionality and capabilities.  The host exercisers can be removed by the designer to release FPGA real estate to accommodate new workload functions. To compile the FIM without host exercisers go to How to compile the FIM in preparation for designing your AFU.
                                                                   
                                                                  OFS is extensible to meet the needs of a broad set of customer applications.  The general use cases listed below are examples where the OFS base design is easily extended to build a custom FIM:
                                                                   
                                                                     
                                                                  1. Use OFS design example as-is
                                                                       
                                                                    • Porting the code to another platform that is identical to OFS reference platform changing targeted FPGA device and pinout
                                                                      •  
                                                                    • Change I/O assignments without changing design
                                                                      •  
                                                                     
                                                                    2.  
                                                                  2. Update the configuration of peripheral IP in OFS design example, not affecting FIM architecture
                                                                       
                                                                    • External memory settings
                                                                      •  
                                                                    • Ethernet Subsystem analog settings
                                                                      •  
                                                                     
                                                                    3.  
                                                                  3. Remove/update peripheral feature in OFS design example, not affecting FIM architecture
                                                                       
                                                                    • External memory speed/width change
                                                                      •  
                                                                    • Change number of VFs supported
                                                                      •  
                                                                     
                                                                    4.  
                                                                  4. Add new features as an extension to OFS design example, not affecting the FIM architecture
                                                                       
                                                                    • Add/remove external memory interface to the design
                                                                      •  
                                                                    • Add/remove user clocks for the AFU
                                                                      •  
                                                                    • Add/remove IP to the design with connection to the AFU
                                                                      •  
                                                                     
                                                                    5.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#13-knowledge-prerequisites","title":"1.3 Knowledge Prerequisites","text":"
                                                                  OFS is an advanced application of FPGA technology. This guide assumes you have the following FPGA logic design-related knowledge and skills:
                                                                   
                                                                     
                                                                  • FPGA compilation flows using Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3.
                                                                    •  
                                                                  • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
                                                                    •  
                                                                  • RTL (System Verilog) and coding practices to create synthesized logic.
                                                                    •  
                                                                  • RTL simulation tools.
                                                                    •  
                                                                  • Intel\u00ae Quartus\u00ae Prime Pro Edition Signal Tap Logic Analyzer tool software.
                                                                    •  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#2-top-level-description","title":"2 Top Level Description","text":"
                                                                  The FIM targets operation in the  Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL the block diagram is shown below.  
                                                                   
                                                                   
                                                                     
                                                                  • Host interface 
                                                                       
                                                                    • PCIe Gen4 x 16
                                                                      •  
                                                                     
                                                                    •  
                                                                  • SoC Interface
                                                                       
                                                                    • PCIe Gen4 x 16
                                                                      •  
                                                                     
                                                                    •  
                                                                  • Network interface
                                                                       
                                                                    • 2 - QSFP28/56 cages
                                                                      •  
                                                                    • Eight Arm\u00ae AMBA\u00ae 4 AXI4-Stream channels of 25G Ethernet interfacing to an E-tile Ethernet Subsystem.
                                                                      •  
                                                                     
                                                                    •  
                                                                  • External Memory - DDR4
                                                                       
                                                                    • Four Fabric DDR4-2400 banks - 4 GB organized as 1Gb x 32 with 1 Gb x 8 ECC (ECC login not implemented in default FIM) 
                                                                      •  
                                                                     
                                                                    •  
                                                                  • Board Management
                                                                       
                                                                    • SPI interface
                                                                      •  
                                                                    • FPGA configuration
                                                                      •  
                                                                     
                                                                    •  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#21-top-level-fpga","title":"2.1 Top Level FPGA","text":"
                                                                  The internal FPGA architecture is shown below:
                                                                   
                                                                   
                                                                  The following Platform Designer IP subsystems are used to implement the following:
                                                                   
                                                                     
                                                                  • P-tile PCIe Subsystem
                                                                    •  
                                                                  • E-Tile Ethernet Subsystem
                                                                    •  
                                                                  • Memory Subsystem
                                                                    •  
                                                                   
                                                                  Documentation on the above Platform Designer IP subsystems is available by request to your Intel support team.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#22-fim-fpga-resource-usage","title":"2.2 FIM FPGA Resource Usage","text":"
                                                                  The provided design includes both required board management and control functions as well as optional interface exerciser logic that both creates transactions and validates operations.  These exerciser modules include:
                                                                   
                                                                     
                                                                  • HE_MEM - this module creates external memory transactions to the DDR4 memory and then verifies the responses.
                                                                    •  
                                                                  • HE_MEM-TG -The memory traffic generator (TG) AFU provides a way for users to characterize local memory channel bandwidth with a variety of traffic configuration features including request burst size, read/write interleave count, address offset, address strobe, and data pattern.
                                                                    •  
                                                                  • HE_HSSI - this module creates ethernet transactions to the HSSI Subsystem and then verifies the responses.
                                                                    •  
                                                                   
                                                                  The FIM uses a small portion of the available FPGA resources.  The table below shows resource usage for a base FIM built with 2 channels of external memory, a small AFU instantiated that has host CSR read/write, external memory test and Ethernet test functionality.
                                                                   
                                                                  Note: The host exerciser modules allow you to evaluate the FIM in hardware and are removed when you begin development. 
                                                                   
                                                                  The AGFC023R25A2E2VR0  FPGA has the following resources available for base FIM design :
                                                                   Resource needed / total on device (%) Logic utilization (ALMs) 229,622 / 782,400 ( 29 % ) M20K blocks 1,241 / 10,464 (12 %) Pins 518 / 742 ( 70 % ) IOPLLs 10 / 15 ( 67 % ) 
                                                                  The resource usage for the FIM base:
                                                                   Entity Name ALMs needed ALM Utilization % M20Ks M20K Utilization % top 229,646.10 29.35 1241 11.86 soc_afu 87,364.80 11.17 273 2.61 soc_pcie_wrapper 37,160.80 4.75 195 1.86 pcie_wrapper 36,233.40 4.63 187 1.79 host_afu 26,462.20 3.38 140 1.34 hssi_wrapper 20,066.30 2.56 173 1.65 pmci_wrapper 8,449.90 1.08 186 1.78 mem_ss_top 7,907.10 1.01 60 0.57 auto_fab_0 2,708.90 0.35 13 0.12 soc_bpf 1,210.20 0.15 0 0.00 qsfp_1 635.50 0.08 4 0.04 qsfp_0 628.70 0.08 4 0.04 fme_top 628.60 0.08 6 0.06 host_soc_rst_bridge 151.40 0.02 0 0.00 rst_ctrl 16.80 0.00 0 0.00 soc_rst_ctrl 16.50 0.00 0 0.00 sys_pll 0.50 0.00 0 0.00 
                                                                  The following example without the he_lb,he_hssi,he_mem,he_mem_tg:
                                                                   Entity Name ALMs needed ALM Utilization % M20Ks M20K Utilization % top 162,010.20 20.71 992 9.48 pcie_wrapper 36,771.70 4.70 195 1.86 soc_afu_top 34,851.30 4.45 85 0.81 pcie_wrapper 33,358.90 4.26 175 1.67 hssi_wrapper 20,109.90 2.57 173 1.65 afu_top 14,084.20 1.80 91 0.87 pmci_wrapper 8,447.90 1.08 186 1.78 mem_ss_top 8,379.70 1.07 60 0.57 alt_sld_fab_0 2,725.10 0.35 13 0.12 bpf_top 1,213.00 0.16 0 0.00 fme_top 638.30 0.08 6 0.06 qsfp_top 626.70 0.08 4 0.04 qsfp_top 619.20 0.08 4 0.04 axi_lite_rst_bridge 147.40 0.02 0 0.00 rst_ctrl 17.40 0.00 0 0.00 rst_ctrl 15.90 0.00 0 0.00 sys_pll 0.50 0.00 0 0.00 
                                                                  The following example without the Ethernet Subsystem (no_hssi):
                                                                   Entity Name ALMs needed ALM Utilization % M20Ks M20K Utilization % top 189,827.00 24.26 980 9.37 soc_afu_top 67,751.40 8.66 197 1.88 pcie_wrapper 36,909.30 4.72 195 1.86 pcie_wrapper 36,077.70 4.61 187 1.79 afu_top 26,549.40 3.39 140 1.34 pmci_wrapper 8,688.10 1.11 186 1.78 mem_ss_top 8,079.00 1.03 60 0.57 alt_sld_fab_0 1,751.90 0.22 9 0.09 bpf_top 1,186.00 0.15 0 0.00 dummy_csr 664.70 0.08 0 0.00 dummy_csr 662.80 0.08 0 0.00 dummy_csr 661.20 0.08 0 0.00 fme_top 649.40 0.08 6 0.06 axi_lite_rst_bridge 161.70 0.02 0 0.00 rst_ctrl 16.30 0.00 0 0.00 rst_ctrl 16.00 0.00 0 0.00 sys_pll 0.50 0.00 0 0.00 
                                                                  The following example without the Ethernet Subsystem (no_hssi) + no host exercisers (he_lb, he_hssi, he_mem, he_mem_tg):
                                                                   Entity Name ALMs needed ALM Utilization % M20Ks M20K Utilization % top 139,105.70 17.78 807 7.71 pcie_wrapper 36,518.80 4.67 195 1.86 pcie_wrapper 33,234.50 4.25 175 1.67 soc_afu_top 32,700.00 4.18 85 0.81 afu_top 14,178.20 1.81 91 0.87 pmci_wrapper 8,693.20 1.11 186 1.78 mem_ss_top 7,999.00 1.02 60 0.57 alt_sld_fab_0 1,758.40 0.22 9 0.09 bpf_top 1,183.50 0.15 0 0.00 dummy_csr 667.20 0.09 0 0.00 dummy_csr 666.30 0.09 0 0.00 dummy_csr 663.10 0.08 0 0.00 fme_top 652.80 0.08 6 0.06 axi_lite_rst_bridge 153.80 0.02 0 0.00 rst_ctrl 18.20 0.00 0 0.00 rst_ctrl 16.50 0.00 0 0.00 sys_pll 0.50 0.00 0 0.00"},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#3-description-of-sub-systems","title":"3 Description of Sub-Systems","text":""},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#31-host-control-and-data-flow","title":"3.1 Host Control and Data Flow","text":"
                                                                  The host control and data flow is shown in the diagram below:
                                                                   
                                                                   
                                                                  The control and data paths are composed of the following:
                                                                   
                                                                     
                                                                  • Host Interface Adapter (PCIe)
                                                                    •  
                                                                  • SoC Interface Adapter (PCIe)
                                                                    •  
                                                                  • Low Performance Peripherals
                                                                       
                                                                    • Slow speed peripherals (JTAG, I2C, Smbus, etc)
                                                                      •  
                                                                    • Management peripherals (FME)
                                                                      •  
                                                                     
                                                                    •  
                                                                  • High Performance Peripherals
                                                                       
                                                                    • Memory peripherals
                                                                      •  
                                                                    • Acceleration Function peripherals (eg. AFUs)
                                                                      •  
                                                                    • HPS Peripheral
                                                                      •  
                                                                     
                                                                    •  
                                                                  • Fabrics
                                                                       
                                                                    • Peripheral Fabric (multi drop)
                                                                      •  
                                                                    • AFU Streaming fabric (point to point)
                                                                      •  
                                                                     
                                                                    •  
                                                                   
                                                                  Peripherals are connected to one another using AXI, either:
                                                                   
                                                                     
                                                                  • Via the peripheral fabric (AXI4-Lite, multi drop)
                                                                    •  
                                                                  • Via the AFU streaming fabric (AXI-S, point to point)
                                                                    •  
                                                                   
                                                                  Peripherals are presented to software as:
                                                                   
                                                                     
                                                                  • OFS managed peripherals that implement DFH CSR structure.  
                                                                    •  
                                                                  • Native driver managed peripherals (i.e. Exposed via an independent PF, VF)
                                                                    •  
                                                                   
                                                                  The peripherals connected to the peripheral fabric are primarily Intel OPAE managed resources, whereas the peripherals connected to the AFU are \u201cprimarily\u201d managed by native OS drivers. The word \u201cprimarily\u201d is used since the AFU is not mandated to expose all its peripherals to Intel OPAE. 
                                                                   
                                                                  OFS uses a defined set of CSRs to expose the functionality of the FPGA to the host software.  These registers are described in Open FPGA Stack Reference Manual - MMIO Regions section.
                                                                   
                                                                  If you make changes to the FIM that affect the software operation, then OFS provides a mechanism to communicate that information to the proper software driver that works with your new hardware.  The Device Feature Header (DFH) structure is followed to provide compatibility with OPAE software.  Please see FPGA Device Feature List (DFL) Framework Overview for a description of DFL operation from the driver perspective.
                                                                   
                                                                  In the default design, the SoC and Host AFUs are isolated from each other. You must develop mechanisms for Host - SoC communication if desired.
                                                                   
                                                                  Note: The default configuration of the Board Peripheral Fabric, there is a connection from the Host Interface to the PMCI-SS, however the PMCI-SS is not in the Host DFL, and is not discovered by Host SW by default. If you want to guarantee that the Host can not access the PMCI-SS, and by extension the Board BMC, you must implement a filtering mechanism, for example, in the Host ST2MM module to prevent access to the PMCI-SS address space.
                                                                   
                                                                  Refer to the following documents for more information on sub-systems:
                                                                   
                                                                     
                                                                  • Intel FPGA PCI Express Subsystem IP User Guide
                                                                    •  
                                                                  • Intel FPGA Memory Subsystem IP User Guide
                                                                    •  
                                                                  • Intel FPGA Ethernet Subsystem IP User Guide
                                                                    •  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#4-high-level-development-flow","title":"4 High Level Development Flow","text":"
                                                                  OFS provides a framework of FPGA synthesizable code, simulation environment, and synthesis/simulation scripts.  FIM designers can use the provided code as-is, modify the provided code, or add new code to meet your specific product requirements. The instructions provided after this point are for you to either evaluate the existing design (the current section) or to modify and generate your own design (described in the Custom FIM Development Flow section). 
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#41-development-pre-requisites","title":"4.1 Development Pre-requisites","text":"
                                                                  The following pre-requisites must be satisfied to go through the development flow.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#411-tutorial-pre-requisites","title":"4.1.1 Tutorial Pre-requisites","text":"
                                                                  To run the FPGA compilation steps covered in this guide, requires the following:
                                                                   
                                                                     
                                                                  1. Workstation or server with a Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 installed on a Intel Quartus Prime Pro-supported Linux distribution.  See Operating System Support.  The Linux distribution known to work with this version of RedHatEnterprise Linux\u00ae (RHEL) 8.6 . Note, Windows is not supported.
                                                                    2.  
                                                                  2. Compilation targeting Intel Agilex 7 devices requires a minimum of 64 GB of RAM.
                                                                    3.  
                                                                  3. Simulation of lower level functionality (not chip level) is supported by Synopsys\u00ae VCS and Mentor Graphics\u00ae QuestaSim SystemVerilog simulators.
                                                                    4.  
                                                                  4. Simulation of chip level requires Synopsys VCS and VIP
                                                                    5.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#412-development-environment","title":"4.1.2 Development Environment","text":"
                                                                  To run the tutorial steps in this guide requires this development environment:
                                                                   Item Version Intel Quartus Prime Pro 23.3 OPAE SDK Branch Tag: release/2.10.0 Simulator Synopsys VCS P-2019.06-SP2-5 or newer for UVM simulation of top level FIM Python 3.6.8 GCC 7.2.0 cmake 3.15 git with git-lfs 1.8.3.1 PERL 5.8.8 
                                                                  Note: Steps to install Intel Quartus Prime Pro are provided in the Installation of OFS section.
                                                                   
                                                                  To install the Git Large File Storage (LFS) extension run the following commands:
                                                                   
                                                                  curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\nsudo dnf install git-lfs\ngit lfs install\n
                                                                   
                                                                  To test FPGA image files on hardware, this version of OFS only targets  Intel IPU Platform F2000X-PL. You may modify the build scripts and pin files to target different boards with Intel Agilex 7 FPGA devices.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#42-installation-of-ofs","title":"4.2 Installation of OFS","text":"
                                                                  In this section you set up a development machine for compiling the OFS FIM. These steps are separate from the setup for a deployment machine where the FPGA acceleration card is installed.  Typically, FPGA development and deployment work is performed on separate machines, however, both development and deployment can be performed on the same server if desired.  Please see the Getting Started Guide: Intel\u00ae Open FPGA Stack for Intel FPGA for instructions on installing software for deployment of your FPGA FIM, AFU and software application on a server.  
                                                                   
                                                                  Building the OFS FIM requires the build machine to have at least 64 GB of RAM.  
                                                                   
                                                                  The following is a summary of the steps to set up for FIM development:
                                                                   
                                                                     
                                                                  1. Install Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 Linux with Agilex device support
                                                                    2.  
                                                                  2. Make sure support tools are installed and meet version requirements
                                                                    3.  
                                                                  3. Clone the  repository
                                                                    4.  
                                                                  4. Install required Intel Quartus Prime Pro patches which are included in the cloned ofs-f2000x repository
                                                                    5.  
                                                                  5. Review the files provided in the repo
                                                                    6.  
                                                                  6. Test installation by building the FIM
                                                                    7.  
                                                                   
                                                                  Intel\u00ae Quartus\u00ae Prime Pro Edition Version 23.3 is the currently verified version of Intel Quartus Prime Pro used for building the FIM and AFU images.  Porting to newer versions of Intel Quartus Prime Pro may be performed by developers, however, you will need to verify operation.
                                                                   
                                                                  The recommended Best Known Configuration (BKC) for development of the OFS FIM is RedHatEnterprise Linux\u00ae (RHEL) 8.6, which is the assumed operating system for this developer guide. 
                                                                   
                                                                     
                                                                  1.  
                                                                    Prior to installing Intel Quartus Prime Pro, perform the following steps to satisfy the required dependencies.
                                                                     
                                                                    sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Apply the following configurations.
                                                                     
                                                                    sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \nsudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \nsudo ln -s /usr/bin/python3 /usr/bin/python\n
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Download Intel\u00ae Quartus\u00ae Prime Pro Edition Linux.
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                                                                     
                                                                    export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                                                                     
                                                                    For example, if the Intel Quartus Prime Pro install directory is /home/intelFPGA_pro/23.3 then the new line is:
                                                                     
                                                                    export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Verify, Intel Quartus Prime Pro is discoverable by opening a new shell:
                                                                     
                                                                    which quartus\n
                                                                     
                                                                    Example output:   
                                                                    /home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    Install Quartus two patches 0.11 and 0.19.  The patch files are included as assets in the OFS-F2000X-PL release.  Scroll to the bottom of the page and download the patches to your Quartus development computer and then install the patches by running the patch script and following the installation items.
                                                                     
                                                                    sudo ./quartus-23.2-0.11-linux-internal-donot-distribute.run\nsudo ./quartus-23.2-0.19-linux-internal-donot-distribute.run\n
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    Verify Quartus the version of Quartus.
                                                                     
                                                                    quartus_sh --version\nVersion 23.2.0 Build 94 06/14/2023 Patches 0.11,0.19 SC Pro Edition\nCopyright (C) 2023  Intel Corporation. All rights reserved.\n
                                                                     
                                                                    8.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#421-clone-the-ofs-git-repo","title":"4.2.1 Clone the OFS Git Repo","text":"
                                                                  Retrieve the OFS FIM source code from the OFS f2000x FIM Github Branch repository. Create a new directory to use as a clean starting point to store the retrieved files.  The following is a short description of each repository, followed by the git commands for cloning.  The instructions section uses the HTTPS git method for cloning repositories.
                                                                   
                                                                     
                                                                  1.  
                                                                    Navigate to the location you want to clone the OFS source files, and create the top-level source directory.
                                                                     
                                                                    mkdir IOFS_BUILD_ROOT\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Clone OFS repositories.
                                                                     
                                                                    cd IOFS_BUILD_ROOT\ngit clone --recurse-submodules https://github.com/OFS/ofs-f2000x-pl\n
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Checkout the proper tag
                                                                     
                                                                    cd ofs-f2000x-pl\ngit checkout --recurse-submodules tags/ofs-2023.3-1\n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Ensure that ofs-common has been cloned as well
                                                                     
                                                                    git submodule status\n
                                                                     
                                                                    Example output:
                                                                     
                                                                    ea585a4f48d50faf3ae7ecfbec82525a8d22c730 ofs-common (ofs-2023.3-1)\n
                                                                     
                                                                    5.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#43-directory-structure-of-ofs","title":"4.3 Directory Structure of OFS","text":"
                                                                  List the directory contents of the cloned repo to verify that the following directories and files are present in $IOFS_BUILD_ROOT/ofs-f2000x directory.  
                                                                   
                                                                  ls -1\n
                                                                   Expected output: 
                                                                  ipss\nLICENSE.txt\nofs-common\nREADME.md  \nSECURITY.md\nsim\nsrc\nsyn\ntools\nverification\n
                                                                   
                                                                  Use the following command to show how the directories are arranged:
                                                                   
                                                                  find . -mindepth 1 -maxdepth 2 -type d -not -path '*/\\.*' -print | sed -e 's/[^-][^\\/]*\\//--/g' -e 's/--/|  /g' -e 's/|-/|   /g'\n
                                                                   Expected output:
                                                                   
                                                                  |  ipss\n|  |  hssi\n|  |  mem\n|  |  pcie\n|  |  pmci\n|  |  qsfp\n|  ofs-common\n|  |  scripts\n|  |  src\n|  |  tools\n|  |  verification\n|  sim\n|  |  bfm\n|  |  common\n|  |  scripts\n|  |  unit_test\n|  src\n|  |  afu_top\n|  |  includes\n|  |  pd_qsys\n|  |  top\n|  syn\n|  |  scripts\n|  |  setup\n|  |  syn_top\n|  tools\n|  |  pfvf_config_tool\n|  verification\n|  |  scripts\n|  |  testbench\n|  |  tests\n|  |  unit_tb\n|  |  verifplan\n
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#44-compiling-the-ofs-fim","title":"4.4 Compiling the OFS FIM","text":"
                                                                  OFS provides a build script with the following FPGA image creation options:
                                                                   
                                                                     
                                                                  • Flat compile which combines the FIM and AFU into one FPGA image that is loaded into the entire FPGA device
                                                                    •  
                                                                  • A PR compile which creates a FPGA image consisting of the FIM that is loaded into the static region of the FPGA and a default AFU that is loaded into dynamic region. The AFU image may be loaded into the dynamic region using partial reconfiguration.  
                                                                    •  
                                                                   
                                                                  The build scripts included with OFS are verified to run in a bash shell. Other shells have not been tested. The full build script typically takes around 3 hours to complete.
                                                                   
                                                                  The build script flow is the primary flow described in this user guide. For instructions on compiling using the Intel Quartus Prime Pro GUI, refer to the Compiling the OFS FIM Using Quartus GUI section.
                                                                   
                                                                  The following sections describe how to set up the environment and build the provided FIM and AFU. Follow these steps as a tutorial to learn the build flow.  You will use this environment and build scripts for the creation of your specialized FIM.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#441-setting-up-required-environment-variables","title":"4.4.1. Setting Up Required Environment Variables","text":"
                                                                  Set required environment variables as shown below.  These environment variables must be set prior to simulation or compilation tasks so creating a simple script to set these variables saves time. 
                                                                   
                                                                  Set the following environment variables based on your environment: 
                                                                  export QUARTUS_MAINPATH=/<YOUR_QUARTUS_DIRECTORY>/23.3 \nexport TOOLS_LOCATION=<YOUR_TOOLS_DIRECTORY> \nexport LM_LICENSE_FILE=<YOUR_LM_LICENSE_PATH>\nexport DW_LICENSE_FILE=<YOUR_DW_LICENSE_PATH>\nexport SNPSLMD_LICENSE_FILE=<YOUR_SNPSLMD_LICENSE_PATH>\n
                                                                   
                                                                  Note: The TOOLS_LOCATION directory is where the Synopsys tools reside. Refer to the UVM_HOME variable below for an example.
                                                                   
                                                                  Then set the remaining environment variables: 
                                                                  export QUARTUS_ROOTDIR=$QUARTUS_MAINPATH/quartus \nexport QUARTUS_HOME=$QUARTUS_ROOTDIR \nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR \nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR \nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip \nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip \nexport INTELFPGAOCLSDKROOT=$QUARTUS_MAINPATH/hld \nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin \nexport IOFS_BUILD_ROOT=$PWD\nexport OFS_ROOTDIR=$IOFS_BUILD_ROOT/ofs-f2000x-pl\nexport WORKDIR=$OFS_ROOTDIR \nexport VERDIR=$OFS_ROOTDIR/verification/ofs-f2000x/common:$OFS_ROOTDIR/verification \nexport OFS_PLATFORM_AFU_BBB=$IOFS_BUILD_ROOT/ofs-platform-afu-bbb \nexport OPAE_SDK_REPO_BRANCH=release/$OPAE_SDK_VERSION\nexport OPAE_PLATFORM_ROOT=$OFS_ROOTDIR/work_dir/build_tree    \nexport LIBRARY_PATH=$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/lib \nexport LD_LIBRARY_PATH=$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/lib64 \nexport OPAE_LOC=/install-opae-sdk \nexport QUARTUS_NUM_PARALLEL_PROCESSORS=8 \nexport DESIGNWARE_HOME=$TOOLS_LOCATION/synopsys/vip_common/vip_Q-2020.03A\nexport UVM_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm \nexport VCS_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel \nexport MTI_HOME=$TOOLS_LOCATION/intelFPGA_pro/questa_fse \nexport PATH=$PATH:$QUARTUS_HOME/bin:$QUARTUS_HOME/qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/bin:$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$DESIGNWARE_HOME/bin:$VCS_HOME/bin\n
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#442-compiling-the-fim","title":"4.4.2 Compiling the FIM","text":"
                                                                  The f2000x  FIM build flow uses the bash script $OFS_ROOTDIR/ofs-common/scripts/common/syn/build_top.sh.  There are several setup files that must be put in place before compilation, which is handled by the build script. If you wish to compile the f2000x FIM using the Intel Quartus Prime Pro GUI, you must at least run the setup portion of the build_top.sh script before compiling with the GUI. For instructions on compiling the FIM using the Quartus GUI, refer to the Compiling the OFS FIM Using Quartus GUI section.
                                                                   
                                                                  The usage of the compile build script is shown below: 
                                                                  ofs-common/scripts/common/syn/build_top.sh/build_top.sh [-p] f2000x[:OPTIONS]  work_dir \n
                                                                   
                                                                  Note: Refer to the $OFS_ROOTDIR/ofs-common/scripts/common/syn/README for detailed information on using this script
                                                                   
                                                                     
                                                                  • To build the provided design using a flat, non-PR build flow use the following commands:
                                                                    cd $OFS_ROOTDIR\n./ofs-common/scripts/common/syn/build_top.sh f2000x:flat  work_dir\n
                                                                     
                                                                    •  
                                                                   
                                                                     
                                                                  • To build the provided design with in-tree PR enabled use the following commands:
                                                                    cd $OFS_ROOTDIR\n./ofs-common/scripts/common/syn/build_top.sh f2000x work_dir\n
                                                                     
                                                                    •  
                                                                   
                                                                     
                                                                  • To build the provided design with a relocatable (out-of-tree) PR directory use the following commands:
                                                                    cd $OFS_ROOTDIR\n./ofs-common/scripts/common/syn/build_top.sh -p f2000x work_dir\n
                                                                     
                                                                    Refer to the Create a Relocatable PR Directory Tree section for more information on out-of-tree PR builds.
                                                                     
                                                                    •  
                                                                   
                                                                  The build takes ~3 hours to complete. A successful build will report the following:
                                                                   
                                                                  Compile work directory:     <$IOFS_BUILD_ROOT>/ofs-f2000x/work_f2000x /syn/syn_top\nCompile artifact directory: <$IOFS_BUILD_ROOT>/ofs-f2000x/work_f2000x /syn/syn_top/output_files\n\n***********************************\n***\n***        OFS_PROJECT: f2000x \n***        OFS_FIM: base\n***        OFS_BOARD: adp\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 0\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                                                   
                                                                  The build script copies the ipss, ofs-common, sim, src,syn and tools directories to the specified work directory and then these copied files are used in the Intel Quartus Prime Pro compilation process.
                                                                   
                                                                  Some key output directories are described in the following table:
                                                                   Directory Description $OFS_ROOTDIR/<WORK_DIR>/syn/syn_top Intel Quartus Prime Pro project (ofs_top.qpf) and other Intel Quartus Prime Pro specific files $OFS_ROOTDIR/<WORK_DIR>/syn/syn_top/output_files Directory with build reports and FPGA programming files 
                                                                  The build script will run PACSign (if installed) and create an unsigned FPGA programming files for both user1 and user2 locations of the f2000x  FPGA flash.  Please note, if the f2000x  has the root entry hash key loaded, then PACsign must be run to add the proper key to the FPGA binary file.
                                                                   
                                                                  The following table provides a detailed description of the generated *.bin files.
                                                                   File Description ofs_top.bin This is an intermediate, raw binary file. This intermediate raw binary file is produced by taking the Intel Quartus Prime Pro generated *.sof file, and converting it to *.pof using quartus_pfg, then converting the *.pof to *.hexout using quartus_cpf, and finally converting the *.hexout to *.bin using objcopy. ofs_top.bin - Raw binary image of the FPGA. ofs_top_page0_unsigned_factory.bin This is the unsigned PACSign output generated for the Factory Image. Unsigned means that the image has been signed with an empty header. ofs_top_page1_user1.bin This is an input file to PACSign to generate ofs_top_page1_unsigned_user1.bin. This file is created by taking the ofs_top.bin file and assigning the User1 or appending factory block information. Unsigned means that the image has been signed with an empty header. ofs_top_page1_unsigned_user1.bin This is the unsigned FPGA binary image generated by the PACSign utility for the User1 Image. This file is used to load the FPGA flash User1 Image using the fpgasupdate tool. Unsigned means that the image has been signed with an empty header. ofs_top_page2_user2.bin This is an input file to PACSign to generate ofs_top_page2_unsigned_user2.bin. This file is created by taking the ofs_top.bin file and assigning the User2 or appending factory block information. ofs_top_page2_unsigned_user2.bin This is the unsigned FPGA binary image generated by the PACSign utility for the User2 Image. This file is used to load the FPGA flash User2 Image using the fpgasupdate tool. Unsigned means that the image has been signed with an empty header. ofs_top.sof This image is used to generate ofs_top.bin, and can also be used to program the Intel Agilex 7 device directly through JTAG 
                                                                  Note: The build/output_files/timing_report directory contains clocks report, failing paths and passing margin reports. 
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#443-create-a-relocatable-pr-directory-tree-from-the-base_x16-fim","title":"4.4.3 Create a Relocatable PR Directory Tree from the base_x16 FIM","text":"
                                                                  If you are developing a FIM to be used by another team developing AFU workload(s), scripts are provided that create a relocatable PR directory tree. ODM and board developers will make use of this capability to enable a broad set of AFUs to be loaded on a board using PR.
                                                                   
                                                                  You can create this relocatable PR directory tree by either:
                                                                   
                                                                     
                                                                  • Build FIM and AFU using ofs-common/scripts/common/syn/build_top.sh followed by running /ofs-common/scripts/common/syn/generate_pr_release.sh
                                                                    •  
                                                                  • Build FIM and AFU using ofs-common/scripts/common/syn/build_top.sh with optional -p switch included
                                                                    •  
                                                                   
                                                                  The generate_pr_release.sh has the following command structure:
                                                                   
                                                                  ofs-common/scripts/common/syn/generate_pr_release.sh -t <work_directory>/build_tree <target_board> <work_directory>\n
                                                                   Where:
                                                                   
                                                                     
                                                                  • <work_directory>/build_tree is the location for your relocatable PR directory tree in the work directory
                                                                    •  
                                                                  • <target_board> is the name of the board target/FIM e.g. f2000x 
                                                                    •  
                                                                  • <work_directory> is the work directory
                                                                    •  
                                                                   
                                                                  Here is an example of running the generate_pr_release.sh script:
                                                                   
                                                                  cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/generate_pr_release.sh -t work_f2000x/build_tree f2000x   work_f2000x \n
                                                                   
                                                                  The resulting relocatable build tree has the following structure:
                                                                   
                                                                  .\n\u251c\u2500\u2500 bin\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 afu_synth\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 build_env_config\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 run.sh -> afu_synth\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 update_pim\n\u251c\u2500\u2500 hw\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 blue_bits\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u251c\u2500\u2500 ofs_top_page0_unsigned_factory.bin\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u251c\u2500\u2500 ofs_top_page1_unsigned_user1.bin\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u251c\u2500\u2500 ofs_top_page2_unsigned_user2.bin\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u2514\u2500\u2500 ofs_top.sof -> ../lib/build/syn/syn_top/output_files/ofs_top.sof\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 lib\n\u2502\u00a0\u00a0     \u251c\u2500\u2500 build\n\u2502\u00a0\u00a0     \u251c\u2500\u2500 fme-ifc-id.txt\n\u2502\u00a0\u00a0     \u251c\u2500\u2500 fme-platform-class.txt\n\u2502\u00a0\u00a0     \u2514\u2500\u2500 platform\n\u2514\u2500\u2500 README\n
                                                                   This build tree can be moved to a different location and used for AFU development of PR-able AFU to be used with this board.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#444-compiling-the-ofs-fim-using-quartus-gui","title":"4.4.4 Compiling the OFS FIM Using Quartus GUI","text":"
                                                                  Perform the following steps to compile the OFS FIM using the Quartus GUI:
                                                                   
                                                                     
                                                                  1.  
                                                                    Set the environment variables as described in the Setting Up Required Environment Variables section.
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Run the setup portion of the build script. This takes a few seconds to complete.
                                                                     
                                                                    ./ofs-common/scripts/common/syn/build_top.sh --stage setup f2000x work_dir\n
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Open the OFS FIM project using the Intel Quartus Prime Pro GUI. The project is located at $OFS_ROOTDIR/work_dir/syn/syn_top/ofs_top.qpf.
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Ensure the checkbox next to Assembler (Generate programming files) is marked.
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Click the arrow next to Compile Design in the Compilation Flow window to start full compilation.
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    After compilation is complete, check the $OFS_ROOTDIR/work_dir/syn/syn_top/output_files directory. This directory will contain the output SOF image generated by Quartus, named ofs_top.sof.
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    Run the finish portion of the build script. This will generate the images that can be programmed to f2000x FPGA flash. Use the optional -p argument to create an out-of-tree PR build.
                                                                     
                                                                    ./ofs-common/scripts/common/syn/build_top.sh --stage finish f2000x work_dir\n
                                                                     
                                                                    8.  
                                                                  8.  
                                                                    Check the $OFS_ROOTDIR/work_dir/syn/syn_top/output_files directory again to see that all output files have been generated.
                                                                     
                                                                    9.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#45-unit-level-simulation","title":"4.5 Unit Level Simulation","text":"
                                                                  Unit level simulation of key components in the FIM is provided. These simulations provide verification of the following areas:
                                                                   
                                                                     
                                                                  • Ethernet
                                                                    •  
                                                                  • PCIe
                                                                    •  
                                                                  • External Memory
                                                                    •  
                                                                  • Core FIM
                                                                    •  
                                                                   
                                                                  The Unit Level simulations work with Synopsys VCS-MX or Mentor Graphics Questasim simulators. Readme files are provided explaining how to run the simulation of each component. 
                                                                   
                                                                  The scripts to run the unit level simulations are located in $OFS_ROOTDIR/sim/unit_test, which contains subdirectories soc_tests and host_tests. The table below lists the tests that are provided.
                                                                   Test Type Test Name SoC Tests csr_test dfh_walker flr he_lb_test he_mem_test hssi_kpi_test hssi_test mem_ss_csr_test mem_ss_rst_test mem_tg_test pf_vf_access_test qsfp_test regress_run.py Host Tests csr_test he_lb_test pcie_csr_test pf_vf_access_test pmci_csr_test pmci_mailbox_test pmci_rd_default_value_test pmci_ro_mailbox_test pmci_vdm_b2b_drop_err_scenario_test pmci_vdm_len_err_scenario_test pmci_vdm_mctp_mmio_b2b_test pmci_vdm_multipkt_error_scenario_test pmci_vdm_multipkt_tlp_err_test pmci_vdm_tlp_error_scenario_test pmci_vdm_tx_rx_all_random_lpbk_test pmci_vdm_tx_rx_all_toggle_test pmci_vdm_tx_rx_lpbk_test regress_run.py"},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#451-run-comprehensive-unit-tests","title":"4.5.1 Run Comprehensive Unit Tests","text":"
                                                                  The regress_run.py script is provided to automatically run all unit tests for either the SoC or the Host. Note that running all tests will take around three hours for the SoC tests and around 2.5 hours for the Host tests to complete.
                                                                   
                                                                  Perform the following steps to run comprehensive tests:
                                                                   
                                                                     
                                                                  1. Set the environment variables as described in the Setting Up Required Environment Variables section.
                                                                    2.  
                                                                  2.  
                                                                    Navigate to the test directory you wish to run from.
                                                                     
                                                                       
                                                                    • SoC Tests
                                                                      cd $OFS_ROOTDIR/sim/unit_test/soc_tests\n
                                                                       
                                                                      •  
                                                                     
                                                                       
                                                                    • Host Tests
                                                                      cd $OFS_ROOTDIR/sim/unit_test/host_tests\n
                                                                       
                                                                      •  
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Run the tests with the regress_run.py. Use the -h argument to display the help menu.
                                                                     
                                                                    For example, to run all tests locally, using VCS, with 8 processors:
                                                                     
                                                                    python regress_run.py -l -n 8 -k all -s vcs\n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Once all tests have completed, the comprehensive test summary will be shown. The following is an example test summary after running the SoC tests:
                                                                     
                                                                    2023-05-14 19:10:55,404: Passing Unit Tests:12/12 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-05-14 19:10:55,404:    csr_test:......... PASS -- Time Elapsed:0:03:57.713154\n2023-05-14 19:10:55,404:    dfh_walker:....... PASS -- Time Elapsed:0:02:46.025067\n2023-05-14 19:10:55,404:    flr:.............. PASS -- Time Elapsed:0:03:26.289900\n2023-05-14 19:10:55,404:    he_lb_test:....... PASS -- Time Elapsed:0:06:41.142643\n2023-05-14 19:10:55,404:    he_mem_test:...... PASS -- Time Elapsed:1:39:01.824096\n2023-05-14 19:10:55,404:    hssi_kpi_test:.... PASS -- Time Elapsed:2:21:33.007328\n2023-05-14 19:10:55,404:    hssi_test:........ PASS -- Time Elapsed:2:16:36.821034\n2023-05-14 19:10:55,404:    mem_ss_csr_test:.. PASS -- Time Elapsed:0:38:45.836540\n2023-05-14 19:10:55,404:    mem_ss_rst_test:.. PASS -- Time Elapsed:0:40:51.065354\n2023-05-14 19:10:55,404:    mem_tg_test:...... PASS -- Time Elapsed:0:54:00.210146\n2023-05-14 19:10:55,404:    pf_vf_access_test: PASS -- Time Elapsed:0:03:25.561919\n2023-05-14 19:10:55,404:    qsfp_test:........ PASS -- Time Elapsed:0:39:29.192304\n2023-05-14 19:10:55,404: Failing Unit Tests: 0/12 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-05-14 19:10:55,404: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-05-14 19:10:55,404:       Number of Unit test results captured: 12\n2023-05-14 19:10:55,404:       Number of Unit test results passing.: 12\n2023-05-14 19:10:55,404:       Number of Unit test results failing.:  0\n2023-05-14 19:10:55,404:     End Unit regression running at date/time................: 2023-05-14 19:10:55.404641\n2023-05-14 19:10:55,404:     Elapsed time for Unit regression run....................: 2:22:39.310455\n
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Output logs for each individual test are saved in the respective test's directory
                                                                     
                                                                    $OFS_ROOTDIR/sim/unit_test/<TEST_TYPE>/<TEST_NAME>/<SIMULATOR>/transcript\n
                                                                     
                                                                    For example, the log for the SoC DFH Walker test using VCS can be found in:
                                                                     
                                                                    $OFS_ROOTDIR/sim/unit_test/soc_tests/dfh_walker/sim_vcs/transcript\n
                                                                     
                                                                    6.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#452-run-individual-unit-tests","title":"4.5.2 Run Individual Unit Tests","text":"
                                                                  The run_sim.sh scripts are provided to run individual unit tests for either the SoC or the Host. Before you can run any unit tests, you must generate the IP simulation files. Note that the regress_run.py script used for comprehensive testing does this step automatically. Perform the following steps to generate the IP simulation files:
                                                                   
                                                                     
                                                                  1. Set the environment variables as described in the Setting Up Required Environment Variables section.
                                                                    2.  
                                                                  2. Generate the IP simulation files for all unit tests:
                                                                    cd `$OFS_ROOTDIR/ofs-common/scripts/common/sim`\nsh gen_sim_files.sh f2000x \n
                                                                     
                                                                    3.  
                                                                   
                                                                  Next, perform the following steps to run individual tests:
                                                                   
                                                                     
                                                                  1.  
                                                                    Navigate to the directory of the test you wish to run
                                                                     
                                                                    cd $OFS_ROOTDIR/sim/unit_test/<TEST_TYPE>/<TEST_NAME>\n
                                                                     
                                                                    For example, to run the DFH walker test for the SoC:
                                                                     
                                                                    cd $OFS_ROOTDIR/sim/unit_test/soc_tests/dfh_walker\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Run the test with your desired simulator:
                                                                     
                                                                       
                                                                    •  
                                                                      Using VCS
                                                                       
                                                                      sh run_sim.sh\n
                                                                       
                                                                      •  
                                                                     
                                                                       
                                                                    •  
                                                                      Using Questasim
                                                                       
                                                                      sh run_sim.sh MSIM=1\n
                                                                       
                                                                      •  
                                                                     
                                                                       
                                                                    •  
                                                                      Using VCS-MX
                                                                       
                                                                      sh run_sim.sh VCSMX=1\n
                                                                       
                                                                      •  
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Once the test has completed, the test summary will be shown. The following is an example test summary after running the SoC DFH Walker Test:
                                                                     
                                                                    Test status: OK\n\n********************\nTest summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/applications.fpga.ofs.fim-f2000x-pl/sim/unit\n_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time            355393750\n        V C S   S i m u l a t i o n   R e p o r t\nTime: 355393750 ps\nCPU Time:     59.870 seconds;       Data structure size:  91.2Mb\nSun May 14 16:54:20 2023\n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    The log for the test is stored in a transcript file in the simulation directory of the test that was run.
                                                                     
                                                                    $OFS_ROOTDIR/sim/unit_test/<TEST_TYPE>/<TEST_NAME>/<SIMULATOR>/transcript\n
                                                                     
                                                                    For example, the log for the SoC DFH Walker test using VCS can be found in:
                                                                     
                                                                    $OFS_ROOTDIR/sim/unit_test/soc_tests/dfh_walker/sim_vcs/transcript\n
                                                                     
                                                                    The simulation waveform database is saved as vcdplus.vpd for post simulation review. You are encouraged to run the additional simulation examples to learn about each key area of the OFS shell.
                                                                     
                                                                    5.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5-custom-fim-development-flow","title":"5 Custom FIM Development Flow","text":"
                                                                  FIM development for a new acceleration card consists of the following steps:
                                                                   
                                                                     
                                                                  1. Installation of OFS and familiarization with scripts and source code
                                                                    2.  
                                                                  2. Development of high level block diagram with your specific functionality
                                                                       
                                                                    1. Determination of requirements and key performance metrics
                                                                      2.  
                                                                    2. Selection of IP cores
                                                                      3.  
                                                                    3. Selection of FPGA device
                                                                      4.  
                                                                    4. Software memory map
                                                                      5.  
                                                                     
                                                                    3.  
                                                                  3. Selection and implementation of FIM Physical interfaces including:
                                                                       
                                                                    1. External clock sources and creation of internal PLL clocks
                                                                      2.  
                                                                    2. General I/O
                                                                      3.  
                                                                    3. Ethernet modules
                                                                      4.  
                                                                    4. External memories
                                                                      5.  
                                                                    5. FPGA programming methodology
                                                                      6.  
                                                                     
                                                                    4.  
                                                                  4. Device physical implementation
                                                                       
                                                                    1. FPGA device pin assignment
                                                                      2.  
                                                                    2. Inclusion of logic lock regions
                                                                      3.  
                                                                    3. Creation of timing constraints
                                                                      4.  
                                                                    4. Create Intel Quartus Prime Pro FIM test project and validate:
                                                                         
                                                                      1. Placement
                                                                        2.  
                                                                      2. Timing constraints
                                                                        3.  
                                                                      3. Build script process
                                                                        4.  
                                                                      4. Review test FIM FPGA resource usage
                                                                        5.  
                                                                       
                                                                      5.  
                                                                     
                                                                    5.  
                                                                  5. Select FIM to AFU interfaces and development of PIM
                                                                    6.  
                                                                  6. FIM design implementation
                                                                       
                                                                    1. RTL coding
                                                                      2.  
                                                                    2. IP instantiation
                                                                      3.  
                                                                    3. Development of test AFU to validate FIM
                                                                      4.  
                                                                    4. Unit and device level simulation
                                                                      5.  
                                                                    5. Timing constraints and build scripts
                                                                      6.  
                                                                    6. Timing closure and build validation
                                                                      7.  
                                                                     
                                                                    7.  
                                                                  7. Creation of FIM documentation to support AFU development and synthesis
                                                                    8.  
                                                                  8. Software Device Feature discovery
                                                                    9.  
                                                                  9. Hardware/software integration, validation and debugging
                                                                    10.  
                                                                  10. High volume production preparation
                                                                    11.  
                                                                   
                                                                  The FIM developer works closely with the hardware design of the target board, software development and system validation.
                                                                   
                                                                  This section describes how to perform specific customizations of areas of the FIM. Each section can be done independently. The following walkthroughs are provided:
                                                                   Section Walkthrough 5.1 How to add a new module to the FIM 5.2 How to debug the FIM with Signal Tap 5.3 How to compile the FIM in preparation for designing your AFU 5.4 How to resize the Partial Reconfiguration Region 5.5 How to modify the Memory Subsystem 5.6 How to compile the FIM with no HSSI 5.7 How to change the PCIe Device ID and Vendor ID 5.8 How to migrate to a different Intel Agilex 7 device number 5.9 How to change the Ethernet interface from 8x25 GbE to 8x10 GbE 5.10 How to change the Ethernet interface from 8x25 GbE to 2x100 GbE 5.11 How to add more transceiver channels to the FIM 5.12 How to modify the PF/VF MUX configuration 5.13 How to create a minimal FIM 
                                                                  In each section, it is assumed that:
                                                                   
                                                                     
                                                                  1. You have a clean, unmodified clone of the OFS repo.  See the Clone the OFS Git Repo section.
                                                                    2.  
                                                                  2. After cloning, you must set various environment variables. See the Setting Up Required Environment Variables section.
                                                                    3.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#51-how-to-add-a-new-module-to-the-fim","title":"5.1 How to add a new module to the FIM","text":"
                                                                  This section provides a walkthrough for adding a custom module to the FIM, simulating the new design, compiling the new design, implementing and testing the new design on hardware, and debugging the new design on hardware.
                                                                   
                                                                  If you intend to add a new module to the FIM area, then you will need to inform the host software of the new module. The FIM exposes its functionalities to host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). This set of CSR registers and their operation is described in FIM MMIO Regions.
                                                                   
                                                                  See FPGA Device Feature List (DFL) Framework Overview for a description of the software process to read and process the linked list of Device Feature Header (DFH) CSRs within a FPGA.
                                                                   
                                                                  The Hello FIM example adds a simple DFH register and 64bit scratchpad register connected to the Board Peripheral Fabric (BPF) that can be accessed by the SoC. You can use this example as the basis for adding a new feature to your FIM. 
                                                                   
                                                                  The Hello FIM design can be verified by Unit Level simulation, Universal Verification Methodology (UVM) simulation, and running in hardware on the f2000x  card. The process for these are described in this section. 
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#511-board-peripheral-fabric-bpf","title":"5.1.1 Board Peripheral Fabric (BPF)","text":"
                                                                  The Hello FIM module will be connected to the Board Peripheral Fabric (BPF), and will be connected such that it can only be mastered by the SoC. The BPF is an interconnect generated by Platform Designer. The figure below shows the APF/BPF Master/Slave interactions, as well as the added Hello FIM module.
                                                                   
                                                                   
                                                                  You can create, modify, and/or generate the BPF manually in Platform Designer or more conveniently by executing a provided script.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#512-soc-mmio-region","title":"5.1.2 SoC MMIO Region","text":"
                                                                  We will add the Hello FIM module to an un-used address space in the SoC MMIO region. The table below shows the MMIO region for the SoC with the Hello FIM module added at base address 0x16000.
                                                                   Offset Feature CSR set 0x00000 FME AFU 0x01000 Thermal Management 0x03000 Global Performance 0x04000 Global Error 0x10000 PCIe Interface 0x12000 QSFP Controller 0 0x13000 QSFP Controller 1 0x14000 HSSI Interface 0x15000 EMIF Interface 0x16000 Hello FIM 0x80000 PMCI Controller 0x100000 ST2MM (Streaming to Memory-Mapped) 0x130000 PR Control & Status (Port Gasket) 0x131000 Port CSRs (Port Gasket) 0x132000 User Clock (Port Gasket) 0x133000 Remote SignalTap (Port Gasket) 0x140000 AFU Errors (AFU Interface Handler) 
                                                                  Refer to the FIM Technical Reference Manual: Interconnect Fabric for more information on the default MMIO region.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#513-hello-fim-csr","title":"5.1.3 Hello FIM CSR","text":"
                                                                  The Hello FIM CSR will consist of the three registers shown in the table below. The DFH and Hello FIM ID registers are read-only. The Scratchpad register supports read and write accesses.
                                                                   Offset Attribute Description Default Value 0x016000 RO DFH(Device Feature Headers) register 0x30000006a0000100 0x016030 RW Scrachpad register 0x0 0x016038 RO Hello FIM ID register 0x6626070150000034"},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#514-files-to-edit-to-support-hello-fim","title":"5.1.4 Files to Edit to Support Hello FIM","text":"
                                                                  The table below shows all files in $OFS_ROOTDIR that will be modified or created in order to implement the Hello FIM module. The build_top.sh script copies files from $OFS_ROOTDIR into the target work directory and then the copied files are used in the Quartus build process. Details on editing these files is given in subsequent sections.
                                                                   Category Status Path File Description Source Modify src/top top.sv Top RTL Modify src/pd_qsys bpf_top.sv BPF top RTL New src/hello_fim hello_fim_top.sv Hello FIM top RTL New src/hello_fim hello_fim_com.sv Hello FIM common RTL Design Files Modify syn/syn_top ofs_top.qsf Quartus setting file Modify syn/syn_top ofs_top_sources.tcl Define sources for top level design New syn/setup hello_fim_design_files.tcl Define RTLs of Hello FIM Modify syn/setup fabric_design_files.tcl Define IPs for fabric Platform Designer Modify src/pd_qsys/fabric bpf.txt Text definition of BPF interconnect Modify src/pd_qsys/fabric bpf.qsys BPF Qsys file Simulation Modify sim/unit_test/soc_tests/dfh_walker test_csr_defs.sv Define CSRs for simulation Verification Modify /ofs-common/verification/fpga_family/agilex/tests/sequences mmio_seq.svh MMIO testbench Modify /ofs-common/verification/fpga_family/agilex/tests/sequences dfh_walking_seq.svh DFH Walking testbench Modify ofs-common/verification/fpga_family/agilex/scripts Makefile_VCS.mk Makefile for VCS"},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#515-pre-requisites-for-adding-hello-fim","title":"5.1.5 Pre-Requisites for Adding Hello FIM","text":"
                                                                  The following pre-requisites must be satisfied before adding the Hello FIM module.
                                                                   
                                                                     
                                                                  1. Clone the design repositories. See the Clone the OFS Git Repo section.
                                                                    2.  
                                                                  2. Set the environment variables. See the Setting Up Required Environment Variables section.
                                                                    3.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#516-file-modification","title":"5.1.6 File Modification","text":"
                                                                  This section describes the steps to add the Hello FIM module to the FIM.  The steps in this simple example are the basis for modifying the FIM for more complex functions.
                                                                   
                                                                     
                                                                  1.  
                                                                    Modify syn/syn_top/ofs_top.qsf
                                                                     
                                                                       
                                                                    1.  
                                                                      Define the INCLUDE_HELLO_FIM Verilog macro to the Verilog Macros section. This will enable instantiation of the Hello FIM module. If this is not set, a dummy register will be instantiated instead.
                                                                       
                                                                      ######################################################\n# Verilog Macros\n######################################################\n.....\nset_global_assignment -name VERILOG_MACRO \"INCLUDE_HELLO_FIM\"     # Includes Hello FIM\n
                                                                       
                                                                      2.  
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Modify syn/syn_top/ofs_top_sources.tcl
                                                                     
                                                                       
                                                                    1.  
                                                                      Add hello_fim_design_files.tcl to the list of subsystems in the Design Files section.
                                                                       
                                                                      ############################################\n# Design Files\n############################################\n...\n# Subsystems\n...\nset_global_assignment -name SOURCE_TCL_SCRIPT_FILE ../setup/hello_fim_design_files.tcl\n
                                                                       
                                                                      2.  
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Create syn/setup/hello_fim_design_files.tcl
                                                                     
                                                                       
                                                                    1.  
                                                                      Create hello_fim_design_files.tcl with the following contents:
                                                                       
                                                                      # Copyright 2023 Intel Corporation.\n#\n# THIS SOFTWARE MAY CONTAIN PREPRODUCTION CODE AND IS PROVIDED BY THE\n# COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED\n# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\n# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR\n# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,\n# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE\n# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,\n# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# Hello FIM Files\n#--------------------\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_com.sv\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_top.sv\n
                                                                       
                                                                      2.  
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Modify /src/pd_qsys/fabric/fabric_design_files.tcl
                                                                     
                                                                       
                                                                    1.  
                                                                      Add bpf_hello_fim_slv.ip to the list of files in the BPF section.
                                                                       
                                                                      #--------------------\n# BPF\n#--------------------\n...\nset_global_assignment -name IP_FILE ../ip_lib/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip\n
                                                                       
                                                                      2.  
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Modify src/top/top.sv
                                                                     
                                                                       
                                                                    1.  
                                                                      Add bpf_hello_fim_slv_if to AXI4-Lite Interfaces:
                                                                       
                                                                      // AXI4-lite interfaces\nofs_fim_axi_lite_if #(.AWADDR_WIDTH(12), .ARADDR_WIDTH(12)) bpf_hello_fim_slv_if();\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Modify the value of NEXT_DFH_OFFSET of mem_ss_top from 24'h6B000 to 24'h1000
                                                                       
                                                                      //*******************************\n// Memory Subsystem\n//*******************************\n`ifdef INCLUDE_DDR4\n   mem_ss_top #(\n      .FEAT_ID          (12'h009),\n      .FEAT_VER         (4'h1),\n      .NEXT_DFH_OFFSET  (24'h1000),\n      .END_OF_LIST      (1'b0)\n   ) mem_ss_top (\n
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Modify the value of NEXT_DFH_OFFSET of the Memory Subsystem dummy_csr from 24'h6B000 to 24'h1000
                                                                       
                                                                      // Placeholder logic if no mem_ss\ndummy_csr #(\n   .FEAT_ID          (12'h009),\n   .FEAT_VER         (4'h1),\n   .NEXT_DFH_OFFSET  (24'h1000),\n   .END_OF_LIST      (1'b0)\n) emif_dummy_csr (\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Add Hello FIM instance and dummy CSR after the Memory Subsystem. Set the NEXT_DFH_OFFSET to 24'h6A000 for both
                                                                       
                                                                      //*******************************\n// Hello FIM Subsystem\n//*******************************\n\n`ifdef INCLUDE_HELLO_FIM\nhello_fim_top #(\n   .ADDR_WIDTH       (12),\n   .DATA_WIDTH       (64),\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (24'h6A000),\n   .END_OF_LIST      (1'b0)\n) hello_fim_top_inst (\n    .clk (clk_csr),\n    .reset(~rst_n_csr),\n    .csr_lite_if    (bpf_hello_fim_slv_if)         \n);\n`else\ndummy_csr #(   \n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (24'h6A000),\n   .END_OF_LIST      (1'b0)\n) hello_fim_dummy (\n   .clk         (clk_csr),\n   .rst_n       (rst_n_csr),\n   .csr_lite_if (bpf_hello_fim_slv_if)\n);\n\n`endif \n
                                                                       
                                                                      5.  
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    Modify /src/pd_qsys/bpf_top.sv
                                                                     
                                                                       
                                                                    1.  
                                                                      Add bpf_hello_fim_slv_if to the interface descriptions
                                                                       
                                                                      ...\nmodule bpf_top (\n...\n//BPF funtions\n...\nofs_fim_axi_lite_if.master bpf_hello_fim_slv_if\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Add bpf_hello_fim_slv_if to the module
                                                                       
                                                                      module bpf_top (\n...\n);\n...\n.bpf_hello_fim_slv_awaddr    (bpf_hello_fim_slv_if.awaddr     ),\n.bpf_hello_fim_slv_awprot    (bpf_hello_fim_slv_if.awprot     ),\n.bpf_hello_fim_slv_awvalid   (bpf_hello_fim_slv_if.awvalid    ),\n.bpf_hello_fim_slv_awready   (bpf_hello_fim_slv_if.awready    ),\n.bpf_hello_fim_slv_wdata     (bpf_hello_fim_slv_if.wdata      ),\n.bpf_hello_fim_slv_wstrb     (bpf_hello_fim_slv_if.wstrb      ),\n.bpf_hello_fim_slv_wvalid    (bpf_hello_fim_slv_if.wvalid     ),\n.bpf_hello_fim_slv_wready    (bpf_hello_fim_slv_if.wready     ),\n.bpf_hello_fim_slv_bresp     (bpf_hello_fim_slv_if.bresp      ),\n.bpf_hello_fim_slv_bvalid    (bpf_hello_fim_slv_if.bvalid     ),\n.bpf_hello_fim_slv_bready    (bpf_hello_fim_slv_if.bready     ),\n.bpf_hello_fim_slv_araddr    (bpf_hello_fim_slv_if.araddr     ),\n.bpf_hello_fim_slv_arprot    (bpf_hello_fim_slv_if.arprot     ),\n.bpf_hello_fim_slv_arvalid   (bpf_hello_fim_slv_if.arvalid    ),\n.bpf_hello_fim_slv_arready   (bpf_hello_fim_slv_if.arready    ),\n.bpf_hello_fim_slv_rdata     (bpf_hello_fim_slv_if.rdata      ),\n.bpf_hello_fim_slv_rresp     (bpf_hello_fim_slv_if.rresp      ),\n.bpf_hello_fim_slv_rvalid    (bpf_hello_fim_slv_if.rvalid     ),\n.bpf_hello_fim_slv_rready    (bpf_hello_fim_slv_if.rready     ),\n...\nendmodule\n
                                                                       
                                                                      3.  
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    Create src/hello_fim
                                                                     
                                                                       
                                                                    1.  
                                                                      Create src/hello_fim directory
                                                                       
                                                                      mkdir $OFS_ROOTDIR/src/hello_fim\n
                                                                       
                                                                      2.  
                                                                     
                                                                    8.  
                                                                  8.  
                                                                    Create src/hello_fim/hello_fim_top.sv
                                                                     
                                                                       
                                                                    1.  
                                                                      Create hello_fim_top.sv with the following contents:
                                                                       
                                                                      // ***************************************************************************\n//                               INTEL CONFIDENTIAL\n//\n//        Copyright (C) 2023 Intel Corporation All Rights Reserved.\n//\n// The source code contained or described herein and all  documents related to\n// the  source  code  (\"Material\")  are  owned  by  Intel  Corporation  or its\n// suppliers  or  licensors.    Title  to  the  Material  remains  with  Intel\n// Corporation or  its suppliers  and licensors.  The Material  contains trade\n// secrets  and  proprietary  and  confidential  information  of  Intel or its\n// suppliers and licensors.  The Material is protected  by worldwide copyright\n// and trade secret laws and treaty provisions. No part of the Material may be\n// used,   copied,   reproduced,   modified,   published,   uploaded,  posted,\n// transmitted,  distributed,  or  disclosed  in any way without Intel's prior\n// express written permission.\n//\n// No license under any patent,  copyright, trade secret or other intellectual\n// property  right  is  granted  to  or  conferred  upon  you by disclosure or\n// delivery  of  the  Materials, either expressly, by implication, inducement,\n// estoppel or otherwise.  Any license under such intellectual property rights\n// must be express and approved by Intel in writing.\n//\n// You will not, and will not allow any third party to modify, adapt, enhance, \n// disassemble, decompile, reverse engineer, change or create derivative works \n// from the Software except and only to the extent as specifically required by \n// mandatory applicable laws or any applicable third party license terms \n// accompanying the Software.\n//\n// -----------------------------------------------------------------------------\n// Engineer     : \n// Create Date  : Nov 2021\n// Module Name  : hello_fim_top.sv\n// Project      : IOFS\n// -----------------------------------------------------------------------------\n//\n// Description: \n// This is a simple module that implements DFH registers and \n// AVMM address decoding logic.\n\nmodule hello_fim_top  #(\n   parameter ADDR_WIDTH  = 12, \n   parameter DATA_WIDTH = 64, \n   parameter bit [11:0] FEAT_ID = 12'h001,\n   parameter bit [3:0]  FEAT_VER = 4'h1,\n   parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n   parameter bit END_OF_LIST = 1'b0\n)(\n   input  logic    clk,\n   input  logic    reset,\n// -----------------------------------------------------------\n//  AXI4LITE Interface\n// -----------------------------------------------------------\n   ofs_fim_axi_lite_if.slave   csr_lite_if\n);\n\nimport ofs_fim_cfg_pkg::*;\nimport ofs_csr_pkg::*;\n\n//-------------------------------------\n// Signals\n//-------------------------------------\n   logic [ADDR_WIDTH-1:0]              csr_waddr;\n   logic [DATA_WIDTH-1:0]              csr_wdata;\n   logic [DATA_WIDTH/8-1:0]            csr_wstrb;\n   logic                               csr_write;\n   logic                               csr_slv_wready;\n   csr_access_type_t                   csr_write_type;\n\n   logic [ADDR_WIDTH-1:0]              csr_raddr;\n   logic                               csr_read;\n   logic                               csr_read_32b;\n   logic [DATA_WIDTH-1:0]              csr_readdata;\n   logic                               csr_readdata_valid;\n   logic [ADDR_WIDTH-1:0]              csr_addr;\n\n   logic [63:0]                        com_csr_writedata;\n   logic                               com_csr_read;\n   logic                               com_csr_write;\n   logic [63:0]                        com_csr_readdata;\n   logic                               com_csr_readdatavalid;\n   logic [5:0]                         com_csr_address;\n\n// AXI-M CSR interfaces\nofs_fim_axi_mmio_if #(\n   .AWID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .AWADDR_WIDTH (ADDR_WIDTH),\n   .WDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH),\n   .ARID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .ARADDR_WIDTH (ADDR_WIDTH),\n   .RDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH)\n) csr_if();\n\n// AXI4-lite to AXI-M adapter\naxi_lite2mmio axi_lite2mmio (\n   .clk       (clk),\n   .rst_n     (~reset),\n   .lite_if   (csr_lite_if),\n   .mmio_if   (csr_if)\n);\n\n//---------------------------------\n// Map AXI write/read request to CSR write/read,\n// and send the write/read response back\n//---------------------------------\nofs_fim_axi_csr_slave #(\n   .ADDR_WIDTH (ADDR_WIDTH),\n   .USE_SLV_READY (1'b1)\n\n   ) csr_slave (\n   .csr_if             (csr_if),\n\n   .csr_write          (csr_write),\n   .csr_waddr          (csr_waddr),\n   .csr_write_type     (csr_write_type),\n   .csr_wdata          (csr_wdata),\n   .csr_wstrb          (csr_wstrb),\n   .csr_slv_wready     (csr_slv_wready),\n   .csr_read           (csr_read),\n   .csr_raddr          (csr_raddr),\n   .csr_read_32b       (csr_read_32b),\n   .csr_readdata       (csr_readdata),\n   .csr_readdata_valid (csr_readdata_valid)\n);\n\n// Address mapping\nassign csr_addr             = csr_write ? csr_waddr : csr_raddr;\nassign com_csr_address      = csr_addr[5:0];  // byte address\nassign csr_slv_wready       = 1'b1 ;\n// Write data mapping\nassign com_csr_writedata    = csr_wdata;\n\n// Read-Write mapping\nalways_comb\nbegin\n   com_csr_read             = 1'b0;\n   com_csr_write            = 1'b0;\n   casez (csr_addr[11:6])\n      6'h00 : begin // Common CSR\n         com_csr_read       = csr_read;\n         com_csr_write      = csr_write;\n      end   \n      default: begin\n         com_csr_read       = 1'b0;\n         com_csr_write      = 1'b0;\n      end\n   endcase\nend\n\n// Read data mapping\nalways_comb begin\n   if (com_csr_readdatavalid) begin\n      csr_readdata       = com_csr_readdata;\n      csr_readdata_valid = 1'b1;\n   end\n   else begin\n      csr_readdata       = '0;\n      csr_readdata_valid = 1'b0;\n   end\nend\n\nhello_fim_com  #(\n   .FEAT_ID          (FEAT_ID),\n   .FEAT_VER         (FEAT_VER),\n   .NEXT_DFH_OFFSET  (NEXT_DFH_OFFSET),\n   .END_OF_LIST      (END_OF_LIST)\n) hello_fim_com_inst (\n   .clk                   (clk                     ),\n   .reset                 (reset                   ),\n   .writedata             (com_csr_writedata       ),\n   .read                  (com_csr_read            ),\n   .write                 (com_csr_write           ),\n   .byteenable            (4'hF                    ),\n   .readdata              (com_csr_readdata        ),\n   .readdatavalid         (com_csr_readdatavalid   ),\n   .address               (com_csr_address         )\n   );\nendmodule\n
                                                                       
                                                                      2.  
                                                                     
                                                                    9.  
                                                                  9.  
                                                                    Create src/hello_fim/hello_fim_com.sv
                                                                     
                                                                       
                                                                    1.  
                                                                      Create hello_fim_com.sv with the following contents:
                                                                       
                                                                      module hello_fim_com #(\n   parameter bit [11:0] FEAT_ID = 12'h001,\n   parameter bit [3:0]  FEAT_VER = 4'h1,\n   parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n   parameter bit END_OF_LIST = 1'b0\n)(\ninput clk,\ninput reset,\ninput [63:0] writedata,\ninput read,\ninput write,\ninput [3:0] byteenable,\noutput reg [63:0] readdata,\noutput reg readdatavalid,\ninput [5:0] address\n);\n\nwire reset_n = !reset;  \nreg [63:0] rdata_comb;\nreg [63:0] scratch_reg;\n\nalways @(negedge reset_n ,posedge clk)  \n   if (!reset_n) readdata[63:0] <= 64'h0; else readdata[63:0] <= rdata_comb[63:0];\n\nalways @(negedge reset_n , posedge clk)\n   if (!reset_n) readdatavalid <= 1'b0; else readdatavalid <= read;\n\nwire wr = write;\nwire re = read;\nwire [5:0] addr = address[5:0];\nwire [63:0] din  = writedata [63:0];\nwire wr_scratch_reg = wr & (addr[5:0]  == 6'h30)? byteenable[0]:1'b0;\n\n// 64 bit scratch register\nalways @( negedge  reset_n,  posedge clk)\n   if (!reset_n)  begin\n      scratch_reg <= 64'h0;\n   end\n   else begin\n   if (wr_scratch_reg) begin \n      scratch_reg <=  din;  \n   end\nend\n\nalways @ (*)\nbegin\nrdata_comb = 64'h0000000000000000;\n   if(re) begin\n      case (addr)  \n        6'h00 : begin\n                rdata_comb [11:0]   = FEAT_ID ;  // dfh_feature_id  is reserved or a constant value, a read access gives the reset value\n                rdata_comb [15:12]  = FEAT_VER ;  // dfh_feature_rev    is reserved or a constant value, a read access gives the reset value\n                rdata_comb [39:16]  = NEXT_DFH_OFFSET ;  // dfh_dfh_ofst is reserved or a constant value, a read access gives the reset value\n                rdata_comb [40]     = END_OF_LIST ;        //dfh_end_of_list\n                rdata_comb [59:40]  = 20'h00000 ;  // dfh_rsvd1     is reserved or a constant value, a read access gives the reset value\n                rdata_comb [63:60]  = 4'h3 ;  // dfh_feat_type  is reserved or a constant value, a read access gives the reset value\n        end\n        6'h30 : begin\n                rdata_comb [63:0]   = scratch_reg; \n        end\n        6'h38 : begin\n                rdata_comb [63:0]       = 64'h6626_0701_5000_0034;\n        end\n        default : begin\n                rdata_comb = 64'h0000000000000000;\n        end\n      endcase\n   end\nend\n\nendmodule\n
                                                                       
                                                                      2.  
                                                                     
                                                                    10.  
                                                                  10.  
                                                                    Modify src/pd_qsys/fabric/bpf.txt
                                                                     
                                                                       
                                                                    1.  
                                                                      Add hello_fim as a slave in the BPF, and enable the SoC as a master for it.
                                                                       
                                                                      #### - '#' means comment\n# NAME   TYPE      BASEADDRESS    ADDRESS_WIDTH    SLAVES\napf         mst     n/a             21             fme,soc_pcie,hssi,qsfp0,qsfp1,emif,pmci,hello_fim\n...\nhello_fim   slv     0x16000         12             n/a\n
                                                                       
                                                                      2.  
                                                                     
                                                                    11.  
                                                                  11.  
                                                                    Execute the helper script to re-generate the BPF design files
                                                                     
                                                                    cd $OFS_ROOTDIR/src/pd_qsys/fabric/\nsh gen_fabrics.sh\n
                                                                     
                                                                    12.  
                                                                  12.  
                                                                    After the shell script finishes, you can find the generated bpf_hello_fim_slv.ip file in $OFS_ROOTDIR/src/pd_qsys/fabric/ip/bpf/. This is the ip variant of the axi4lite shim that bridges the Hello FIM module with the BPF. The updated bpf.qsys file is located in $OFS_ROOTDIR/src/pd_qsys/fabric. You can view the updated bpf file in Platform designer as follows.
                                                                     
                                                                    cd $OFS_ROOTDIR/src/pd_qsys/fabric\nqsys-edit bpf.qsys --quartus-project=$OFS_ROOTDIR/syn/syn_top/ofs_top.qpf\n
                                                                     
                                                                    The image below shows the BPF that integrates the bpf_hello_fim_slv axi4lite shim at address 0x16000, generated through the helper script gen_fabrics.sh.
                                                                     
                                                                     
                                                                    13.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#517-unit-level-simulation-of-hello-fim-design","title":"5.1.7 Unit Level Simulation of Hello FIM Design","text":"
                                                                  The following section describes the file modifications that need to be made to perform unit level simulations of the Hello FIM design, followed by instructions for running the unit level simulations.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5171-unit-level-simulation-file-modification","title":"5.1.7.1 Unit Level Simulation File Modification","text":"
                                                                  Perform the following steps to modify the Unit Level simulation files to support the Hello FIM design.
                                                                   
                                                                     
                                                                  1.  
                                                                    Modify $OFS_ROOTDIR/sim/unit_test/soc_tests/dfh_walker/test_csr_defs.sv
                                                                     
                                                                       
                                                                    1.  
                                                                      Add a HELLO_FIM_IDX entry to the t_dfh_idx enumeration:
                                                                       
                                                                      typedef enum {\n   FME_DFH_IDX,\n   THERM_MNGM_DFH_IDX,\n   GLBL_PERF_DFH_IDX,\n   GLBL_ERROR_DFH_IDX,\n   QSFP0_DFH_IDX,\n   QSFP1_DFH_IDX,\n   HSSI_DFH_IDX,\n   EMIF_DFH_IDX,\n   HELLO_FIM_DFH_IDX,\n   PMCI_DFH_IDX,\n   ST2MM_DFH_IDX,\n   PG_PR_DFH_IDX,\n   PG_PORT_DFH_IDX,\n   PG_USER_CLK_DFH_IDX,\n   PG_REMOTE_STP_DFH_IDX,\n   AFU_ERR_DFH_IDX,\n   MAX_DFH_IDX\n} t_dfh_idx;\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Add an entry for HELLO_FIM_IDX into the get_dfh_names() function:
                                                                       
                                                                      function automatic dfh_name[MAX_DFH_IDX-1:0] get_dfh_names();\n   dfh_name[MAX_DFH_IDX-1:0] dfh_names;\n\n   dfh_names[FME_DFH_IDX]         = \"FME_DFH\";\n   dfh_names[THERM_MNGM_DFH_IDX]  = \"THERM_MNGM_DFH\";\n   dfh_names[GLBL_PERF_DFH_IDX]   = \"GLBL_PERF_DFH\";\n   dfh_names[GLBL_ERROR_DFH_IDX]  = \"GLBL_ERROR_DFH\";\n   dfh_names[QSFP0_DFH_IDX]       = \"QSFP0_DFH\";\n   dfh_names[QSFP1_DFH_IDX]       = \"QSFP1_DFH\";\n   dfh_names[HSSI_DFH_IDX]        = \"HSSI_DFH\";\n   dfh_names[EMIF_DFH_IDX]        = \"EMIF_DFH\";\n   dfh_names[HELLO_FIM_DFH_IDX]   = \"HELLO_FIM_DFH\";\n   dfh_names[PMCI_DFH_IDX]        = \"PMCI_DFH\";\n   dfh_names[ST2MM_DFH_IDX]       = \"ST2MM_DFH\";\n   dfh_names[PG_PR_DFH_IDX]       = \"PG_PR_DFH\";\n   dfh_names[PG_PORT_DFH_IDX]     = \"PG_PORT_DFH\";\n   dfh_names[PG_USER_CLK_DFH_IDX] = \"PG_USER_CLK_DFH\";\n   dfh_names[PG_REMOTE_STP_DFH_IDX] = \"PG_REMOTE_STP_DFH\";\n   dfh_names[AFU_ERR_DFH_IDX] = \"AFU_ERR_DFH\";\n\n   return dfh_names;\nendfunction\n
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Modify the expected DFH value of the EMIF from 64'h3_00000_06B000_1009 to 64'h3_00000_001000_1009 and add the expected value for HELLO_FIM as 64'h3_00000_06A000_0100:
                                                                       
                                                                      function automatic [MAX_DFH_IDX-1:0][63:0] get_dfh_values();\n   logic[MAX_DFH_IDX-1:0][63:0] dfh_values;\n\n   dfh_values[FME_DFH_IDX]        = 64'h4000_0000_1000_0000;\n   dfh_values[THERM_MNGM_DFH_IDX] = 64'h3_00000_002000_0001;\n   dfh_values[GLBL_PERF_DFH_IDX]  = 64'h3_00000_001000_0007;\n   dfh_values[GLBL_ERROR_DFH_IDX] = 64'h3_00000_00e000_1004;\n   dfh_values[QSFP0_DFH_IDX]      = 64'h3_00000_001000_0013;\n   dfh_values[QSFP1_DFH_IDX]      = 64'h3_00000_001000_0013;\n   dfh_values[HSSI_DFH_IDX]       = 64'h3_00000_001000_100f;\n   dfh_values[EMIF_DFH_IDX]       = 64'h3_00000_001000_1009;\n   dfh_values[HELLO_FIM_DFH_IDX]  = 64'h3_00000_06A000_0100;\n   dfh_values[PMCI_DFH_IDX]       = 64'h3_00000_080000_1012;\n   dfh_values[ST2MM_DFH_IDX]      = 64'h3_00000_030000_0014;\n   dfh_values[PG_PR_DFH_IDX]      = 64'h3_00000_001000_1005;\n   dfh_values[PG_PORT_DFH_IDX]     = 64'h4_00000_001000_1001;\n   dfh_values[PG_USER_CLK_DFH_IDX] = 64'h3_00000_001000_1014;\n   dfh_values[PG_REMOTE_STP_DFH_IDX] = 64'h3_00000_00d000_2013;\n   dfh_values[AFU_ERR_DFH_IDX] = 64'h3_00001_000000_2010;\n\n   return dfh_values;\nendfunction\n
                                                                       
                                                                      4.  
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Regenerate the simulation files
                                                                     
                                                                    cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\nsh gen_sim_files.sh f2000x \n
                                                                     
                                                                    3.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5172-run-dfh-walker-simulation","title":"5.1.7.2 Run DFH Walker Simulation","text":"
                                                                  After the simulation files have been re-generated, run the DFH Walker test to ensure the Hello FIM module can be accessed by the SoC through the BPF.
                                                                   
                                                                     
                                                                  1.  
                                                                    Run the DFH Walker test
                                                                     
                                                                    cd $OFS_ROOTDIR/sim/unit_test/soc_tests/dfh_walker\nsh run_sim.sh\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Check the output for the presence of the HELLO_FiM module at address 0x16000:
                                                                     
                                                                    ********************************************\nRunning TEST(0) : test_dfh_walking\n********************************************\n\n...\n\nREAD64: address=0x00015000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x3000000010001009\n\nEMIF_DFH\n   Address   (0x15000)\n   DFH value (0x3000000010001009)\n\nREAD64: address=0x00016000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x30000006a0000100\n\nHELLO_FIM_DFH\n   Address   (0x16000)\n   DFH value (0x30000006a0000100)\n\nREAD64: address=0x00080000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x3000000800001012\n\nPMCI_DFH\n   Address   (0x80000)\n   DFH value (0x3000000800001012)\n\n...\n\nTest status: OK\n\n********************\nTest summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n
                                                                     
                                                                    3.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#518-uvm-verfication-of-the-hellofim","title":"5.1.8 UVM Verfication of the HelloFIM","text":"
                                                                  The following section describes the file modifications that need to be made to perform UVM verifaction of the Hello FIM design, followed by instructions for running the UVM simulations.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5181-uvm-simulation-file-modification","title":"5.1.8.1 UVM Simulation File Modification","text":"
                                                                  Perform the following steps to modify the UVM simulation files to support the Hello FIM design.
                                                                   
                                                                     
                                                                  1.  
                                                                    Modify $OFS_ROOTDIR/verification/tests/sequences/dfh_walking_seq.svh
                                                                     
                                                                       
                                                                    1.  
                                                                      Modify the dfh_offset_array to insert the Hello FIM.
                                                                       
                                                                      dfh_offset_array = new[16];\ndfh_offset_array[ 0] = tb_cfg0.PF0_BAR0;                    // FME_DFH                0x8000_0000\ndfh_offset_array[ 1] = dfh_offset_array[ 0] + 64'h0_1000;   // THERM_MNGM_DFH         0x8000_1000\ndfh_offset_array[ 2] = dfh_offset_array[ 1] + 64'h0_2000;   // GLBL_PERF_DFH          0x8000_3000\ndfh_offset_array[ 3] = dfh_offset_array[ 2] + 64'h0_1000;   // GLBL_ERROR_DFH         0x8000_4000\ndfh_offset_array[ 4] = dfh_offset_array[ 3] + 64'h0_E000;   // QSFP0_DFH              0x8001_2000\ndfh_offset_array[ 5] = dfh_offset_array[ 4] + 64'h0_1000;   // QSFP1_DFH              0x8001_3000\ndfh_offset_array[ 6] = dfh_offset_array[ 5] + 64'h0_1000;   // HSSI_DFH               0x8001_4000\ndfh_offset_array[ 7] = dfh_offset_array[ 6] + 64'h0_1000;   // EMIF_DFH               0x8001_5000\ndfh_offset_array[ 8] = dfh_offset_array[ 7] + 64'h0_1000;   // HELLO_FIM_DFH          0x8001_6000\ndfh_offset_array[ 9] = dfh_offset_array[ 8] + 64'h6_a000;   // PMCI_DFH               0x8008_0000\ndfh_offset_array[ 10] = dfh_offset_array[ 9] + 64'h8_0000;  // ST2MM_DFH              0x8010_0000\ndfh_offset_array[ 11] = dfh_offset_array[10] + 64'h3_0000;  // PG_PR_DFH_IDX          0x8013_0000\ndfh_offset_array[ 12] = dfh_offset_array[11] + 64'h0_1000;  // PG_PORT_DFH_IDX        0x8013_1000\ndfh_offset_array[ 13] = dfh_offset_array[12] + 64'h0_1000;  // PG_USER_CLK_DFH_IDX    0x8013_2000\ndfh_offset_array[ 14] = dfh_offset_array[13] + 64'h0_1000;  // PG_REMOTE_STP_DFH_IDX  0x8013_3000\ndfh_offset_array[ 15] = dfh_offset_array[14] + 64'h0_D000;  // PG_AFU_ERR_DFH_IDX     0x8014_0000\n
                                                                       
                                                                      2.  
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Modify $OFS_ROOTDIR/verification/tests/sequences/mmio_seq.svh
                                                                     
                                                                       
                                                                    1.  
                                                                      Add test code related to the Hello FIM. This code will verify the scratchpad register at 0x16030 and read only the register at 0x16038.
                                                                       
                                                                      // HELLO_FIM_Scratchpad 64 bit access\n`uvm_info(get_name(), $psprintf(\"////Accessing PF0 HELLO_FIM_Scratchpad Register %0h+'h16030////\", tb_cfg0.PF0_BAR0), UVM_LOW)\n\nassert(std::randomize(wdata));\naddr = tb_cfg0.PF0_BAR0+'h1_6000+'h30;\n\nmmio_write64(.addr_(addr), .data_(wdata));\nmmio_read64 (.addr_(addr), .data_(rdata));\n\nif(wdata !== rdata)\n    `uvm_error(get_name(), $psprintf(\"Data mismatch 64! Addr = %0h, Exp = %0h, Act = %0h\", addr, wdata, rdata))\nelse\n    `uvm_info(get_name(), $psprintf(\"Data match 64! addr = %0h, data = %0h\", addr, rdata), UVM_LOW)\n\naddr = tb_cfg0.PF0_BAR0+'h1_6000+'h38;\nwdata = 64'h6626_0701_5000_0034;\nmmio_read64 (.addr_(addr), .data_(rdata));\nif(wdata !== rdata)\n    `uvm_error(get_name(), $psprintf(\"Data mismatch 64! Addr = %0h, Exp = %0h, Act = %0h\", addr, wdata, rdata))\nelse\n    `uvm_info(get_name(), $psprintf(\"Data match 64! addr = %0h, data = %0h\", addr, rdata), UVM_LOW)\n
                                                                       
                                                                      Note: uvm_info and uvm_error statements will put a message into log file.
                                                                       
                                                                      2.  
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Modify $OFS_ROOTDIR/verification/scripts/Makefile_VCS.mk
                                                                     
                                                                       
                                                                    1.  
                                                                      Add INCLUDE_HELLO_FIM define option to enable Hello FIM on UVM
                                                                       
                                                                      VLOG_OPT += +define+INCLUDE_HELLO_FIM\n
                                                                       
                                                                      2.  
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Re-generate the UVM files
                                                                     
                                                                       
                                                                    1.  
                                                                      Navigate to the verification scripts directory
                                                                       
                                                                      cd $VERDIR/scripts\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Clean the output of previous builds
                                                                       
                                                                      gmake -f Makefile_VCS.mk clean\n
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Compile the IP files
                                                                       
                                                                      gmake -f Makefile_VCS.mk cmplib_adp\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Build the RTL and Test Benches
                                                                       
                                                                      gmake -f Makefile_VCS.mk build_adp DUMP=1 DEBUG=1 \n
                                                                       
                                                                      Note: Using the DEBUG option will provide more detail in the log file for the simulation.
                                                                       
                                                                      5.  
                                                                     
                                                                    5.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5182-run-uvm-dfh-walker-simulation","title":"5.1.8.2 Run UVM DFH Walker Simulation","text":"
                                                                  Perform the following steps to run the UVM DFH Walker Simulation.
                                                                   
                                                                     
                                                                  1.  
                                                                    Run the DFH Walker simulation
                                                                     
                                                                    cd $VERDIR/scripts\ngmake -f Makefile_VCS.mk run TESTNAME=dfh_walking_test DUMP=1 DEBUG=1\n
                                                                     
                                                                    Note: Using the DEBUG option will provide more detail in the log file for the simulation.
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    The output logs are stored in the $VERDIR/sim/dfh_walking_test directory. The main files to note are described in the table below:
                                                                     File Name Description runsim.log A log file of UVM trans.log A log file of transactions on PCIe bus inter.vpd A waveform for VCS 
                                                                    3.  
                                                                  3.  
                                                                    Run the following command to quickly verify- that the Hello FIM module was successfully accessed. In the example below, the message DFH offset Match! Exp = 80016000 Act = 80016000 shows that the Hello FIM module was successfully accessed.
                                                                     
                                                                    cd $VERDIR/sim/dfh_walking_test\ncat runsim.log | grep \"DFH offset\"\n
                                                                     
                                                                    Expected output:
                                                                     
                                                                    UVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 111950000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp = 80000000 Act = 80000000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 112586000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80001000 Act = 80001000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 113222000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80003000 Act = 80003000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 113858000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80004000 Act = 80004000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 114494000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80012000 Act = 80012000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 115147000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80013000 Act = 80013000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 115801000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80014000 Act = 80014000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 116628000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80015000 Act = 80015000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 117283000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80016000 Act = 80016000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 117928000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80080000 Act = 80080000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 118594000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80100000 Act = 80100000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 119248000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80130000 Act = 80130000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 119854000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80131000 Act = 80131000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 120460000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80132000 Act = 80132000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 121065000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80133000 Act = 80133000\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/dfh_walking_seq.svh(73) @ 121672000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80140000 Act = 80140000\n
                                                                     
                                                                    4.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5183-run-uvm-mmio-simulation","title":"5.1.8.3 Run UVM MMIO Simulation","text":"
                                                                  Perform the following steps to run the UVM MMIO Simulation.
                                                                   
                                                                     
                                                                  1.  
                                                                    Run the MMIO test
                                                                     
                                                                    cd $VERDIR/scripts\ngmake -f Makefile_VCS.mk run TESTNAME=mmio_test DUMP=1\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Run the following commands to show the result of the scratchpad register and Hello FIM ID register. You can see the \"Data match\" message indicating that the registers are successfuly verified.
                                                                     
                                                                    cd $VERDIR/sim/mmio_test\ncat runsim.log | grep \"Data\" | grep 1603\n
                                                                     
                                                                    Expected output:
                                                                     
                                                                    UVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/mmio_seq.svh(68) @ 115466000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] Data match 64! addr = 80016030, data = 880312f9558c00e1\nUVM_INFO /home/applications.fpga.ofs.fim-f2000x-pl/verification/tests/sequences/mmio_seq.svh(76) @ 116112000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] Data match 64! addr = 80016038, data = 6626070150000034\n
                                                                     
                                                                    3.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#519-compile-the-f2000x-design-with-hello-fim","title":"5.1.9 Compile the f2000x Design with Hello FIM","text":"
                                                                  Perform the following to compile the Hello FIM design.
                                                                   
                                                                     
                                                                  1. Ensure the pre-requesites described in the Pre-Requisites for Adding Hello FIM section are satisfied.
                                                                    2.  
                                                                  2. Ensure that Intel Quartus Prime Pro is in your $PATH
                                                                    3.  
                                                                  3.  
                                                                    Compile the design
                                                                     
                                                                    $OFS_ROOTDIR/ofs-common/scripts/common/syn/build_top.sh -p f2000x work_hello_fim\n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Once compilation is complete, the output files can be found in the $OFS_ROOTDIR/work_hello_fim/syn/syn_top/output_files directory.
                                                                     
                                                                    5.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5110-program-the-f2000x-with-the-hellofim-design","title":"5.1.10 Program the f2000x with the HelloFIM Design","text":"
                                                                  Perform the following steps to program the f2000x with the HelloFIM design generated in the previous section. This flow uses the RSU feature, which requires that an OPAE enabled design is already loaded on the FPGA. All OPAE commands are run from the SoC, and the new image will be updated from the SoC.
                                                                   
                                                                     
                                                                  1.  
                                                                    Use the fpgainfo fme command to obtain the PCIe s:b:d.f associated with your board. In this example, the PCIe s:b:d.f is 0000:15:00.0.
                                                                     
                                                                    fpgainfo fme\n
                                                                     
                                                                    Example output:
                                                                     
                                                                    Intel IPU Platform f2000x\nBoard Management Controller NIOS FW version: 1.2.4 \nBoard Management Controller Build version: 1.2.4 \n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010302A97C08A3\nBitstream Version                : 5.0.1\nPr Interface Id                  : fb25ff1d-c31a-55d8-81d8-cbcedcfcea17\nBoot Page                        : user1\nUser1 Image Info                 : a566ceacaed810d43c60b0b8a7145591\nUser2 Image Info                 : a566ceacaed810d43c60b0b8a7145591\nFactory Image Info               : None\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Navigate to the output directory in the Hello FIM work directory that contains the output files from compilation. 
                                                                     
                                                                    cd $OFS_ROOTDIR/work_hello_fim/syn/syn_top/output_files\n
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Initiate the User Image 1 update by running fpgasupdate from a shell in the SoC. This will update User Image 1 stored in FPGA Flash. Remember to use the PCIe BDF associated with your board.
                                                                     
                                                                    sudo fpgasupdate ofs_top_page1_unsigned_user1.bin 0000:15:00.0\n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Run the rsu command to re-configure the FPGA with User Image 1 from FPGA Flash.
                                                                     
                                                                    sudo rsu fpga --page=user1 0000:15:00.0\n
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Run the fpgainfo fme command again to verify the User1 Image Info has been updated.
                                                                     
                                                                    Example Output:
                                                                     
                                                                    Intel IPU Platform f2000x\nBoard Management Controller NIOS FW version: 1.2.4 \nBoard Management Controller Build version: 1.2.4 \n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010302A97C08A3\nBitstream Version                : 5.0.1\nPr Interface Id                  : fb25ff1d-c31a-55d8-81d8-cbcedcfcea17\nBoot Page                        : user1\nUser1 Image Info                 : 81d8cbcedcfcea17fb25ff1dc31a55d8\nUser2 Image Info                 : a566ceacaed810d43c60b0b8a7145591\nFactory Image Info               : None\n
                                                                     
                                                                    6.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5111-verify-the-hello-fim-design-on-the-f2000x-using-opae","title":"5.1.11 Verify the Hello FIM Design on the f2000x Using OPAE","text":"
                                                                  This section will describe how to access the Hello FIM registers using the opae.io tool.
                                                                   
                                                                     
                                                                  1.  
                                                                    Confirm the driver software on 0000:15:00.0
                                                                     
                                                                    opae.io ls\n
                                                                     
                                                                    Example output:
                                                                     
                                                                    [0000:15:00.0] (0x8086:0xbcce 0x8086:0x17d4) Intel IPU Platform f2000x (Driver: dfl-pci)\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Initialize the opae.io tool
                                                                     
                                                                    opae.io init -d 15:00.0\n
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Confirm the driver software on 0000:15:00.0 has been updated
                                                                     
                                                                    opae.io ls\n
                                                                     
                                                                    Example Output:
                                                                     
                                                                    [0000:15:00.0] (0x8086:0xbcce 0x8086:0x17d4) Intel IPU Platform f2000x (Driver: vfio-pci)\n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Run the DFH Walker test to verify there is a module at offset 0x16000
                                                                     
                                                                    opae.io walk -d 15:00.0\n
                                                                     
                                                                    Example output:
                                                                     
                                                                    offset: 0x0000, value: 0x4000000010000000\n   dfh: id = 0x0, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x4\noffset: 0x1000, value: 0x3000000020000001\n    dfh: id = 0x1, rev = 0x0, next = 0x2000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x3000, value: 0x3000000010000007\n    dfh: id = 0x7, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x4000, value: 0x30000000e0001004\n    dfh: id = 0x4, rev = 0x1, next = 0xe000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x12000, value: 0x3000000010000013\n    dfh: id = 0x13, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x13000, value: 0x3000000010000013\n    dfh: id = 0x13, rev = 0x0, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x14000, value: 0x3000000010002015\n    dfh: id = 0x15, rev = 0x2, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x15000, value: 0x3000000010001009\n    dfh: id = 0x9, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x16000, value: 0x30000006a0000100\n    dfh: id = 0x100, rev = 0x0, next = 0x6a000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x80000, value: 0x3000000800002012\n    dfh: id = 0x12, rev = 0x2, next = 0x80000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x100000, value: 0x3000000300000014\n    dfh: id = 0x14, rev = 0x0, next = 0x30000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x130000, value: 0x3000000010001005\n    dfh: id = 0x5, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x131000, value: 0x4000000010001001\n    dfh: id = 0x1, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x4\noffset: 0x132000, value: 0x3000000010001014\n    dfh: id = 0x14, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x133000, value: 0x30000000d0002013\n    dfh: id = 0x13, rev = 0x2, next = 0xd000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x140000, value: 0x3000010000002010\n    dfh: id = 0x10, rev = 0x2, next = 0x0, eol = 0x1, reserved = 0x0, feature_type = 0x3\n
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Read all of the registers in the Hello FIM module
                                                                     
                                                                       
                                                                    1.  
                                                                      Read the DFH Register
                                                                       
                                                                      opae.io -d 15:00.0 -r 0 peek 0x16000\n
                                                                       
                                                                      Example Output:
                                                                       
                                                                      0x30000006a0000100\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Read the Scratchpad Register
                                                                       
                                                                      opae.io -d 15:00.0 -r 0 peek 0x16030\n
                                                                       
                                                                      Example Output:
                                                                       
                                                                      0x0\n
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Read the ID Register
                                                                       
                                                                      opae.io -d 15:00.0 -r 0 peek 0x16038\n
                                                                       
                                                                      Example Output:
                                                                       
                                                                      0x6626070150000034\n
                                                                       
                                                                      4.  
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    Verify the scratchpad register at 0x16030 by writing and reading back from it.
                                                                     
                                                                       
                                                                    1.  
                                                                      Write to Scratchpad register
                                                                       
                                                                      opae.io -d 0000:15:00.0 -r 0 poke 0x16030 0x123456789abcdef\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Read from Scratchpad register
                                                                       
                                                                      opae.io -d 15:00.0 -r 0 peek 0x16030\n
                                                                       
                                                                      Expected output:
                                                                       
                                                                      0x123456789abcdef\n
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Write to Scratchpad register
                                                                       
                                                                      opae.io -d 15:00.0 -r 0 poke 0x16030 0xfedcba9876543210\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Read from Scratchpad register
                                                                       
                                                                      opae.io -d 15:00.0 -r 0 peek 0x16030\n
                                                                       
                                                                      Expected output:
                                                                       
                                                                      0xfedcba9876543210\n
                                                                       
                                                                      5.  
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    Release the opae.io tool
                                                                     
                                                                    opae.io release -d 15:00.0\n
                                                                     
                                                                    8.  
                                                                  8.  
                                                                    Confirm the driver has been set back to dfl-pci
                                                                     
                                                                    opae.io ls\n
                                                                     
                                                                    Example output:
                                                                     
                                                                    [0000:15:00.0] (0x8086:0xbcce 0x8086:0x17d4) Intel IPU Platform f2000x (Driver: dfl-pci)\n
                                                                     
                                                                    9.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#52-how-to-debug-the-fim-with-signal-tap","title":"5.2 How to Debug the FIM with Signal Tap","text":"
                                                                  For debugging issues within the FIM, Signal Tap can be used to gain internal visibility into your design. This section describes the process of adding a Signal Tap instance to the Hello FIM design example described in the How to add a new module to the FIM section, however the process can be used for any design.
                                                                   
                                                                  For more detailed information on Signal Tap please see refer to Quartus Prime Pro Edition User Guide: Debug Tools (RDC Document ID 683819).
                                                                   
                                                                  Signal Tap uses the Intel FPGA Download Cable II USB device to provide access.  Please see Intel FPGA Download Cable II for more information. This device is widely available via distributors for purchase.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#521-adding-signal-tap-to-the-hello-fim-example","title":"5.2.1 Adding Signal Tap to the Hello FIM example","text":"
                                                                  The following steps guide you through the process of adding a Signal Tap instance to your design. The added Signal Tap instance provides hardware to capture the desired internal signals and connect the stored trace information via JTAG. Please be aware that the added Signal Tap hardware will consume FPGA resources and may require additional floorplanning steps to accommodate these resources. Some areas of the FIM use logic lock regions and these regions may need to be re-sized. These steps assume the use of the  Intel IPU Platform F2000X-PL.  
                                                                   
                                                                  The steps below use the hello_fim example to add Signal Tap, however the general process can be used for any design.
                                                                   
                                                                     
                                                                  1.  
                                                                    The design must be synthesized before adding Signal Tap.
                                                                     
                                                                       
                                                                    •  
                                                                      If you are using the previously built Hello FIM design, copy the work directory and rename it so that we have a work directory dedicated to the Hello FIM Signal Tap design.
                                                                       
                                                                      cp -r $OFS_ROOTDIR/work_hello_fim $OFS_ROOTDIR/work_hello_fim_with_stp\n
                                                                       
                                                                      •  
                                                                     
                                                                       
                                                                    •  
                                                                      If you are adding signal tap to a new design that has not yet been synthesized, perform the following steps to synthesize the design.
                                                                       
                                                                         
                                                                      1. Set the environment variables as described in the Setting Up Required Environment Variables section.
                                                                        2.  
                                                                      2.  
                                                                        Run the setup portion of the build script to create a working directory based on the original source files.
                                                                         
                                                                        ./ofs-common/scripts/common/syn/build_top.sh --stage setup f2000x <YOUR_STP_WORK_DIR>\n
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Open the project in the Intel Quartus Prime Pro GUI. The Intel Quartus Prime Pro project is named ofs_top.qpf and is located in the work directory $OFS_ROOTDIR/<YOUR_STP_WORK_DIR>/syn/syn_top/ofs_top.qpf.
                                                                         
                                                                        4.  
                                                                      4. In the Compilation Flow window, run Analysis & Synthesis.
                                                                        5.  
                                                                      5. Once Synthesis has completed, you may skip to Step 3.
                                                                        6.  
                                                                       
                                                                      •  
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Open the Hello FIM Signal Tap project in the Intel Quartus Prime Pro GUI. The Intel Quartus Prime Pro project is named ofs_top.qpf and is located in the work directory $OFS_ROOTDIR/work_hello_fim_with_stp/syn/syn_top/ofs_top.qpf.
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Select Tools > Signal Tap Logic Analyzer to open the Signal Tap GUI.
                                                                     
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Accept the \"Default\" selection and click \"Create\".
                                                                     
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    This opens the Signal Tap Logic Analyzer window.
                                                                     
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    Set up the clock for the STP instance. This example instruments the hello_fim_top module previously intetegrated into the FIM. If unfamiliar with code, it is helpful to use the Intel Quartus Prime Pro Project Navigator to find the block of interest and open the design instance for review. For example, see the image below using Project Navigator to open the top module where hello_fim_top_inst is instantiated.
                                                                     
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    Assign the clock for sampling the Signal Tap instrumented signals of interest. Note, that the clock selected should be associated with the signals you want to view for best trace fidelity. Different clocks can be used, however, there maybe issues with trace inaccuracy due to sampling time differences. Ensure that all signals that are to be sampled are on the same clock domain as the clock you select here. In the middle right of the Signal Tap window, under Signal Configuration, Clock:, select \"\u2026\" as shown below:
                                                                     
                                                                     
                                                                    8.  
                                                                  8.  
                                                                    In the Node Finder tool that popped up, input \"hello_fim_top_inst|clk\" into the \"Named:\" textbox and click \"Search\". Select \"clk\" in the Matching Nodes list and click the \">\" button to select this clock as shown below. Click \"OK\" to close the Node Finder dialog.
                                                                     
                                                                     
                                                                    9.  
                                                                  9.  
                                                                    Update the sample depth and other Signal Tap settings as needed for your debugging criteria.
                                                                     
                                                                     
                                                                    10.  
                                                                  10.  
                                                                    In the Signal Tap GUI add the nodes to be instrumented by double-clicking on the \"Double-click to add nodes\" legend.
                                                                     
                                                                     
                                                                    11.  
                                                                  11.  
                                                                    This brings up the Node Finder to add the signals to be traced. In this example we will monitor the memory mapped interface to the Hello FIM. Select the signals that appear from the  search patterns hello_fim_top_inst|reset and hello_fim_top_inst|csr_lite_if*. Click Insert and close the Node Finder dialog.
                                                                     
                                                                     
                                                                    12.  
                                                                  12.  
                                                                    To provide a unique name for your Signal Tap instance, select \"auto signaltap_0\", right-click, and select Rename Instance (F2). Provide a descriptive name for your instance, for example, \"STP_For_Hello_FIM\".
                                                                     
                                                                     
                                                                    13.  
                                                                  13.  
                                                                    Save the newly created Signal Tap file, and give it the same name as the instance. Ensure that the Add file to current project checkbox is ticked.
                                                                     
                                                                     
                                                                    14.  
                                                                  14.  
                                                                    In the dialog that pops up, click \"Yes\" to add the newly created Signal Tap file to the project settings files.
                                                                     
                                                                     
                                                                    This will aurtomatically add the following lines to $OFS_ROOTDIR/work_hello_fim_with_stp/syn/syn_top/ofs_top.qsf:
                                                                     
                                                                    set_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE STP_For_Hello_FIM.stp\nset_global_assignment -name SIGNALTAP_FILE STP_For_Hello_FIM.stp\n
                                                                     
                                                                    15.  
                                                                  15.  
                                                                    Close all Quartus GUIs.
                                                                     
                                                                    16.  
                                                                  16.  
                                                                    Compile the project with the Signal Tap file added to the project. Use the -k switch to perform the compilation using the files in a specified working directory and not the original ones from the cloned repository. 
                                                                     
                                                                    ofs-common/scripts/common/syn/build_top.sh -p -k f2000x work_hello_fim_with_stp\n
                                                                     
                                                                    Alternatively, you can copy the ofs_top.qsf and STP_For_Hello_FIM.stp files from the Hello FIM with STP work directory to replace the original files in the cloned OFS repository. In this scenario, all further FIM compilation projects will include the Signal Tap instance integrated into the design. Execute the following commands for this alternative flow:
                                                                     
                                                                    Copy the modified file \"work_hello_fim_with_stp/syn/syn_top/ofs_top.qsf\" over to the source OFS repository, into \"syn/syn_top/\".
                                                                     
                                                                    cd $OFS_ROOTDIR/work_hello_fim_with_stp/syn/syn_top\ncp ofs_top.qsf $OFS_ROOTDIR/syn/syn_top\ncp STP_For_Hello_FIM.stp $OFS_ROOTDIR/syn/syn_top\n
                                                                     
                                                                    Compile the FIM using the files from the OFS repository to create a new work directory.
                                                                     
                                                                    cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x work_hello_fim_with_stp_from_src_repo\n
                                                                     
                                                                    17.  
                                                                  17.  
                                                                    Ensure that the compile completes successfully and meets timing:
                                                                     
                                                                    ***********************************\n***\n***        OFS_PROJECT: f2000x\n***        OFS_FIM: base\n***        OFS_BOARD: adp\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 0\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                                                     
                                                                    18.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#522-configuring-the-fpga-with-a-sof-image-via-jtag","title":"5.2.2 Configuring the FPGA with a SOF Image via JTAG","text":"
                                                                  Every successful run of build_top.sh script creates the file $OFS_ROOTDIR/<WORK_DIRECTORY>/syn/syn_top/output_files/ofs_top.sof which can be used with the Intel FPGA Download Cable II to load the image into the FPGA using the f2000x  JTAG access connector. 
                                                                   
                                                                  Perform the steps described in the following sections to load the ofs_fim.sof image created in the previous section into the Intel Agilex 7 FPGA using the Intel FPGA Download Cable II. You will also use the Intel FPGA Download Cable II to access the Signal Tap instance via JTAG.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5221-connecting-to-intel-fpga-download-cable","title":"5.2.2.1 Connecting to Intel FPGA Download Cable","text":"
                                                                  The f2000x  has a 10 pin JTAG header on the top side of the board.  This JTAG header provides access to either the Intel Agilex 7 FPGA or Cyclone\u00ae 10 BMC device.  Perform the followign steps to connect the Itel FPGA Download Cable II and target the Intel Agilex 7 device:
                                                                   
                                                                     
                                                                  1.  
                                                                    Locate SW2 and SW3 on the f2000x board as shown in the following figure.
                                                                     
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Set the switches described in the following table:
                                                                     Switch Position SW2 ON SW3.3 ON 
                                                                    3.  
                                                                  3.  
                                                                    Connect the Intel FPGA Download Cable to the JTAG header of the f2000x as shown in the figure below.
                                                                     
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Depending on your server, install the card in a slot that allows room for the JTAG cable. The figure below shows the f2000x installed in a Supermicro server slot.
                                                                     
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    There are two JTAG modes that exist. Short-chain mode is when only the Cyclone 10 device is on the JTAG chain. Long-chain mode is when both the Cyclone 10 and the Intel Agilex 7 FPGA are on the JTAG chain. Check which JTAG mode the f2000x board is in by running the following command.
                                                                     
                                                                    $QUARTUS_ROOTDIR/bin/jtagconfig\n
                                                                     
                                                                       
                                                                    •  
                                                                      Example output when in short-chain mode (only Cyclone 10 detected):
                                                                       
                                                                      1) USB-BlasterII [3-4]\n020F60DD    10CL080(Y|Z)/EP3C80/EP4CE75\n
                                                                       
                                                                      •  
                                                                     
                                                                       
                                                                    •  
                                                                      Example output when in long-chain mode (both Cyclone 10 and Intel Agilex 7 detected):
                                                                       
                                                                      1) USB-BlasterII [3-4]\n020F60DD   10CL080(Y|Z)/EP3C80/EP4CE75\n234150DD   AGFC023R25A(.|AE|R0)\n
                                                                         If the Intel Agilex 7 device does not appear on the chain, ensure that the switches described in Step 1 have been set correctly and power cycle the board. Also ensure that the JTAG Longchain bit is set to 0 in BMC Register 0x378. The BMC registers are accessed through SPI control registers at addresses 0x8040C and 0x80400 in the PMCI. Use the following commands to clear the JTAG Longchain bit in BMC register 0x378. 
                                                                       
                                                                      •  
                                                                     
                                                                    Note: These commands must be executed as root user from the SoC.
                                                                     
                                                                    Note: You may find the PCIe BDF of your card by running fpgainfo fme.
                                                                     
                                                                    opae.io init -d <BDF>\nopae.io -d <BDF> -r 0 poke 0x8040c 0x000000000\nopae.io -d <BDF> -r 0 poke 0x80400 0x37800000002\nopae.io release -d <BDF>\n
                                                                     For example, for a board with PCIe BDF 15:00.0: 
                                                                    opae.io init -d 15:00.0\nopae.io -d 15:00.0 -r 0 poke 0x8040c 0x000000000\nopae.io -d 15:00.0 -r 0 poke 0x80400 0x37800000002\nopae.io release -d 15:00.0\n
                                                                     
                                                                    6.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#5222-programming-the-intel-agilex-7-fpga-via-jtag","title":"5.2.2.2 Programming the Intel Agilex 7 FPGA via JTAG","text":"
                                                                  Perform the following steps to program the Intel Agilex 7 FPGA via JTAG.
                                                                   
                                                                     
                                                                  1.  
                                                                    The generated SOF file is located in the work directory $OFS_ROOTDIR/work_hello_fim_with_stp/syn/syn_top/output_files/ofs_top.sof. If the target FPGA is on a different server, then transfer ofs_top.sof and STP_For_Hello_FIM.stp files to the server with the target FPGA.
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    You can use a Full Intel Quartus Prime Pro Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the f2000x  is installed or on a separte machine such as a laptop.
                                                                     
                                                                    Note: You can download the Quartus Prime Programmer and Tools by clicking on the \"Additional Software\" tab on the FPGA download page. The Quartus Prime Programmer and Tools come with Quartus programmer as well as System Console which are needed to program the flash devices.
                                                                     
                                                                    Note: If using the Intel FGPA download Cable on Linux, add the udev rule as described in Intel FPGA Download Cable (formerly USB-Blaster) Driver for Linux.
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Temporarily disable the PCIe AER feature and remove the PCIe port for the board you are going to update.. This is required because when you program the FPGA using JTAG, the f2000x PCIe link goes down for a moment causing a server surprise link down event. To prevent this server event, temporarily disable PCIe AER and remove the PCIe port using the following commands:
                                                                     
                                                                    Note: enter the following commands as root.
                                                                     
                                                                       
                                                                    1.  
                                                                      Find the PCIe BDF and Device ID of your board from the SoC. You may use the OPAE command fpaginfo fme on the SoC to display this information. Run this command on the SoC.
                                                                       
                                                                      fpgainfo fme\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      Intel IPU Platform F2000X-PL\nBoard Management Controller NIOS FW version: 1.2.3\nBoard Management Controller Build version: 1.2.3\n//****** FME ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x50103024BF5B5B1\nBitstream Version                : 5.0.1\nPr Interface Id                  : e7926956-9b1b-5ea1-b02c-307f1cb33446\nBoot Page                        : user1\nUser1 Image Info                 : b02c307f1cb33446e79269569b1b5ea1\nUser2 Image Info                 : b02c307f1cb33446e79269569b1b5ea1\nFactory Image Info               : b02c307f1cb33446e79269569b1b5ea1\n
                                                                       
                                                                      In this case, the PCIe BDF for the board on the SoC is 15:00.0, and the Device ID is 0xBCCE.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      From the SoC, use the pci_device OPAE command to \"unplug\" the PCIe port for your board using the PCIe BDF found in Step 3.a. Run this command on the SoC.
                                                                       
                                                                      pci_device <B:D.F> unplug\n
                                                                       
                                                                      For example:
                                                                       
                                                                      pci_device 15:00.0 unplug\n
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Find the PCIe BDF of your board from the Host. Use lspci and grep for the device ID found in Step 3.a to get the PCIe BDF. Run this command on the Host.
                                                                       
                                                                      For example:
                                                                       
                                                                      lspci | grep bcce\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      31:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n31:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                                                                       
                                                                      In this case, the board has PCIe BDF 31:00.0 from the Host.
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      From the Host, use the pci_device OPAE command to \"unplug\" the PCIe port for your board using the PCIe BDF found in Step 3.c. Run this command on the Host.
                                                                       
                                                                      pci_device <B:D.F> unplug\n
                                                                       
                                                                      For example:
                                                                       
                                                                      pci_device 31:00.0 unplug\n
                                                                       
                                                                      5.  
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Launch \"Quartus Prime Programmer\" software from the device which the Intel FPGA Programmer is connected.
                                                                     
                                                                    $QUARTUS_ROOTDIR/bin/quartus_pgmw\n
                                                                     
                                                                    Click on Hardware Setup, select USB-BlasterII in the Current Selected Hardware list, and ensure the JTAG Hardware Frequency is set to 16Mhz (The default is 24MHz).
                                                                     
                                                                     
                                                                    Alternatively, use the following command from the command line to change the clock frequency:
                                                                     
                                                                    jtagconfig \u2013setparam \u201cUSB-BlasterII\u201d JtagClock 16M\n
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Click Auto Detect and make sure the Intel Agilex 7 Device is shown in the JTAG chain. Select the Cyclone 10 and Intel Agilex 7 devices if prompted.
                                                                     
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    Right-click on the cell in the File column for the Intel Agilex 7 device and click on Change file
                                                                     
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    Select the generated ofs_top.sof file for the Intel Agilex 7 FPGA with the Signal Tap instrumented Hello FIM example. Remember that the output files are located under work_hello_fim_with_stp/syn/syn_top/output_files/.
                                                                     
                                                                    8.  
                                                                  8.  
                                                                    Tick the checkbox below \"Program/Configure\" column and click on Start to program this .sof file.
                                                                     
                                                                     
                                                                    9.  
                                                                  9.  
                                                                    After successful programming, you can close the \"Quartus Prime Programmer\" software. You can answer 'No' if a dialog pops up asking to save the 'Chain.cdf' file. This completes the Intel Agilex 7 .sof programming.
                                                                     
                                                                    10.  
                                                                  10.  
                                                                    Re-plug the PCIe ports on both the SoC and Host.
                                                                     
                                                                    Note: enter the following commands as root.
                                                                     
                                                                       
                                                                    1.  
                                                                      From the SoC, use the pci_device OPAE command to \"plug\" the PCIe port for your board using the PCIe BDF found in Step 3.a. Run this command on the SoC.
                                                                       
                                                                      pci_device <B:D.F> plug\n
                                                                       
                                                                      For example:
                                                                       
                                                                      pci_device 15:00.0 plug\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      From the Host, use the pci_device OPAE command to \"plug\" the PCIe port for your board using the PCIe BDF found in Step 3.c. Run this command on the Host.
                                                                       
                                                                      pci_device <B:D.F> plug\n
                                                                       
                                                                      For example:
                                                                       
                                                                      pci_device 31:00.0 plug\n
                                                                       
                                                                      3.  
                                                                     
                                                                    11.  
                                                                  11.  
                                                                    Verify the f2000x is present by checking expected bitstream ID using fpaginfo fme on the SoC:
                                                                     
                                                                    fpgainfo fme\n
                                                                     
                                                                    Example output: 
                                                                    Intel IPU Platform f2000x-PL\nBoard Management Controller NIOS FW version: 1.1.9\nBoard Management Controller Build version: 1.1.9\n//****** FME ******//\nObject Id                        : 0xF000000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010302A97C08A3\nBitstream Version                : 5.0.1\nPr Interface Id                  : cf00eed4-a82b-5f07-be25-0528baec3711\nBoot Page                        : user1\nUser1 Image Info                 : 9e3db8b6a4d25a4e3e46f2088b495899\nUser2 Image Info                 : 9e3db8b6a4d25a4e3e46f2088b495899\nFactory Image Info               : None\n
                                                                     
                                                                    Note: The Image Info fields will not change, because these represent the images stored in flash. In this example, we are programming the Intel Agilex 7 FPGA directly, thus only the Bitstream ID should change.
                                                                     
                                                                    12.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#523-signal-tap-trace-acquisition-of-hello-fim-signals","title":"5.2.3 Signal Tap trace acquisition of Hello FIM signals","text":"
                                                                     
                                                                  1.  
                                                                    Once the instrumented HelloFIM SOF file is downloaded into the Intel Agilex 7 FPGA, start the Quartus Signal Tap GUI.
                                                                     
                                                                    $QUARTUS_ROOTDIR/bin/quartus_stpw\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    In the Signal Tap GUI, open your STP file. Your STP file settings will load. In this example we used STP_For_Hello_FIM.stp.
                                                                     
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    In the right pane of the Signal Tap GUI, in the Hardware: selection box select the cable USB-BlasterII. In the Device: selection box select the Intel Agilex 7 device.
                                                                     
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    If the Intel Agilex 7 Device is not displayed in the Device: list, click the 'Scan Chain' button to re-scan the JTAG device chain.
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    If not already set, you can create the trigger conditions. In this example, we will capture data on a rising edge of the Read Address Valid signal.
                                                                     
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    Start analysis by selecting the 'STP_For_Hello_FIM' instance and pressing 'F5' or clicking the Run Analysis icon in the toolbar. You should see a green message indicating the Acquisition is in progress. Then, move to the Data Tab to observe the signals captured.
                                                                     
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    To generate traffic in the csr_lite_if signals of the Hello FIM module, go back to the terminal and walk the DFH list or peek/poke the Hello FIM registers as was done during the creation of the Hello FIM design example.
                                                                     
                                                                    opae.io init -d 0000:15:00.0\nopae.io walk -d 0000:15:00.0\nopae.io release -d 0000:15:00.0\n
                                                                     
                                                                    The signals should be captured on the rising edge of arvalid in this example. Zoom in to get a better view of the signals.
                                                                     
                                                                     
                                                                    8.  
                                                                  8.  
                                                                    The PCIe AER feature is automatically re-enabled by rebooting the server. 
                                                                     
                                                                    9.  
                                                                   
                                                                  This concludes the example on how to instrument an OFS FIM with the Quartus Prime Signal Tap Logic Analyzer.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#53-how-to-compile-the-fim-in-preparation-for-designing-your-afu","title":"5.3 How to compile the FIM in preparation for designing your AFU","text":"
                                                                  To save area, the default Host Excercisers in the FIM can be replaced by a \"he_null\" block during compile-time. There are a few things to note:
                                                                   
                                                                     
                                                                  • \"he_null\" is a minimal block with registers that responds to PCIe MMIO request. MMIO responses are required to keep PCIe alive (end points enabled in PCIe-SS needs service downstream requests).
                                                                    •  
                                                                  • If an exerciser with other I/O connections such as \"he_mem\" or \"he_hssi\" is replaced, then then those I/O ports are simply tied off.
                                                                    •  
                                                                  • The options supported are null_he_lb, null_he_hssi, null_he_mem and null_he_mem_tg. Any combination, order or all can be enabled at the same time. 
                                                                    •  
                                                                  • Finer grain control is provided for you to, for example, turn off only the exercisers in the Static Region in order to save area.
                                                                    •  
                                                                   
                                                                  To compile a FIM for where the exercisers are removed and replaced with a he_null module and keeping the PF/VF multiplexor connections, execute the following command.
                                                                   
                                                                  cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x:null_he,null_he_hssi,null_he_mem,null_he_mem_tg work_null_he\n
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#54-how-to-resize-the-partial-reconfiguration-region","title":"5.4 How to Resize the Partial Reconfiguration Region","text":"
                                                                  To take advantage of the available resources in the Intel Agilex 7 FPGA for an AFU design, you can adjust the size of the AFU PR partition. An example reason for the changing the size of PR region is if you add more logic to the FIM region, then you may need to reduce the size of the PR region to fit the additional logic into the static region.  Similiarly, if you reduce logic in the FIM region, then you can increase the size of the PR region to provide more logic resources for the AFU.
                                                                   
                                                                  After the compilation of the FIM, the resources usage broken down by partitions is reported in the Logic Lock Region Usage Summary sections of following two files:
                                                                   
                                                                     
                                                                  • $OFS_ROOTDIR/<YOUR_WORK_DIRECTORY>/syn/syn_top/output_files/ofs_top.fit.place.rpt
                                                                    •  
                                                                  •  
                                                                    $OFS_ROOTDIR/<YOUR_WORK_DIRECTORY>/syn/syn_top/output_files/ofs_top.fit.rpt
                                                                     
                                                                     
                                                                    •  
                                                                   
                                                                  In this case, the default size for the afu_top|port_gasket|pr_slot|afu_main PR partition is large enough to comfortably hold the logic of the default AFU, which is mainly occupied by the Host Exercisers. However, larger designs might require additional resources.
                                                                   
                                                                  Perform the following steps to customize the resources allocated to the AFU in the PR regions:
                                                                   
                                                                     
                                                                  1.  
                                                                    The $OFS_ROOTDIR/syn/setup/pr_assignments.tcl TCL file defines the Logic Lock Regions in the design, including the PR partition where the AFU is allocated.
                                                                     
                                                                    The default design uses the the following Logic Lock Regions:
                                                                     
                                                                    set TOP_MEM_REGION    \"X115 Y310 X219 Y344\"\nset BOTTOM_MEM_REGION \"X0 Y0 X294 Y20\"\nset SUBSYSTEM_REGION  \"X0 Y0 X60 Y279; X0 Y0 X300 Y39; X261 Y0 X300 Y129;\"\n\nset AFU_PLACE_REGION  \"X61 Y40 X260 Y309; X220 Y130 X294 Y329; X12 Y280 X114 Y329;\"\nset AFU_ROUTE_REGION  \"X0 Y0 X294 Y329\"\n
                                                                     
                                                                    Each region is made up of rectangles defined by the origin (X0,Y0) and the top right corner (X1,Y1).
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Use Quartus Chip Planner to identify the locations of the resources available within the Intel Agilex 7 device for placement and routing your AFU. The image below shows the default floorplan for the f2000x Intel Agilex 7 device.
                                                                     
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Make changes to the $OFS_ROOTDIR/syn/setup/pr_assignments.tcl file based on your findings in Quartus Chip Planner. You can modify the size and location of existing Logic Lock Regions or add new ones and assign them to the AFU PR partition. You will need to modify the coordinates of other regions assigned to the FIM accordingly to prevent overlap.
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Recompile your FIM and create the PR relocatable build tree using the following commands:
                                                                     
                                                                    cd $OFS_ROOTDIR    \nofs-common/scripts/common/syn/build_top.sh -p f2000x  <YOUR_WORK_DIRECTORY>\n
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Analyze the resource utilization report per partition produced after recompiling the design.
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    Make further modifications to the PR regions until the results are satisfactory. Make sure timing constraints are met.
                                                                     
                                                                    7.  
                                                                   
                                                                  Refer to the following documentation for more information on how to optimize the floor plan of your Partial Reconfiguration design:
                                                                   
                                                                     
                                                                  • Analyzing and Optimizing the Design Floorplan
                                                                    •  
                                                                  • Partial Reconfiguration Design Flow - Step 3: Floorplan the Design
                                                                    •  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#55-how-to-modify-the-memory-subsystem","title":"5.5 How to modify the Memory Subsystem","text":"
                                                                  In this example we will modify the Memory Subsystem to enable ECC on all of the existing memory interfaces. You may make different modifications to meet your own design requirements. Perform the following steps to make this change.
                                                                   
                                                                     
                                                                  1.  
                                                                    Clone the design repositories or use an existing design. See the Clone the OFS Git Repo section.
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Set the environment variables as described in the Setting Up Required Environment Variables section.
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Navigate to the directory containing the Memory Subsystem IP file mem_ss_fm.ip.
                                                                     
                                                                    cd $OFS_ROOTDIR/ipss/mem/qip/mem_ss/ \n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Open the Memory Subsystem IP file in Platform Designer to perform the required edits. 
                                                                     
                                                                    qsys-edit mem_ss_fm.ip\n
                                                                     
                                                                    The IP Parameter Editor GUI opens as shown below.
                                                                     
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Select the \"Memory Interfaces\" tab to view the current configuration of the Memory Interfaces. You may make edits to the configuration of any of the interfaces as needed. The figure below shows the default configuration of Interface 3.
                                                                     
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    In this example we will enable ECC for all four interfaces. In Interface tabs 0 through 3, change the drop-down selection for Memory DQ width from 32 (no ECC) to 40 (with ECC). The figure below shows this change for Interface 3.
                                                                     
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    Generate the HDL code by clicking the Generate HDL... button at the bottom right corner of the Platform Designer window. In the dialog box that appears next, review the HDL generation options and click the Generate button at the bottom right corner of the dialog box. Save the system if prompted to do so. Once the generation process is finished, close the Platform designer windows. 
                                                                     
                                                                    8.  
                                                                  8.  
                                                                    Edit the $OFS_ROOTDIR/ipss/mem/rtl/mem_ss_pkg.sv file to change the DDR4_DQ_WIDTH from 32 to 40.
                                                                     
                                                                    // DDR PARAMS\n...\nlocalparam DDR4_DQ_WIDTH      = 40;\n
                                                                     
                                                                    9.  
                                                                  9.  
                                                                    Edit the $OFS_ROOTDIR/syn/setup/emif_loc.tcl file to assign the pins required for ECC enabled interfaces.
                                                                     
                                                                       
                                                                    1.  
                                                                      Uncomment the DQS4 pin assignments for all memory interfaces
                                                                       
                                                                      #---------------------------------------------------------\n# EMIF CH0\n#---------------------------------------------------------\n...\n# # CH0 DQS4 (ECC)\nset_location_assignment PIN_A39 -to ddr4_mem[0].dq[32]\nset_location_assignment PIN_J35 -to ddr4_mem[0].dq[33]\nset_location_assignment PIN_C38 -to ddr4_mem[0].dq[34]\nset_location_assignment PIN_G34 -to ddr4_mem[0].dq[35]\nset_location_assignment PIN_G38 -to ddr4_mem[0].dq[36]\nset_location_assignment PIN_C34 -to ddr4_mem[0].dq[37]\nset_location_assignment PIN_J39 -to ddr4_mem[0].dq[38]\nset_location_assignment PIN_A35 -to ddr4_mem[0].dq[39]\nset_location_assignment PIN_C36 -to ddr4_mem[0].dqs[4]\nset_location_assignment PIN_A37 -to ddr4_mem[0].dqs_n[4]\nset_location_assignment PIN_G36 -to ddr4_mem[0].dbi_n[4]\n\n#---------------------------------------------------------\n# EMIF CH1\n#---------------------------------------------------------\n...\n# # CH1 DQS4 (ECC)\nset_location_assignment PIN_N7  -to ddr4_mem[1].dq[32]\nset_location_assignment PIN_L12 -to ddr4_mem[1].dq[33]\nset_location_assignment PIN_L6  -to ddr4_mem[1].dq[34]\nset_location_assignment PIN_U14 -to ddr4_mem[1].dq[35]\nset_location_assignment PIN_U7  -to ddr4_mem[1].dq[36]\nset_location_assignment PIN_W12 -to ddr4_mem[1].dq[37]\nset_location_assignment PIN_W6  -to ddr4_mem[1].dq[38]\nset_location_assignment PIN_N14 -to ddr4_mem[1].dq[39]\nset_location_assignment PIN_L9  -to ddr4_mem[1].dqs[4]\nset_location_assignment PIN_N10 -to ddr4_mem[1].dqs_n[4]\nset_location_assignment PIN_W9  -to ddr4_mem[1].dbi_n[4]\n\n#---------------------------------------------------------\n# EMIF CH2\n#---------------------------------------------------------\n...\n# CH2 DQS4 (ECC)\nset_location_assignment PIN_GC37 -to ddr4_mem[2].dq[32]\nset_location_assignment PIN_GC41 -to ddr4_mem[2].dq[33]\nset_location_assignment PIN_FY37 -to ddr4_mem[2].dq[34]\nset_location_assignment PIN_GE40 -to ddr4_mem[2].dq[35]\nset_location_assignment PIN_FV36 -to ddr4_mem[2].dq[36]\nset_location_assignment PIN_FY41 -to ddr4_mem[2].dq[37]\nset_location_assignment PIN_GE36 -to ddr4_mem[2].dq[38]\nset_location_assignment PIN_FV40 -to ddr4_mem[2].dq[39]\nset_location_assignment PIN_GE38 -to ddr4_mem[2].dqs[4]\nset_location_assignment PIN_GC39 -to ddr4_mem[2].dqs_n[4]\nset_location_assignment PIN_FV38 -to ddr4_mem[2].dbi_n[4]\n\n#---------------------------------------------------------\n# EMIF CH3\n#---------------------------------------------------------\n...\n# # CH3 DQS4 (ECC)\nset_location_assignment PIN_FP46 -to ddr4_mem[3].dq[32]\nset_location_assignment PIN_FT43 -to ddr4_mem[3].dq[33]\nset_location_assignment PIN_FH47 -to ddr4_mem[3].dq[34]\nset_location_assignment PIN_FP42 -to ddr4_mem[3].dq[35]\nset_location_assignment PIN_FT47 -to ddr4_mem[3].dq[36]\nset_location_assignment PIN_FH43 -to ddr4_mem[3].dq[37]\nset_location_assignment PIN_FK46 -to ddr4_mem[3].dq[38]\nset_location_assignment PIN_FK42 -to ddr4_mem[3].dq[39]\nset_location_assignment PIN_FP44 -to ddr4_mem[3].dqs[4]\nset_location_assignment PIN_FT45 -to ddr4_mem[3].dqs_n[4]\nset_location_assignment PIN_FK44 -to ddr4_mem[3].dbi_n[4]\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Change the pin assignment for ddr4_mem[1].dbi_n[4] from PIN_G12 to PIN_W9
                                                                       
                                                                      set_location_assignment PIN_W9  -to ddr4_mem[1].dbi_n[4]\n
                                                                       
                                                                      3.  
                                                                     
                                                                    10.  
                                                                  10.  
                                                                    Compile the design.
                                                                     
                                                                    cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x <YOUR_WORK_DIRECTORY>\n
                                                                     
                                                                    11.  
                                                                  11.  
                                                                    You may need to adjust the floorplan of the design in order to meet timing after a design change such as this. Refer to the How to Resize the Partial Reconfiguration Region section for information regarding modifications to the floorplan.
                                                                     
                                                                    12.  
                                                                   
                                                                  The configuration edits described here were made to the original source files of the cloned OFS repository. Therefore, these modifications will present in subsequent FIM compilations. This is because the FIM compilation process links and copies source files from the cloned OFS repository to the FIM compilation work directory.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#56-how-to-compile-the-fim-with-no-hssi","title":"5.6 How to compile the FIM with No HSSI","text":"
                                                                  In this example we will compile f2000x with the Ethernet subsystem removed. To perform the flat compile of the FIM with no Ethernet subsystem, pass the no_hssi and flat options to the build_top.sh script:
                                                                   
                                                                  cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh f2000x :flat,no_hssi <YOUR_WORK_DIRECTORY>\n
                                                                   
                                                                  If you wish to build a PR enabled design, you may adjust the Logic Lock regions to allocate more resources to the PR region since the Ethernet subsystem has been removed from the FIM. Refer to the How to Resize the Partial Reconfiguration Region section for information regarding modifications to the floorplan.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#57-how-to-change-the-pcie-device-id-and-vendor-id","title":"5.7 How to change the PCIe device ID and Vendor ID","text":"
                                                                  The PCIe configuration registers contains the Vendor, Device and Subsystem Vendor ID registers which are used in PCIe add-in cards to uniquely identify the card for assignment to software drivers.  OFS has these registers set with Intel values for out of the box usage.  If you are using OFS for a PCIe add in card that your company manufactures, then update the PCIe Subsytem Subsystem ID and Vendor ID registers as described below and change OPAE provided software code to properly operate with your company's register values.
                                                                   
                                                                  The Vendor ID is assigned to your company by the PCI-SIG (Special Interest Group). The PCI-SIG is the only body officially licensed to give out IDs. You must be a member of PCI-SIG to request your own ID. Information about joining PCI-SIG is available here: PCI-SIG. You select the Subsystem Device ID for your PCIe add in card.
                                                                   
                                                                  Follow the instructions below to customize the PCIe device ID and Vendor ID of the f2000x  PLPlatform.
                                                                   
                                                                  You can display the current settings using the command lspci -nvmms <PCIe B.D.f> as shown below:
                                                                   
                                                                  lspci -nvmms 98:00\n
                                                                   Example Output:
                                                                   
                                                                  Slot:   98:00.0\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nPhySlot:        1\nRev:    01\nNUMANode:       1\n\nSlot:   98:00.1\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nPhySlot:        1\nNUMANode:       1\n\nSlot:   98:00.2\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nPhySlot:        1\nNUMANode:       1\n\nSlot:   98:00.3\nClass:  1200\nVendor: 1af4\nDevice: 1000\nSVendor:        1af4\nSDevice:        1771\nPhySlot:        1\nNUMANode:       1\n\nSlot:   98:00.4\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nPhySlot:        1\nNUMANode:       1\n
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#571-changing-the-pcie-subsystem-device-id-and-vendor-id","title":"5.7.1 Changing the PCIe Subsystem Device ID and Vendor ID","text":"
                                                                  You will use IP Parameter Editor to modify the PCIe configuration registers.
                                                                   
                                                                     
                                                                  1.  
                                                                    Navigate to the PCIe Subsystem IP file and bring up IP Parameter Editor to change values.
                                                                     
                                                                    Note: Both the Host and SoC PCIe subsystems use the same IP module, so the following changes to Device ID and Vendor ID will be reflected in both the Host and SoC PCIe Subsystems.
                                                                     
                                                                    cd $OFS_ROOTDIR/ipss/pcie/qip/ss\nqsys-edit pcie_ss.ip\n
                                                                     The IP Parameter Editor GUI will open.  Close any tool pop-ups.
                                                                     
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Scroll down through the PCIe subsystem settings tabs to the PCIe Interfaces 0 Ports Settings tab as shown below:
                                                                     
                                                                     
                                                                    Select the PCIe0 Device Identification Registers tab. You can edit the values of Vendor ID, Device ID, Subsystem Vendor ID and Subsystem Device ID for each PF/VF in use.
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Once you have made changes, click Generate HDL and save. 
                                                                     
                                                                    4.  
                                                                  4. Make sure the environment variables are set as described in the Setting Up Required Environment Variables section.
                                                                    5.  
                                                                  5.  
                                                                    Build your new FPGA image with build_top.sh script
                                                                     
                                                                    cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x work_pcie_vid\n
                                                                     
                                                                    6.  
                                                                   
                                                                  Be aware that OPAE FPGA management commands require recognition of the FPGA PCIe Device ID for control.  If there is a problem between OPAE management recognition of FPGA PCIe values, then control of the card will be lost.  For this reason, you are strongly encouraged to initially confiugre the FPGA via JTAG to load the test FPGA image. Instructions for thes process are given in the Configuring the FPGA with a SOF Image via JTAG section. If there is a problem with the SOF image working with your host software that is updated for the new PCIe settings, then you can load a known good SOF file to recover.  Once you sure that both the software and FPGA work properly, you can load the FPGA into FPGA flash using the OPAE command fpgasupdate.
                                                                   
                                                                  The changes to software required to work with new PCIe settings are described in [Software Reference Manual: Open FPGA Stack] 
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#58-how-to-migrate-to-a-different-intel-agilex-7-device-number","title":"5.8 How to migrate to a different Intel Agilex 7 device number","text":"
                                                                  The following instructions enable you to change the Intel Agilex 7 FPGA device part number of the f2000x, for example, to migrate to a device with a larger density. Be aware that this release works with Intel Agilex 7 devices that have P-tile for PCIe and E-tile for Ethernet.
                                                                   
                                                                  The default device for the Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL is AGFC023R25A2E2VR0
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#581-migrating-to-a-larger-device-with-the-same-package","title":"5.8.1 Migrating To a Larger Device With the Same Package","text":"
                                                                  Perform the following steps to change the device to a larger density with the same package.
                                                                   
                                                                     
                                                                  1.  
                                                                    Clone the design repository. See the Clone the OFS Git Repo section.
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Set the environment variables as described in the Setting Up Required Environment Variables section.
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Navigate to the OFS Root Directory
                                                                     
                                                                    cd $OFS_ROOTDIR\n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Use the following command to change the device part number throughout the OFS Root directory heirarchy, replacing <DEFAULT_OPN> and <NEW_OPN> with the part numbers specific to your update:
                                                                     
                                                                    grep -rli '<DEFAULT_OPN>' * | xargs -i@ sed -i 's/<DEFAULT_OPN>/<NEW_OPN>/g' @\n
                                                                     
                                                                    For example, use the following command to change from part AGFC023R25A2E2VR0 to part AGFA027R25A2E2VR0:
                                                                     
                                                                    grep -rli 'AGFC023R25A2E2VR0'* | xargs -i@ sed -i 's/AGFC023R25A2E2VR0/AGFA027R25A2E2VR0/g' @\n
                                                                     
                                                                    This changes all occurrences of the default device (AGFC023R25A2E2VR0) in the $OFS_ROOTDIR directory to the new device number (AGFA027R25A2E2VR0).
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Compile the flat (non-PR) design to verify the compilation is successful with the new part. The flat design is compiled without any Logic Lock constraints.
                                                                     
                                                                    cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh f2000x:flat  <YOUR_WORK_DIRECTORY>\n
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    To enable the PR region, use Quartus Chip Planner to analyze the compiled flat design and adjust the Logic Lock constraints defined in $OFS_ROOTDIR/syn/setup/pr_assignments.tcl for the new device layout. Refer to the How to Resize the Partial Reconfiguration Region section for instructions. Re-compile the design with the out-of-tree PR region enabled.
                                                                     
                                                                    cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x  <YOUR_WORK_DIRECTORY>\n
                                                                     
                                                                    7.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#59-how-to-change-ethernet-interface-from-8x25-gbe-to-8x10-gbe","title":"5.9 How to change Ethernet interface from 8x25 GbE to 8x10 GbE","text":"
                                                                  This section describes steps to change the Ethernet interface from 8x25 GbE to 8x10 GbE. 
                                                                   
                                                                     
                                                                  1.  
                                                                    Edit the HSSI IP Subsystem $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss_8x25g.ip to be 8x10 GbE using IP Platform Editor.
                                                                     
                                                                    cd $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss\nqsys-edit hssi_ss_8x25g.ip\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    The IP Prameter Editer comes up - expect 2-5 minutes for this process to complete.  When the pop-up indicates Open System Completed, click Close.  When the General Configuration window comes up, scroll down and switch ports 0 through 7 from 25GbE to 10GbE as shown below:
                                                                     
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Click the IP Configuration tab and note the default settings of OFF for AN/LT and SYNCE.  You may optionally change these settings based on your application needs. The settings for P0 IP cover ports 0 to 3.  The settings for P4 cover ports 4 to 7.
                                                                     
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Click \"P0 Configuration\" tab and note the default settings for maximum frame size.  You may optionally change these settings based on your application needs.  Set \"P4 Configuration\" as needed. 
                                                                     
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Leave other settings at default values.
                                                                     
                                                                    6.  
                                                                  6. Click File and Save As hssi_ss_8x10g.ip  Click Generate HDL in the bottom right hand corner of IP Editor and enable simulation support.
                                                                    7.  
                                                                  7.  
                                                                    Edit $OFS_ROOTDIR/ipss/hssi/eth_design_files.tcl to comment out 8x25g and add in 8x10g.ip
                                                                     
                                                                    #-----------------\n# HSSI SS IP\n#-----------------\n#set_global_assignment -name IP_FILE               ../ip_lib/ipss/hssi/qip/hssi_ss/hssi_ss_8x25g.ip\nset_global_assignment -name IP_FILE               ../ip_lib/ipss/hssi/qip/hssi_ss/hssi_ss_8x10g.ip\nset_global_assignment -name IP_FILE               ../ip_lib/ipss/hssi/qip/ptp_iopll/ptp_sample_clk_pll.ip\n
                                                                     
                                                                    8.  
                                                                  8.  
                                                                    Edit $OFS_ROOTDIR/syn/syn_top/ofs_top.qsf and $OFS_ROOTDIR/syn/syn_top/ofs_pr_afu.qsf to add new macro definition:
                                                                     
                                                                    set_global_assignment -name VERILOG_MACRO \"ETH_10G\"    # Change Ethernet from 8x25 to 8x10 GbE\n
                                                                     
                                                                    9.  
                                                                  9.  
                                                                    Build new 8x10G FIM
                                                                     
                                                                    cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x work_8x10gbe\n
                                                                     
                                                                    10.  
                                                                  10.  
                                                                    You may need to adjust the floorplan of the design in order to meet timing after a design change such as this. Refer to the How to Resize the Partial Reconfiguration Region section for information regarding modifications to the floorplan.
                                                                     
                                                                    11.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#510-how-to-change-ethernet-interface-from-8-x-25-gbe-to-2-x-100-gbe","title":"5.10 How to change Ethernet interface from 8 X 25 GbE to 2 X 100 GbE","text":"
                                                                  This section describes steps to change the Ethernet interface from 8 X 25 GbE to 2 x 100 GbE. 
                                                                   
                                                                     
                                                                  1.  
                                                                    Edit HSSI IP Subsystem $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss/hssi_ss_8x25g.ip to be 2 X 100 GbE using IP Platform Editor.
                                                                     
                                                                    cd $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss\nqsys-edit hssi_ss_8x25g.ip\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    The IP Prameter Editer comes up - expect 2-5 minutes for this process to complete.  When pop-up indicates Open System Completed, click 'Close'.  The General Configuration window comes up, change ports to 2 and set \"PORT0_PROFILE\" and \"PORT4_PROFILE\" to \"100GCAUI-4\" as shown below:
                                                                     
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Click the IP Configuration tab and note the default settings of OFF for AN/LT and SYNCE.  You may optionally change these settings based on your application needs.
                                                                     
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Click \"P0 Configuration\" tab and note the default settings for maximum frame size.  You may optionally change these settings based on your application needs.  Set \"P4 Configuration\" as needed. 
                                                                     
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Leave other settings at default values.
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    Click File and Save As hssi_ss_2x100g.  Click Generate HDL in the bottom right hand corner of IP Editor and enable simulation support.
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    Edit $OFS_ROOTDIR/ipss/hssi/eth_design_files.tcl to comment out 8x25g and add in 2x100g.ip
                                                                     
                                                                    #-----------------\n# HSSI SS IP\n#-----------------\n#set_global_assignment -name IP_FILE               ../ip_lib/ipss/hssi/qip/hssi_ss/hssi_ss_8x25g.ip\nset_global_assignment -name IP_FILE               ../ip_lib/ipss/hssi/qip/hssi_ss/hssi_ss_2x100g.ip\nset_global_assignment -name IP_FILE               ../ip_lib/ipss/hssi/qip/ptp_iopll/ptp_sample_clk_pll.ip\n
                                                                     
                                                                    8.  
                                                                  8.  
                                                                    Edit $OFS_ROOTDIR/syn/syn_top/ofs_top.qsf and $OFS_ROOTDIR/syn/syn_top/ofs_pr_afu.qsf to add new macro definition:
                                                                     
                                                                    set_global_assignment -name VERILOG_MACRO \"ETH_100G\"      # Change Ethernet from 8x25 to 2x100 GbE\n
                                                                     
                                                                    9.  
                                                                  9.  
                                                                    Update $OFS_ROOTDIR/syn/setup/eth_top.sdc:
                                                                     
                                                                    #Timing for 100G\nset_false_path -from [get_clocks {sys_pll|iopll_0_clk_100m}] -to [get_clocks {hssi_wrapper|hssi_ss|hssi_ss_0|U_hssi_ss_ip_wrapper|U_hssi_ss_ip_top_p*|alt_ehipc3_fm_0|alt_ehipc3_fm_top_p*|alt_ehipc3_fm_hard_inst|E100GX4_FEC.altera_xcvr_native_inst|xcvr_native_s10_etile_0_example_design_4ln_ptp|tx_clkout|ch0}]; \n\nset_false_path -from [get_clocks {hssi_wrapper|hssi_ss|hssi_ss_0|U_hssi_ss_ip_wrapper|U_hssi_ss_ip_top_p*|alt_ehipc3_fm_0|alt_ehipc3_fm_top_p*|alt_ehipc3_fm_hard_inst|E100GX4_FEC.altera_xcvr_native_inst|xcvr_native_s10_etile_0_example_design_4ln_ptp|tx_clkout|ch0}] -to [get_clocks {sys_pll|iopll_0_clk_100m}];   \n
                                                                     
                                                                    10.  
                                                                  10.  
                                                                    Build new 2x100G FIM.
                                                                     
                                                                    cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x work_2x100gbe\n
                                                                     
                                                                    11.  
                                                                  11.  
                                                                    You may need to adjust the floorplan of the design in order to meet timing after a design change such as this. Refer to  for information regarding modifications to the floorplan.
                                                                     
                                                                    12.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#511-how-to-add-more-transceiver-channels-the-ethernet-subsystem","title":"5.11 How to add more Transceiver channels the Ethernet Subsystem","text":"
                                                                  This section describes how to add 4 extra Ethernet channels to the existing f2000x FIM design which uses the 8x25G (2x4x25G) as the default ethernet configuration.
                                                                   
                                                                  In this exercise we will add 4 extra channels to make a total of 12 channels. This configuration will be called 12x25G.
                                                                   
                                                                     
                                                                  1. Clone the design repository or use an existing design. See the Clone the OFS Git Repo section.
                                                                    2.  
                                                                  2. Set the environment variables. See the Setting Up Required Environment Variables section.
                                                                    3.  
                                                                  3.  
                                                                    Navigate to the directory containing the existing Ethernet Subsystem IP hssi_ss_8x25g.ip.
                                                                     
                                                                    cd $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss\n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Create a copy of the existing 8x25G IP and name it hssi_ss_12x25g.ip.
                                                                     
                                                                    cp hssi_ss_8x25g.ip hssi_ss_12x25g.ip\n
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Open the newly created hssi_ss_12x25g.ip file in Platform Designer.
                                                                     
                                                                    qsys-edit hssi_ss_12x25g.ip\n
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    In the HSSI Subsystem > Device 0 Configuration > Main Configuration tab, change the NUM_ENABLED_PORTS value from 8 to 12. 
                                                                     
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    In the HSSI Subsystem > Device 0 Configuration > Main Configuration tab, enable Ports 8 through 11, using the same configuration as the original 8 transceiver ports.
                                                                     
                                                                     
                                                                     
                                                                    8.  
                                                                  8.  
                                                                    Click Generate HDL. Close Platform Designer once generation is complete with no errors.
                                                                     
                                                                    9.  
                                                                  9.  
                                                                    Edit $OFS_ROOTDIR/ipss/hssi/eth_design_files.tcl
                                                                     
                                                                       
                                                                    1.  
                                                                      Comment out the old 8x25G IP and and the new 12x25G IP in the HSSI SS IP section
                                                                       
                                                                      #set_global_assignment -name IP_FILE ../ip_lib/ipss/hssi/qip/hssi_ss/hssi_ss_8x25g.ip\nset_global_assignment -name IP_FILE ../ip_lib/ipss/hssi/qip/hssi_ss/hssi_ss_12x25g.ip\n
                                                                       
                                                                      2.  
                                                                     
                                                                    10.  
                                                                  10.  
                                                                    Edit $OFS_ROOTDIR/ipss/hssi/rtl/hssi_wrapper.sv
                                                                     
                                                                       
                                                                    1.  
                                                                      In the HSSI SS Instantiation section, replace the existing 8x25G IP instantiation with the new 12x25G IP instantiation.
                                                                       
                                                                      //hssi_ss_8x25g\nhssi_ss_12x25g\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      In the Serial signal mapping to QSFP section, after the else statement, add the 4 new Ethernet ports:
                                                                       
                                                                      `ifdef INCLUDE_HSSI_PORT_8\nassign serial_rx_p[PORT_8] = qsfp_serial[2].rx_p[0];\nassign serial_rx_n[PORT_8] = 1'b0;\nassign qsfp_serial[2].tx_p[0] = serial_tx_p[PORT_8];\n`endif\n`ifdef INCLUDE_HSSI_PORT_9\nassign serial_rx_p[PORT_9] = qsfp_serial[2].rx_p[1];\nassign serial_rx_n[PORT_9] = 1'b0;\nassign qsfp_serial[2].tx_p[1] = serial_tx_p[PORT_9];\n`endif\n`ifdef INCLUDE_HSSI_PORT_10\nassign serial_rx_p[PORT_10] = qsfp_serial[2].rx_p[2];\nassign serial_rx_n[PORT_10] = 1'b0;\nassign qsfp_serial[2].tx_p[2] = serial_tx_p[PORT_10];\n`endif\n`ifdef INCLUDE_HSSI_PORT_11\nassign serial_rx_p[PORT_11] = qsfp_serial[2].rx_p[3];\nassign serial_rx_n[PORT_11] = 1'b0;\nassign qsfp_serial[2].tx_p[3] = serial_tx_p[PORT_11];\n`endif\n
                                                                       
                                                                      3.  
                                                                     
                                                                    11.  
                                                                  11.  
                                                                    Edit $OFS_ROOTDIR/ipss/hssi/rtl/inc/ofs_fim_eth_plat_defines.svh
                                                                     
                                                                       
                                                                    1.  
                                                                      Define the new port macros in the section configuring 25G with no CVL
                                                                       
                                                                      `define INCLUDE_HSSI_PORT_8\n`define INCLUDE_HSSI_PORT_9\n`define INCLUDE_HSSI_PORT_10\n`define INCLUDE_HSSI_PORT_11\n
                                                                       
                                                                      2.  
                                                                     
                                                                    12.  
                                                                  12.  
                                                                    Edit $OFS_ROOTDIR/ipss/hssi/rtl/inc/ofs_fim_eth_plat_if_pkg.sv
                                                                     
                                                                       
                                                                    1.  
                                                                      Change the parameter defining the number of QSFP ports from 2 to 3
                                                                       
                                                                      localparam NUM_QSFP_PORTS = 3; // QSFP cage on board\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Change the number of ethernet channels parameter for the 25G with no CVL configuration from 8 to 12
                                                                       
                                                                      ```verilog localparam NUM_ETH_CHANNELS = 12; // Ethernet Ports ````
                                                                       
                                                                      3.  
                                                                     
                                                                    13.  
                                                                  13.  
                                                                    Edit $OFS_ROOTDIR/syn/setup/eth_loc.tcl
                                                                     
                                                                       
                                                                    1.  
                                                                      Edit the pinout file to assign pins for the new QSFP. In this example we are using Channels 8-11 in the E-tile.
                                                                       
                                                                      set_location_assignment PIN_DL1  -to qsfp_serial[2].tx_p[0]\nset_location_assignment PIN_DN4  -to qsfp_serial[2].tx_p[1]\nset_location_assignment PIN_DY1  -to qsfp_serial[2].tx_p[2]\nset_location_assignment PIN_EB4  -to qsfp_serial[2].tx_p[3]\n\nset_location_assignment PIN_DL8  -to qsfp_serial[2].rx_p[0]\nset_location_assignment PIN_DN13 -to qsfp_serial[2].rx_p[1]\nset_location_assignment PIN_DY8  -to qsfp_serial[2].rx_p[2]\nset_location_assignment PIN_EB13 -to qsfp_serial[2].rx_p[3]\n
                                                                       
                                                                      2.  
                                                                     
                                                                    14.  
                                                                  14.  
                                                                    Compile the design
                                                                     
                                                                    cd $OFS_ROOTDIR\nofs-common/scripts/common/syn/build_top.sh -p f2000x <YOUR_WORK_DIRECTORY>\n
                                                                     
                                                                    15.  
                                                                  15.  
                                                                    You may need to adjust the floorplan of the design in order to meet timing after a design change such as this. Refer to the How to Resize the Partial Reconfiguration Region section for information regarding modifications to the floorplan.
                                                                     
                                                                    16.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#512-how-to-modify-the-pfvf-mux-configuration","title":"5.12 How to modify the PF/VF MUX configuration","text":"
                                                                  The PF/VF Configuration Tool allows you to easily reconfigure the default number of PFs and VFs on both the SoC and Host side of your design. To modify the PF/VF configuration, you must:
                                                                   
                                                                     
                                                                  1.  
                                                                    Decide which PCIe PF/VFs require modification.  If you are modifying host side PF/VF configuration, you must edit file pcie_host.ofss file found in $OFS_ROOTDIR/tools/pfvf_config_tool.  If you want to modify SoC-side PF/VF configuration, edit the pcie_soc.ofss file found in the same location.  
                                                                     
                                                                    The code given below show the default Host *.ofss file:
                                                                     
                                                                    [ProjectSettings]\nplatform = f2000x \nfamily = Agilex\nfim = base_x16\nPart = AGFC023R25A2E2VR0\nIpDeployFile = pcie_ss.sh\nIpFile = pcie_ss.ip\nOutputName = pcie_ss\nComponentName = pcie_ss\nis_host = True\n\n[pf0]\n\n[pf1]\n
                                                                     
                                                                    This default configuration is made up of two physical functions (PF), and neither of them has any virtual functions (VF). 
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Modify the OFSS file with the new PF/VF configuration.
                                                                     
                                                                    An example modification to the OFSS file is shown below.  In this example we have changed the configuration to: 6 PFs in total, 4 VFs in PF0, 1 VF in PF2, and 2 VFs on PF3.  You can add up to 8 PFs and could conceivably add up to the number of VFs supported by the PCIe IP. Note that more PFs/VFs will use more FPGA resources, which may cause fitter challenges.
                                                                     
                                                                    [ProjectSettings]\nplatform = f2000x \nfamily = Agilex\nfim = base_x16\nPart = AGFC023R25A2E2VR0\nIpDeployFile = pcie_ss.sh\nIpFile = pcie_ss.ip\nOutputName = pcie_ss\nComponentName = pcie_ss\nis_host = True\n\n[pf0]\nnum_vfs = 4\n\n[pf1]\n\n[pf2]\nnum_vfs = 1\n\n[pf3]\nnum_vfs = 2\n\n[pf4]\n\n[pf5]\n
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Run the gen_ofs_settings.py script found in $OFS_ROOTDIR/ofs-fim-common/tools/pfvf_config_tool.
                                                                     
                                                                    ./gen_ofs_settings.py\u00a0 --ini $OFS_ROOTDIR/tools/pfvf_config_tool/<PCIE_SOURCE>.ofss --platform <PLATFORM>\n
                                                                     
                                                                    For example, execute the following command to generate Host settings in an f2000x design:
                                                                     
                                                                    ./gen_ofs_settings.py\u00a0 --ini $OFS_ROOTDIR/tools/pfvf_config_tool/pcie_host.ofss --platform f2000x\n
                                                                     
                                                                    This script reconfigures the FIM by:
                                                                     
                                                                       
                                                                    1. Updating the PF/VF Mux Package APIs:
                                                                         
                                                                      • $OFS_ROOTDIR/src/afu_top/mux/top_cfg_pkg.sv
                                                                        •  
                                                                       
                                                                      2.  
                                                                    2. Adding/removing AFU endpoints
                                                                         
                                                                      • PF0 VFs - afu_main.port_afu_instances
                                                                        •  
                                                                      • All other functions: afu_top.fim_afu_instances
                                                                        •  
                                                                      • New AFUs will be instantiated as HE-NULL (he_null.sv) AFUs
                                                                        •  
                                                                       
                                                                      3.  
                                                                    3. Updating the pcie_ss.sh \"ip-deploy\" file
                                                                         
                                                                      • Generating the new pcie_ss.ip file ($OFS_ROOTDOR/ipss/pcie/ss/pcie_ss.ip)
                                                                        •  
                                                                      • Adding scratch register reads/writes to sim/csr_test for added functions
                                                                        •  
                                                                      • Updating the simulation filelists ($OFS_ROOTDIR/sim/common/pfvf_sim_pkg.sv)
                                                                        •  
                                                                       
                                                                      4.  
                                                                     
                                                                    If the port gasket is enabled in the OFSS file, all functions in the PR region must be a virtual function (VF) on physical function 0 (PF0) and are routed to Port AFU Instances (port_afu_instances.sv) in the port gasket.  You can enable the port gasket in the ini (*.ofss file) by adding pg_enable = True under the num_vfs in PF0. In the 2023.3 OFS SoC Attach Release for Intel IPU Platform F2000X-PL the PR region is in the SoC AFU, so the SoC OFSS file has the port gasket enabled by default:
                                                                     
                                                                    [ProjectSettings]\nplatform = f2000x \nfamily = Agilex\nfim = base_x16\nPart = AGFC023R25A2E2VR0\nIpDeployFile = pcie_ss.sh\nIpFile = pcie_ss.ip\nOutputName = pcie_ss\nComponentName = pcie_ss\nis_host = False\n\n[pf0]\npg_enable = True\nnum_vfs = 3\n
                                                                     
                                                                    If the port gasket is disabled, the virtual functions on PF0 are routed to FIM AFU Instances (fim_afu_instances.sv) in the static region.  All physical functions and virtual functions not on PF0 are routed to the FIM AFU Instances module (fim_afu_instances.sv) in afu_top.
                                                                     
                                                                    After you run the gen_ofs_settings.py script you should see a final success report:
                                                                     
                                                                    2022.07.28.21:46:38 Info: Regenerate these scripts whenever you make any change to any Quartus-generated IP in your project.\n2022.07.28.21:46:38 Info: Finished: Create simulation script\nsh: /home/applications.fpga.ofs.rtl/env_not_shipped/f2000x/update_sim.sh: No such file or directory\nSuccess!  Thank you for using the IP-Deploy Tool\n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Recompile the FIM using the build_top.sh script described in the Compiling the FIM section of this guide. 
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Verify the correct functionality of new the PF/VF Mux configuration.
                                                                     
                                                                    New PF/VF are seamlessly connected to their own CSR stub, which can be read at DFH Offset 0x0. You can bind to the function and perform opae.io peek commands to read from the stub CSR. Similarly, perform opae.io poke commands to write into the stub CSRs. Use this mechanism to verify that the new PF/VF Mux configuration allows to write and read back values from the stub CSRs. 
                                                                     
                                                                    The GUID for every new PF/VF CSR stub is the same.   
                                                                     
                                                                       
                                                                    • NULL_GUID_L           = 64'haa31f54a3e403501
                                                                      •  
                                                                    • NULL_GUID_H           = 64'h3e7b60a0df2d4850
                                                                      •  
                                                                     
                                                                    Limitations: Setting 0 virtual functions on SoC PF0 is not supported. This is because the PR region cannot be left unconnected. A loopback may need to be instantiated in this special case. 
                                                                     
                                                                    Load the newly compiled FIM to the card to test the functionality of the new PF/VF functions. Use the following commands to verify the number of PFs/VFs created:
                                                                     
                                                                    sudo lspci -vvv -s b1:00.0 | grep VF\n
                                                                     Example output: 
                                                                    Initial VFs: 4, Total VFs: 4, Number of VFs: 0, Function Dependency Link: 00\nVF offset: 6, stride: 1, Device ID: bccf               \n
                                                                     
                                                                    Note: The PCIe B:D.F associated with your board may be different. Use the fpgainfo fme command to see the PCIe B:D:F for your board.
                                                                     
                                                                    6.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#513-how-to-create-a-minimal-fim","title":"5.13 How to Create a Minimal FIM","text":"
                                                                  In this example, the exercisers and Ethernet subsystem are removed and a new AFU PR area is used to make use of the added area from the removed components.  This minimal FIM is useful for HDL applications.
                                                                   
                                                                  To create this minimal FIM, perform the following steps:
                                                                   
                                                                     
                                                                  1.  
                                                                    Change the PF/VF configuration to support only 1 PF with 1 VF. Edit the following files in the $OFS_ROOTDIR/tools/pfvf_config_tool/ directory as shown below:
                                                                     
                                                                       
                                                                    1.  
                                                                      $OFS_ROOTDIR/tools/pfvf_config_tool/pcie_host.ofss
                                                                       
                                                                      [ProjectSettings]\nplatform = f2000x \nfamily = Agilex\nfim = base_x16\nPart = AGFC023R25A2E2VR0\nIpDeployFile = pcie_ss.sh\nIpFile = pcie_ss.ip\nOutputName = pcie_ss\nComponentName = pcie_ss\nis_host = True\n\n[pf0]\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      $OFS_ROOTDIR/tools/pfvf_config_tool/pcie_soc.ofss 
                                                                       
                                                                      [ProjectSettings]\nplatform = f2000x \nfamily = Agilex\nfim = base_x16\nPart = AGFC023R25A2E2VR0\nIpDeployFile = pcie_ss.sh\nIpFile = pcie_ss.ip\nOutputName = soc_pcie_ss\nComponentName = pcie_ss\nis_host = False\n\n[pf0]\nnum_vfs = 1\npg_enable = True\n
                                                                       
                                                                      3.  
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Save the modified ofs_dev.ofss file and build a new configuration.
                                                                     
                                                                    python3 gen_ofs_settings.py --ini $OFS_ROOTDIR/tools/pfvf_config_tool/pcie_host.ofss --platform f2000x\n\npython3 gen_ofs_settings.py --ini $OFS_ROOTDIR/tools/pfvf_config_tool/pcie_soc.ofss --platform f2000x\n
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Compile the new FIM with exercisers removed.
                                                                     
                                                                    cd $OFS_ROOTDIR\n./ofs-common/scripts/common/syn/build_top.sh -p f2000x:null_he,null_he_lp,null_he_hssi,null_he_mem,null_he_mem_tg,no_hssi work_null_he_no_hssi\n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    The build will complete with reduced resources as compared to the base version. You may review the floorplan in Quartus Chip Planner and modify the Logic Lock regions to allocate more resources to the PR region if desired. Refer to the How to Resize the Partial Reconfiguration Region section for information regarding modifications to the floorplan. 
                                                                     
                                                                    5.  
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#6-single-event-upset-reporting","title":"6 Single Event Upset Reporting","text":"
                                                                  A Single Event Upset (SEU) is the change in state of a storage element inside a device or system. They are caused by ionizing radiation strikes that discharge the charge in storage elements, such as configuration memory cells, user memory and registers.
                                                                   
                                                                  Error Detection CRC (EDCRC) circuitry in the Card BMC is used to detect SEU errors. The CRC function is enabled in Intel Quartus Prime Pro to enable CRC status to be reported to the FM61 via the dedicated CRC_ERROR pin.
                                                                   
                                                                  With the EDCRC there is no method to determine the severity of an SEU error i.e. there is not way to distinguish between non-critical and catastrophic errors. Hence once the SEU error is detected, the Host system must initiate the Card BMC reset procedure.
                                                                   
                                                                  SEU errors can be read from either the Card BMC SEU Status Register or the PMCI Subsystem SEU Error Indication Register. The processes to read these registers are described in greater detail in the BMC User Guide. Contact your Intel representative for access to the BMC User Guide.
                                                                   
                                                                  Additionally, refer to the [Intel Agilex SEU Mitigation User Guide] for more information on SEU detection and mitigation.
                                                                  "},{"location":"hw/f2000x/dev_guides/fim_dev/ug_dev_fim_ofs/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                  Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                   
                                                                  OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                  "},{"location":"hw/f2000x/doc_modules/Glossary/","title":"Glossary","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to user space."},{"location":"hw/f2000x/doc_modules/Notices_%26_Disclaimers/","title":"Notices & Disclaimers","text":""},{"location":"hw/f2000x/doc_modules/Notices_%26_Disclaimers/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                  Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Performance varies by use, configuration and other factors. Your costs and results may vary. You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for the safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/","title":"FPGA Interface Manager Technical Reference Manual for Intel Agilex 7 SoC Attach: Open FPGA Stack","text":"
                                                                  Last updated: February 03, 2024 
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1-overview","title":"1 Overview","text":""},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#11-about-this-document","title":"1.1 About this Document","text":"
                                                                  This document describes the hardware architecture for the SoC Attach reference FIM of the Open FPGA Stack (OFS) targeting the Intel\u00ae Agilex 7 FPGA.  After reviewing this document you should understand the features and functions of the components that comprise the FPGA Interface Manager (FIM), also known as the \"shell.\"
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#12-glossary","title":"1.2 Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#13-introduction-to-open-fpga-stack","title":"1.3 Introduction to Open FPGA Stack","text":"
                                                                  The Open FPGA Stack (OFS) is a modular infrastructure of hardware platform components, open source unstreamed software, and broad ecosystem support that enables an efficient path to develop a custom FPGA platform.  OFS Provides a framework of FPGA synthesizable code, simulation environment and synthesis/simulation scripts. 
                                                                   
                                                                  The key components of OFS include:
                                                                   
                                                                     
                                                                  • Target development platforms such as Intel-branded  Acceleration Development Platforms (ADPs) and third-party platforms.
                                                                    •  
                                                                   
                                                                     
                                                                  • Board Management Controller RTL and firmware that supports telemetry monitoring and capability for remote configuration updates.
                                                                    •  
                                                                   
                                                                     
                                                                  • Source accessible, modular FPGA Interface  manager (FIM) RTL with a UVM infrastructure unit tests that can be leveraged for your own custom FIM design. The FIM can be thought of as the FPGA shell that provides the I/O ring and timing closed management components for the FPGA.
                                                                    •  
                                                                   
                                                                     
                                                                  • Basic building blocks for interconnect and PF/VF translation and arbitration; Platform Interface Manager (PIM) which provides Avalon\u00ae and Arm\u00ae AMBA\u00ae 4 AXI4 bus compliant interfaces.
                                                                    •  
                                                                   
                                                                     
                                                                  • AFU examples both in the git repository and workload examples provided by 3rd party vendors.
                                                                    •  
                                                                   
                                                                     
                                                                  • Unit level simulation test suite
                                                                    •  
                                                                   
                                                                     
                                                                  • System level simulation through a unified verification methodology (UVM)
                                                                    •  
                                                                   
                                                                     
                                                                  • OPAE software development kit (APIs, up-streamed Linux drivers and software tools)
                                                                    •  
                                                                   
                                                                     
                                                                  • Support for other frameworks to be built on top of the OPAE such as DPDK 
                                                                    •  
                                                                   
                                                                  These components are available in a two GitHub locations:
                                                                   
                                                                     
                                                                  • OFS hardware GitHub site
                                                                    •  
                                                                  • OPAE software GitHub site
                                                                    •  
                                                                   
                                                                  The OFS hardware repository supports hardware development and simulation.  Repositories for OFS high-level design support and board management controller RTL and firmware source code are also provided.  These repositories can be found in the Intel Opensource Technology GitHub location, which requires entitlement access.  To request access, please contact your local Intel sales representative.
                                                                   
                                                                  Table 1-2 OFS Hardware Repositories
                                                                   Repository Contains OFS f2000x FIM Github Branch Contains FIM or shell RTL, automated compilation scripts, and unit tests and UVM framework. 
                                                                  The OPAE software GitHub site is fully opensource and contains resources for both software and workload developers.
                                                                   
                                                                  Table 1-3  OFS Software Repositories
                                                                   OPAE Git Repository Folder Contains OPAE SDK Repo Contains the files for building and installing OPAE SDK from source. Linux DFL Contains OFS Linux drivers that are being upstreamed to the Linux kernel. ofs-platform-afu-bbb Contains the files and scripts to build the platform interface manager. opae-sim Contains the files for an AFU developer to build the Accelerator Functional Unit Simulation Environment (ASE) for workload development. 
                                                                  Providing the hardware and software source code and supporting test frameworks in a GitHub repository allows you to customize your designs with the latest versions easily.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14-ofs-features","title":"1.4 OFS Features","text":"
                                                                  The OFS architecture within the FPGA comprises two partitions:
                                                                   
                                                                     
                                                                  • FPGA Interface Manager (FIM)
                                                                    •  
                                                                   
                                                                     
                                                                  • Accelerator Functional Unit (AFU)     -   AFU SoC Dynamic Region and Static Region      -   AFU Host Static Region 
                                                                    •  
                                                                   
                                                                  The FIM or shell provides platform management functionality, clocks, resets, and interface access to the host and peripheral features of the acceleration platform. 
                                                                   
                                                                  The FIM architecture along with the supporting OPAE software supports features such as partial reconfiguration and virtualization.
                                                                   
                                                                  The FIM provides a standard Arm\u00ae AMBA\u00ae 4 AXI4 datapath interface. The FIM resides in the static region of the FPGA.
                                                                   
                                                                  The AFU partition is provided for custom acceleration workloads and may contain both static and partial reconfiguration regions.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#141-fpga-interface-manager-fim","title":"1.4.1 FPGA Interface Manager (FIM)","text":"
                                                                  The updated OFS architecture for Intel\u00ae Agilex\u00ae 7 FPGA devices improves upon the modularity, configurability and scalability of the first release of the OFS architecture while maintaining compatibility with the original design.  The primary components of the FPGA Interface Manager or shell of this reference design are: 
                                                                   
                                                                     
                                                                  • P-tile PCIe Subsystem
                                                                    •  
                                                                  • E-Tile Ethernet Subsystem
                                                                    •  
                                                                  • Memory Subsystem
                                                                    •  
                                                                  • Reset Controller
                                                                    •  
                                                                  • FPGA Management Engine (FME)
                                                                    •  
                                                                  • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                                                                    •  
                                                                  • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                                                                    •  
                                                                  • Platform Management Controller Interface (PMCI) to the board management controller
                                                                    •  
                                                                   
                                                                  The AFU Region provides design space for custom workloads and contains both static and partial reconfiguration regions.  Partial reconfiguration allows you to update your specific logic blocks or entire workload while the rest of your static design is still in operation.
                                                                   
                                                                  Note that as discussed previously, the BMC RTL and firmware, the OFS OPAE software stack and support for building your own customer board support package are also provided in separate OFS repositories.
                                                                   
                                                                  Figure 1-1 OFS for OFS FIM Platform Block Diagram
                                                                   
                                                                   
                                                                  The table provides an overview of the OFS features targeting the Intel\u00ae Agilex\u00ae 7 FPGA.  This reference FIM (shell) is a starting point for your custom FPGA design.  With this initial starting point, you can add or subtract interfaces or ports to different Intel Agilex 7 devices.
                                                                   
                                                                  Table 1-4 OFS FIM for Intel\u00ae Agilex\u00ae 7 FPGA Features
                                                                   Key Feature Description PCIe P-tile PCIe* Gen4x16 to the HostP-tile PCIe* Gen4x16 to the SoC (IceLake-D) Virtualization Host: 2 physical functions SoC:   1 physical function and 3 Virtual functions Memory Four Fabric DDR4 banks, x40 (optional ECC, be configured as x32 and ECC x8 ), 1200 MHz, 4GB Ethernet Eight Arm\u00ae AMBA\u00ae 4 AXI4-Stream channels of 25G Ethernet interfacing to an E-tile Ethernet Subsystem. Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration) * Platform Controller Management Interface (PMCI) Module contained within the Intel Agilex 7 FPGA that interfaces through Avalon-Streaming x8 QuadSPI and SPI to a Board Management Controller Partial Reconfiguration Partial Reconfiguration region supported in hardware and software Software Support * Linux DFL drivers targeting OFS FIMs * OPAE Software Development Kit * OPAE Tools"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#subsystem-interfaces","title":"Subsystem Interfaces","text":"
                                                                  The PCIe, Memory and Ethernet interfaces in this design use a new flexible subsystem design that provides a standard Arm\u00ae AMBA\u00ae 4 AXI4 interface.  To access these Intel FPGA IP Subsystem documents. Please go here and search for the following ID numbers: * 690604: PCIe Subsystem IP User Guide (Note: you must login to myIntel and request entitled access) * 686148: Memory Subsystem IP User Guide (Note: you must login to myIntel and request entitled access) * 773413: [Ethernet Subsystem Intel FPGA IP] (public document)
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#fpga-management-engine-fme","title":"FPGA Management Engine (FME)","text":"
                                                                  The FIM contains only one FME, regardless of the number of host interfaces to the FIM. The FME provides management features for the platform and controls reset and loading of the AFU into the partial reconfiguration region of the FPGA.
                                                                   
                                                                  Any feature, such as a memory interface or global error control that you want to control through FME, must expose its capability to host software drivers.  New features are exposed to the FME by adding a device feature header (DFH) register at the beginning of the feature's control status register (CSR) space. The FME CSR maps to physical function 0 (PF0) Base address register 0 (BAR0) so that software can access it through a single PCIe link.  For more information about DFHs, refer to the FPGA Device Feature List Framework Overview
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#streaming-datapath","title":"Streaming Datapath","text":"
                                                                  The FIM implements an Arm\u00ae AMBA\u00ae 4 AXI4-Stream bus protocol for data transfer in the FIM. Arm\u00ae AMBA\u00ae 4 AXI4-Stream channels send data packets to and from the host channel IP without data abstraction. Memory-mapped I/O (MMIO) CSR accesses are routed to the ST2MM module, which converts the Arm\u00ae AMBA\u00ae 4 AXI4-Stream to an Arm\u00ae AMBA\u00ae 4 AXI4 memory-mapped protocol.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#virtualization","title":"Virtualization","text":"
                                                                  This design supports virtualization by making use of the virtualization functionality in the PCIe Hard IP and mapping packets to the appropriate physical or virtual function through a PF/VF multiplexer.  
                                                                   
                                                                  This reference FIM example supports 2 PFs for the host and 1PF, 3VFs for the SoC; however, you may extend your configuration to whatever the PCIe Hard IP can support or your application requires.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#142-afu","title":"1.4.2 AFU","text":"
                                                                  An AFU is an acceleration workload that interfaces with the FIM. The AFU boundary in this design comprises both static and partial reconfiguration (PR) regions.  You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region.  A port gasket within the design provides all the PR-specific modules and logic required for partial reconfiguration. Only one partial reconfiguration region is supported in this design for SoC AFU.
                                                                   
                                                                  Similar to the FME, the port gasket exposes its capability to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space.  
                                                                   
                                                                  You can compile your design in one of the following ways:  * Your AFU (workload) for Host (Static Area) and SoC resides in the partial reconfiguration area.  * Your AFU is part of the static region and is compiled from a flat design. 
                                                                   
                                                                  In this design, the AFU region is comprised of: * AFU Interface handler to verify transactions coming from AFU region. * PF/VF Mux to route transactions to and from corresponding AFU components:    * Host:     * ST2MM module.     * PCIe loopback host exerciser (HE-LPBK) .
                                                                   
                                                                  * SoC:     * ST2MM module.     * Ethernet Subsystem (previous HSSSI) host exerciser (HE-HSSI).     * Memory Host Exerciser (HE-MEM).     * Traffic Generator to memory (HE-MEM-TG).     * Port Gasket (PRG).
                                                                   
                                                                     
                                                                  • Arm\u00ae AMBA\u00ae 4 AXI4 Streaming to Memory Map (ST2MM) Module that routes MMIO CSR accesses to FME and board peripherals.
                                                                    •  
                                                                  • Host exercisers to test PCIe, memory and Ethernet Interfaces (these can be removed from the AFU region after your FIM design is complete to provide more resource area for workloads)
                                                                    •  
                                                                  • Port gasket and partial reconfiguration support.
                                                                    •  
                                                                  • Component for handling PLDM over MCTP over PCIe Vendor Defined Messages (VDM)
                                                                    •  
                                                                   
                                                                  For this design the PF/VF Mux provides the following mappings (found in /src/afu_top/mux/top_cfg_pkg.sv and /src/afu_top/mux/soc_top_cfg_pkg.sv):
                                                                   
                                                                  Table 1-5 PF/VF Mapping
                                                                   
                                                                  SoC (IceLake-D)
                                                                   PF VF Feature BAR Bar Size PF0 OFS managed peripherals BAR 0 2MB AFU peripherals  Board peripherals BAR 0 256KB  768KB PF0 VF0 HE-MEM BAR 0 2MB VF1 HE-HSSI BAR 0 2MB VF2 HE-MEM TG BAR 0 2MB 
                                                                  Xeon Host
                                                                   PF VF Feature BAR Bar Size PF0 OFS managed peripherals BAR 0 1MB AFU peripherals  Board peripherals 256KB  768KB PF1 HE-LPBK BAR 0 16KB 
                                                                  Figure 1-2 AFU Diagram 
                                                                   
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#143-platform-interface-manager","title":"1.4.3 Platform Interface Manager","text":"
                                                                  The PIM provides a way to abstract the Arm\u00ae AMBA\u00ae 4 AXI4-Stream interface to the AFU by providing a library of shims that convert the host channel native packet into other protocols such as Arm\u00ae AMBA\u00ae 4 AXI4 memory-mapped, Avalon\u00ae streaming (Avalon-ST) or Avalon\u00ae memory-mapped (Avalon-MM). 
                                                                   
                                                                  The FPGA or AFU developer implements these interface abstractions in the AFU regions (afu_top and soc_afu_top) of the design.  
                                                                   
                                                                  For more information, refer to the AFU Developer's Guide.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#144-platform-feature-discovery","title":"1.4.4 Platform Feature Discovery","text":"
                                                                  This reference design comes with specific Intel FPGA drivers that are upstreamed to linux-dfl.  These drivers abstract the hardware and operating system specific details of the platform to the host. 
                                                                   
                                                                  The FIM implements a device feature header (DFH) at the beginning of each host-discoverable feature in a linked list format that allows an FPGA platform driver running on the host to discover FME, port, and AFU features. 
                                                                   
                                                                  You must implement a 64-bit DFH Device Feature Header register at the beginning (first 8B aligned address) of the feature CSR space for a new feature to be discovered or enumerated by a driver.  
                                                                   
                                                                  During host discovery, the driver traverses the DFH of the first feature from the first address on PF0 BAR0. Based on the information in the DFH, a driver can determine the CSR address range of the feature and other associated details. The end of the DFH contains a \"next DFH offset\" field that points the driver to the DFH of the next feature. 
                                                                   
                                                                  The software must continue traversing the linked list until it sees the EOL (End-Of-List) bit set to 1 in the \"next DFH offset\" field it inspects.  A 1 indicates this is the last feature in the feature set. The figure below gives a simple illustration of the feature discovery by traversing the DFH registers. This model is similar to how PCIe enumeration occurs.
                                                                   
                                                                  Figure 1-3 Device Feature Header Linked List Traversal
                                                                   
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#145-ofs-reference-design","title":"1.4.5 OFS Reference Design","text":"
                                                                  OFS provides FIM designs you can use as a starting point for your own custom design. These designs target a specific programmable acceleration card or development kit and exercise key FPGA device interfaces. 
                                                                   
                                                                  The Intel Agilex\u00ae 7 code line for OFS targets the Intel IPU Platform F2000X-PL.  FIM designs are released to for evaluation and use. 
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#146-fim-simulation","title":"1.4.6 FIM Simulation","text":"
                                                                  OFS provides unit tests and a UVM environment for the FIM and a framework for new feature verification. UVM provides a modular, reusable, and scalable testbench structure by providing an API framework that can be deployed across multiple projects. 
                                                                   
                                                                  The FIM testbench is UVM compliant and integrates third-party verification IPs from Synopsys that require a license. 
                                                                   
                                                                  Verification components include:
                                                                   
                                                                     
                                                                  • FIM monitor to detect correct design behavior
                                                                    •  
                                                                   
                                                                     
                                                                  • FIM assertions for signal level integrity testing
                                                                    •  
                                                                   
                                                                     
                                                                  • Arm\u00ae AMBA\u00ae 4 AXI4 scoreboards to check data integrity
                                                                    •  
                                                                   
                                                                     
                                                                  • FIM coverage to collect functional data
                                                                    •  
                                                                   
                                                                  The verification infrastructure can be found here for evaluation and use. Please refer to the Simulation User Guide for more information.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#2-ofs-high-level-architecture","title":"2 OFS High Level Architecture","text":"
                                                                  OFS provides distinct data paths that simplify the design and integration process for adding or removing interface modules:
                                                                   
                                                                     
                                                                  • High Bandwidth data path for AFU-attached high-performance peripherals (Ethernet Subsystem, Memory,  workload).
                                                                    •  
                                                                  • Low Bandwidth data path for OFS management and slow peripheral components (JTAG, I2C, SMBus).
                                                                    •  
                                                                  • AFU Peripheral Fabric (APF) to Board Peripheral Fabric (BPF) path to communicate with interface control and status registers (CSRs) and board components.
                                                                    •  
                                                                  • Peer-to-peer datapath between AFU components.
                                                                    •  
                                                                  • Peer-to-peer datapath between BPF components.
                                                                    •  
                                                                   
                                                                  Depending on your design goals, you can present peripherals to software as:
                                                                   
                                                                     
                                                                  • OFS managed peripherals with a device feature header that is part of a device feature list.
                                                                    •  
                                                                  • Native driver managed peripherals that are exposed through an independent physical function or virtual function.
                                                                    •  
                                                                   
                                                                  Figure 2-1 OFS Datapath Structure 
                                                                   
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#3-pcie-subsystem","title":"3 PCIe Subsystem","text":"
                                                                  The FIM's PCIe Subsystem is a hierarchical design that targets the P-tile PCIe* hard IP and is configured to support Gen4 speeds and Arm\u00ae AMBA\u00ae 4 AXI4-Stream Data Mover functional mode.  The IP supports SR-IOV and is configured to provide  2 PFs for the host and 1PF, 3VFs for the SoC.  Native PCIe TLP packets are sent through the PCIe usingArm\u00ae AMBA\u00ae 4 AXI4 Stream Protocol.  Before they reach the AFU, the packets go through an adapter in the subsystem that converts any headers to a data mover format.  Tag allocation and management for packets sent from the application to the host are done by the PF/VF Mux which is part of the AFU region.
                                                                   
                                                                  Figure 3-1 OFS FIM RX-TX Datapath
                                                                   
                                                                   
                                                                  Some key features of the PCIe interface are:
                                                                   Feature OFS for Intel Intel Agilex 7 FPGA SoC Attach Subsystem Configuration Mode Host: PCIe Gen4x16SoC: PCIe Gen4x16 Tile P-Tile Port Mode Native Endpoint SR-IOV Host: 2 PFs, No VFsSoC:  1 PFs, 3 VFs MSI-X Support Yes Functional Mode Data Mover Profile Basic TLP Bypass No Header Packing Scheme Simple Data Width 512-bit (64-byte) Arm\u00ae AMBA\u00ae 4 AXI4-ST Clock Frequency 500 MHz Tags Supported 128 Reordering Enabled with buffer 64 KB Maximum Payload Size 512 Bytes Memory Requests Supported 1CL, 2CL, 4CL MMIO transaction Size 4B, 8B Control Shadow Interface Enabled Completion Timeout Interface Enabled 
                                                                  The PCIe PF, VF and Base Address Register (BAR) configuration can be modified in the PCIe Subsystem Platform Designer GUI interface.  The current implementation for the OFS FIM for Intel IPU Platform F2000X-PL is as follows:
                                                                   
                                                                  Table 3-1 Function and BAR Table for OFS for Intel IPU Platform F2000X-PL
                                                                   
                                                                  SoC (IceLake-D)
                                                                   PF VF Feature BAR Bar Size PF0 OFS managed peripherals BAR 0 2MB AFU peripherals  Board peripherals BAR 0 256KB  768KB PF0 VF0 HE-MEM BAR 0 2MB VF1 HE-HSSI BAR 0 2MB VF2 HE-MEM TG BAR 0 2MB 
                                                                  Xeon Host
                                                                   PF VF Feature BAR Bar Size PF0 OFS managed peripherals BAR 0 1MB AFU peripherals  Board peripherals 256KB  768KB PF1 HE-LPBK BAR 0 16KB"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#31-pcie-subsystem-header-format","title":"3.1 PCIe Subsystem Header Format","text":"
                                                                  The first 32 bytes of the TLP from the PCIe subsystem denotes the PCIe header. There are two types of header format supported \u2013 Power User Mode header and Data Mover mode header. The tuser_vendor[0] bit on the Arm\u00ae AMBA\u00ae 4 AXI4-Stream channel indicates the header format of the TLP; tuser_vendor[0] =0 indicates Power User Mode header and tuser_vendor[0] =1 indicates Data Mover Mode header.
                                                                   
                                                                  The OFS FIM for Intel Intel Agilex 7 FPGA implements the Data Mover Functional Mode.  With this implementation, the application has the flexibility to use either mode for PCIe transaction, as shown in the following table. For more detailed information about the PCIe Subsystem, see the PCIe Subsystem Intel FPGA User Guide. 
                                                                   
                                                                  Table 3-2 PCIe Subsystem Header Format Support for OFS for Intel Agilex 7 FPGA
                                                                   Direction Type Power User Data Mover Host to Endpoint  MWr, MRd Yes No CPL/CPLd Yes Yes Msg Yes No Endpoint to Host MWr, MRd Yes Yes Intr Yes (MWr)  Yes CPL/CPLd Yes Yes Msg Yes Yes"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#311-power-user-header-format","title":"3.1.1 Power User Header Format","text":"
                                                                  The Power User Format provides user complete control over PCIe Hard IP. The user can implement functionality of interest with finer control over PCIe Transaction Layer Packet (TLP), credit handling and various mode provided by HIP. 
                                                                   
                                                                  The lower 16 bytes of the Power User Format are standard PCIe header as defined by PCIe specification, and the upper 16 bytes are specific to the PCIe Subsystem Intel FPGA IP. 
                                                                   
                                                                  Table 3-3 Power User Header Format
                                                                   
                                                                   
                                                                  The mapping of the PCIe defined header to the lower 16 bytes of the Arm\u00ae AMBA\u00ae 4 AXI4-Stream data channel is shown in the figure below. Each double word (DW) or 4 bytes in the PCIe header is mapped from big-endian to little-endian on Arm\u00ae AMBA\u00ae 4 AXI4-S data channel.  
                                                                   
                                                                  Figure 3-2 Power User Header Mapping to Arm\u00ae AMBA\u00ae 4 AXI4 Channel
                                                                   
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#312-data-mover-header-format","title":"3.1.2 Data Mover Header Format","text":"
                                                                  The data mover mode allows high bandwidth data transfers to and from Host memory. It hides the complexity of handling PCIe TLPs. This format provides a simple interface for reading and writing to Host Memory. The data mover checks link partner credits before transmitting packets. It also provides MSI-X interrupt generation capability. 
                                                                   
                                                                  In Data Mover Mode, the lower 16 bytes are data mover specific and do not follow a PCIe standard header.  
                                                                   
                                                                  Table 3-4 Data Mover Header Format
                                                                   
                                                                   
                                                                   
                                                                  The mapping of the data mover header to the lower 16 bytes of the Arm\u00ae AMBA\u00ae 4 AXI4-S data channel is shown below. Each byte in the data mover header is mapped directly to the Arm\u00ae AMBA\u00ae 4 AXI4-S data channel.  
                                                                   
                                                                  Figure 3-3 Data Mover Header Mapping to Arm\u00ae AMBA\u00ae 4 AXI4 Channel
                                                                   
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#32-pcie-subsystem-interface-module","title":"3.2 PCIe Subsystem Interface Module","text":"
                                                                  The PCIe Subsystem Interface module (/ipss/pcie/rtl/pcie_ss_if.sv), provides the supporting interface between software and the PCIe subsystem.  
                                                                   
                                                                  The interface module provides the following:
                                                                   
                                                                     
                                                                  • Device Feature Header Registers
                                                                    •  
                                                                  • Control and Status Registers
                                                                    •  
                                                                  • Indirect access to PCIe subsystem CSR registers through a CSR mailbox in the PCIe Subsystem Interface.
                                                                    •  
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#33-data-mover-request-cycles","title":"3.3 Data Mover Request Cycles","text":"
                                                                  For Host read request cycles using the OFS FIM for Intel Agilex 7 FPGA: * All requests in the RX direction will be MMIO.  * Requester ID from the request does get sent to the AFU. It is the AFU's responsibility to send back a completion to the host with the correct completer ID.  * Prefix is not supported.  * Memory Mapped (MM) Mode is not supported. * Slot Number is 0. * Base address is not sent to the AFU. * Local Address field is not used.
                                                                   
                                                                  For AFU/application request cycles using the OFS FIM for Intel Agilex 7 FPGA: * All requests in the TX direction will be Memory Read/Write.  * The tag must be generated by the AFU/application. * Prefix is not supported.  * MM Mode is not supported. * Slot Number is 0 (non-0 only for switch) * VF Active, VF number and PF number are obtained from Data Mover Header Packet.
                                                                   
                                                                  Figure 3-4 Data Mover Request Cycles
                                                                   
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#34-data-mover-completion-cycles","title":"3.4 Data Mover Completion Cycles","text":"
                                                                  For Host completion cycles using the OFS FIM for Intel Agilex 7 FPGA: * All completions in the RX direction will be Data Completions.  * Prefix is not supported. * MM Mode is not supported. * Slot Number is 0. * Data packet responses (for Memory Read requests from AFU) from the PCIe SS may come out of order when the size is >64B.
                                                                   
                                                                  For AFU/application completion cycles using the OFS FIM for Intel Agilex 7 FPGA: * All requests in the TX direction will be Memory Read/Write.  * Requester ID is generated within the FIM. * That tag must be generated by the AFU/application. * Prefix is not supported.  * MM Mode is not supported. * Slot Number is 0. * VF Active, VF Number and PF number are obtained from the Data Mover Header Packet. 
                                                                   
                                                                  Figure 3-5 Data Mover Completion Cycles
                                                                   
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#4-platform-interface-manager","title":"4 Platform Interface Manager","text":"
                                                                  The FIM interfaces to an application in the AFU region through Arm\u00ae AMBA\u00ae 4 AXI4-Stream channels.  This format allows the AFU to access the host channel's raw interface without any translation. 
                                                                   
                                                                  As a FIM developer, you have the option to provide the raw data format associated with the host interface channel to the workload or AFU developer or you can provide an intermediate protocol using Platform Interface Manager Components or your own custom interface.
                                                                   
                                                                  If you expose the raw Arm\u00ae AMBA\u00ae 4 AXI4-Stream interface of the FIM, workload developers also have the option to convert to a desired protocol using the PIM resources as well.  
                                                                   
                                                                  Refer to the AFU Developer Guide and the FPGA Interface Manager Developer Guide for more information on using the PIM in your design.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#5-interconnect-fabric","title":"5 Interconnect Fabric","text":"
                                                                  There are three types of interconnect fabric in the OFS FIM design:  * Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF mux/demux fabric  * AFU Peripheral Fabric (APF)  * Board Peripheral Fabric (BPF)
                                                                   
                                                                  Figure 5-1 Interconnect Fabric Diagram
                                                                   
                                                                   
                                                                  TLP packets sent from upstream PCIe Subsystem on Arm\u00ae AMBA\u00ae 4 AXI4-Stream channel are demultiplexed in the Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF mux/demux fabric and routed to the respective PF/VF function based on the PF/VF information in the TLP header, such as vf_active or the PF/VF number. In the opposite direction, TLP packets from downstream PF/VF function are muxed in the fabric and sent to PCIe subsystem over Arm\u00ae AMBA\u00ae 4 AXI4-Stream channel.
                                                                   
                                                                  All host MMIO requests targeting PF0 BAR0 are routed to the ST2MM module. The ST2MM converts MMIO TLP packets into Arm\u00ae AMBA\u00ae 4 AXI4-Lite memory requests and places the requests onto AFU Peripheral Fabric (APF). AFU peripherals, such as OFS managed AFU features and ST2MM, and Board Peripheral Fabric (BPF) are interconnected by APF. The BPF is the interconnect fabric one hierarchy below APF which connects all the board peripherals. Both APF and BPF allow multiple Arm\u00ae AMBA\u00ae 4 AXI4-Lite master and slave interconnect topology.
                                                                   
                                                                  If you are modifying the APF or BPF connections, you must use Platform Designer to generate the fabrics directly.  Please refer to the FPGA Interface Manager Developer Guide for directions on what files must be modified and how to generate the interconnect.
                                                                   
                                                                  For modifying the PF/VF mux you must update parameters in these files: * src/includes/top_cfg_pkg.sv * src/common/pf_vf_mux.sv Then make the corresponding update to AFU top level instantiation and connections: * src/afu_top/soc_afu_top.sv(SoC_AFU) and src/afu_top/afu_top.sv (HOST_AFU)
                                                                   
                                                                  For details on these modifications, please refer to the FIM Interface Manager Developer Guide.  
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#51-afu-peripheral-fabric-apf","title":"5.1 AFU Peripheral Fabric (APF)","text":"
                                                                  The AFU Peripheral Fabric (APF) is a 64-bit Arm\u00ae AMBA\u00ae 4 AXI4-lite compliant interconnect fabric that connects AFU peripheral modules to board peripheral modules through the Board Peripheral Fabric (BPF). 
                                                                   
                                                                  The fabric is clocked by clk_csr and has a read allowance and write allowance of 1, i.e. only 1 active write/read is allowed in the fabric at any single time. 
                                                                   
                                                                  The address mapping for components interconnected by the APF is listed below. All components are mapped to PF0 BAR0 and implement Arm\u00ae AMBA\u00ae 4 AXI4-lite slave interface. Note that none of the features in the APF mapping are designed to act as a master.
                                                                   
                                                                  Table 5-1 APF Address Mapping
                                                                   
                                                                  SoC PF0 BAR 0
                                                                   Address Size (Byte) Feature 0x00000\u20130xFFFFF 1024K Board Peripherals (See BPF address mapping) AFU Peripherals 0x100000 \u2013 0x10FFFF 64K ST2MM 0x130000 \u2013 0x13FFFF 64K PR Gasket: 4K= PR Gasket DFH, control and status 4K= Port DFH 4K=User Clock 52K=Remote STP 0x140000 \u2013 0x14FFFF 64K AFU Error Reporting (Protocol checker) 
                                                                  Host PF0 BAR 0
                                                                   Address Size (Byte) Feature 0x00000\u20130xFFFFF 1024K Board Peripherals (See BPF address mapping) AFU Peripherals 0x100000 \u2013 0x10FFFF 64K ST2MM 0x140000 \u2013 0x14FFFF 64K AFU Error Reporting (Protocol checker)"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#52-board-peripheral-fabric-bpf","title":"5.2 Board Peripheral Fabric (BPF)","text":"
                                                                  The Board Peripheral Fabric is the 64-bit Arm\u00ae AMBA\u00ae 4 AXI4-Lite compliant interconnect fabric that connects board peripheral modules to APF. The fabric is clocked by clk_csr and has a read allowance and write allowance of 1, i.e. only 1 active write/read is allowed in the fabric at any single time.
                                                                   
                                                                  The address mapping for components interconnected by BPF is listed below. All components are mapped to PF0 BAR0 and implement Arm\u00ae AMBA\u00ae 4 AXI4-lite slave interface. The Master column indicates if a component also implements Arm\u00ae AMBA\u00ae 4 AXI4-lite master interface which can send requests to the BPF.
                                                                   
                                                                  Table 5-2 BPF Address Mapping
                                                                   
                                                                  SoC PF0 BAR 0
                                                                   Address Size (Byte) Feature Master 0x00000 \u2013 0x0FFFF 64K FME (FME, Error, etc) No 0x10000 \u2013 0x10FFF 4K SoC PCIe Interface No 0x11000 \u2013 0x11FFF 4K Reserved - 0x12000 \u2013 0x12FFF 4K QSFP Controller 0 No 0x13000 \u2013 0x13FFF 4K QSFP Controller 1 No 0x14000 \u2013 0x14FFF 4K Ethernet Subsystem - 0x15000 \u2013 0x15FFF 4K Memory Subsystem 0x80000 \u2013 0xFFFFF 512K PMCI Controller Yes 
                                                                  Host PF0 BAR 0
                                                                   Address Size (Byte) Feature Master 0x00000 \u2013 0x00FFF 4K Host PCIe Interface No 0x80000 \u2013 0xFFFFF 512K PMCI Controller Yes"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#53-arm-amba-4-axi4-stream-pfvf-muxdemux","title":"5.3 Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF Mux/Demux","text":"
                                                                  The Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF Mux/Demux routes the PCIe TLP packets from the PCIe subsystem Arm\u00ae AMBA\u00ae 4 AXI4-Stream RX channel to downstream PF/VF based on the pf_num and vf_num information in the PCIe TLP header.
                                                                   
                                                                  The Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF mux arbitrates PCIe TLP packets from downstream PF/VF to the PCIe SS Arm\u00ae AMBA\u00ae 4 AXI4-S TX channel. The PF/VF Mux/Demux is an M X N switch that allows any M port to target any N port, and any N port to target any M port, where M is the number of host/upstream ports, and N is the numbers functions/downstream ports.   
                                                                   
                                                                  The fpga top package file, /src/afu_top/mux/top_cfg_pkg.sv and soc_top_cfg_pkg.sv and, contains the PF/VF parameters and mappings.
                                                                   
                                                                  Figure 5-2 PF/VF Mux
                                                                   
                                                                   
                                                                  The PF/VF mux integration is part of afu_top (src/afu_top/afu_top.sv and /soc_afu_top).  There are two independent TX PF/VF MUX trees, labeled \"A\" and \"B\".
                                                                   
                                                                  Both an A and a B port are passed to each AFU component with a unique PF/VF. You can design your AFU components to send all requests to the primary A port or partition requests across both A and B ports. A typical high-performance AFU sends read requests to the B port and everything else to the A port, giving the arbiter freedom to keep both the host TX and RX channels busy.
                                                                   
                                                                  In the reference FIM provided for Intel Agilex 7 OFS, the A and B TX trees have been multiplexed down to a single channel for A and another for B. The A/B multiplexer merges them into a single TX stream that will be passed to the tag remapper.
                                                                   
                                                                  The tag remapper provides unique tags as required by the PCIe specification.  Tags are not provided by the PCIe Subsystem FPGA IP.  When creating your own AFU you can leverage this module to generate unique tags.
                                                                   
                                                                  Note that the primary PF/VF Mux A supports RX and TX ports.  For the secondary PF/VF Mux B only TX ports are supported and the RX input to the Mux is tied off.  
                                                                   
                                                                  The default mapping is shown below:
                                                                   
                                                                  Table 5-3 PF/VF Mapping
                                                                   
                                                                  SoC (IceLake-D)
                                                                   PF VF Feature BAR Bar Size PF0 OFS managed peripherals BAR 0 2MB AFU peripherals  Board peripherals BAR 0 256KB  768KB PF0 VF0 HE-MEM BAR 0 2MB VF1 HE-HSSI BAR 0 2MB VF2 HE-MEM TG BAR 0 2MB 
                                                                  Xeon Host
                                                                   PF VF Feature BAR Bar Size PF0 OFS managed peripherals BAR 0 1MB AFU peripherals  Board peripherals 256KB  768KB PF1 HE-LPBK BAR 0 16KB 
                                                                  For information on how to modify the PF/VF mapping for your own design, refer to the OFS FIM Developer User Guide.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#54-afu-interface-handler","title":"5.4 AFU Interface Handler","text":"
                                                                  The AFU Interface Handler resides inline between the PCIe Arm\u00ae AMBA\u00ae 4 AXI4-Stream Adapter and the Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF Demux/Mux logic. Its main function is to provide: * Unique PCIe tags \u2013 Each PCIe transaction shares the 512 tags across all VFs in the AFU region * AFU error logging for all VFs in the AFU region
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#541-unified-tag-remapping","title":"5.4.1 Unified Tag Remapping","text":"
                                                                  When a FPGA function sends out a read cycle, it allocates a unique tag which is subsequently used to identify the read completion.  The tag is considered busy; it cannot be assigned to another read cycle until read completion.  While a tag may be unique within a unit, two different units could unknowingly send out two read cycles of the same tag.  The PCIe subsystem requires unique tags for all read cycles irrespective of their origins.  Therefore, a mechanism is needed to uniquify tag globally across different units.
                                                                   
                                                                  OFS contains a tag remapper (tag_remap.sv) that intercepts the read cycle, finds a globally unique tag, and replaces the original tag value.  It also restores the original tag value when returning completion to the read requester.  tag_remap is placed between the Arm\u00ae AMBA\u00ae 4 AXI4-Stream interface of the PCIE subsystem and the PF/VF Mux/Demux.
                                                                   
                                                                  The logic is described as follows:
                                                                   
                                                                     
                                                                  1. A sub-module (ofs_fim_tag_pool) maintains a pool of available tags.
                                                                    2.  
                                                                  2. TX read requests are held until a tag is available from the pool by setting tvalid=0 to the host, and tready=0 to the PF/VF Mux/Demux.
                                                                    3.  
                                                                  3. When a TX read is dispatched, the tag is marked busy in the pool.
                                                                    4.  
                                                                  4. The original tag is stored in tag_reg, so it can be recovered when returning a completion to the unit/function.
                                                                    5.  
                                                                  5. Because completion to a read request can split into multiple smaller transfer sizes, responses are monitored and the final completion is detected using PCIe TLP rules.
                                                                    6.  
                                                                  6. Tags are released in the pool only when all requested data are transferred.
                                                                    7.  
                                                                  7. When the completion returns, the original tag is restored from tag_reg.
                                                                    8.  
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#542-afu-error-handling","title":"5.4.2 AFU Error Handling","text":"
                                                                  In this OFS design, the AFU Interface Handler handles error logging for all VFs in the AFU. Errors handled are as follows
                                                                   
                                                                  Table 5-4 AFU Error Descriptions
                                                                   Checker Field Description AFU protocol checker (PCIe TLP) Malformed TLP AFU PCIe TLP contains unsupported format type MaxPayloadError AFU memory write payload size exceeds max_payload_length limit MaxReadReqSizeError AFU memory read payload size exceeds max_read_request_size limit MaxTagError AFU memory read request tag value exceeds the maximum supported tag count UnalignedAddrErr The address field in AFU memory write/read request TLP is not DW-aligned. UnexpMMIOResp AFU is sending a MMIO read response with no matching MMIO read request. MMIOTimedOutAFU is not responding to a MMIO read request within the pre-defined response timeout period.  MMIODataPayloadOverrunThe number of data payload sent by AFU for a MMIO response (cplD) is more than the data length specified in the response. MMIOInsufficientDataThe number of data payload sent by AFU for a MMIO response (cplD) is less than the data length specified in the response. TxMWrDataPayloadOverrun    The number of data payload sent by AFU for a memory write request is more than the data length specified in the request. TxMWrInsufficientData  The number of data payload sent by AFU for a memory write request is less than the data length specified in the request. AFU Protocol Checker (Arm\u00ae AMBA\u00ae 4 AXI4-Stream)TxValidViolationThree checkers are implemented in the FIM to catch errors and protocol violations."},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#55-tlp-to-arm-amba-4-axi4-lite-memory-mapped-bridge-st2mm","title":"5.5 TLP to Arm\u00ae AMBA\u00ae 4 AXI4-Lite Memory Mapped Bridge (ST2MM)","text":"
                                                                  ST2MM implements the following key features: * Host MMIO bridge     * Maps MMIO TLP packets received from the PCIe Subsystem over streaming interface to Arm\u00ae AMBA\u00ae 4 AXI4-Lite memory-mapped request. The memory-mapped request is sent to AFU or Board peripherals over APF and BPF.     * Maps Arm\u00ae AMBA\u00ae 4 AXI4-lite MM response received from AFU or Board peripherals to TLP packets and send the packets over ST streaming channel to host HIA subsystem. * Sends MMIO response of all 0\u2019s for MMIO read to unused BAR region. * Interrupt     * Sends interrupt packets to the PCIe subsystem when interrupt requests are received from the peripherals. Interrupts can be requested by a peripheral through a memory write to interrupt CSR registers in the ST2MM.
                                                                   
                                                                  Figure 5-3 ST2MM Module
                                                                   
                                                                   
                                                                  ST2MM implements both Arm\u00ae AMBA\u00ae 4 AXI4-lite master and slave interfaces that are connected to the designated slave and master port on APF. Host memory requests are sent on the ST2MM master interface to AFP where the requests are routed to the targeted peripherals. 
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#6-mmio-regions","title":"6 MMIO Regions","text":"
                                                                  The FIM and AFU expose their functionalities to the host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). An MMIO region is an address space within a base address register (BAR) region to which features are memory mapped.  
                                                                   
                                                                  For example, when a feature is mapped to an MMIO region, the CSR registers of that feature are located within the address range of that region. There can be multiple MMIO regions within a BAR region. 
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#61-feature-region","title":"6.1 Feature Region","text":"
                                                                  A group of related CSRs can be categorized as a feature region. For example, a DMA engine has queue management function and quality of service (QoS) function; these are two different features of the DMA engine. A feature region is contained within a single PCIe BAR and cannot span across two BAR region boundaries. 
                                                                   
                                                                  A Device Feature Header (DFH) register marks the start of the feature region and sub-feature region, and you must place it at the first address of the region. Each DFH starts at 4KB boundary. A DFH register contains information that OPAE software requires to enumerate the feature. It also has an offset field that points to the next DFH in a feature list. OPAE software traverses the linked list of DFHs in each BAR region to discover all the features implemented on the platform. 
                                                                   
                                                                  The EOL field in a DFH marks the end of a DFH list and is only set in the DFH of the last feature in the feature list. The feature type field in the DFH is used to differentiate between the different types of feature region. Basic building blocks (BBB) and private features are always a child of an AFU or FPGA Interface Unit (FIU) and must be contained within an AFU or FIU, respectively.  
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#611-device-feature-header-dfh-structure","title":"6.1.1 Device Feature Header (DFH) Structure","text":"
                                                                  All DFHs must follow the following structure to be compatible with OPAE software.  Note that only features you want to be exposed to the OPAE software must have a DFH.
                                                                   
                                                                  Table 6-1: DFH Register Structure
                                                                   Bitfield Name Range Access Description FeatureType 63:60 RO 4\u2019b0000 \u2013 Reserved 4\u2019b0001 \u2013 AFU4\u2019b0010 \u2013 BBB4\u2019b0011 \u2013 Private Feature4'b0100 \u2013 FIM Reserved 59:41 Rsvd Reserved EOL 40 RO End of DFH List1'b0=No other feature header beyond this one1'b1=This is the last feature header NextDFHByteOffset 39:16 RO Next DFH byte offsetNext DFH Address= Current DFH address + Next DFH byte offset.  You can also use this value as an indication of the maximum size of the MMIO region occupied by this feature. FeatureRev 15:12 RO For AFU Feature type= AFU major version number that is user defined.All other feature types= Feature revision number FeatureID 11:0 RO For AFU feature type= CoreFIM version numberFor BBB feature type= Intel defined ID for BBBFor private feature type= User-defined ID to identify within an AFU For FIU type=ID for FIU unit (ex. 0 for FME, 1 for Port) 
                                                                  You must increment a feature revision number if a feature changes.  This change requires a corresponding change in the software to detect the new version and report mismatches between the hardware and software revision number. 
                                                                   
                                                                  When indicating \"FeatureType\" in your DFH Structure, use the following guidance: * FIM= Any static region feature you want discovered through software.  FIM features must implement mandatory FIM registers, including a FIM feature ID. * AFU= User Application region; may be PR region.  AFU feature must implement mandatory AFU register including an AFU ID. * Private Feature = Linked list of features within the FIM or AFU which are enumerated and managed by separate software and do not require an ID. * Basic Building Blocks (BBB) = Special features within the AFU or FIM that are meant to be reusable basic building blocks.  They are enumerated and called by separate software services.  
                                                                   
                                                                  Table 6-2: Mandatory FIM and AFU Registers
                                                                   
                                                                  The following table details the registers required for a FIM feature or AFU to be discovered by OPAE software.  The ID_H and ID_L fields allow you to create a unique feature or AFU identifier that remains unchanged while the FeatureRev in the DFH register increments.
                                                                   Byte Address offset with respect to DFH Register Name 0x0000 DFH with corresponding FeatureType field implemented 0x0008 ID_L: Lower 64 bits of unique ID number (GUID) 0x0010 ID_H: Upper 64 bits of unique ID number (GUID) 0x0018 Next AFU 
                                                                  Table 6-3: Next AFU Register Definition
                                                                   
                                                                  The following table details the \"Next AFU Register Definition.\"  This register is used if you have multiple AFUs that can be used with your FIM.  
                                                                   Bit Attribute Description 63:24 Reserved Reserved 23:0 RO Next AFU DFH Byte Offset Next AFU DFH address =current address + offset; where a value of 0 implies this is the last AFU in the list. Example: AFU0 at address 0x0: Next AFU offset= 0x100  AFU1@ address 0x100: Next AFU offset = 0x100 AFU2 at address 0x200: Next AFU offset = 0x0 (End of AFU List) 
                                                                  6.2 Control and Status Registers
                                                                   
                                                                  All the Control and Status Registers (CSRs) in the FIM are 64-bit registers with the following MMIO write, and MMIO read support.
                                                                   
                                                                  Table 6-4: CSR MMIO Read and Write Support
                                                                   Request Memory Attribute Payload size Memory Ordering MMIO Write UC 4B or 8B Strongly ordered MMIO Read UC 4B or 8B Strongly ordered 
                                                                  The FIM does not reorder the MMIO requests or responses. For MMIO writes, there is no reordering of requests in FIM, and UC ordering rules are followed. Similarly, for MMIO reads, there is no re-ordering of requests or responses in the FIM. An AFU may opt to re-order the MMIO read responses but the FIM does not enforce read response ordering. 
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#621-software-access-to-registers","title":"6.2.1 Software Access to Registers","text":"
                                                                     
                                                                  • Software accesses 64-bit registers as aligned quadwords. For example, to modify a field (bit or byte) in a 64-bit register, the entire quadword is read, the appropriate field(s) are modified, and the entire quadword is written back.
                                                                    •  
                                                                  • When updating registers through multiple accesses (whether in software or due to hardware disassembly), certain registers may have specific requirements on how the accesses must be ordered for proper behavior. These are documented as part of the respective register descriptions.
                                                                    •  
                                                                  • For compatibility with future extensions or enhancements, software must assign the last read value to all \u201cReserved and Preserved\u201d (RsvdP) fields when written. In other words, any updates to a register must be read so that the appropriate merge between the RsvdP and updated fields occurs. Also, software must assign a value of zero for \u201cReserved and Zero\u201d (RsvdZ) fields when written.
                                                                    •  
                                                                  • PCIe locked operations to FPGA hardware registers are not supported. Software must not issue locked operations to access FPGA hardware registers.
                                                                    •  
                                                                   
                                                                  In the following two cases, the FIM terminates MMIO Read requests by sending a completion with the data (CplD) specified below: * MMIO Timeout:  This occurs when the AFU does not respond within a set timeout. The timeout value is currently configured to 512 pclks (clk_2x). In this case, the FIM returns all 1s.
                                                                   
                                                                     
                                                                  • Illegal MMIO Accesses: This occurs when the read is accessing undefined registers in the FIM or if an AFU access violation.  An example of an access violation is when a PF attempts to access the AFU when it is set to VF mode, or when a VF attempts to access the AFU when it is set to PF mode. In this case, the FME will returns all 0s.
                                                                    •  
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#62-register-attribute-definition","title":"6.2 Register Attribute Definition","text":"
                                                                  Table 6-5: OFS Register Attribute Definitions
                                                                   Attribute Expansion Description RW Read/Write This bit can be read or written by software. RO Read Only The bit is set by hardware only. Software can only read this bit. Writes do not have any effect. RW1C Read/ Write 1 to Clear Software can read or clear this bit. The software must write 1 to clear this bit. Writing zero to RW1C bit has no effect. Note that a multi-bit RW1C field may exist. In this case, all bits in the field are cleared if a 1 is written to any of the bits. RW1S Read/ Write 1 to Set Software can read this bit. Writing a 1 to the bit sets it to 1. Writing a 0 has no effect. It is not possible for software to set this bit to 0. The 1 to 0 transition can only be performed by HW. RW1CS Read/Write 1 to Clear Sticky Software can read and clear this bit. Writing a 1 to a bit clears it, while writing a 0 to a bit has no effect. This bit is only reinitialized to its default value by a power-on reset. RWD Read/Write Sticky across Hard Reset The bit can be read or written by SW. This bit is sticky or unchanged by any reset type, including Hard Reset. The bit gets cleared only with power on. *S Sticky across Soft Reset The bit will be sticky or unchanged by soft reset. These bits are only re-initialized to their default value by a power-on reset. *D Sticky across Hard Reset The bit is sticky or unchanged by or unchanged by any reset type, including hard reset. The bit gets cleared only with power on. Rsvd Reserved Reserved for future definitions. Currently don\u2019t care bits. RsvdP Reserved and Protected Reserved for future RW implementations. The software must preserve the value of this bit by read modify write. RsvdZ Reserved and Zero Reserved for future RW1C implementations. The software must write zero to this bit."},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#623-csr-offset-in-bars","title":"6.2.3 CSR Offset in BARs","text":"
                                                                  The table below captures the FIM and AFU features in the supported BAR regions. The highlighted offset indicates the first DFH in the DFH list of a BAR region where device driver starts the DFH traversal.
                                                                   
                                                                  Table 6-6: PF0 BAR0 Features SoC AFU
                                                                   Offset Feature CSR set 0x0000 FME AFU 0x10000 PCIe Interface 0x12000 QSFP Controller 0 0x13000 QSFP Controller 1 0x14000 Ethernet Interface 0x15000 Memory Interface 0x80000 PMCI 0x100000 St2MM 0x130000 PR Control &  Status (Port Gasket) 0x131000 Port CSRs (Port Gasket) 0x132000 User Clock (Port  Gasket) 0x133000 Remote SignalTap (Port Gasket) 0x140000 AFU Errors (Protocol Checker) 
                                                                  Table 6-7: PF0 BAR0 Features Host AFU
                                                                   Offset Feature CSR  set 0x0000 PCIe Interface"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#7-clocks","title":"7 Clocks","text":"
                                                                  The following table provides external clock sources which correspond to pins on the FPGA that drive blocks of internal logic or are used as a reference clock for internal PLLs.  The names in the table are used in the top.sv or are the logical clock names used by their respective functional blocks.  
                                                                   
                                                                  Table 7-1: External Clock Source 
                                                                   Clock Frequency Description SYS_REFCLK 100MHz Reference clock to system IOPLL (sys_pll) which provides FIM system clocks. PCIE_REFCLK0 100MHz PCIe  Refclk 0 PCIE_REFCLK1 100MHz PCIe Refclk 1 SOC_PCIE_REFCLK0 100MHz PCIe Refclk 0 on the SOC side SOC_PCIE_REFCLK1 100MHz PCIe Refclk 1 on the SOC side qsfp_ref_clk 156.25MHz Ethernet  Refclk ddr4[0].ref_clk 150MHz Refclk  for EMIF channel 0 ddr4[1].ref_clk 150MHz Refclk for EMIF channel 1 ddr4[2].ref_clk 150MHz Refclk  for EMIF channel 2 ddr4[3].ref_clk 150MHz Refclk for EMIF channel 3 sdm_config_clk 125MHz SDM  Config Clock altera_reserved_tck 10MHz Default  JTAG Clock 
                                                                  Table 7-2: Internal Clocks
                                                                   Clock Frequency Description clk_sys 400MHz System clock. Primary clock for PCIe datapath. clk_100m 100MHz 100MHz clock. For RemoteSTP and user  clock, or any logic that requires a 100MHz clock. clk_csr 100MHz Arm\u00ae AMBA\u00ae 4 AXI4-lite  interconnect fabric and CSR clock. Driven by clk_100m. clk_2x 400MHz 2x clock to AFU in PR slot. Driven by clk_sys. clk_1x 171.43MHz 1x clock to AFU in PR slot. uclk_usr 312.50  MHz Configurable  \u201cH\u201d clk for AFU. Default 312.50 MHz uclk_usr_div2 156.25 MHz Configurable  \u201cL\u201d clk for AFU. Default 156.25 MHz"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#8-reset","title":"8 Reset","text":""},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#81-reset-signals","title":"8.1 Reset Signals","text":"
                                                                  The FIM system reset signals are driven according to a their respective requirements derived from their use cases.  The behavioral code defining the reset operation is located in the file rst_ctrl.sv. The FIM System Reset Drivers table below provides a list of the input and feedback signals that compose the various resets.
                                                                   
                                                                  Table 8-1: FIM System Reset Drivers
                                                                   Reset Description nPERST pin Active low PCIe reset pin from the PCIe card edge that may be set by the host machine for cold or warm resets. nCONFIG pin Active low input to the FPGA that causes the FPGA to lose its configuration data, enter a reset state, and tri-state all I/O pins. Host software must reload the FPGA FIM after nCONFIG is activated. ninit_done Active low signal indicating FPGA core configuration is complete and FPGA has entered usermode.  This signal is provided by the configuration monitor block in rst_ctrl.sv. pcie_reset_status Active high reset status from PCIe hard IP. When driven high, this signal indicates that the PCIe IP core is not ready for usermode. locked Active high SYS IOPLL locked signal pcie_cold_rst_ack_n Reset Acknowledge signal from the PCIe subsystem that lets the Reset Controller know that the Cold Reset sequence has been completed. pcie_warm_rst_ack_n Reset Acknowledge signal from the PCIe subsystem that lets the Reset Controller know that the Warm Reset sequence has been completed. 
                                                                  Upon power-on, the reset module in the FIM holds the FIM in reset until all the reset conditions are de-activated:
                                                                   
                                                                     
                                                                  • nPERST signal is asserted.
                                                                    •  
                                                                  • The INIT_DONE pin is driven high to indicate core configuration is complete.
                                                                    •  
                                                                  • The SYS IOPLL is locked.
                                                                    •  
                                                                  • The reset status from PCIe hard IP is de-asserted indicating the IP is ready for transfer. 
                                                                    •  
                                                                   
                                                                  The reset module places the FIM back into reset if any of these conditions becomes active again. The only way to invoke a system reset to the FIM after power-up is to deassert the nPERST pin either by performing a warm reboot or through PCIe driver intervention. There are soft reset signals set aside to allow software to reset the Port, AFU and partial reconfiguration IP. 
                                                                   
                                                                  The following table lists the reset outputs from the rst_ctrl.sv block.
                                                                   
                                                                  \u200b                                                      Table 8-2: FIM System Reset Outputs
                                                                   Reset Description rst_n_sys pin System general reset synchronous to clk_sys.  This is a warm reset of the FPGA logic.  Sticky bits in the FME and other CSR registers will not be reset by this signal. rst_n_100m pin System general reset synchronous to clk_100m. ninit_done Active low signal indicating FPGA core configuration is complete and FPGA has entered usermode.  This signal is provided by the configuration monitor block in rst_ctrl.sv. pwr_good_n Hardware reset conditioned by ninit_done and the pll_locked signal.  The signal is generally set when power has been cycled or a physical hardware fault has occurred, requiring a reset of the FPGA logic.   This signal is synchronous to clk_sys. pcie_cold_rst_ack_n Hardware reset conditioned by the pcie_reset_status which is a summary reset driven by the PCIe Hard IP logic tile on the FPGA die.  This signal is synchronous to clk_sys. pcie_warm_rst_ack_n Soft reset conditioned by nPERST when the pcie_reset_status is not asserted, meaning a warm reset is desired.  Cold reset sticky bits in the PCIe subsystem will not be reset by this signal."},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#9-interrupts","title":"9 Interrupts","text":"
                                                                  The OFS platform supports interrupts through MSI-X feature implemented in the PCIe SS. The MSI-X Interrupt feature handles FME and AFU interrupts. 
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#91-msi-x","title":"9.1 MSI-X","text":"
                                                                  FME interrupts are primarily used to notify the host of error events happened in the FIM. When any of the bits in the FME error status registers is set, an interrupt request is sent to the MSI-X module. There are FME error status registers for various FPGA features such as RAS Error, Temperature monitoring etc.\u202fIf any of those error registers log an error, we trigger an FME interrupt.
                                                                   
                                                                  An AFU sends an interrupt to the MSI-X module in the PCIE SS on the AXI interrupt request channel. 
                                                                   
                                                                  The MSI-X table entries and PBA vectors are implemented in the PCIE SS. The PCIE SS supports upto 4096 vectors in Static MSI-X mode
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#92-msi-x-entry-table","title":"9.2 MSI-X Entry Table","text":"
                                                                  Each entry in the MSI-X table stores the message address, message data, and vector control for one interrupt vector. The address, data, and vector control information are populated by software (usually done by the PCIe SRIOV driver) after reading the PCIe endpoint configuration space.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#93-msi-x-pba-table","title":"9.3 MSI-X PBA Table","text":"
                                                                  The PBA table contains a corresponding bit for each MSI-X interrupt vector. This PBA bit is set if that interrupt is triggered but is masked. When the interrupt is unmasked, the PBA bit is unset, and an interrupt is sent to the Host. When the Application generates an interrupt from an IRQ source, it sets the corresponding PBA bit. 
                                                                   
                                                                  When the interrupt is triggered by an IRQ Source, the IRQ Processor checks the masking of that interrupt to determine if the PBA bit should be set. If unmasked, the IRQ Processor looks up MSI-X address and data for the interrupt vector and generates the interrupt request over the PCIe EP.
                                                                   
                                                                  The FIM implements the pending bit array as per the MSI-X specification.  When interrupts are enabled the FIM immediately generates an interrupt in response to a suitable event.  When interrupt vectors are masked, the pending bit array entry for the corresponding interrupt vector is set without issuing an interrupt.  When interrupt vectors are re-enabled, any pending interrupt entries are issued to the PCIe EP
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#94-interrupts-supported","title":"9.4 Interrupts Supported","text":"
                                                                  Table 9-1: OFS for Intel Agilex 7 FPGA Interrupts Supported
                                                                   PF/VF Vector Assignment PF0 0-5 Reserved 6 FME Error Interrupt PF0.VF00-3User AFU Interrupt"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#95-msi-x-table-locations","title":"9.5 MSI-X Table Locations","text":"
                                                                  The MSI-X vector tables are at BAR4, address 0x2000. The MSI-X PBA tables are at BAR4, address 0x3000.
                                                                   
                                                                  Table 9-2: PF0 BAR4 Features
                                                                   Offset Feature CSR set 0x02000 MSI-X Vector Tables 0x03000 MSI-X PBA Tables"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#10-external-memory-interface-emif","title":"10 External Memory Interface (EMIF)","text":"
                                                                  There are 4 EMIF channels (4 DDR4 banks) on the f2000x platform which is targeted OFS FIM  for Intel Agilex 7 FPGA. The HE-MEM exerciser module in AFU. ECC is not implemented in this design. Both memory subsystem and HE-MEM implement Arm\u00ae AMBA\u00ae 4 AXI4-MM interface.
                                                                   
                                                                  **Table 10-1 Memory Subsystem Configuration on the Intel IPU Platform F2000X-PL platform **
                                                                   EMIF  Channel # Width ECC Size (GB) Speed  (MT/s) DDR4 Device FPGA Bank 0 x32 x8 4 2400 Three 1Gx16 3D 1 x32 x8 4 2400 Three 1Gx16 3A 2 x32 x8 4 2400 Three 1Gx16 2F 3 x32 x8 4 2400 Three 1Gx16 2E 
                                                                  Figure 10-1: EMIF Interfaces
                                                                   
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#101-emif-csr","title":"10.1 EMIF CSR","text":"
                                                                  TheMIF is implemented as an external FME feature in OFS EA and the CSR for the EMIF feature is memory mapped to the FME BAR region. The following table captures the EMIF CSR registers.
                                                                   
                                                                  Table 10-2: EMIF CSR Registers
                                                                   EMIF_DFH 0x000 0x300000001000100F EMIF Management DFH FIELD NAME RANGE ACCESS DEFAULT DESCRIPTION FeatureType [63:60] RO 0x3 Feature Type = Private Feature Reserved40 [59:40] RsvdZ 0x0 Reserved NextDfhByteOffset [39:16] RO 0x1000 Next DFH Byte  offset FeatureRev [15:12] RO 0x1 Feature Revision FeatureID [11:0] RO 0x00F Feature Id 
                                                                  \u200b               
                                                                   EMIF_STAT 0x008 0x0000000000000000 EMIF Calibration Status Register FIELD NAME RANGE ACCESS DEFAULT DESCRIPTION Reserved [63:8] RsvdZ 0x0 Reserved CalFaliure [7:4] RO 0x0 EMIF PHY Calibration Failure (1 bit per interface) CalSuccess [3:0] RO 0x0 EMIF PHY  Calibration Successful (1 bit per  interface) EMIF_CAPABILI TY 0x010 0x000000000000000F EMIF Capabliity Register FIELD NAME RANGE ACCESS DEFAULT DESCRIPTION Reserved [63:4] RsvdZ 0x0 Reserved EMIFCap [3:0] RO 0xF Attached Memory  Capability (1 bit per  interface)"},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#11-ethernet-interface-subsystem","title":"11 Ethernet Interface Subsystem","text":""},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#111-ethernet-interface-overview","title":"11.1 Ethernet Interface Overview","text":"
                                                                  The Ethernet Subsystem provides portability to different Ethernet configurations across platforms and generations and reusability of the hardware framework and software stack. This reference FIM implements an E-tile Ethernet Subsystem IP in a 2x4x25GbE configuration and provides a Linux driver that can be leveraged for customization. Each group of 4x25GbE routes to a QSFP.
                                                                   
                                                                  For more information about how to reconfigure the Ethernet Subsystem please refer to the [Ethernet Subsystem Intel FPGA IP].
                                                                   
                                                                  Table 11-1: Ethernet Configurations for example OFS FIMs for Intel Agilex 7 FPGAs
                                                                   Configuration/Mode Base Fim IP file name hssi_ss_8x25g Number of ports enabled 8 Enabled ports Port 0-7 Port{x} profile 25GbE Port{x} subprofile MAC+PCS Port{x} RSFEC True Port{x} PTP False Enable AN/LT False (for simulation efficiency) Enable NPDME True Enable JTAG to Avalon master bridge False Enable Tx packet classifier NA PTP accuracy mode NA Enable iCAL and pCAL recipe at power True TX/RX maximum frame size 1518 Enforce maximum frame size False Link fault generation mode Bidirectional Stop TX traffic when link partner sends PAUSE Yes Bytes to remove from RX frames Remove CRC bytes Forward RX PAUSE requests False Use source address insertion False Enable TX/RX VLAN detection True Enable asynchronous adapter clocks False Enable preamble Passthrough False Enable asynchronous adapter clocks False Enable strict preamble check False Enable strict SFD check False Average IPG 12 Enable adaptation load soft IP True Additional IPG removed as per AM period 0 PMA adaptation select NRZ_28Gbps_VSR 
                                                                  To determine which Transceiver Subsystem port maps to the QSFP A and B lanes, please refer to the following table:
                                                                   
                                                                  Table 11-2: Transceiver Subsystem Port Mapping
                                                                   Port Number base 2x4x25G 0 QSFP-A Lane-0 1 QSFP-A Lane-1 2 QSFP-A Lane-2 3 QSFP-A Lane-3 4 QSFP-B Lane-0 5 QSFP-B Lane-1 6 QSFP-B Lane-2 7 QSFP-B Lane-3 8 NA 9 NA 10 NA 11 NA 12 NA 13 NA 14 NA 15 NA 
                                                                  Figure 11-1: Transceiver Subsystem Block Diagram
                                                                   
                                                                   
                                                                  A host exerciser, named he-hssi, is provided in the pr_slot of the AFU partition. The Transceiver susbystem interface to the AFU has an Arm\u00ae AMBA\u00ae 4 AXI4-Stream data and sideband interface.  
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#112-ofs-ethernet-subsystem-interface-module","title":"11.2 OFS  Ethernet Subsystem Interface Module","text":"
                                                                  A wrapper around the  Ethernet Subsystem integrates the following features:
                                                                   
                                                                     
                                                                  • DFH registers 
                                                                    •  
                                                                  • Control & status registers 
                                                                    •  
                                                                  • Indirect access to Transceiver SS CSR registers via CSR mailbox in the Ethernet Subsystem interface 
                                                                    •  
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1121-ethernet-subsystem-control-and-status-register-csr-map","title":"11.2.1 Ethernet Subsystem Control and Status Register (CSR) Map","text":"
                                                                  The Ethernet Subsystem connects to the APF which is memory mapped to PF0 BAR0. The Ethernet Subsystem CSR space in the FIM consists of two parts:
                                                                   
                                                                     
                                                                  • Ethernet Subsystem CSRs assigned from offset 0x000 to 0x7FF
                                                                    •  
                                                                  • Additional CSRs are added to FIM at offset 0x800 to 0xFFF
                                                                    •  
                                                                   
                                                                  The PCIe subsystem uses Arm\u00ae AMBA\u00ae 4 AXI4 Memory Mapped accesses to read and write Ethernet Subsystem Control and Status Registers in the FIM. The Ethernet Subsystem CSR Map structure is designed to scale according to IP capabilities.
                                                                   
                                                                  The Ethernet Subsystem CSR Map can be found ipss/hssi/HSSI_SS_CSR.xls.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#114-ethernet-subsytem-software","title":"11.4 Ethernet Subsytem Software","text":"
                                                                  There are two pieces of software related to running the Ethernet Subsystem: The Linux* dfl network driver and the user space OPAE Tools.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1141-ethernet-subsystem-linux-driver","title":"11.4.1 Ethernet Subsystem Linux Driver","text":"
                                                                  The Ethernet subystem is exposed as a feature in the PCIe PF BAR0 region. It has a Device Feature Header (DFH) specifying the interface.
                                                                   
                                                                  The primary functionality of the driver is to interact with the ethernet MAC and PHY through an indirect register access mailbox implemented by the HSSI_RCFG_CMD0, HSSI_RCFG_DATA0 registers described above. To aid in RTL bringup, the driver provides a debugfs interface directly to the indirect register access mailbox. For each HSSI interface in the system there would be a directory with the following form containing two files, regaddr and regval: /sys/kernel/debug/dfl-fme.X.Y
                                                                   
                                                                  To read a register offset in the MAC/PHY write the offset to the regaddr file as a C hex string (e.g. 0x04) and then read the value as string out of regval file. To write a register offset in the MAC/PHY write the offset to the regaddr file as a C hex string (e.g. 0x04) and then write the value as a C hex string to regval file.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1142-ethernet-subsystem-opae-user-space-tool","title":"11.4.2 Ethernet Subsystem OPAE User Space Tool","text":"
                                                                  User space OPAE Tools that are part of OPAE SDK provide support for the Ethernet Subsystem. You can use the --help option to print more information about each of these commands:
                                                                   
                                                                     
                                                                  • hssi: Provides a means of interacting with the 10G and 100G HSSI AFUs through the host exerciser.
                                                                    •  
                                                                  • hssiloopback: Enables and disables Ethernet loopback.
                                                                    •  
                                                                  • hssistats: Provides MAC statistics.
                                                                    •  
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#12-partial-reconfiguration","title":"12 Partial Reconfiguration","text":"
                                                                  Partial Reconfiguration (PR) is an Intel FPGA technology that allows users to reconfigure parts of the FPGA device dynamically, while the remainder of the device continues to operate. In a non-partial reconfiguration flow, any change to the design requires full reprogramming of the entire configuration RAM (CRAM) arrays in the device. With partial reconfiguration, you can dynamically reprogram one or more CRAM frames. A partial reconfiguration design has a static region, and a PR regions, which can be modified to implement new logic. The portion of the CRAM on the chip to be reconfigured is contained within a PR region. For the PR flow, your design must be partitioned into static region and reconfigurable region. The static region is the area of your FPGA that is not reconfigured without reprogramming the entire FPGA. An area of the chip that you plan to partially reconfigure is a PR region.
                                                                   
                                                                  The Port Gasket contains all the PR specific modules and logic, such as PR slot reset/freeze control, user clock, remote STP etc. For this reference example only one PR slot is supported.
                                                                   
                                                                  The Figure below shows the fundamental concepts required to support PR in OFS platform. There are 4 OFS management DFH \u2013 PR, Port, User Clock and Remote STP in Port Gasket that is exposed to OFS software. These platform capabilities are generally used in conjunction to Partial Reconfiguration. The Figure below also demonstrates the concepts of adding a new interface to the PR region.
                                                                   
                                                                  Figure 12-1 Partial Reconfiguration Gasket
                                                                   
                                                                   
                                                                  The isolation logic is provided on the output signals of the PR region to ensure they don\u2019t glitch and affect the interfacing logic in the Static Region (SR). The isolation logic is controlled by the PR Freeze logic during PR operation.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#121-user-clock-support","title":"12.1 User Clock Support","text":"
                                                                  The reference platform provides two user configurable clock (uclk_usr, uclk_usr_div2) for use by the AFU. These clocks are generated by a reconfigurable IOPLL. The control and status of these clocks is expose through two pairs of command and status registers (USR_CLK_FREQ_CMD0 / USR_CLK_FREQ_STS0 & USR_CLK_FREQ_CMD1 / USR_CLK_FREQ_STS1). The first pair controls the fPLL reconfiguration functionality. The second pair controls a clock frequency measurement block.
                                                                   
                                                                  The following are the default values of the userclk.
                                                                   
                                                                  uclk_usr: 312.5 MHz
                                                                   
                                                                  uclk_usr_div2: 156.2 MHz
                                                                   
                                                                  Table 12-1 usr_clk_* Acceptable Programming Range
                                                                   Fabric Frequency Range uclk_usr (H) uclk_usr_div2 (L) Details 0-400 MHz 0-800 MHz 0-400 MHz Clocks set on 2x relationship for L<400 MHz 400-800 MHz 400-800 MHz 400-800 MHz Clks are equal for L>400 MHz 
                                                                  PLL Reconfiguration
                                                                   
                                                                  The blue bit stream hardware exposes the low level IOPLL reconfiguration interfaces directly to software control. Through the USR_CLK_FREQ_CMD0 register software can select the reference clock, assert the PLL power down pin and issue reads and writes on the IOPLL Avalon-mm reconfiguration bus. Through the USR_CLK_FREQ_STS0 register software can query the IOPLL active reference clock, locked status and readdata returned from the IOPLL AVMM interface for read requests.
                                                                   
                                                                  Clock Frequency Counter
                                                                   
                                                                  The user clocks, generated by the reconfigurable IOPLL, are connected to a frequency measurement circuit. Software selects which of the two clocks to measure by setting the clock select bit in the USER_CLK_FREQ_CMD1 register. After 10ms software can read the frequency, in 10 KHz units, from the USER_CLK_FREQ_STS1 register. Reading this register before 10ms has passed will return undefined data but otherwise has no effect.
                                                                   
                                                                  Configuring User Clocks
                                                                   
                                                                  To program the user clock to the desired frequency, user will set the clock-frequency-low and clock-frequency-high fields in the PR AFU GBS .json file to the desired frequency value. During PR, SW will try to provide the closest possible frequency to the value specified in the .json file.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#13-host-exercisers","title":"13 Host Exercisers","text":"
                                                                  The Host Exerciser (HE) modules are responsible for generating traffic with the intent of exercising the specific interface they are designed to connect to. They are intended to test the interface in simulation and hardware, enable measurement of bandwidth and other performance KPIs and, in some cases, provide an example of data movement between interfaces (PCIe to DDR for e.g.) for adoption into a customer application.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#131-he-lb-and-he-mem-host-exerciser","title":"13.1 HE-LB and HE-MEM Host Exerciser","text":"
                                                                  The Host Exerciser Loopback module is a traffic generator that can be attached both to host memory, over PCIe (HE-LB), and to local memory, such as board-attached DDR (HE-MEM). The Host Exerciser (HE) is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth as well as demonstrating data path correctness. The FIM picks up the HE-LB module behind the PIM (Platform Interface Manager). The PIM converts the Arm\u00ae AMBA\u00ae 4 AXI4 with TLP out of the PCIe SS to standard Arm\u00ae AMBA\u00ae 4 AXI4 (MM for completions and Lite for CSR) interfaces. The figure below shows how the HE-LB is instantiated in the FIM.
                                                                   
                                                                  Figure 13-1 HE-LB Dataflow Diagram 
                                                                   
                                                                   
                                                                  The exerciser has two primary modes: loopback, in which read data is fed back as writes, and read/write mode, in which reads and writes are generated independently. Furthermore, the host_exerciser software provided with OPAE that drives the RTL has two major modes: \"lpbk\" and \"mem\". These modes only select the UUID of the HE AFU, with lpbk selecting a version configured with no local memory (56e203e9-864f-49a7-b94b-12284c31e02b) and mem seeking a version with local memory (8568ab4e-6ba5-4616-bb65-2a578330a8eb). The behavior of each engine with and without local memory is described below.
                                                                   
                                                                  Figure 13-2 HE Engine Heirarchy 
                                                                   
                                                                  flowchart TB\n    classDef gr fill:green,color:white;\n    classDef ol fill:olive,color:white;\n    classDef rd fill:maroon,color:white;\n\n    he_lb(\"he_lb_top\"):::gr --- he_lb_main\n    he_mem(\"he_mem_top\"):::gr --- he_lb_main\n    he_lb_main:::gr ---- he_lb_engines\n    he_lb_engines:::gr ---- mode_rdwr:::rd\n    he_lb_engines:::gr ---- mode_lpbk:::rd\n    he_lb_engines:::gr ---- mode_atomic:::rd\n    he_lb_main --- he_lb_csr:::ol
                                                                   
                                                                  For more details of HE-LB and HE-MEM IP, please refer to ofs-fim-common/src/common/he_lb/README.md
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#132-memory-traffic-generator-he-mem-tg","title":"13.2 Memory Traffic Generator  (HE-MEM-TG)","text":"
                                                                  The memory traffic generator (TG) AFU provides a way for users to characterize local memory channel bandwidth with a variety of traffic configuration features including request burst size, read/write interleave count, address offset, address strobe, and data pattern.
                                                                   
                                                                  The AFU consists of a series of configurable Arm\u00ae AMBA\u00ae 4 AXI4-MM traffic generators connected to the available local memory channels not attached to HE-MEM. It has a separate CSR block for AFU feature information and high-level status, including a TG_CAPABILITY register that describes the available traffic generators with a 1 bit active high mask and a TG_STAT register that provides the 4-bit, one-hot status of each TG in adjacent fields. The status is decoded left to right as follows: pass, fail, timeout, active. For additional detail, refer to the MEM_TG CSR table ofs-fim-common/src/common/mem_tg/MEM_TG_CSR.xls
                                                                   
                                                                  Each traffic generator is configured through a separate Avalon-MM interface at incremental offsets of 0x1000 from the AFU DFH. For details on the TG2 configuration space, refer to the MEM_TG_CSR.xls. The default configuration for each TG performs a single-word write followed by a read at address 0. Triggering the start of the test on a TG will initiate a counter to measure the duration of the test which is recorded in the AFU CSR block and used to report memory channel bandwidth.
                                                                   
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#132-hssi-host-exerciser-he-hssi","title":"13.2 HSSI Host Exerciser (HE-HSSI)","text":"
                                                                  HE-HSSI is an Ethernet AFU that handles client side ethernet traffic. The reference HE-HSSI has following features:
                                                                   
                                                                     
                                                                  • HE-HSSI provides an E-tile compatible interface with the Transceiver Subsystem.
                                                                    •  
                                                                  • Includes a traffic generator and checker or monitor
                                                                    •  
                                                                  • Provides pause signals to the Transceiver Subsystem for XON and XOFF generation
                                                                    •  
                                                                  • Generates traffic or incoming traffic that can be looped back into transmit path by enabling loopback mode, which will bypass traffic generator
                                                                    •  
                                                                  • At the HE-HSSI interface boundary the Ethernet data interface is Arm\u00ae AMBA\u00ae 4 AXI4-Stream with 64-bit data at eth_clk clock
                                                                    •  
                                                                  • The Traffic generator and checker modules have a 64-bit data interface at eth_clk clock.
                                                                    •  
                                                                  • The traffic generator supports the following modes:
                                                                       
                                                                    • Fixed length or Random Length
                                                                      •  
                                                                    • Incremental pattern or Random pattern
                                                                      •  
                                                                     
                                                                    •  
                                                                  • Traffic checker does a 32-bit crc check in 10/25G. In the 100G configuration, there is no data integrity check, only a packet counter.
                                                                    •  
                                                                  • The CSR of this AFU is accessible through Arm\u00ae AMBA\u00ae 4 AXI4-Stream PCIe TLP interface
                                                                    •  
                                                                  • The PCIe TLP to CSR Interface Conversion module converts PCIe TLPs into simple CSR interface
                                                                    •  
                                                                  • The CSR space of the traffic generator and checker modules are accessed in an indirect way using mailbox registers
                                                                    •  
                                                                  • If used for more than one channel, each channel has a separate traffic generator and traffic checker with separate CSR space.
                                                                    •  
                                                                  • Reads and Writes to individual traffic controller CSR spaces can be done by selecting that particular channel using channel select register.
                                                                    •  
                                                                   
                                                                  The HE-HSSI Ethernet block diagram is below.
                                                                   
                                                                  Figure 13-3: HE-HSSI Block Diagram Block Diagram
                                                                   
                                                                   
                                                                  The diagram below shows the path followed depending on if you enable loopback mode in HE-HSSI or not. In loopback mode, traffic is looped back into the transmit path which will bypass traffic. Alternatively, the traffic generates traffic.
                                                                   
                                                                  Figure 13-4: HE-HSSI Operation Mode Traffic Patterns 
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1322-he-hssi-csr-map","title":"13.2.2 HE-HSSI CSR Map","text":"
                                                                  The reference HSSI AFU contains the following registers and a similar arrangement of register space can be implemented for other use case-specific HSSI AFUs.
                                                                   
                                                                     
                                                                  • AFU DFH Register: Device feature header for the AFU (AFU_DFH)
                                                                    •  
                                                                  • AFU ID Registers: 128-bit UUID for the AFU which occupies two 64-bit registers (AFU_ID_L, AFU_ID_H)
                                                                    •  
                                                                  • Mailbox Registers: Command and Data register for traffic controller register access. It follows the standard access method defined for OFS. Access method and implementation is same as Reconfiguration Interface defined for the HSSI FIM. (TRAFFIC_CTRL_CMD, TRAFFIC_CTRL_DATA)
                                                                    •  
                                                                  • Channel Select Register: Channel select register for traffic controller mailbox access. It is used in cases where more than one channel is in the AFU, else it defaults to zero, meaning channel-0 is selected. (TRAFFIC_CTRL_PORT_SEL)
                                                                    •  
                                                                  • Scratchpad Register: Scratchpad register for CSR access checking. (AFU_SCRATCHPAD)
                                                                    •  
                                                                   
                                                                  The CSR excel for HE-HSSI module can be found at ofs-fim-common/src/common/he_hssi/HE_HSSI_CSR.xls.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#133-he-null-overview","title":"13.3 HE-Null Overview","text":"
                                                                  This module is a simple stub that is used to replace various HE and other blocks in the FIM whenever they are bypassed using the qsf compiler directive such as null_he_lb, null_he_hssi, null_he_mem and null_he_mem_tg. To find out more about these compiler directives, refer to the [Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs].
                                                                   
                                                                  Table 13-1 HE-Null DFH
                                                                   REGISTER NAME ADDRESS ACCESS DEFAULT DESCRIPTION DFH 0x0000 RO 0x0000_0000_1000_0000 DFH register GUID_L 0x0008 RO 0xaa31f54a3e403501 Lower 64b of GUID GUID_H 0x0010 RO 0x3e7b60a0df2d4850 Upper 64b of GUID SCRATCHPAD 0x0018 RW 0x0 Scratchpad 
                                                                  Figure 13-5: HE-Null Block Diagram
                                                                   
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14-reliability-accessibility-serviceability-ras-and-error-handling","title":"14 Reliability, Accessibility, Serviceability (RAS) and Error Handling","text":"
                                                                     
                                                                  1. Downstream AFU checker: Identifies AFU violations.  For example, these checker flags violations of the interface specification. 
                                                                    2.  
                                                                  2. Upstream software or PCIe link checker: Identifies invalid traffic from PCIe that violates FIM or PCIe specifications. For example, this checker flags an application sending traffic if it violates the FIM specification or creates a PCIe link issue by causing completion timeout or malformed TLP. 
                                                                    3.  
                                                                  3. FIM - Checks for bugs in the FIM fabric.
                                                                    4.  
                                                                   
                                                                  Errors reported by the checker are logged in either the FME error registers port error registers, or both, as shown in the table below.  For more details on each of the registers, please refer to FME_CSR.xls
                                                                   
                                                                  Table 14-1: Error Registers
                                                                   MMIO Region Area Register Description FME CoreFIM FME_ERROR FME Error Status Register 0.  Registers parity errors, underflow or overflow errors and access mismatches. FME CoreFIM FME_ERROR0_MASK FME Error Mask Register 0.  Write a 0 to mask errors in the FME Error Status Register 0. FME External PCIE0_ERROR PCIe0 Error Status Register. FME External PCIE0_ERROR_MASK PCIe0 Error Mask Register 0.  Write a 0 to mask errors in the PCIe0 Error Status Register 0. FME CoreFIM FME_FIRST_ERROR First FME Error Register. FME CoreFIM FME_NEXT_ERROR FME Next Error Register. FME CoreFIM RAS_NOFAT_ERR_STAT Reliability/Accessibility/Serviceability (RAS) Non-Fatal Error Status Register. FME CoreFIM RAS_NOFAT_ERR_MASK RAS Non-Fatal Error Mask Register. Write 0 to mask error fields in RAS_NOFAT_ERR_STAT Register. FME CoreFIM RAS_CATFAT_ERR_STAT RAS Catastrophic and Fatal Errors Status Register. FME CoreFIM RAS_CATFAT_ERR_MASK RAS Catastrophic and Fatal Errors Mask Register. Write 0 to mask error fields in the RAS_CATFAT_ERR_STAT Register. FME CoreFIM RAS_ERROR_INJ RAS error Injection Register. PORT CoreFIM PORT_ERROR Port Error Status Register. PORT CoreFIM PORT_FIRST_ERROR Port First Error Register . PORT CoreFIM PORT_MALFORMED_REQ0 Port Malformed Request Register 0.  Provides malformed request header LSBs. PORT CoreFIM PORT_MALFORMED_REQ1 Port Malformed Request Register 1.  Provides malformed request header MSBs."},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#141-fme-errors","title":"14.1 FME Errors","text":""},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1411-fme_error0","title":"14.1.1 FME_ERROR0","text":"
                                                                  The FME_ERROR0 register flags CoreFIM FME errors in the Global Error (GLBL_ERROR) private feature. The error bits in this register are sticky bits. You can only clear these bits through software or through a system reset. Writing a 1 to the error field in the register clears the corresponding error bit. Writing a 1 to the corresponding bit in FME_ERROR0_MASK register masks the error.
                                                                   
                                                                  Table 14-2: FME Error Types
                                                                   Error Type Description Fabric errors FIFO underflow/overflow condition in CoreFIM. These errors only occur if you have introduced bugs into the FIM or during very rare single event upset (SEU) or SEU-like events. Invalid port access A port can either be mapped to a PF or a VF, and not both. The checker reads the AfuAccessCtrl field in the FME CSR PORT0_OFFSET register to determine the access type of the Port. If it finds a PF is trying to access a port that is mapped to a VF or vice-versa, an error will be reported. Invalid AFU access An AFU can either be mapped to a PF or a VF, and not both. The checker reads the AfuAccessCtrl field in the FME CSR PORT0_OFFSET register to determine the access type of the AFU associated with the Port. If it finds a PF is trying to access an AFU that is mapped to a VF or vice-versa, an error is reported and a fake response is sent back to the requester to avoid a completion timeout on the host."},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1412-pcie0_error","title":"14.1.2 PCIE0_ERROR","text":"
                                                                  The PCIe Avalon-ST to Arm\u00ae AMBA\u00ae 4 AXI4-Stream bridge monitors the PCIe link for errors and logs any such errors in the PCIE0_ERROR register (in PCIE0 feature region) and PCIE0_ERROR register in the GLBL_ERR private feature.    The error bits in the PCIE0_ERROR register are sticky bits that you can only clear through software or through a system reset. Writing a 1 to the error field in the register clears the corresponding error bit. Writing a 1 to the corresponding bit in PCIE0_ERROR0_MASK masks the error.  
                                                                   
                                                                  If you have other external FME features, you can add similar _ERROR registers to this space. Please refer to the following spreadsheet: https://github.com/OFS/ofs-f2000x-pl/ipss/pcie/rtl/PCIE_CSR.xls or the SystemVerilog file: https://github.com/OFS/ofs-f2000x-pl/ipss/pcie/rtl/pcie_csr.sv for more details on this register.  
                                                                  NOTE
                                                                   
                                                                  The PCIE0_ERROR register is located in both the Global Error external feature memory space and a separate PCIe external feature memory space.  OPAE software supports the PCIe external feature memory space beginning at offset 0x40000 for OFS EA and going forward.  PCIe registers beginning at 0x4000 in the Global Error external feature memory space are there for backward compatibility to the Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1413-fme_first_error-fme_next_error","title":"14.1.3 FME_FIRST_ERROR, FME_NEXT_ERROR","text":"
                                                                  The FME_FIRST_ERROR register flags which of the FME error reporting registers, such as FME_ERROR0, PCIE0_ERROR0, has reported the first error occurrence. The error fields of the first error register are then continuously logged into the FME_FIRST_ERROR register until a system reset or software clears all the errors in that first error register. Likewise, the FME_NEXT_ERROR indicates which of the FME error reporting registers (except the first error register) has reported the next occurrence of error after the first error register. The error fields of the next error register are continuously logged into the FME_NEXT_ERROR register until a system reset or software clears all the errors in the second error register.
                                                                   
                                                                  Please refer to fme_csr.sv for individual register field descriptions or the SystemVerilog file: fme_csr.sv
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1414-reliability-accessibility-serviceability-ras-error-status","title":"14.1.4 Reliability, Accessibility, Serviceability (RAS) Error Status","text":"
                                                                  The RAS feature in CoreFIM labels errors as non-fatal, fatal or catastrophic based on their impact to the system.  * A non-fatal error usually originates from software or an AFU.  With a non-fatal error, the user application may still be able to recover from the error by performing a soft reset on the AFU, fixing the user application software if necessary, and clearing the error. On the other hand, a fatal or catastrophic error is non-recoverable and requires the platform to be reset. * Non-fatal errors are logged in the RAS_NOFAT_ERR_STAT register and fatal or catastrophic errors are logged in the RAS_CATFAT_ERR_STAT register.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14141-non-fatal-errors","title":"14.1.4.1 Non-Fatal Errors","text":"
                                                                  The RAS_NOFAT_ERR_STAT is a read-only status register that is specifically added for the RAS feature. It logs the high-level status of non-fatal errors in the hardware.  Unlike the error bits in the PCIE0_ERROR and FME_ERROR0 registers which are RW1C (software can write a 1 to clear the error), the error bits in this register are read-only and can only be cleared by system reset. Software has an option to mask the error using RAS_NOFAT_ERR_MASK.
                                                                   
                                                                  Please refer to FME_CSR.xls for individual register field descriptions or the SystemVerilog file: fme_csr.sv
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14142-catastrophic-fatal-errors","title":"14.1.4.2 Catastrophic & Fatal Errors","text":"
                                                                  The RAS_CATFAT_ERR_STAT is a read-only status register that is specifically added for the RAS feature. It captures the high-level status of errors that can only be recovered with a system reset. Therefore, the error bits in the RAS_CATFAT_ERR_STAT register are read-only and can only be cleared by system reset or masked through RAS_CATFAT_ERR_MASK. Please refer to FME_CSR.xls for individual register field descriptions or the SystemVerilog file: fme_csr.sv
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1415-ras-error-injection","title":"14.1.5 RAS Error Injection","text":"
                                                                  For software testing purposes, you can inject non-fatal, fatal, and catastrophic errors into the platform through the RAS_ERROR_INJ register.  These errors are reported in the RAS_CATFAT_ERR_STAT and RAS_NOFAT_ERR_STAT registers. Please refer to FME_CSR.xls for individual register field descriptions or the SystemVerilog file: fme_csr.sv
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1416-fme-error-interrupts","title":"14.1.6 FME Error Interrupts","text":"
                                                                  In an event of an FME error, the MSI-X module in the FIM generates an interrupt so the host can decide on the next course of action. The FIM does not stall upstream and downstream traffic after the FME error. However, a port error triggered by invalid request from a user AFU stalls all the traffic going from AFU to PCIe. The interrupt capability is discoverable by querying the NumbSuppInterrupt field of the PORT_CAPABILITY register in the Port private feature.  The MSI-X vector number is recorded in the InterruptVectorNumber field of the GLBL_ERROR_CAPABILITY register of the Global Error external feature.
                                                                   
                                                                  An FME error interrupt is generated in response to the FME_ERROR0, PCIE0_ERROR0, RAS_NOFAT_ERR_STAT, or RAS_CATFAT_ERR_STAT registers recording a new, unmasked, error per the rules defined by CoreFIM interrupt feature.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14161-msi-x-masking-pending-bit-array-pba-clearing","title":"14.1.6.1 MSI-X Masking & Pending Bit Array (PBA) Clearing","text":"
                                                                  If the MSI-X vector corresponding to the FME error interrupt is masked, events are recorded in the PBA array.  Clearing the FME error status registers clears the corresponding MSI-X PBA entries.  If only some events are cleared, the normal interrupt triggering rules apply and a new pending interrupt is registered.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1417-fme-error-handling","title":"14.1.7 FME Error Handling","text":"
                                                                  When the host receives an FME error interrupt, it must take the recommended actions described below to bring the system back to its normal operation.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14171-catastrophicfatal-error","title":"14.1.7.1 Catastrophic/Fatal Error","text":"
                                                                  A system reset is mandatory for any catastrophic or fatal error.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#14172-non-fatal-error","title":"14.1.7.2 Non-Fatal Error","text":"
                                                                  When software receives a non-fatal error interrupt which does not require a system reset, it can take the following steps to clear the error after software handles the error: 1.  Set the *_ERROR_MASK register to all 1\u2019s to mask all errors 2. Clear the *_FIRST_ERROR register 3. Clear the *_ERROR register 4. Set *_ERROR_MASK register to all 0\u2019s to enable all errors
                                                                   
                                                                     
                                                                  • Result: The *_ERROR & *_FIRST_ERROR registers begin capturing new errors.
                                                                    •  
                                                                   
                                                                  NOTE
                                                                   
                                                                  A system reset can only clear RAS Error status registers.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#143-mmio-requests","title":"14.3 MMIO Requests","text":"
                                                                  The FIM is designed to gracefully handle MMIO request scenarios.  
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1431-unsupported-functions-and-bar","title":"14.3.1 Unsupported Functions and BAR","text":"
                                                                  The PCIe hard IP in the FIM guarantees that only TLP packets for the functions and BARs supported by the FIM (as configured in PCIe HIP IP instantiation) are sent to the FIM.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1432-mmio-request-decoding","title":"14.3.2 MMIO Request Decoding","text":"
                                                                  The packet router and memory decoder in the FIM ensure that only legal MMIO requests are forwarded to the targeted MMIO region. Full address and BAR decoding is done both in the packet router and the memory decoder to ensure the requests are forwarded to the designated CSR region as defined in the MMIO Regions chapter.  Any unsolicited/illegal MMIO request is dropped, and an error is reported back to host through the FME error register.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1433-unused-fmeport-csr-regions","title":"14.3.3 Unused FME/Port CSR Regions","text":"
                                                                  All the CSR slaves in FIM which are mapped to the FME or Port CSR regions must always respond to MMIO read requests targeting its associated CSR region. A CSR slave must return all 0s for MMIO reads to its unused CSR region such as a reserved space or a region where no CSR register is implemented for the address.
                                                                   
                                                                  The FIM ensures MMIO reads to FME or Port CSR regions that are not mapped to any CSR slave always gets a response of all 0s. The memory decoder module and fake responder module in the FIM provide this guaranteed response.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1434-unsupported-mmio-request","title":"14.3.4 Unsupported MMIO Request","text":"
                                                                  Any MMIO request targeting FME or Port CSR regions with a length or address alignment that are not  supported by the FIM is dropped, and an error is logged in PCIE0_ERROR register. The MMIO checker module in the FIM guarantees this response. When an unsupported MMIO read request to the FIM CSR region is detected, the FIM sends back a CPL (completion without data) with error status (UR) back to host.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1435-afu-access-violation","title":"14.3.5 AFU Access Violation","text":"
                                                                  AFU access violations refer to the scenarios where a PF is attempting to access the MMIO region of an AFU bound to a VF (virtualization enabled), or when a VF is trying to access the MMIO region of an AFU bound to a PF (virtualization disabled). When such a violation is detected, the FIM drops the request and logs the error in the FME_ERROR0 register. If the request is an MMIO read request, the FIM returns a fake response to the host.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#1436-afu-mmio-response-timeout","title":"14.3.6 AFU MMIO Response Timeout","text":"
                                                                  An AFU MMIO Response timeout functions in the same manner described in the MMIO Response Timeout section.
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#15-ofs-design-hierarchy","title":"15 OFS Design Hierarchy","text":"
                                                                  Files for design, build and unit test simulation are found at https://github.com/OFS/ofs-f2000x-pl, release branch release/ofs-2023.3.  
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#151-design-guidance","title":"15.1 Design Guidance","text":"
                                                                  The OFS FIM is designed with configurability and scalability in mind.  At a high level, these are the necessary steps for a user to customize the design.  Please refer to the FPGA Interface Manager Developer's Guide
                                                                   
                                                                  Table 15-2 Features
                                                                   Step Description Comments 1 Re-configure PCIe HIP for additional VFs (if necessary) * PF0 is mandatory for running OPAE software* Only modification of the VFs (added or subtracted) by the user is recommended. * Default configuration supports 1 PF and 3 VFs. 2 Update Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF MUX-DEMUX configuration (if necessary) * The PF/VF MUX-DEMUX is parameterized for flexibility. You can change, add or delete PF or VF functions by updating the top_cfg_pkg.sv file. * You also have the option of keeping the default configuration and tying off the unused VFs if needed. 3 Update top level and AFU level as necessary * If you integrating additional external interfaces, make the edits at the top level (ofs_top.sv) and propagate the interface down to the AFU level (afu_top.sv and soc_afu_top.sv) 4 Add user implemented function(s) in AFU * All of your implemented functions must have the required Arm\u00ae AMBA\u00ae 4 AXI4-Stream interface for both the data path and the MMIO control path to CSRs. * All CSRs in the user-implemented function must have the required DFH layout. * See host exerciser CSRs for reference. 5 Update UVM testbench * The OFS full chip UVM environment is coded specifically for verifying the default configuration containing the host exercisers for the PCIe, memory, and Ethernet. * User must update the UVM testbench to reflect new RTL behavior for any customization changes. See The Simulation User Guide 
                                                                  For more information on modifying the FIM, refer to the FPGA Interface Manager Developer's Guide
                                                                  "},{"location":"hw/f2000x/reference_manuals/ofs_fim/mnl_fim_ofs/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                  Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                   
                                                                  OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/","title":"Platform Evaluation Script: Open FPGA Stack for Intel Agilex 7 SoC Attach FPGAs","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to user space."},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#1-overview","title":"1 Overview","text":""},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#11-about-this-document","title":"1.1 About this Document","text":"
                                                                  This document serves as a set-up and user guide for the checkout and evaluation of an Intel IPU Platform F2000X-PL development platform using Open FPGA Stack (OFS). After reviewing the document, you will be able to:
                                                                   
                                                                     
                                                                  • Set-up and modify the script to the your environment
                                                                    •  
                                                                   
                                                                     
                                                                  • Compile and simulate an OFS reference design
                                                                    •  
                                                                   
                                                                     
                                                                  • Run hardware and software tests to evaluate the complete OFS flow
                                                                    •  
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#table-1-2-software-version-summary","title":"Table 1-2: Software Version Summary","text":"Component Version Description FPGA Platform Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL Intel platform you can use for your custom board development OFS FIM Source Code Branch: https://github.com/OFS/ofs-f2000x-pl, Tag: ofs-2023.3-1 OFS Shell RTL for Intel Agilex 7 FPGA (targeting https://github.com/OFS/ofs-f2000x-pl) OFS FIM Common Branch: https://github.com/OFS/ofs-fim-common/releases/tag/ofs-2023.3-2, Tag: https://github.com/OFS/ofs-fim-common/releases/tag/ofs-2023.3-2 Common RTL across all OFS-based platforms AFU Examples Branch: examples-afu , Tag:ofs-examples-https://github.com/OFS/examples-afu/releases/tag/ofs-2023.3-2 Tutorials and simple examples for the Accelerator Functional Unit region (workload region) OPAE SDK Branch: 2.10.0-1, Tag: 2.10.0-1 Open Programmable Acceleration Engine Software Development Kit Kernel Drivers Branch: ofs-2023.3-6.1-3, Tag: ofs-2023.3-6.1-3 OFS specific kernel drivers OPAE Simulation Branch: opae-sim, Tag: 2.10.0-1 Accelerator Simulation Environment for hardware/software co-simulation of your AFU (workload) Intel Quartus Prime Pro Edition Design Software 23.3 Intel\u00ae Quartus\u00ae Prime Pro Edition Linux Software tool for Intel FPGA Development Operating System Ubuntu 22.04 Operating system on which this script has been tested 
                                                                  A download page containing the release and already-compiled FIM binary artifacts that you can use for immediate evaluation on the Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL can be found on the OFS ofs-2023.3-1 official release drop on GitHub.
                                                                   
                                                                   Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#2-introduction-to-ofs-evaluation-script","title":"2 Introduction to OFS Evaluation Script","text":"
                                                                  By following the setup steps and using the OFS evaluation script you can quickly evaluate many features that the OFS framework provides and also leverage this script for your own development.  
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#21-pre-requisites","title":"2.1 Pre-Requisites","text":"
                                                                  This script uses the following set of software tools which should be installed using the directory structure below. Tool versions can vary.
                                                                   
                                                                     
                                                                  • Intel Quartus\u00ae Prime Pro Software
                                                                    •  
                                                                  • Synopsys\u00ae VCS Simulator
                                                                    •  
                                                                  • Siemens\u00ae Questa\u00ae Simulator
                                                                    •  
                                                                   
                                                                  Figure 2-1 Folder Hierarchy for Software Tools
                                                                   
                                                                   
                                                                     
                                                                  1.  
                                                                    You must create a directory named \"ofs-X.X.X\" where the X represents the current release number, for example ofs-2023.3-1. 
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    You must clone the required OFS repositories as per Figure 2-2.  Please refer to the BKC table for locations. Please go [OFS Getting Started User Guide] for the instructions for the BKC installation.
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Once the repositories are cloned, contact your intel representative to receive the eval script evaluation script (ofs_f2000x_eval.sh) and copy it to the ofs-2023.3-1 directory location as shown in the example below:
                                                                     
                                                                    4.  
                                                                   
                                                                  Figure 2-2 Directory Structure for OFS Project
                                                                   
                                                                  ## ofs-2023.3-1\n##  -> examples-afu\n##  -> linux-dfl\n##  -> ofs-f2000x-pl\n##  -> opae-sdk\n##  -> opae-sim\n##  -> ofs_f2000x_eval.sh\n
                                                                   
                                                                     
                                                                  1. Contact your intel representative to receive the README file named (README_ofs-agx7-pcie-attach_eval.txt) and copy it to the ofs-2023.3-1 directory location. The README informs the user which sections to modify in the script prior to building the FIM and running hardware, software and simulation tests.
                                                                    2.  
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#22-f2000x-evaluation-script-modification","title":"2.2 f2000x Evaluation Script modification","text":"
                                                                  To adapt this script to the user environment please follow the instructions below which explains which line numbers to change in the ofs_f2000x_eval.sh script 
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#user-directory-creation","title":"User Directory Creation","text":"
                                                                  The user must create the top-level source directory and then clone the OFS repositories
                                                                   
                                                                  mkdir ofs-2023.3-1\n
                                                                   
                                                                  In the example above we have used ofs-2023.3-1 as the directory name
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#set-up-proxy-server-lines-67-69","title":"Set-Up Proxy Server (lines 67-69)","text":"
                                                                  Please enter the location of your proxy server to allow access to external internet to build software packages.
                                                                   
                                                                  Note: Failing to add proxy server will prevent cloning of repositories and the user will be unable to build the OFS framework.
                                                                   
                                                                  export http_proxy=<user_proxy>\nexport https_proxy=<user_proxy>\nexport no_proxy=<user_proxy>\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#license-files-lines-72-74","title":"License Files (lines 72-74)","text":"
                                                                  Please enter the the license file locations for the following tool variables
                                                                   
                                                                  export LM_LICENSE_FILE=<user_license>\nexport DW_LICENSE_FILE=<user_license>\nexport SNPSLMD_LICENSE_FILE=<user_license>\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#tools-location-line-87-88-89-90","title":"Tools Location (line 87, 88, 89, 90)","text":"
                                                                  Set Location of Quartus, Synopsys and Questasim
                                                                   
                                                                  export QUARTUS_TOOLS_LOCATION=/home\nexport SYNOPSYS_TOOLS_LOCATION=/home\nexport QUESTASIM_TOOLS_LOCATION=/home\nexport ONEAPI_TOOLS_LOCATION=/opt\n
                                                                   
                                                                  In the example above /home is used as the base location of Quartus, Synopsys and Questasim tools, /opt is used for the oneAPI tools
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#quartus-tools-version-line-93","title":"Quartus Tools Version (line 93)","text":"
                                                                  Set version of Quartus
                                                                   
                                                                  export QUARTUS_VERSION=23.3\n
                                                                   
                                                                  In the example above \"23.3\" is used as the Quartus tools version
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#opae-tools-line-108","title":"OPAE Tools (line 108)","text":"
                                                                  change OPAE SDK VERSION
                                                                   
                                                                  export OPAE_SDK_VERSION=2.10.0-1\n
                                                                   
                                                                  In the example above \"2.10.0-1\" is used as the OPAE SDK tools version
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#pcie-bus-number","title":"PCIe (Bus Number)","text":"
                                                                  The Bus number must be entered by the user after installing the hardware in the chosen server, in the example below \"b1\" is the Bus Number for a single card as defined in the evaluation script.
                                                                   
                                                                  export OFS_CARD0_BUS_NUMBER=b1\n
                                                                   
                                                                  The evaluation script uses the bus number as an identifier to interrogate the card. The command below will identify the accelerater card plugged into a server. 
                                                                   
                                                                  lspci | grep acc\n\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                                                                   
                                                                  The result identifies the card as being assigned \"b1\" as the bus number so the entry in the script changes to
                                                                   
                                                                  export OFS_CARD0_BUS_NUMBER=b1\n
                                                                   
                                                                  The user can also run the following command on the ofs_f2000x_eval.sh script to automatically change the bus number to b1 in the ofs_f2000x_eval.sh script.
                                                                   
                                                                  grep -rli 'b1' * | xargs -i@ sed -i 'b1' @
                                                                   
                                                                  if the bus number is 85 for example 
                                                                   
                                                                  85:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n
                                                                   
                                                                  the command to change to 85 in the evaluation script would be
                                                                   
                                                                  grep -rli 'b1' * | xargs -i@ sed -i '85' @
                                                                   
                                                                  The ofs_f2000x_eval.sh script has now been modified to the server set-up and the user can proceed to build, compile and simulate the OFS stack
                                                                   
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#3-f2000x-evaluation-script","title":"3 f2000x Evaluation Script","text":""},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#31-overview","title":"3.1 Overview","text":"
                                                                  The figure below shows a snapshot of the full evaluation script menu showing all 62 options and each one one of 11 sub-menus which focus on different areas of evaluation. Each of these menu options are described in the next section.
                                                                   
                                                                  Figure 3-1 ofs_f2000x_eval.sh Evaluation Menu
                                                                   
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#311-ofs-tools-menu","title":"3.1.1 OFS TOOLS MENU","text":"
                                                                  By selecting \"List of Documentation for OFS f2000x Project,\" a list of links to the latest OFS documentation appears. Note that these links will take you to documentation for the most recent release which may not correspond to the release version you are evaluating. To find the documentation specific to your release, ensure you clone the intel-ofs-docs tag that corresponds to your OFS version.
                                                                   
                                                                  By selecting \"Check Versions of Operating System and Quartus Premier Design Suite\", the tool verifies correct Operating System, Quartus version, kernel parameters, license files and paths to installed software tools.
                                                                   
                                                                   Menu Option Example Output 1 - List of Documentation for OFS f2000x Project  Open FPGA Stack Overview                  Guides you through the setup and build steps to evaluate the OFS solution                 https://ofs.github.io 2 - Check versions of Operating System and Quartus Premier Design Suite (QPDS) Checking Linux release                 Linux version 5.15.92-dfl-66b0076c2c-lts (oe-user@oe-host) (x86_64-ese-linux-gcc (GCC) 11.3.0, GNU ld (GNU Binutils) 2.38.20220708) #1 SMP PREEMPT Wed Feb 8 21:21:27 UTC 2023                  Checking RedHat release                 cat: /etc/redhat-release: No such file or directory                  Checking Ubuntu release                 cat: /etc/lsb-release: No such file or directory                  Checking Kernel parameters                 initrd=\\microcode.cpio LABEL=Boot root=PARTUUID=2beb0add-07bf-4736-bc31-9b60e7b78375  rootfstype=ext4  console=ttyS5,115200 iomem=relaxed intel_iommu=on pci=realloc default_hugepagesz=1G hugepagesz=1G hugepages=4 rootwait console=ttyS0,115200 console=tty0                  Checking Licenses                 LM_LICENSE_FILE is set to port@socket number:port@socket number                 DW_LICENSE_FILE is set to port@socket number:port@socket number                 SNPSLMD_LICENSE_FILE is set to port@socket number:port@socket number                  Checking Tool versions                 QUARTUS_HOME is set to /home/intelFPGA_pro/23.3/quartus                 QUARTUS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus                 IMPORT_IP_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../ip                 QSYS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../qsys/bin                  Checking QPDS Patches                 Quartus Prime Shell                 Version 23.3 Build XXX XX/XX/XXXX Patches X.XX SC Pro Edition                 Copyright (C) XXXX  Intel Corporation. All rights reserved."},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#312-ofs-hardware-menu","title":"3.1.2 OFS HARDWARE MENU","text":"
                                                                  Identifies card by PCIe number, checks power, temperature and current firmware configuration. 
                                                                   
                                                                   Menu Option Example Output 3 - Identify Acceleration Development Platform (OFS) f2000x Hardware via PCIe PCIe card detected as                 15:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)                 Host Server is connected to SINGLE card configuration 4 - Identify the Board Management Controller (BMC) Version and check BMC sensors Intel IPU Platform F2000X-PL                 Board Management Controller NIOS FW version: 1.1.9                 Board Management Controller Build version: 1.1.9                 //****** BMC SENSORS ******//                 Object Id                        : 0xF000000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x5010302EB5E4B14                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 3e46f208-8b49-5899-9e3d-b8b6a4d25a4e 5 - Identify the FPGA Management Engine (FME) Version Intel IPU Platform F2000X-PL                 Board Management Controller NIOS FW version: 1.1.9                 Board Management Controller Build version: 1.1.9                 //****** FME ******//                 Object Id                        : 0xF000000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x5010302EB5E4B14                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 3e46f208-8b49-5899-9e3d-b8b6a4d25a4e                 Boot Page                        : user1                 User1 Image Info                 : 9e3db8b6a4d25a4e3e46f2088b495899                 User2 Image Info                 : 9e3db8b6a4d25a4e3e46f2088b495899                 Factory Image Info               : None 6 - Check Board Power and Temperature Intel IPU Platform F2000X-PL                 Board Management Controller NIOS FW version: 1.1.9                 Board Management Controller Build version: 1.1.9                 //****** POWER ******//                 Object Id                        : 0xF000000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x5010302EB5E4B14                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 3e46f208-8b49-5899-9e3d-b8b6a4d25a4e                 ( 1) QSFP (Primary) Supply Rail Voltage                 : N/A                 etc ......................                  Intel IPU Platform F2000X-PL                 Board Management Controller NIOS FW version: 1.1.9                 Board Management Controller Build version: 1.1.9                 //****** TEMP ******//                 Object Id                        : 0xF000000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x5010302EB5E4B14                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 3e46f208-8b49-5899-9e3d-b8b6a4d25a4e                 ( 1) FPGA FABRIC Remote Digital Temperature#3           : 33.00 Celsius                 etc ...................... 7 - Check Accelerator Port status //****** PORT ******//                 Object Id                        : 0xEF00000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00 8 - Check MAC and PHY status Intel IPU Platform F2000X-PL                 Board Management Controller NIOS FW version: 1.1.9                 Board Management Controller Build version: 1.1.9                 //****** MAC ******//                 Object Id                        : 0xF000000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x5010302EB5E4B14                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 3e46f208-8b49-5899-9e3d-b8b6a4d25a4e                 Number of MACs                   : 255                 mac info is not supported                  Intel IPU Platform F2000X-PL                 Board Management Controller NIOS FW version: 1.1.9                 Board Management Controller Build version: 1.1.9                 //****** PHY ******//                 Object Id                        : 0xF000000                 PCIe s:b:d.f                     : 0000:15:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x17D4                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x5010302EB5E4B14                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 3e46f208-8b49-5899-9e3d-b8b6a4d25a4e                 QSFP0                            : Connected                 QSFP1                            : Connected                  //****** HSSI information ******//                 HSSI version                     : 1.0                 Number of ports                  : 8                 Port0                            :25GbE        DOWN                 Port1                            :25GbE        DOWN                 Port2                            :25GbE        DOWN                 Port3                            :25GbE        DOWN                 Port4                            :25GbE        DOWN                 Port5                            :25GbE        DOWN                 Port6                            :25GbE        DOWN                 Port7                            :25GbE        DOWN"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#313-ofs-pfvf-mux-and-ofss-menu","title":"3.1.3 OFS PF/VF MUX and OFSS MENU","text":"
                                                                  This menu reports the number of PF/VF functions in the reference example and also allows you to reduce the number to 1PF and 1VF to reduce resource utilisation and create larger area for your workload development. This selection is optional and if the user wants to implement the default number of PF's and VF's then option 9, 10 and 11 should not be used.  Additionally the code used to make the PF/VF modification can be leveraged to increase or modify the number of PF/VFs in the existing design within the limits that the PCIe Subsystem supports (8PF/2K VFs)
                                                                   
                                                                   Menu Option Description 9 - Check PF/VF Mux and OFSS Configuration for OFS f2000x Project  This menu selection displays the current configuration of all f2000x ofss files located in the following directory $OFS_ROOTDIR/tools/ofss_config               Check f2000x base config OFSS set up             [ip]             type = ofs              [settings]             platform = f2000x             fim = base_x16             family = agilex             part = AGFC023R25A2E2VR0             device_id = 6100               Check f2000x hssi_2x100 OFSS set up             [ip]             type = hssi              [settings]             output_name = hssi_ss             num_channels = 2             data_rate = 100GCAUI-4               Check f2000x hssi_2x100_caui2 OFSS set up             [ip]             type = hssi              [settings]             output_name = hssi_ss             num_channels = 2             data_rate = 100GAUI-2               Check f2000x hssi_8x10 OFSS set up             [ip]             type = hssi              [settings]             output_name = hssi_ss             num_channels = 8             data_rate = 10GbE              Check f2000x hssi_8x25 OFSS set up             [ip]             type = hssi              [settings]             output_name = hssi_ss             num_channels = 8             data_rate = 25GbE               Check f2000x IOPLL OFSS set up             [ip]             type = iopll              [settings]             output_name = sys_pll             instance_name = iopll_0              [p_clk]             freq = 470              Check f2000x Memory OFSS set up             [ip]             type = memory              [settings]             output_name = mem_ss_fm             preset = f2000x              Check f2000x PCIe Host OFSS set up             [ip]             type = pcie              [settings]             output_name = pcie_ss              [pf0]             bar0_address_width = 21              [pf1]              Check f2000x PCIe SoC OFSS set up             [ip]             type = pcie              [settings]             output_name = soc_pcie_ss              [pf0]             num_vfs = 3             bar0_address_width = 21             vf_bar0_address_width = 21  10 - Modify PF/VF Mux and OFSS Configuration for OFS f2000x Project As an example this menu selection modifies the pcie_host.ofss and pcie_soc.ofss file to 1 PF located in the following directory $OFS_ROOTDIR/tools/ofss_config/pcie             This option also displays the the modified pcie_host.ofss and pcie_soc.ofss files 11 - Build PF/VF Mux and OFSS Configuration for OFS f2000x Project If option 10 is not used then the default number of PF's and VF's is used to build the FIM, if option 10 is selected then only 1 VF is built to reduce logic utilisation"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#314-ofs-fimpr-build-menu","title":"3.1.4 OFS FIM/PR BUILD MENU","text":"
                                                                  Builds FIM, Partial Reconfiguration Region and Remote Signal Tap
                                                                   
                                                                   Menu Option Description 12 - Check OFS software versions for OFS f2000x Project OFS_ROOTDIR is set to /home/user_area/ofs-X.X.X/ofs-f2000x-pl                 OPAE_SDK_REPO_BRANCH is set to release/X.X.X                 OPAE_SDK_ROOT is set to /home/user_area/ofs-X.X.X/ofs-f2000x-pl/../opae-sdk                 LD_LIBRARY_PATH is set to /home/user_area/ofs-X.X.X/ofs-f2000x-pl/../opae-sdk/lib64: 13 - Build FIM for f2000x Hardware This option builds the FIM based on the setting for the $OFS_PLATFORM, $FIM_SHELL environment variable. Check this variable in the following file ofs_f2000x_eval.sh 14 - Check FIM Identification of FIM for f2000x Hardware The FIM is identified by the following file fme-ifc-id.txt located at $OFS_ROOTDIR/$FIM_WORKDIR/syn/syn_top/ 15 - Build Partial Reconfiguration Tree for f2000x Hardware This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development 16 - Build Base FIM Identification(ID) into PR Build Tree template This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU workloads 17 - Build Partial Reconfiguration Tree for f2000x Hardware with Remote Signal Tap This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development for the Remote Signal Tap flow 18 - Build Base FIM Identification(ID) into PR Build Tree template with Remote Signal Tap This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree for Remote Signal Tap to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU workloads"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#315-ofs-hardware-programmingdiagnostic-menu","title":"3.1.5 OFS HARDWARE PROGRAMMING/DIAGNOSTIC MENU","text":"
                                                                  The following submenu allows you to: * Program and check flash  * Perform a remote system update (RSU) of the FPGA image into the FPGA * Bind virtual functions to VFIO PCIe driver  * Run host exerciser (HE) commands such as loopback to test interfaces VFIO PCI driver binding * Read the control and status registers (CSRs) for bound modules that are part of the OFS reference design.
                                                                   
                                                                   Menu Option Description 19 - Program BMC Image into f2000x Hardware The user must place a new BMC flash file in the following directory $OFS_ROOTDIR/bmc_flash_files. Once the user executes this option a new BMC image will be programmed. A remote system upgrade command is initiated to store the new BMC image 20 - Check Boot Area Flash Image from f2000x Hardware This option checks which location area in FLASH the image will boot from, the default is user1              Boot Page : user1 21 - Program FIM Image into user1 area for f2000x Hardware This option programs the FIM image \"ofs_top_page1_unsigned_user1.bin\" into user1 area in flash 22 - Initiate Remote System Upgrade (RSU) from user1 Flash Image into f2000x Hardware This option initiates a Remote System Upgrade and soft reboots the server and re-scans the PCIe bus for the new image to be loaded              2022-11-10 11:26:24,307 - [[pci_address(0000:b1:00.0), pci_id(0x8086, 0xbcce)]] performing RSU operation             2022-11-10 11:26:24,310 - [[pci_address(0000:b0:02.0), pci_id(0x8086, 0x347a)]] removing device from PCIe bus             2022-11-10 11:26:24,357 - waiting 10 seconds for boot             2022-11-10 11:26:34,368 - rescanning PCIe bus: /sys/devices/pci0000:b0/pci_bus/0000:b0             2022-11-10 11:26:35,965 - RSU operation complete 23 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 24 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 25 - Create Virtual Functions (VF) and bind driver to vfio-pci f2000x Hardware This option creates vfio-pci driver binding for the PF's and VF's             Once the VF's have been bound to the driver the user can select menu option 23 to check that the new drivers are bound 26 - Verify FME Interrupts with hello_events The hello_events utility is used to verify FME interrupts. This tool injects FME errors and waits for error interrupts, then clears the errors 27 - Run HE-LB Test This option runs 5 tests              1) checks and generates traffic with the intention of exercising the path from the AFU to the Host at full bandwidth             2) run a loopback throughput test using one cacheline per request             3) run a loopback read test using four cachelines per request             4) run a loopback write test using four cachelines per request             5) run a loopback throughput test using four cachelines per request 28 - Run HE-MEM Test This option runs 2 tests              1) Checking and generating traffic with the intention of exercising the path from FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host             2) run a loopback throughput test using one cacheline per request 29 - Run HE-HSSI Test This option runs 1 test              HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs, and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic              1) Send traffic through the 10G AFU              30 - Run Traffic Generator AFU Test This option runs 3 tests              TG AFU has an OPAE application to access & exercise traffic, targeting a specific bank              1) Run the preconfigured write/read traffic test on channel 0             2) Target channel 1 with a 1MB single-word write only test for 1000 iterations             3) Target channel 2 with 4MB write/read test of max burst length for 10 iterations 31 - Read from CSR (Command and Status Registers) for f2000x Hardware This option reads from the following CSR's             HE-LB Command and Status Register Default Definitions             HE-MEM Command and Status Register Default Definitions             HE-HSSI Command and Status Register Default Definitions"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#316-ofs-hardware-afu-testing-menu","title":"3.1.6  OFS HARDWARE AFU TESTING MENU","text":"
                                                                  This submenu tests partial reconfiguration by building and loading an memory-mapped I/O example AFU/workload, executes software from host, and tests remote signal tap.
                                                                   
                                                                   Menu Option Description 32 - Build and Compile host_chan_mmio example This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS (Green Bit Stream) ready for hardware programming 33 - Execute host_chan_mmio example This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test 34 - Modify host_chan_mmio example to insert Remote Signal Tap This option inserts a pre-defined host_chan_mmio.stp Signal Tap file into the OFS code to allow a user to debug the host_chan_mmio AFU example 35 - Build and Compile host_chan_mmio example with Remote Signal Tap This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS(Green Bit Stream) ready for hardware programming with Remote Signal tap enabled 36 - Execute host_chan_mmio example with Remote Signal Tap This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test. The user must open the Signal Tap window when running the host code to see the transactions in the Signal tap window"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#317-ofs-hardware-afu-bbb-testing-menu","title":"3.1.7 OFS HARDWARE AFU BBB TESTING MENU","text":"
                                                                  This submenu tests partial reconfiguration using a hello_world example AFU/workload, executes sw from host
                                                                   
                                                                   Menu Option Description 37 - Build and Compile hello_world example This option builds the hello_ world example from the following repo $FPGA_BBB_CCI_SRC/samples/tutorial/afu_types/01_pim_ifc/$AFU_BBB_TEST_NAME, where AFU_BBB_NAME=hello_world. This produces a GBS(Green Bit Stream) ready for hardware programming 38 - Execute hello_world example This option builds the host code for hello_world example and programs the GBS file and then executes the test"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#319-ofs-unit-test-project-menu","title":"3.1.9 OFS UNIT TEST PROJECT MENU","text":"
                                                                  Builds, compiles and runs standalone simulation block tests. More unit test examples are found at the following location ofs-f2000x-pl/sim/unit_test 
                                                                   
                                                                   Menu Option Result 39 - Generate Simulation files for Unit Test This option builds the simulation file set for running a unit test simulation 40 - Simulate Unit Test dfh_walker and log waveform This option runs the dfh_walker based on the environment variable \"UNIT_TEST_NAME=dfh_walker\" in the evaluation script. A user can change the test being run by modifying this variable"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#3110-ofs-uvm-project-menu","title":"3.1.10 OFS UVM PROJECT MENU","text":"
                                                                  Builds, compiles and runs full chip simulation tests. The user should execute the options sequentially ie 41,42, 43 and 44
                                                                   
                                                                   Menu Option Description 41 - Check UVM software versions for f2000x Project DESIGNWARE_HOME is set to /home/synopsys/vip_common/vip_Q-2020.03A                 UVM_HOME  is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm                 VCS_HOME is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel                 VERDIR is set to /home/user_area/ofs-X.X.X/fim-f2000x-pl/verification                 VIPDIR is set to /home/user_area/ofs-X.X.X/fim-f2000x-pl/verification 42 - Compile UVM IP This option compiles the UVM IP  43 - Compile UVM RTL and Testbench This option compiles the UVM RTL and Testbench 44 - Simulate UVM dfh_walking_test and log waveform This option runs the dfh_walking test based on the environment variable \"UVM_TEST_NAME=dfh_walking_test\" in the evaluation script. A user can change the test being run by modifying this variable 45 - Simulate all UVM test cases (Regression Mode) This option runs the f2000x regression mode, cycling through all UVM tests defined in verification/tests/test_pkg.svh file"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#3111-ofs-build-all-project-menu","title":"3.1.11 OFS BUILD ALL PROJECT MENU","text":"
                                                                  Builds the complete OFS flow, good for regression testing and overnight builds
                                                                   
                                                                  For this menu a user can run a sequence of tests (compilation, build and simulation) and executes them sequentially. After the script is successfully executed, a set of binary files is produced which a you can use to evaluate your hardware. Log files are also produced which checks whether the tests passed.
                                                                   
                                                                  A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 46 from the main menu the script will execute 24 tests ie (main menu options 2, 9, 12, 13, 14, 15, 16, 17, 18, 32, 34, 35, 37, 39, 40, 41, 42, 43 and 44. These 19 menu options are chosen to build the complete OFS flow covering build, compile and simulation.
                                                                   
                                                                   Menu Option Result 46 - Build and Simulate Complete f2000x Project Generating Log File with date and timestamp                 Log file written to /home/guest/ofs-2.3.1/log_files/f2000x_log_2022_11_10-093649/ofs_f2000x_eval.log"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#definition-of-multi-test-set-up","title":"Definition of Multi-Test Set-up","text":"
                                                                  Menu Option 46 above in the evaluation script can be refined to tailor it to the users need and is principally defined by the variable below
                                                                   
                                                                  MULTI_TEST[A,B]=C
                                                                   
                                                                  where
                                                                   
                                                                  A= Total Number of menu options in script B= Can be changed to a number to select the test order C= Menu Option in Script
                                                                   
                                                                  Example 1 MULTI_TEST[46,0]=2
                                                                   
                                                                  A= 46 is the total number of options in the script B= 0 indicates that this is the first test to be run in the script C= Menu option in Script ie 2- List of Documentation for OFS f2000x Project
                                                                   
                                                                  Example 2 MULTI_TEST[46,0]=2 MULTI_TEST[46,1]=9
                                                                   
                                                                  In the example above two tests are run in order ie 0, and 1 and the following menu options are executed ie 2- List of Documentation for OFS f2000x Project and 9 - Check OFS software versions for OFS f2000x Project
                                                                   
                                                                  The user can also modify the build time by de-selecting options they do not wish to use, see below for a couple of use-case scenarios.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#default-user-case","title":"Default User Case","text":"
                                                                  A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 46 from the main menu the script will execute 24 tests ie (main menu options 2, 9, 12, 13, 14, 15, 16, 17, 18, 32, 34, 35, 37, 39, 40, 41, 42, 43 and 44. All other tests with an \"X\" indicates do not run that test
                                                                   
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#user-case-for-ofs-fimpr-build-menu","title":"User Case for OFS FIM/PR BUILD MENU","text":"
                                                                  In the example below when the user selects option 46 from the main menu the script will only run options from the OFS FIM/PR BUILD MENU (7 options, main menu options 12, 13, 14, 15, 16, 17 and 18). All other tests with an \"X\" indicates do not run that test.
                                                                   
                                                                   
                                                                  "},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#4-f2000x-common-test-scenarios","title":"4 f2000x Common Test Scenarios","text":"
                                                                  This section will describe the most common compile build scenarios if a user wanted to evaluate an acceleration card on their server. The Pre-requisite column indcates the menu comamnds that must be run befere executing the test eg To run Test 5 then a user needs to have run option 13, 15 and 16 before running options 23, 24, 25, 32 and 33.
                                                                   Test Test Scenario Pre-Requisite Menu Option Menu Option Test 1 FIM Build - 13 Test 2 Partial Reconfiguration Build 13 15, 16 Test 3 Program FIM and perform Remote System Upgrade 13 21, 22 Test 4 Bind PF and VF to vfio-pci drivers - 23, 24, 25 Test 5 Build, compile and test AFU on hardware 13, 15, 16 23, 24, 25, 32, 33 Test 6 Build, compile and test AFU Basic Building Blocks on hardware 13, 15, 16 23, 24, 25, 37, 38 Test 7 Build and Simulate Unit Tests - 39, 40 Test 8 Build and Simulate UVM Tests - 42, 43, 44, 45"},{"location":"hw/f2000x/user_guides/ug_eval_ofs/ug_eval_script_ofs_f2000x/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                  Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                   
                                                                  OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/","title":"OFS Getting Started Guide for Intel Agilex 7 SoC Attach FPGAs","text":"
                                                                  Last updated: February 03, 2024 
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#10-about-this-document","title":"1.0 About this Document","text":"
                                                                  The purpose of this document is to help users get started in evaluating the 2023.3 version of the SoC Attach release targeting the Intel\u00ae Infrastructure Processing Unit (Intel\u00ae IPU) Platform F2000X-PL. After reviewing this document, a user shall be able to:
                                                                   
                                                                     
                                                                  • Set up their server environment according to the Best Known Configuration (BKC)
                                                                    •  
                                                                  • Build a Yocto image with the OPAE SDK and Linux DFL Drivers included
                                                                    •  
                                                                  • Load and verify firmware targeting the SR and PR regions of the board, the BMC, and the ICX-D SoC NVMe and BIOS
                                                                    •  
                                                                  • Verify full stack functionality offered by the SoC Attach OFS solution
                                                                    •  
                                                                  • Know where to find additional information on other SoC Attach ingredients
                                                                    •  
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#11-audience","title":"1.1 Audience","text":"
                                                                  The information in this document is intended for customers evaluating the Intel IPU Platform F2000X-PL. The card is an acceleration development platform (ADP) intended to be used as a starting point for evaluation and development. This document will cover key topics related to initial bring up and development of the Intel IPU Platform F2000X-PL, with links for deeper dives on the topics discussed therein.
                                                                   
                                                                  Note: Code command blocks are used throughout the document. Commands that are intended for you to run are preceded with the symbol '$', and comments with '#'. Full command output may not be shown.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-1-terminology","title":"Table 1: Terminology","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to user space."},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-2-related-documentation","title":"Table 2: Related Documentation","text":"Name Location Platform Evaluation Script: Open FPGA Stack for Intel Agilex 7 FPGA link FPGA Interface Manager Technical Reference Manual for Intel Agilex 7 SoC Attach: Open FPGA Stack link Software Reference Manual: Open FPGA Stack link Intel\u00ae FPGA Interface Manager Developer Guide: Intel Agilex 7 SoC Attach: Open FPGA Stack link Intel\u00ae Accelerator Functional Unit Developer Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 FPGAs SoC Attach link Security User Guide: Intel Open FPGA Stack link Virtual machine User Guide: Open FPGA Stack + KVM link Docker User Guide: Intel\u00ae Open FPGA Stack link Release Notes link - under \"Important Notes\" Board Management Controller User Guide v1.1.9 (Pre-Release): Intel IPU Platform F2000X-PL Work with your Intel sales representative for access"},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-3-related-repositories","title":"Table 3: Related Repositories","text":"Name Location Intel-FPGA-BBB https://github.com/OPAE/intel-fpga-bbb.git OFS-PLATFORM-AFU-BBB https://github.com/OFS/ofs-platform-afu-bbb.git, tag: ofs-2023.3-1 AFU-EXAMPLES https://github.com/OFS/examples-afu.git, tag: ofs-2023.2-1 OPAE-SDK https://github.com/OFS/opae-sdk.git, tag: 2.10.0-1 LINUX-DFL https://github.com/OFS/linux-dfl.git, tag: ofs-2023.1-6.15-1 META-OFS https://github.com/OFS/meta-ofs, tag: ofs-2023.1-4"},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-4-software-and-firmware-component-version-summary-for-soc-attach","title":"Table 4: Software and Firmware Component Version Summary for SoC Attach","text":"Component Version Download link (where applicable) Available FIM Version(s) PR Interface ID: 3dac7126-3ce7-5fe8-b629-932096abb09b 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell Host Operating System Ubuntu 22.04 Official Release Page Host OPAE SDK 2.10.0-1 https://github.com/OFS/opae-sdk/releases/tag/2.5.0-3 SoC OS meta-intel-ese Reference Distro 1.0-ESE (kirkstone) 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell SoC Kernel Version 5.15.92-dfl-66b0076c2c-lts 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell SoC OPAE SDK 2.10.0-1 https://github.com/OFS/opae-sdk/releases/tag/2.5.0-3 SoC Linux DFL ofs-2023.1-6.15-1 https://github.com/OFS/linux-dfl/releases/tag/ofs-2023.1-5.15-1 SoC BMC and RTL 1.2.4 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell SoC BIOS 0ACRH608_REL 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell 
                                                                  Not all components shown in Table 4 will have an update available upon release. The OPAE SDK and Linux DFL software stacks are incorporated into a Yocto image and do not need to be downloaded separately. Updates required for the 2023.3 OFS SoC Attach Release for Intel IPU Platform F2000X-PL are shown under Table 9 in section 2.0 Updating the Intel IPU Platform F2000X-PL.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#12-server-requirements","title":"1.2 Server Requirements","text":"
                                                                  The following requirements must be met when purchasing a server to support the Intel IPU Platform F2000X-PL.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#121-host-server-specifications","title":"1.2.1 Host Server Specifications","text":"
                                                                  The host server must meet the following minimal specifications:
                                                                   
                                                                     
                                                                  • The server platform must contain at least 64GiB of RAM to to compile the Yocto or FIM images
                                                                    •  
                                                                  • If using the server platform to build a Yocto image, it is recommended to have at least 200 GB of free storage space
                                                                    •  
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#122-host-bios","title":"1.2.2 Host BIOS","text":"
                                                                  Te Host BIOS settings known to work with the Intel IPU Platform F2000X-PL:
                                                                   
                                                                     
                                                                  • PCIe slot width must be x16
                                                                    •  
                                                                  • PCIe slot speed must be 4
                                                                    •  
                                                                  • PCIe slot must have iommu enabled
                                                                    •  
                                                                  • Intel VT for Directed I/O (VT-d) must be enabled
                                                                    •  
                                                                   
                                                                  Specific BIOS paths are not listed here, as they can differ between BIOS vendors and versions.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#123-host-server-kernel-and-grub-configuration","title":"1.2.3 Host Server Kernel and GRUB Configuration","text":"
                                                                  While many host Linux kernel and OS distributions may work with this design, only the following configuration(s) have been tested:
                                                                   
                                                                     
                                                                  • Ubuntu 22.04, 6.1-lts
                                                                    •  
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#13-server-forced-air-cooling","title":"1.3 Server Forced Air Cooling","text":"
                                                                  The Intel IPU Platform F2000X-PL is a high-performance processing card with a passive heat sink to dissipate device heat and must be installed in a server with sufficient forced airflow cooling to keep all devices operating below maximum temperature. The table below lists the thermal terms and descriptions used in thermal analysis.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-5-thermal-terms-and-descriptions","title":"Table 5: Thermal Terms and Descriptions","text":"Term Description Cubic Feet per Minute (CFM) Volumetric airflow rate, in cubic feet per minute, of air passing through faceplate. Tj FPGA Junction Temperature TLA Local Ambient temperature. Temperature of forced air as it enters the Intel IPU Platform F2000X-PL. \u00a0 Note: In many systems, this is higher than the room ambient due to  heating effects of chassis components. 
                                                                  Note: The FPGA junction temperature must not exceed 100\u00b0C. The case temperature of the QSFP modules must meet the module vendor's specification.
                                                                   
                                                                  Note: The table below provides the thermal targets for which the Intel IPU Platform F2000X-PL was designed. As a card manufacturer, you must qualify your own production cards.
                                                                   
                                                                  The maximum card inlet air temperatures must support continuous operation under the worst-case power scenario of 150W TDP.
                                                                   
                                                                  The airflow requirements for optimal heat sink performance at minimum is characteristic of CAT 3 servers or PCIe SIG Level 7 thermal profiles, in both, forward & reverse flow, see figure below:
                                                                   
                                                                   
                                                                   
                                                                  As the Intel IPU Platform F2000X-PL is a development platform, it is not integrated into the server baseband management controller closed loop cooling control. It is strongly recommended that you set your server's fan settings to run constantly at 100% with the server chassis lid closed to prevent unwanted Intel IPU Platform F2000X-PL thermal shutdown.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#14-external-connections","title":"1.4 External Connections","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-1-external-connections","title":"Figure 1: External Connections","text":"
                                                                  The items listed Table 6 in are known to work for external connectivity. Specific links are given for convenience, other products may be used but have not been tested.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-6-external-connection-cables","title":"Table 6: External Connection Cables","text":"Item Part Number Link to source RS-232 to USB Adapter DTECH FTDI USB to TTL Serial Adapter, 3 m USB to TTL Serial USB to Ethernet Adapter, Aluminum 3 Port USB 3.0 Teknet USB Hub with Ethernet adapter Flash Drive, 64 GB or larger SanDisk QSFP DAC Cable \u00a0FS.com Generic 2m 100G QSP28 Passive Direct Attach Copper QSFP28 DAC (optional) Intel FPGA Download Cable II PL-USB2-BLASTER USB-Blaster II"},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#15-preparing-the-intel-ipu-platform-f2000x-pl-for-installation","title":"1.5 Preparing the Intel IPU Platform F2000X-PL for Installation","text":"
                                                                  Turn the board over to back side and remove the Kapton tape covering switches SW2 and SW3 and make sure the switches are set as shown in Figure 1.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-7-switch-settings","title":"Table 7: Switch Settings","text":"Name Value SW3.1 off SW3.2 off SW3.2 on SW3.2 off SW2 off"},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-2-board-switch-settings","title":"Figure 2: Board Switch Settings","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#151-usb-to-serial-adapter","title":"1.5.1 USB to Serial Adapter","text":"
                                                                  The Intel IPU Platform F2000X-PL has a serial UART for access located on back edge of the board. This connection is useful for making BIOS and boot settings and for monitoring the SoC. In most servers, you will need to remove a riser card and route the USB to serial cable and (optional) Intel FPGA USB Blaster through an unused PCIe slot above or below where the IPU is installed. See Figure 3 for an example of cable routing.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-3-cable-routing","title":"Figure 3: Cable Routing","text":"
                                                                  The USB to serial connection is shown in Figure 4 where the White wire is TXD, Black wire is ground and Green wire is RXD.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-4-usb-to-serial-adapter-connection","title":"Figure 4: USB to Serial Adapter connection","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#152-ipu-jtag","title":"1.5.2 IPU JTAG","text":"
                                                                  The Intel IPU Platform F2000X-PL provides a 10 pin JTAG header for FPGA and Cyclone 10 Board Management Controller development work using a Intel FPGA Download Cable II. This JTAG connection is optional for initial bring-up but is useful for manual image reprogramming and debug. See Figure 5 noting the orientation of the connection. The orientation of the USB Blaster II requires careful installation in a PCIe bay that has additional room in the adjacent bay. This may require you to either install the board over the PSU of the server, or to temporarily remove an adjacent riser while programming.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-5-usb-blaster-ii-connection","title":"Figure 5: USB Blaster II Connection","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-6-usb-blaster-ii-installation","title":"Figure 6: USB Blaster II Installation","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#153-power","title":"1.5.3 Power","text":"
                                                                  The Intel IPU Platform F2000X-PL must receive power from both the 12 V and 3.3V PCIe slot and the 12 V Auxiliary 2\u00d74 power connector. The board does not power up if any of the 12 V and 3.3 V PCIe slot, or 12 V Auxiliary power sources are disconnected.
                                                                   
                                                                  PCIe specifications define 12 V Auxiliary power connector pin assignment. The Intel IPU Platform F2000X-PL implements an 8-position right angle (R/A) through-hole PCB header assembly on the top right side of the board as depicted in the picture below.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-7-12v-pcie-aux-connector-location","title":"Figure 7: 12V PCIe AUX Connector Location","text":"
                                                                  Refer the table below for pinout details.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-8-12v-2x3-aux-connector-pin-out","title":"Table 8: 12V (2x3) AUX Connector Pin Out","text":"Pin Description 1 +12V 2 +12V 3 +12V 4 Sense 1 5 Ground 6 Sense 0 7 Ground 8 Ground 
                                                                  See Auxiliary power connection in Figure 8.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-8-auxiliary-power-connection","title":"Figure 8: Auxiliary Power Connection","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#154-usb-hub-connection","title":"1.5.4 USB Hub Connection","text":"
                                                                  The USB Hub is connected to the USB type A connector on the front panel. Additionally, attach a network connected Ethernet connection to the USB hub. See Figure 9.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#figure-9-usb-hub-connection","title":"Figure 9: USB Hub Connection","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#155-creating-a-bootable-usb-flash-drive-for-the-soc","title":"1.5.5 Creating a Bootable USB Flash Drive for the SoC","text":"
                                                                  Connect your flash drive to an available Linux host. In this section the USB will set up to be used as a secondary boot source for the SoC and will also be used to update the NVMe from which the ICX-D SoC boots in section 2.1 Updating the F2000X-PL ICX-D SoC NVMe.
                                                                   
                                                                  You will load the latest pre-compiled Yocto core-image-minimal WIC image into USB flash. This image can be downloaded from 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell, under assets, or compiled from meta-ofs. Compilation is discussed in section 4.0 Compiling a Custom Yocto SoC Image.
                                                                   
                                                                     
                                                                  1.  
                                                                    Insert a 64 GB or larger USB Flash Drive into the USB slot of a computer/server you can use to format the drive. The following instructions assume you are using some flavor of GNU+Linux. You need sudo access privileges on this machine.
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    In a terminal window, find the device name of the USB flash drive and unmount the device:
                                                                     
                                                                    $ lsblk\n\nNAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT\nsda 8:0 0 1.8T 0 disk\n\u251c\u2500sda1 8:1 0 600M 0 part /boot/efi\n\u251c\u2500sda2 8:2 0 1G 0 part /boot\n\u2514\u2500sda3 8:3 0 1.8T 0 part\n\u251c\u2500rhel-root 253:0 0 50G 0 lvm /\n\u251c\u2500rhel-swap 253:1 0 4G 0 lvm \\[SWAP\\]\n\u2514\u2500rhel-home 253:6 0 1.7T 0 lvm /home\nsdb 8:16 0 447.1G 0 disk\n\u251c\u2500sdb1 8:17 0 600M 0 part\n\u251c\u2500sdb2 8:18 0 1G 0 part\n\u2514\u2500sdb3 8:19 0 445.5G 0 part\n\u251c\u2500fedora_localhost\\--live-swap 253:2 0 4G 0 lvm\n\u251c\u2500fedora_localhost\\--live-home 253:3 0 301G 0 lvm\n\u251c\u2500fedora_localhost\\--live-root 253:4 0 70G 0 lvm\n\u2514\u2500fedora_localhost\\--live-centos_root 253:5 0 70.5G 0 lvm\nsdd 8:48 1 57.3G 0 disk\n\u2514\u2500sdd1 8:49 1 57.3G 0 part\n
                                                                     
                                                                    In the above example, the 64 GB USB Flash device is designated sdd. Note, your device file name may be different. You are looking for an entry that matches the size of   your USB Flash. You can also check the output of dmesg after manually plugging in your USB Flash device to view the name the kernel has given it in an auto-generated event.
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Unmount the USB flash (if not already unmounted).
                                                                     
                                                                    $ sudo umount /dev/sdd1\numount: /dev/sdd1: not mounted.\n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Download the Yocto WIC image. To prevent boot errors that may arise when using the same boot image loaded in both USB flash and on-board NVMe, you must choose an older version of the Yocto WIC to load onto the USB. Browse the tagged Yocto release images on GitHub and choose the second newest release image as the temporary USB boot target. In this example we will use the OFS 2023.1 RC3 release. You will also need to download the newest Yocto release image (core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic.gz).
                                                                     
                                                                    # Download an older version of the Yocto release image to use as the USB boot target, version 2023.1 RC3 shown here\n$ wget https://github.com/OFS/meta-ofs/releases/download/ofs-2023.1-3/core-image-full-cmdline-intel-corei7-64-20230505161810.rootfs.wic.gz\n$ wget https://github.com/OFS/meta-ofs/releases/download/ofs-2023.1-3/core-image-full-cmdline-intel-corei7-64-20230505161810.rootfs.wic.gz.sha256\n# Verify the checksum of the downloaded image\n$ sha256sum -c https://github.com/OFS/meta-ofs/releases/download/ofs-2023.1-3/core-image-full-cmdline-intel-corei7-64-20230505161810.rootfs.wic.gz.sha256\n# Uncompress the package\n$ gzip -d core-image-full-cmdline-intel-corei7-64-20230505161810.rootfs.wic.gz\n\n\n\n# Download the most recent Yocto release image, which will overwrite on-board NVMe\n$ wget https://github.com/OFS/meta-ofs/releases/download/ofs-2023.1-4/core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic\n$ wget https://github.com/OFS/meta-ofs/releases/download/ofs-2023.1-4/core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic.sha256\n# Verify the checksum of the downloaded image\nsha256sum -c core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic.sha256\n# Uncompress the package\n$ gzip -d core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic\n
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Copy core-image-full-cmdline-intel-corei7-64-20230505161810.rootfs.wic to the USB flash. This process may take several minutes.
                                                                     
                                                                    $ sudo dd if=core-image-full-cmdline-intel-corei7-64-20230505161810.rootfs.wic of=/dev/sdd1 bs=512k status=progress conv=sync \n$ sgdisk -e /dev/sdd\n
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    Create a partition to store the Yocto image, which will be used to overwrite on-board NVMe as the default boot target.
                                                                     
                                                                        $ sudo fdisk /dev/sdd\n    Command (m for help): p\n    Command (m for help): n\n    Partition number (4-128, default 4): <<press enter>>\n    First sector (14617908-125045390, default 14618624): <<press enter>>\n    Last sector, +/-sectors or +/-size{K,M,G,T,P} (14618624-125045390, default\n    125045390): <<press enter>>\n    Created a new partition 4 of type 'Linux filesystem' and of size 92 GiB.\n    Command (m for help): p\nCommand (m for help): w\n
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    Verify USB flash is partitioned.
                                                                     
                                                                    $ lsblk\n    NAME        MAJ:MIN RM   SIZE RO TYPE MOUNTPOINTS\n    sdd           8:0    1 114.6G  0 disk\n    |-sdd1        8:1    1   300M  0 part\n    |-sdd2        8:2    1  22.1G  0 part\n    |-sdd3        8:3    1    44M  0 part\n    `-sdd4        8:4    1  92.2G  0 part\n
                                                                     
                                                                    8.  
                                                                  8.  
                                                                    Format the new partition.
                                                                     
                                                                    $ mkfs -t ext4 /dev/sdd4\n$ mount /dev/sda4 /mnt\n
                                                                     
                                                                    9.  
                                                                  9.  
                                                                    Copy compressed core-image-minimal WIC into /mnt.
                                                                     
                                                                    $ cp core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic /mnt\n
                                                                     
                                                                    10.  
                                                                   
                                                                  Remove the USB flash from the Linux computer and install the flash drive in the USB hub attached to the Intel IPU Platform F2000X-PL.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#20-updating-the-intel-ipu-platform-f2000x-pl","title":"2.0 Updating the Intel IPU Platform F2000X-PL","text":"
                                                                  Every Intel IPU Platform F2000X-PL ships with pre-programmed firmware for the FPGA user1, user2, and factory images, the Cyclone 10 BMC RTL and FW, the SoC NVMe, and the SoC BIOS. The combination of FW images in Table 4 compose the official 2023.3 OFS SoC Attach Release for Intel IPU Platform F2000X-PL. Upon initial receipt of the board from manufacturing you will need to update two of these regions of flash to conform with the best known configuration for SoC Attach.  As shown in Table 9, not all devices require firmware updates. To instruct users in the process of updating on-board Intel IPU Platform F2000X-PL firmware, examples are provided in this guide illustrating the firmware update process for all devices.  
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-9-intel-ipu-platform-f2000x-pl-fw-components","title":"Table 9: Intel IPU Platform F2000X-PL FW Components","text":"HW Component File Name Version Update Required (Yes/No) Download Location FPGA SR Image1 user1: ofs_top_page1_unsigned_user1.binuser2: ofs_top_page2_unsigned_user2.binfactory: ofs_top_page0_unsigned_factory.bin PR Interface ID: 3dac7126-3ce7-5fe8-b629-932096abb09b Yes 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell FPGA PR IMAGE2 ofs_pr_afu.green_region_unsigned.gbs N/A Yes Compiled with FIM ICX-D NVMe core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic N/A Yes 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell BMC RTL and FW AC_BMC_RSU_user_retail_1.2.4_unsigned.rsu 1.2.4 No 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell BIOS Version 0ACRH608_REL.BIN 0ACRH608_REL Yes 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell 
                                                                  If a component does not have a required update, it will not have an entry on 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell.
                                                                   
                                                                  1To check the PR Interface ID of the currently programmed FIM and the BMC RTL and FW version, use fpgainfo fme from the SoC. 2Must be programmed if using AFU-enabled exercisers, not required otherwise.
                                                                   
                                                                  $ fpgainfo fme\nIntel IPU Platform F2000X-PL\n**Board Management Controller NIOS FW version: 1.2.4**\n**Board Management Controller Build version: 1.2.4**\n//****** FME ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\n**Bitstream Id                     : 360572756485162679**\nBitstream Version                : 5.0.1\n**Pr Interface Id                  : 3dac7126-3ce7-5fe8-b629-932096abb09b**\nBoot Page                        : user1\nUser1 Image Info                 : 9e3db8b6a4d25a4e3e46f2088b495899\nUser2 Image Info                 : None\nFactory Image Info               : None\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#21-updating-the-f2000x-pl-icx-d-soc-nvme","title":"2.1 Updating the F2000X-PL ICX-D SoC NVMe","text":"
                                                                  The Intel IPU Platform F2000X-PL ships with a Yocto image pre-programmed in NVMe, which is not the same as the SoC Attach OFS image that we will be using. The latter provides only the OPAE SDK and Linux DFL drivers and is fully open source. This section will show how you can use an attached USB drive to load a new image into flash. You will use a serial terminal to install the new image - Minicom and PuTTy terminal emulators have both been tested. minicom is used for demonstration purposes as the serial terminal to access the ICX-D SoC UART connection in this section. Information on compiling your own Yocto image for use with the Intel IPU Platform F2000X-PL is discussed in section 4.0 Compiling a Custom Yocto SoC Image.
                                                                   
                                                                  Note: Username and password for the default SoC NVMe boot image are \"root\" and \"root@123\".
                                                                   
                                                                     
                                                                  1.  
                                                                    First, make sure to complete the steps in section 1.5.5 Creating a Bootable USB Flash Drive for the SoC, and attach the USB drive either directly into the rear of the Intel IPU Platform F2000X-PL, or into a USB Hub that itself is connected to the board.
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Ensure your Minicom terminal settings match those shown below. You must direct Minicom to the USB device created in /dev associated with your RS-232 to USB Adapter cable. This cable must be attached to a server that is separate from the one housing your Intel IPU Platform F2000X-PL. Check the server logs in dmesg to figure out which device is associated with your cable: [    7.637291] usb 1-4: FTDI USB Serial Device converter now attached to ttyUSB0. In this example the special character file /dev/ttyUSB0 is associated with our cable, and can be connected to using the following command: sudo minicom --color=on -D /dev/ttyUSB0.
                                                                     
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Change the SoC boot order to boot from USB first. Reboot the server. From your     serial Minicom terminal, watch your screen and     press 'ESC' key to go into BIOS setup mode. Once BIOS setup comes     up as shown below, click the right arrow key six times to move from     'Main' set up menu to 'Boot' setup:
                                                                     
                                                                    Main setup menu:
                                                                     
                                                                     
                                                                     
                                                                    Your order of boot devices may differ. You need to move the USB flash    up to Boot Option #1 by first using the down arrow key to highlight    the USB device then use '+' key to move the USB device to #1 as    shown below:
                                                                     
                                                                     
                                                                    Press 'F4' to save and exit.
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    You will boot into Yocto automatically. Log in with username root and an empty password using Minicom. Take note of the IP address of the board, you can use this to log in without needing the serial cable.
                                                                     
                                                                     
                                                                    Verify that you have booted from the USB and not the on-board NVMe lsblk -no pkname $(findmnt -n / | awk '{ print $2 }'). You should see a device matching /dev/sd*, and not nvme*n*p*. If you see nvme*n*p*, then review the previous steps.
                                                                     
                                                                    Record the IP address of the SoC at this time ip -4 addr. This will be used to log in remotely using SSH.
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Check that 4 partitions created in 1.5.5 Creating a Bootable USB Flash Drive for the SoC are visible to the SoC in /dev/sd*:
                                                                     
                                                                    $ lsblk -l\nroot@intel-corei7-64:/mnt# lsblk -l\nNAME      MAJ:MIN RM   SIZE RO TYPE MOUNTPOINTS\nsda         8:0    1 117.2G  0 disk\nsda1        8:1    1  38.5M  0 part /boot\nsda2        8:2    1    20G  0 part /\nsda3        8:3    1    44M  0 part [SWAP]\nsda4        8:4    1  97.1G  0 part /mnt\nnvme0n1   259:0    0  59.6G  0 disk\nnvme0n1p1 259:1    0   300M  0 part\nnvme0n1p2 259:2    0  22.1G  0 part\nnvme0n1p3 259:3    0    44M  0 part\n
                                                                     
                                                                    Mount partition 4, and cd into it.
                                                                     
                                                                    $ mount /dev/sda4 /mnt`\n$ cd /mnt\n
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    Install the Yocto release image in the SoC NVMe.
                                                                     
                                                                    $ dd if=core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic of=/dev/nvme0n1 bs=1M status=progress conv=sync\n$ sync\n$ sgdisk -e /dev/nvme0n1\n
                                                                     
                                                                    The transfer from USB to NVMe will take several minutes.
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    Reboot the SoC and update the SoC BIOS to boot from NVMe. Follow steps 2 and 3 from this section again, and this time move the NVME back to the front of the boot order. The NVMe is named UEFI OS (PCIe SSD) by the BIOS. Press F4 to save and exit.
                                                                     
                                                                    You can use wget to retrieve a new version of the Yocto release image from meta-ofs once the SoC's network connection is up. Use wget to copy the image to the SoC over the network under /mnt. You may need to delete previous Yocto images to save on space: $ wget --no-check-certificate --user <Git username> --ask-password https://github.com/OFS/meta-ofs/releases/download/ofs-2023.3-2/core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic.gz. Uncompress the newly retrieved file: gzip -d core-image-full-cmdline-intel-corei7-64-20231129215057.rootfs.wic.gz. This may take several minutes.
                                                                     
                                                                    8.  
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#211-setting-the-time","title":"2.1.1 Setting the Time","text":"
                                                                     
                                                                  1.  
                                                                    Use Linux command to set system time using format: date --set=\"STRING\".
                                                                     
                                                                    date -s \"26 APRIL 2023 18:00:00\"\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Set HWCLOCK to current system time:
                                                                     
                                                                    hwclock --systohc\n
                                                                     
                                                                    Verify time is set properly
                                                                     
                                                                    date\n...\nhwclock --show\n...\n
                                                                     
                                                                    3.  
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#30-updating-the-intel-ipu-platform-f2000x-pl","title":"3.0 Updating the Intel IPU Platform F2000X-PL","text":""},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#31-preparing-the-intel-ipu-platform-f2000x-pl-soc-for-updates","title":"3.1 Preparing the Intel IPU Platform F2000X-PL SoC for Updates","text":"
                                                                  Updating the Intel IPU Platform F2000X-PL firmware often requires reboots of the SoC or reconfiguration of the FPGA region. If there are processes connected from the host to the SoC that do not expect the downtime to occur, or if the host is not tolerant to a surprise PCie link down, the following instructions can be used to properly orchestrate updates with the host when reboots occur.
                                                                   
                                                                  Note: Intel IPU Platform F2000X-PL FPGA and BMC updates are initiated by commands issued on the IPU SoC. Issue the following commands from the host to remove any processes that would be impacted by this update. The instructions on properly removing the IPU from PCIe bus require the OPAE SDK to be installed on the host. Refer to section 6.0 Setting up the Host for this process.
                                                                   
                                                                     
                                                                  1.  
                                                                    From a host terminal shell, find PCIe Bus/Device/Function (BDF) address of your Intel IPU Platform F2000X-PL. Run the command lspci | grep bcce to print all boards with a DID that matches bcce.
                                                                     
                                                                    $ lspci | grep bcce\n31:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\n31:00.1 Processing accelerators: Intel Corporation Device bcce (rev 01)\n# In this example, 31:00.0 is the proper PCIe BDF of our device\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Shut down all VMs and software applications attached to any PFs/VFs on the host
                                                                     
                                                                    3.  
                                                                  3. Issue the command sudo pci_device <<PCIe BDF>> unplug on the host to remove the PCIe device from the PCIe bus
                                                                    4.  
                                                                  4. Shut down all software applications on the SoC accessing non-management PFs/VFs
                                                                    5.  
                                                                  5. Issue your update command on the SoC, which will cause an SoC reboot and surprise PCIe link down on the host (ex. reboot, rsu bmc/bmcimg/fpga)
                                                                    6.  
                                                                  6. Once you have completed all firmware updates, you may restart application software on the SoC
                                                                    7.  
                                                                  7. Issue command sudo pci_device <<PCIe BDF>> plug on the host to rescan the PCIe bus and rebind the device to its native driver
                                                                    8.  
                                                                  8. Restart software applications on the host
                                                                    9.  
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#32-updating-fim-bmc-and-afu-with-fpgasupdate","title":"3.2 Updating FIM, BMC and AFU with fpgasupdate","text":"
                                                                  The fpgasupdate tool updates the Intel\u00ae C10 10 BMC image and firmware, root entry hash, FPGA Static Region (SR) and user image (PR). The fpgasupdate will only accept images that have been formatted using PACSign. If a root entry hash has been programmed onto the board, then the image will also need to be signed using the correct keys. Please refer to the Security User Guide: Intel Open FPGA Stack for information on creating signed images and on programming and managing the root entry hash. This repository requires special permissions to access - please reach out to your Intel representative to request. The fpgasupdate tool is used to program images into flash memory. The rsu tool is used to configure the FPGA/BMC with an image that is already stored in flash memory, or to switch between user1 and user2 images. All images received from an official Intel release will be \"unsigned\", as described in the Security User Guide: Intel Open FPGA Stack.
                                                                   
                                                                  Note: 'Unsigned' in this context means the image has passed through PACsign and has had the proper security blocks prepended using a set of 'dummy' keys. FIMs with image signing enabled will require all programmable images to pass through PACsign even if the currently programmed FIM/BMC do not require specific keys to authenticate.
                                                                   
                                                                  There are two regions of flash you may store FIM images for general usage, and one backup region. These locations are referred to as user1, user2, and factory. The factory region is not programmed by default and can only be updated once keys have been provisioned. The BMC FW and RTL will come pre-programmed with version 1.1.9.
                                                                   
                                                                  Updating the FIM from the SoC requires the SoC be running a Yocto image that includes the OPAE SDK and Linux DFL drivers. Updating the FIM using fpgasupdate also requires an OFS enabled FIM to be configured on the F2000X-PL, which it will ship with from manufacturing. You need to transfer any update files to the SoC over SSH. The OPAE SDK utility fpgasupdate will be used to update all ofthe board's programmable firmware . This utility will accept files of the form *.rsu, *.bin, and *.gbs, provided the proper security data blocks have been prepended by PACSign. The default configuration the IPU platform ships with will match the below:
                                                                   
                                                                  $ fpgainfo fme\nIntel IPU Platform F2000X-PL\n**Board Management Controller NIOS FW version: 1.2.4**\n**Board Management Controller Build version: 1.2.4**\n//****** FME ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\n**Bitstream Id                     : 360572756485162679**\nBitstream Version                : 5.0.1\n**Pr Interface Id                  : 3dac7126-3ce7-5fe8-b629-932096abb09b**\nBoot Page                        : user1\nUser1 Image Info                 : 9e3db8b6a4d25a4e3e46f2088b495899\nUser2 Image Info                 : None\nFactory Image Info               : None\n
                                                                   
                                                                  To load a new update image, you need to pass the IPU's PCIe BDF and the file name to fpgasupdate on the SoC. The below example will update the user1 image in flash:
                                                                   
                                                                  $ fpgasupdate ofs_top_page1_unsigned_user1.bin 15:00.0\n
                                                                   
                                                                  After loading an update image, rsu fpga/bmc/bmcimg can be used to reload the firmware and apply the change, as discussed below. An RSU of the BMC will always cause a reload of both the BMC and FPGA images.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#33-loading-images-with-rsu","title":"3.3 Loading images with rsu","text":"
                                                                  RSU performs a Remote System Update operation on an Intel IPU Platform F2000X-PL, given its PCIe address. An rsu operation sends an instruction to the device to trigger a power cycle of the card only if run with bmcimg. rsu will force reconfiguration from flash for either the BMC or FPGA. PCIe Advanced Error Reporting (AER) is temporarily disabled for the card when RSU is in progress
                                                                   
                                                                  The Intel IPU Platform F2000X-PL contains two regions of flash you may store FIM images. These locations are referred to as user1 and user2. After an image has been programmed to either of these regions in flash using fpgasupdate, you may perform an rsu to reconfigure the Agilex 7 FPGA with the new image stored in flash. This operation will indicate to the BMC which region to configure the FPGA device from after power-on.
                                                                   
                                                                  If the factory image has been updated, Intel strongly recommends you immediately RSU to the factory image to ensure the image is functional.
                                                                   
                                                                  You can determine which region of flash was used to configure their FPGA device using the command fpgainfo fme and looking at the row labelled Boot Page.
                                                                   
                                                                  When loading a new FPGA SR image, use the command rsu fpga. When loading a new BMC image, use the command rsu bmc. When using the RSU command, you may select which image will be configured to the selected device. For example, when performing an RSU for the Intel Agilex 7 FPGA, you may select to configure the user1, user2, or factory image. When performing an RSU for the C10 BMC, you may select to configure the user or factory image. You may also use RSU to reconfigure the SDM on devices that support it. The RSU command sends an instruction to the BMC to reconfigure the selected device from an image stored in flash.
                                                                   
                                                                  $ rsu fpga --page=user1 15:00.0\n
                                                                   
                                                                  Useage:
                                                                   
                                                                  rsu bmc --page=(user|factory) [PCIE_ADDR]\nrsu fpga --page=(user1|user2|factory) [PCIE_ADDR]\nrsu sdm [PCIE_ADDR]\n
                                                                   
                                                                  You can use RSU to change which page in memory the FPGA will boot from by default.
                                                                   
                                                                  Synopsis:
                                                                   
                                                                  rsu fpgadefault --page=(user1|user2|factory) --fallback=<csv> 15:00.0\n
                                                                   
                                                                  Use to set the default FPGA boot sequence. The --page option determines the primary FPGA boot image. The --fallback option allows a comma-separated list of values to specify fallback images.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#34-updating-the-icx-d-soc-bios","title":"3.4 Updating the ICX-D SoC BIOS","text":"
                                                                  The ICX-D SoC NVMe comes pre-programmed with BIOS v7 (0ACRH007). This version will need to be replaced with 0ACRH608_REL. BIOS update files come in the form 0ACRH\\<\\<version>>.bin, and can be downloaded on 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell. This update process is in-band, and requires you to download and install a BIOS UEFI utility from AMI called \"APTIO V AMI Firmware Update Utility\", available here. This package will install a utility in the UEFI shell called AfuEfix64.efi which will be used to overwrite the ICX-D BIOS.
                                                                   
                                                                     
                                                                  1.  
                                                                    Check your BIOS Version. Reboot the SoC and wait for the BIOS version to be shown. In this example, the BIOS will need to be updated.
                                                                     
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Download both the ICX-D SoC Update image and APTIO V AMI Firmware Update Utility. Unzip the BIOS update image and locate your BIOS update binary. Unzip Aptio_V_AMI_Firmware_Update_Utility.zip, and then navigate to Aptio_V_AMI_Firmware_Update_Utility\\afu\\afuefi\\64 and unzip AfuEfi64.zip. The file we need from this package is called AfuEfix64.efi.
                                                                     
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Copy both files over to the SoC into /boot/EFI using the SoC's IP.
                                                                     
                                                                    $ scp 0ACRH608_REL.BIN root@XX.XX.XX.XX:/boot/EFI\n$ scp AfuEfix64.efi root@XX.XX.XX.XX:/boot/EFI\n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Reboot the SoC from a TTY Serial terminal. Watch your screen and press 'ESC' key to go into BIOS setup mode. Select 'Built-in EFI Shell'.
                                                                     
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    At EFI prompt enter the following:
                                                                     
                                                                    Shell> FS0:\nFS0:> cd EFI\nFS0:\\EFI\\> AfuEfix64.efi 0ACRH608_REL.BIN /p /n /b\n
                                                                     
                                                                    Press 'E'. When the update has completed type 'reset -w' to reboot.
                                                                     
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    Watch your screen and press 'ESC' key to go into BIOS setup mode. Verify BIOS version matches expectation.
                                                                     
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    Click the right arrow key six times to move from 'Main' set up menu to 'Boot' setup. Select NVMe as the primary boot source.
                                                                     
                                                                     
                                                                    8.  
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#40-compiling-a-custom-yocto-soc-image","title":"4.0 Compiling a Custom Yocto SoC Image","text":"
                                                                  Current Yocto image architecture for SoC Attach is based off of the IOTG Yoct-based ESE BSP and substitutes the Linux DFL kernel including the latest DFL drivers for FPGA devices along with the OPAE SDK user space. The image targets x86_64 SoC FPGA devices but should boot on most UEFI-based machines. The source code and documentation for this image is hosted on the meta-ofs repository.
                                                                   
                                                                  Build requirements exceed 100 GiB of disk space, depending on which image is built. As a reference point, on a system with two Intel(R) Xeon(R) E5-2699 v4 for a total of 44 CPU cores, the initial, non-incremental build takes less than an hour of wall time.
                                                                   
                                                                  The repo tool is needed to clone the various Yocto layer repositories used in this example.
                                                                   
                                                                  Note: If you are behind a firewall that prevents you from accessing references using the git:// protocol, you can use the following to redirect Git to use the corresponding https repositories for Yocto only: git config --global url.https://git.yoctoproject.org/.insteadOf git://git.yoctoproject.org/
                                                                   
                                                                  To compile the image as-is, use the following steps (as provided in meta-ofs):
                                                                   
                                                                     
                                                                  1.  
                                                                    Create and initialize the source directory.
                                                                     
                                                                    mkdir ofs-yocto && cd ofs-yocto\ngit clone --recurse-submodules --shallow-submodules https://github.com/OFS/meta-ofs\ncd meta-ofs\ngit checkout tags/ofs-2023.3-2\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Build packages and create an image.
                                                                     
                                                                    cd examples/iotg-yocto-ese\nTEMPLATECONF=$PWD/conf source openembedded-core/oe-init-build-env build\nbitbake mc:x86-2022-minimal:core-image-full-cmdline\n
                                                                     
                                                                    3.  
                                                                   
                                                                  The resulting GPT disk image is available in uncompressed (.wic) and compressed form (.wic.gz) in meta-ofs/examples/iotg-yocto-ese/build/tmp-x86-2021-minimal-glibc/deploy/images/intel-corei7-64/. With no changes the uncompressed image size is ~21 GB.
                                                                   
                                                                  The image type core-image-full-cmdline includes the familiar GNU core utilities, as opposed to core-image-minimal which uses BusyBox instead.
                                                                   
                                                                  The example build configuration files under build/conf/ are symlinked from examples/iotg-yocto-ese/. To customise the image, start by modifying local.conf and bblayers.conf.
                                                                   
                                                                  The uncompressed Yocto image can be loaded onto a flash drive as discussed in section 1.5.5 Creating a Bootable USB Flash Drive for the SoC and written to NVMe as the default boot target for the SoC as demonstrated in section 2.1 Updating the F2000X-PL ICX-D SoC NVMe.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#50-verifying-the-icx-d-soc-opae-sdk","title":"5.0 Verifying the ICX-D SoC OPAE SDK","text":"
                                                                  The reference SoC Attach FIM and unaltered FIM compilations contain Host Exerciser Modules (HEMs). These are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Full supported functionality of the HEMs is documented in the host_exerciser opae.io GitHub page. SoC Attach supports HEMs run both with and without an AFU image programmed into the board's one supported PR region. This image is available on the offial SoC Attach GitHub Page, and is programmed using fpgasupdate as shown in section 3.2. A few select examples run from the SoC and their expected results will be shown below.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#51-checking-telemetry-with-fpgainfo","title":"5.1 Checking Telemetry with fpgainfo","text":"
                                                                  The fpgainfo utility displays FPGA information derived from sysfs files.
                                                                   
                                                                  The command argument is one of the following: errors, power, temp, port, fme, bmc, phy, mac, and security. Some commands may also have other arguments or options that control their behavior.
                                                                   
                                                                  For systems with multiple FPGA devices, you can specify the BDF to limit the output to the FPGA resource with the corresponding PCIe configuration. If not specified, information displays for all resources for the given command.
                                                                   
                                                                  An example output for fpgainfo fme is shown below. Your IDs may not match what is shown here:
                                                                   
                                                                  $ fpgainfo fme\nIntel IPU Platform F2000X-PL\nBoard Management Controller NIOS FW version: 1.1.9\nBoard Management Controller Build version: 1.1.9\n//****** FME ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:15:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x17D4\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x50103023DFBFC8E\nBitstream Version                : 5.0.1\nPr Interface Id                  : bf74e494-ad12-5509-98ab-4105d27979f3\nBoot Page                        : user1\nUser1 Image Info                 : 98ab4105d27979f3bf74e494ad125509\nUser2 Image Info                 : None\nFactory Image Info               : None\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#52-host-exercisers","title":"5.2 Host Exercisers","text":"
                                                                  Of these five tests listed below, the first three do not require an AFU be loaded into the board's PR region. They exercise data paths that pass exclusively through the FIM. The latter three tests exercise data through the AFU data path, and require SoC Attach release AFU Image to be configured using fpgasupdate.
                                                                   
                                                                     
                                                                  • Run HE-MEM with 2 cachelines per request in mem mode, exercising the FPGA's connection to DDR. No AFU required.
                                                                    Note: If you see the error message Allocate SRC Buffer, Test mem(1): FAIL, then you may need to manually allocate 2MiB Hugepages using the following: echo 20 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
                                                                     
                                                                    # Create VF device\n$ pci_device <PCIe Bus>00.0 vf 3\n# Bind VF0 to vfio-pci\n$ opaei.io init -d <PCIe Bus>:00.1 <username>:<username>\n# Check for HE-MEM Accelerator GUID 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n$ fpgainfo port -B <PCIe Bus>:00.0\n# Run desired HE-MEM test(s)\n$ host_exerciser --cls cl_1 --mode lpbk mem\nstarting test run, count of 1\nAPI version: 2\nAFU clock: 470 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Host Exerciser numPendEmifReads: 0\n    Host Exerciser numPendEmifWrites: 0\n    Number of clocks: 9948\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1024\n    Bandwidth: 3.096 GB/s\n    Test mem(1): PASS\n
                                                                     
                                                                    •  
                                                                   
                                                                     
                                                                  • Generate traffic with HE-HSSI. No AFU required.
                                                                    # Create VF device\n$ pci_device <PCIe Bus>00.0 vf 3\n# Bind VF2 to vfio-pci\n$ opaei.io init -d <PCIe Bus>:00.2 <username>:<username>\n# Check for HE-HSSI Accelerator GUID 823c334c-98bf-11ea-bb37-0242ac130002\n$ fpgainfo port -B <PCIe Bus>:00.0\n# Show number of configured ports\n$ fpgainfo phy | grep Port\n# Generate traffic for specific port number\n$ hssi --pci-address <PCIe Bus>:00.2 hssi_10g --num-packets 100 --port <port number>\n10G loopback test\n  Tx/Rx port: 1\n  Tx port: 1\n  Rx port: 1\n  eth_loopback: on\n  he_loopback: none\n  num_packets: 100\n  packet_length: 64\n  src_address: 11:22:33:44:55:66\n    (bits): 0x665544332211\n  dest_address: 77:88:99:aa:bb:cc\n    (bits): 0xccbbaa998877\n  random_length: fixed\n  random_payload: incremental\n...\n
                                                                     
                                                                    This command will generate a log file in the directory is was run from. Check TxPackets and RxPackets for any loss. Two more supported HSSI commands are hssistats, which provides MAC statistics, and hssimac, which provides maximum TX and RX frame size.
                                                                     
                                                                    •  
                                                                   
                                                                     
                                                                  • Test memory traffic generation using MEM-TG. This will exercise and test available memory channels with a configurable memory pattern. Does not require an AFU image.
                                                                    # Create VF device\n$ pci_device <PCIe Bus>00.0 vf 3\n# Bind VF2 to vfio-pci\n$ opaei.io init -d <PCIe Bus>:00.3 <username>:<username>\n# Check for MEM-TG Accelerator GUID 4dadea34-2c78-48cb-a3dc-5b831f5cecbb\n$ fpgainfo port -B <PCIe Bus>:00.0\n# Example MEM-TG Test\n$  mem_tg --loops 500 -w 1000 -r 0 -b 0x1 --stride 0x1 -m 0 tg_test\n
                                                                     
                                                                    •  
                                                                   
                                                                     
                                                                  • HE-LPBK is designed to demo how AFUs move data between host memory and the FPGA. Will check latency, MMIO latency, MMIO bandwidth, and PCIe bandwidth.  LPBK workload requires the SoC Attach AFU be loaded into the board's PR slot.
                                                                    # Create VF device\n$ pci_device <PCIe Bus>00.0 vf 3\n# Bind VF1 to vfio-pci\n$ opaei.io init -d <PCIe Bus>:00.1 <username>:<username>\n# Check for LPBK GUID 56e203e9-864f-49a7-b94b-12284c31e02b\n$ fpgainfo port -B <PCIe Bus>:00.0\n# Example loopback test\n$  host_exerciser --mode lpbk lpbk\n
                                                                     
                                                                    •  
                                                                   
                                                                     
                                                                  • Exercise He-HSSI subsystem from the AFU. This test will generate and receieve packets from any of the 8 available ports. This HSSI workload requires the SoC Attach AFU be loaded into the board's PR slot.
                                                                    # Create VF device\n$ pci_device <PCIe Bus>00.0 vf 3\n# Bind VF6 to vfio-pci\n$ opaei.io init -d <PCIe Bus>:00.2 <username>:<username>\n# Check for HSSI GUID 823c334c-98bf-11ea-bb37-0242ac130002\n$ fpgainfo port -B <PCIe Bus>:00.0\n# Geneate traffic for specific port\n$  hssi --pci-address <bus_num>:00.6 hssi_10g --num-packets 100 --port <port_num>\n
                                                                     
                                                                    This command will generate a log file in the directory is was run from. Check TxPackets and RxPackets for any loss. Two more supported HSSI commands are hssistats, which provides MAC statistics, and hssimac, which provides maximum TX and RX frame size.
                                                                     
                                                                    •  
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#53-additional-opae-sdk-utilities","title":"5.3 Additional OPAE SDK Utilities","text":"
                                                                  This section will discuss OPAE SDK utilities that were not covered by previous sections. These commands are all available on the ICX-D SoC Yocto image by default. A full description and syntax breakdown for each command is located on the official OPAE SDK github.io repo.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#table-6-opae-sdk-utilities","title":"Table 6: OPAE SDK Utilities","text":"Utility Description fpgasupdate 3.2 Updating FIM, BMC and AFU with fpgasupdate rsu Section 3.3 Loading images with rsu host_exerciser Section 5.2 Host Exercisers hssi Section 5.2 Host Exercisers hssistats Section 5.2 Host Exercisers hssimac Section 5.2 Host Exercisers mem_tg Section 5.2 Host Exercisers usrclk userclk tool is used to set high and low clock frequency to an AFU. mmlink Remote signaltap is software tool used for debug RTL (AFU), effectively a signal trace capability that Quartus places into a green bitstream. Remote Signal Tap provides access the RST part of the Port MMIO space, and then runs the remote protocol on top. opaevfio The opaevfio command enables the binding/unbinding of a PCIe device to/from the vfio-pci device driver. See https://kernel.org/doc/Documentation/vfio.txt for a description of vfio-pci. opae.io An interactive Python environment packaged on top of libopaevfio.so, which provides user space access to PCIe devices via the vfio-pci driver. The main feature of opae.io is its built-in Python command interpreter, along with some Python bindings that provide a means to access Configuration and Status Registers (CSRs) that reside on the PCIe device. bitstreaminfo Prints bitstream metadata. fpgaconf Lower level programming utility that is called automatically by fpgasupdate. fpgaconf writes accelerator configuration bitstreams (also referred to as \"green bitstreams\") to an FPGA device recognized by OPAE. In the process, it also checks the green bitstream file for compatibility with the targeted FPGA and its current infrastructure bitstream (the \"blue bistream\"). fpgad Periodically monitors/reports the error status reflected in the device driver's error status sysfs files. Establishes the channel by which events are communicated to the OPAE application. Programs a NULL bitstream in response to AP6 event. fpgad is required to be running before API calls fpgaRegisterEvent and fpgaUnregisterEvent will succeed."},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#60-setting-up-the-host","title":"6.0 Setting up the Host","text":"
                                                                  External SoC Attach supports testing Host to FPGA latency, MMIO latency, and MMIO bandwidth. This testing is accomplished using the utility host_exerciser on the host, whichis included as a part of OPAE. This section will cover the installation and verification flow for a host interacting with the SoC Attach workload.
                                                                   
                                                                  Review Section 1.2 Server Requirements for a list of changes required on the host to support an Intel IPU Platform F2000X-PL and for a list of supported OS distributions. Installation will require an active internet connetion to resolve dependencies.
                                                                   
                                                                  The following software checks may be run on the host to verify the FPGA has been detected and has auto-negotatiated the correct PCIe link width/speed. These commands do not require any packages to be installed. We are using PCIe BDF b1:00.0 as an example address.
                                                                   
                                                                  # Check that the board has enumerated successfully.\n# Your PCIe BDF may differ from what is shown below.\n$ lspci | grep accel\nb1:00.0 Processing accelerators: Intel Corporation Device bcce \nb1:00.1 Processing accelerators: Intel Corporation Device bcce\n\n# Check PCIe link status and speed. Width should be x16, and speed whould be 16GT/s\nsudo lspci -s b1:00.0 -vvv | grep LnkSta | grep -o -P 'Width.{0,4}'\nsudo lspci -s b1:00.0 -vvv | grep LnkSta | grep -o -P 'Speed.{0,7}'\n\nsudo lspci -s b1:00.1 -vvv | grep LnkSta | grep -o -P 'Width.{0,4}'\nsudo lspci -s b1:00.1 -vvv | grep LnkSta | grep -o -P 'Speed.{0,7}'\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#61-installing-the-opae-sdk-on-the-host","title":"6.1 Installing the OPAE SDK On the Host","text":"
                                                                  The OPAE SDK software stack sits in user space on top of the OFS kernel drivers. It is a common software infrastructure layer that simplifies and streamlines integration of programmable accelerators such as FPGAs into software applications and environments. OPAE consists of a set of drivers, user-space libraries, and tools to discover, enumerate, share, query, access, manipulate, and reconfigure programmable accelerators. OPAE is designed to support a layered, common programming model across different platforms and devices. To learn more about OPAE, its documentation, code samples, an explanation of the available tools, and an overview of the software architecture, visit the opae.io page.
                                                                   
                                                                  The OPAE SDK source code is contained within a single GitHub repository hosted at the OPAE Github. This repository is open source and does not require any permissions to access.
                                                                   
                                                                     
                                                                  1.  
                                                                    Before Installing the newest version of OPAE you must remove any prior OPAE framework installations.
                                                                     
                                                                    $ sudo apt-get remove opae*\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    The following system and Python3 package dependencies must be installed before OPAE may be built.
                                                                     
                                                                    $ sudo apt-get install bison flex git ssh pandoc devscripts debhelper cmake python3-dev libjson-c-dev uuid-dev libhwloc-dev doxygen libtbb-dev libncurses-dev libspdlog-dev libspdlog1 python3-pip libedit-dev pkg-config libcli11-dev libssl-dev dkms libelf-dev gawk openssl libudev-dev libpci-dev  libiberty-dev autoconf llvm\n\n$ python3 -m pip install setuptools pybind11 jsonschema\n
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Clone the OPAE SDK repo. In this example we will use the top level directory OFS for our package installs.
                                                                     
                                                                    $ mkdir OFS && cd OFS\n$ git init\n$ git clone https://github.com/OFS/opae-sdk.git\n$ cd opae-sdk\n$ git checkout tags/2.10.0-1\n\n# Verifying we are on the correct release tag\n$ git describe --tags\n2.10.0-1\n
                                                                     
                                                                    4.  
                                                                  4.  
                                                                    Navigate to the automatic DEB package build script location and execute.
                                                                     
                                                                    $ cd OFS/opae-sdk/packaging/opae/deb \n$ ./create\n\n# Verify all packages are present\n$ ls | grep opae.*.deb\nopae_2.5.0-1_amd64.deb\nopae-dbgsym_2.5.0-1_amd64.ddeb\nopae-devel_2.5.0-1_amd64.deb\nopae-devel-dbgsym_2.5.0-1_amd64.ddeb\nopae-extra-tools_2.5.0-1_amd64.deb\nopae-extra-tools-dbgsym_2.5.0-1_amd64.ddeb\n
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Install your newly built OPAE SDK packages.
                                                                     
                                                                    $ cd OFS/opae-sdk/packaging/opae/deb\n$ sudo dpkg -i opae*.deb\n
                                                                     
                                                                    The OPAE tools installed on the host are identical to those installed on the SoC as shown in sections 5.0 through 5.3. A set of pre-compiled OPAE SDK artifacts are included in this release. These can be downloaded from 2023.3 OFS Release for Intel Agilex 7 SoC Attach Reference Shell and installed without building/configuring.
                                                                     
                                                                    $ tar xf opae-*.tar.gz\n$ sudo dpkg -i opae*.deb\n
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    Enable iommu=on, pcie=realloc, and set hugepages as host kernel parameters.
                                                                     
                                                                    # Check if parameters are already enabled\n$ cat /proc/cmdline\n
                                                                     
                                                                    If you do not see intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200, then add them manually.
                                                                     
                                                                    $ sudo vim /etc/default/grub\n# Edit the value for GRUB_CMDLINE_LINUX, add the values at the end of the variable inside of the double quotes. Example: GRUB_CMDLINE_LINUX=\"crashkernel=auto resume=/dev/mapper/rhel00-swap rd.lvm.lv=rhel00/root rd.lvm.lv=rhel00/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\"\n# Save your changes, then apply them with the following\n$ sudo grub2-mkconfig\n$ sudo reboot now\n
                                                                     
                                                                    7.  
                                                                   
                                                                  After rebooting, check that proc/cmdline reflects your changes.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#62-verifying-the-soc-attach-solution-on-the-host","title":"6.2 Verifying the SoC Attach Solution on the Host","text":"
                                                                  The SoC Attach workload supports testing MMIO HW and Latency and PCIe BW and latency out of box. Execution of the host_exerciser binary on the host requires the OPAE SDK to be installed as shown in section 6.1 Installing the OPAE SDK On the Host. You will also need to have a proper SoC Attach FIM configured on your board as shown in section 3.2 Updating FIM, BMC and AFU with fpgasupdate.
                                                                   
                                                                     
                                                                  1.  
                                                                    Initialize PF attached to HSSI LPBK GUID with vfio-pci driver.
                                                                     
                                                                    $ sudo opae.io init -d 0000:b1:00.0 <username>:<username>\n$ sudo opae.io init -d 0000:b1:00.1 <username>:<username>\n
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Run host_exerciser loopback tests (only lpbk is supported). There are more methods of operation than are shown below - read the HE help message for more information.
                                                                     
                                                                    3.  
                                                                   
                                                                  # Example lpbk tests.\n$ sudo host_exerciser lpbk\n$ sudo host_exerciser --mode lpbk lpbk\n$ sudo host_exerciser --cls cl_4  lpbk\n$ sudo host_exerciser --perf true --cls cl_4  lpbk\n# Number of cachelines per request 4.\n$ sudo host_exerciser --mode lpbk --cls cl_4 lpbk\n$ sudo host_exerciser --perf true --mode lpbk --cls cl_4 lpbk\n# vNumber of cachelines per request 4.\n$ sudo host_exerciser --mode read --cls cl_4 lpbk\n$ sudo host_exerciser --perf true --mode read --cls cl_4 lpbk\n# Number of cachelines per request 4.\n$ sudo host_exerciser --mode write --cls cl_4 lpbk\n$ sudo host_exerciser --perf true --mode write --cls cl_4 lpbk\n# Number of cachelines per request 4.\n$ sudo host_exerciser --mode trput --cls cl_4 lpbk\n$ sudo host_exerciser --perf true --mode trput --cls cl_4 lpbk\n# Enable interleave requests in throughput mode\n$ sudo host_exerciser --mode trput --interleave 2 lpbk\n$ sudo host_exerciser --perf true --mode trput --interleave 2 lpbk\n#with delay option.\n$ sudo host_exerciser --mode read --delay true lpbk\n$ sudo host_exerciser --mode write --delay true lpbk\n# Test all modes of operation\n$ host_exerciser --testall=true lpbk\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#63-fpga-device-access-permissions","title":"6.3 FPGA Device Access Permissions","text":"
                                                                  Access to FPGA accelerators and devices is controlled using file access permissions on the Intel\u00ae FPGA device files, /dev/dfl-fme.* and /dev/dfl-port.*, as well as to the files reachable through /sys/class/fpga_region/.
                                                                   
                                                                  In order to allow regular (non-root) users to access accelerators, you need to grant them read and write permissions on /dev/dfl-port.* (with * denoting the respective socket, i.e. 0 or 1). E.g.:
                                                                   
                                                                  sudo chmod a+rw /dev/dfl-port.0\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#64-memlock-limit","title":"6.4 Memlock limit","text":"
                                                                  Depending on the requirements of your application, you may also want to increase the maximum amount of memory a user process is allowed to lock. The exact way to do this depends on your Linux distribution.
                                                                   
                                                                  You can check the current memlock limit using
                                                                   
                                                                  ulimit -l\n
                                                                   
                                                                  A way to permanently remove the limit for locked memory for a regular user is to add the following lines to your /etc/security/limits.conf:
                                                                   
                                                                  user1    hard   memlock           unlimited\nuser1    soft   memlock           unlimited\n
                                                                   
                                                                  This removes the limit on locked memory for user user1. To remove it for all users, you can replace user1 with *:
                                                                   
                                                                  *    hard   memlock           unlimited\n*    soft   memlock           unlimited\n
                                                                   
                                                                  Note that settings in the /etc/security/limits.conf file don't apply to services. To increase the locked memory limit for a service you need to modify the application's systemd service file and add the line:
                                                                   
                                                                  [Service]\nLimitMEMLOCK=infinity\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_qs_ofs_f2000x/ug_qs_ofs_f2000x/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                  Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                   
                                                                  OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/","title":"Simulation User Guide: Open FPGA Stack for Intel Agilex 7 SoC Attach FPGAs","text":"Term Description AER Advanced Error Reporting, The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. AFU Accelerator Functional Unit, Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance. Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region BBB Basic Building Block, Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. BKC Best Known Configuration, The exact hardware configuration Intel has optimized and validated the solution against. BMC Board Management Controller, Acts as the Root of Trust (RoT) on the Intel FPGA PAC platform. Supports features such as power sequence management and board monitoring through on-board sensors. CSR Command/status registers (CSR) and software interface, OFS uses a defined set of CSR's to expose the functionality of the FPGA to the host software. DFL Device Feature List, A concept inherited from OFS. The DFL drivers provide support for FPGA devices that are designed to support the Device Feature List. The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration. FIM FPGA Interface Manager, Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FME FPGA Management Engine, Provides a way to manage the platform and enable acceleration functions on the platform. HEM Host Exerciser Module, Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Intel VT-d Intel Virtualization Technology for Directed I/O, Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. IOCTL Input/Output Control, System calls used to manipulate underlying device parameters of special files. JTAG Joint Test Action Group, Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. MMIO Memory Mapped Input/Output, Users may map and access both control registers and system memory buffers with accelerators. OFS Open FPGA Stack, A modular collection of hardware platform components, open source software, and broad ecosystem support that provides a standard and scalable model for AFU and software developers to optimize and reuse their designs. OPAE SDK Open Programmable Acceleration Engine Software Development Kit, A collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. PAC Programmable Acceleration Card: FPGA based Accelerator card PIM Platform Interface Manager, An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. PR Partial Reconfiguration, The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. In the context of Intel FPGA PAC, a PR bitstream refers to an Intel FPGA PAC AFU. Refer to Partial Reconfiguration support page. RSU Remote System Update, A Remote System Update operation sends an instruction to the Intel FPGA PAC D5005 device that triggers a power cycle of the card only, forcing reconfiguration. SR-IOV Single-Root Input-Output Virtualization, Allows the isolation of PCI Express resources for manageability and performance. TB Testbench, Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. UVM Universal Verification Methodology, A modular, reusable, and scalable testbench structure via an API framework. VFIO Virtual Function Input/Output, An IOMMU/device agnostic framework for exposing direct device access to user space."},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#1-overview","title":"1 Overview","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#11-about-this-document","title":"1.1 About this Document","text":"
                                                                  This document serves as a set-up and user guide for the UVM simulation tool using OFS. After reviewing the document, you will be able to:
                                                                   
                                                                     
                                                                  • Set-up the UVM verification tool suite
                                                                    •  
                                                                   
                                                                     
                                                                  • Run pre-existing UVM unit tests and also create new UVM tests for your design
                                                                    •  
                                                                   
                                                                   Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#2-introduction-to-uvm","title":"2 Introduction to UVM","text":"
                                                                  The Functional Verification Environment for OFS is UVM (Universal Verification Methodology) compliant and provides configurable setup for verifying various FIM features in simulation.
                                                                   
                                                                  The purpose of this document is to demonstrate a full chip level and unit level UVM based verification environment for the current base shell FIM architecture as well as providing examples to extend the setup for different OFS variants by reusing the existing architecture and infrastructure for UVM based verification.
                                                                   
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#3-universal-testbench-architecture","title":"3 Universal Testbench Architecture","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#31-overview","title":"3.1 Overview","text":"
                                                                  The main idea behind UVM is to develop modular, reusable, and a scalable testbench structure by providing an API framework that can be deployed across multiple projects.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#32-core-verification-concepts","title":"3.2 Core Verification Concepts","text":"
                                                                  The following is the list of verification components that will be used to design a UVM testbench architecture:
                                                                   
                                                                  \u2022   Sequencer \u2022   Driver \u2022   Monitor \u2022   Scoreboard \u2022   Virtual Sequencer \u2022   Interface \u2022   Verification Environment \u2022   TOP Testbench
                                                                   
                                                                  Figure 1 provides the general UVM Testbench and the verification components involved in the top-level architecture.
                                                                   
                                                                   
                                                                  Figure 1 Typical UVM Testbench
                                                                   
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4-ofs-testbench-architecture","title":"4 OFS Testbench Architecture","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#41-overview","title":"4.1 Overview","text":"
                                                                  OFS (Open FPGA Stack) provides a UVM (Universal Verification Methodology) environment for the FIM with a modular, reusable, and scalable testbench structure via an API framework.
                                                                   
                                                                  The framework consists of a FIM Testbench which is UVM compliant and integrates third party VIPs from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this Testbench. UVM RAL(Register Abstraction Level) is used for CSR (Command and Status Registers) verification. 
                                                                   
                                                                  The qualified verification IPs will help to detect incorrect protocol behavior, help to focus on FIM features and accelerate the verification process.
                                                                   
                                                                  Verification components include:
                                                                   
                                                                  \u2022 FIM monitor to detect correct design behavior\n\u2022 FIM assertions for signal level integrity testing\n\u2022 Arm AMBA Arm\u00ae AMBA\u00ae 4 AXI4 scoreboards to check data integrity\n\u2022 FIM coverage to collect functional data\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#42-base-fim-dut","title":"4.2 Base FIM DUT","text":"
                                                                  The hardware architecture of an Intel Agilex 7 FIM is based on the OFS hardware architecture. The following is the list of features and subsystems supported in the base shell.
                                                                   
                                                                  \u2022   PCIe Subsystem\n\u2022   HSSI Subsystem\n\u2022   Memory Subsystem\n\u2022   HPS Subsystem\n\u2022   FME\n\u2022   AFU with PR support\n\u2022   QSFP Controllers\n\u2022   PMCI Controller, MCTP\n
                                                                   
                                                                   
                                                                  Figure 2 DUT Base Shell Diagram
                                                                   
                                                                  Figure 2 shows the high level architecture of an Intel Agilex 7 Base Shell. It has a Gen4x16, 100G Ethernet Datapath in a 2x4x25G configuration. The Intel Agilex 7 Base Shell is a shell that will enable a user to build other shell variants for a custom configuration. For the f2000x board there is one shell variant
                                                                   
                                                                  base_x16
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#43-full-chip-level-verification-archiecture-for-fim","title":"4.3 Full Chip Level Verification Archiecture for FIM","text":"
                                                                  Figure 3 shows a graphical representation a full chip testbench that includes major RTL blocks depicted in a OFS Intel Agilex 7 based UVM environment 
                                                                   
                                                                   
                                                                  Figure 3 OFS FIM Testbench
                                                                   
                                                                  The major connection is the interface between the Xeon CPU and the FPGA where the PCIe Verification IP is connected to PCIe Subsystem. Therefore, as a full chip simulation environment, PCIe host VIP is the sole VIP/BFM used. PCIe host VIP connects to PCIe device which resides in FPGA in serial mode.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#431-testbench-components","title":"4.3.1 Testbench components","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4311-tb-top","title":"4.3.1.1 TB TOP","text":"
                                                                  TOP is the top level testbench and consists of a FIM DUT instance and top-level UVM Verification Environment instance. It binds RTL inputs with the verification environmnet interfaces to drive stimulus. It also has clock generation and reset driving logic.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4312-fim-verification-environment","title":"4.3.1.2 FIM Verification Environment","text":"
                                                                  This is the top most verification environment class and consists of the protocol specific PCI Express and AXI UVM environment VIP instances, Monitors, Scoreboards, Assertions, Functional coverage Modules and other verification components. It instantiates Virtual sequencers to control stimuli for FIM traffic from different sequencers of the VIPs.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4313-synopsys-vips","title":"4.3.1.3 Synopsys VIPs","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#43131-pci-vip-as-host","title":"4.3.1.3.1 PCI VIP as Host","text":"
                                                                  This is Synopsys Licensed PCI Express Gen4 VIP and acts as Root Port. The PCI Express link is connected to the DUT using TX-RX lanes. Agent is an active component and includes a sequencer to generate TLPs, Driver to drive it on a PCI Express link and Monitor to check the protocol correctness.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#43132-axi-streaming-vip-monitors","title":"4.3.1.3.2 AXI-Streaming VIP Monitors","text":"
                                                                  This is Synopsys Licensed AXI streaming interface Verification IP used as a Passive Agent to monitor AXI-ST links at various points. Please refer to Figure 3 to see all the AXI-ST monitors connected to different modules.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#43133-axi4-memory-mapped-vip-monitors","title":"4.3.1.3.3 AXI4-Memory Mapped VIP Monitors","text":"
                                                                  This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 memory mapped interface Verification IP used in passive mode to observe memory requests. For each master-slave pair, the verification environment has a VIP instantiated in passive mode for monitoring protocol violations and performing data integrity checks.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#43134-axi4-lite-vip-monitors","title":"4.3.1.3.4 AXI4-Lite VIP Monitors","text":"
                                                                  This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4 Lite interface Verification IP used in passive mode to observe MMIO requests. For each master-slave pair, the verification environment has a VIP instantiated in passive mode for monitoring protocol violations and perfoming data integrity checks. Please refer to Figure 3 to see all the Arm\u00ae AMBA\u00ae 4 AXI4-Lite monitors connected to different modules.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#43135-axi4-lite-vip-as-pmci-master","title":"4.3.1.3.5 AXI4-Lite VIP as PMCI Master","text":"
                                                                  This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 Lite interface Verification IP used to drive and observe MMIO requests as PMCI master. For each master-slave pair, the verification environment has a VIP instantiated in active mode and includes a sequencer to generate MMIO requests, driver to drive these requests on AXI-4 Lite interface to BPF and a monitor for observing protocol violations and data integrity checks. 
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4314-axi4-s-scoreboard","title":"4.3.1.4 AXI4-S Scoreboard","text":"
                                                                  The Arm\u00ae AMBA\u00ae 4 AXI4-S scoreboard checks data integrity of source and sink components. It has input transactions from VIP monitors and a TLP to AXI converter for PCIe TLP packets. It makes sure the valid TLPs or AXI transactions are not missed while traversing from Host to AFU and reverse. The scoreboard will be disabled for error testing especially for invalid TLP requests and UR responses.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4315-virtual-sequencer","title":"4.3.1.5 Virtual Sequencer","text":"
                                                                  The virtual sequencer is the top-level sequencer which controls Enumeration, MMIO Requests, downstream and Upstream traffic as well as HSSI and Memory transactions. It makes sure that the transactions are ordered correctly as per the design specification while running a UVM test simulation.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4316-fim-monitor","title":"4.3.1.6 FIM Monitor","text":"
                                                                  The FIM monitor is used for checking the correctness of a specific FIM feature. As an example, a user can add interrupt related checks, error generation and clearing checks, Protocol checker impacts etc. in this component. 
                                                                   
                                                                  This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4317-fim-assertions","title":"4.3.1.7 FIM Assertions","text":"
                                                                  The assertion component is used for checking the signal level integrity of a specific FIM feature or behavior. As an example, we can add interrupt signal triggering and clear assertions, Error detection to error register bit assertions, protocol checker and clearing logic, FLR behavior assertion etc. in this top-level module. There are alos assertions to make sure the inputs are driven correctly to catch errors in a users simulation. 
                                                                   
                                                                  This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#4318-fim-functional-coverage","title":"4.3.1.8 FIM Functional Coverage","text":"
                                                                  The FIM functional coverage component will have the functional coverage of each feature like interrupts, CSR's, MMIO requests, Byte align transactions, error reporting etc. to demonstrate a variety of FIM features. This will help us to find holes in the design as well as wide verification coverage to make sure thorough testing of a design. It will also provide a user what the FIM is capable of and how much code coverage they are testing.
                                                                   
                                                                  This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                                                                   
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#5-uvm-verification-set-up","title":"5 UVM Verification set-up","text":"
                                                                  To run the tutorial steps in this guide requires the following development environment:
                                                                   Item Version Intel Quartus Prime Pro Intel Quartus Prime Pro 23.3 Simulator Synopsys VCS P-2019.06-SP2-5 or newer for UVM simulation of top level FIM Simulator (Questasim) Questasim 2023.2 or newer for UVM simulation of top level FIM"},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#51-uvm-prerequisite","title":"5.1 UVM Prerequisite","text":"
                                                                  Retrieve OFS repositories.
                                                                   
                                                                  The OFS FIM source code is included in the OTCShare GitHub repository. Create a new directory to use as a clean starting point to store the retrieved files.  The following is a short description of each repository, followed by the git commands for cloning.  The instructions section uses the HTTPS git method for cloning repositories.  Cloning the repo using the HTTPS method requires a personal access token.  Please see this blog post for information about obtaining a personal access token Token authentication requirements for Git operations.
                                                                   
                                                                  Navigate to location for storage of OFS source, create the top-level source directory and clone OFS repositories.
                                                                   
                                                                  $ mkdir release/ofs-2023.3\n$ cd release/ofs-2023.3\n$ export OFS_BUILD_ROOT=$PWD\n$ git clone --branch --recurse-submodules https://github.com/ofs-f2000x-pl.git\n\nCloning into 'ofs-f2000x-pl'...'\nUsername for 'https://github.com': <<Enter your git hub username>>\nPassword for 'https://<<Your username>>': <<Enter your personal access token>>\nremote: Enumerating objects:  ....\n...\n...\nResolving deltas  ..., done.\n\n$ cd ofs-f2000x-pl\n$ git checkout tags/ofs-2023.3-1\n
                                                                   
                                                                  Verify that the correct tag/branch have been checked out        
                                                                   
                                                                  $ git describe --tags\n\n$ ofs-2023.3-1\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#52-license-requirements","title":"5.2 License Requirements","text":"
                                                                  The FIM Testbench is UVM compliant and integrates third party VIP's from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this TB. UVM RAL (Register Abstraction Layer) is used for CSR Verification.
                                                                   
                                                                  The Qualified Verification IPs will help to detect incorrect protocol behavior easily, help to focus on FIM features and accelerate the verification process.
                                                                   
                                                                  \u2022 VCS & DVE\n\u2022 SNPS-Assertions\n\u2022 Verdi\n\u2022 VerdiCoverage\n\u2022 VerdiSimDB\n\u2022 VerdiTransactionDebugUltra\n\u2022 VIP-AMBA-AXI-SVT\n\u2022 VIP-AMBA-STREAM-SVT\n\u2022 VIP-PCIE-SVT\n\u2022 VIP-PCIE-TS-SVT\n\u2022 VIP-PCIE-G3-OPT-SVT\n\u2022 VIP-Ethernet-SVT\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#53-software-tools-requirements","title":"5.3 Software Tools Requirements","text":"
                                                                  The following tools are required for successful UVM set-up
                                                                   
                                                                     
                                                                  • Python 3.6.8
                                                                    •  
                                                                  • Synopsys PCIE and AMBA AXI UVM VIP Q-2020.03A License
                                                                    •  
                                                                  • VCS R-2020.12-SP2 License       Note: Makefile can be modified to use DVE instead of Verdi
                                                                    •  
                                                                   
                                                                     
                                                                  • VCS R-2020.12-SP2 License 
                                                                    •  
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#54-creating-a-software-tools-script","title":"5.4 Creating a Software Tools Script","text":"
                                                                  The UVM tool set-up is best done by creating a simple set-up script so all applicable tools are sourced before running the tests.
                                                                   
                                                                  The following environment variables can be pasted into a script and used prior to running the UVM verification environment
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#license-files","title":"License Files","text":"
                                                                  export LM_LICENSE_FILE=\nexport SNPSLMD_LICENSE_FILE=\n
                                                                   
                                                                  The license environment variables LM_LICENSE_FILE and SNPSLMD_LICENSE_FILE can point to a server license on your system.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#general-environment-variables","title":"General Environment Variables","text":"
                                                                  export IOFS_BUILD_ROOT=$PWD\nexport OFS_ROOTDIR=<user_path>/ofs-f2000x-pl\nexport WORKDIR=$OFS_ROOTDIR\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#quartus-tools","title":"Quartus Tools","text":"
                                                                  export QUARTUS_HOME=<user_path>/intelFPGA_pro/23.3/quartus\nexport QUARTUS_ROOTDIR=$QUARTUS_HOME\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$PATH\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-verification-tools","title":"Synopsys Verification Tools","text":"
                                                                  export DESIGNWARE_HOME=<user_path>/synopsys/vip_common/vip_Q-2020.03A\nexport PATH=$DESIGNWARE_HOME/bin:$PATH\nexport UVM_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport PATH=$VCS_HOME/bin:$PATH\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-verification-tools","title":"QuestaSIM Verification Tools","text":"
                                                                  export MTI_HOME=<user_path>/mentor/questasim/2023.2/linux64\nexport PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\nexport QUESTA_HOME=$MTI_HOME\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#6-running-a-uvm-simulation-test-and-analysing-results","title":"6 Running a UVM Simulation Test and Analysing Results","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#61-simulation","title":"6.1 Simulation","text":"
                                                                  The default simulator used in the simulation script is Synopsys VCS-MX. Users can refer to the options and adopt the options for other simulators. The script is a makefile that calls vlogan, vcs and simv for compilation, elaboration and simulation, respectively.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#62-file-structure","title":"6.2 File Structure","text":"
                                                                  After cloning the repo, the verification and ofs-common directories contain all UVM verification related files. The directory structure is shown in Figure 4 below.
                                                                   
                                                                   
                                                                  Figure 4 UVM Verification Directory File Structure
                                                                   
                                                                  ofs-f2000x-pl/verification/testbench has a testbench, uvm env, virtual sequencer, RAL etc. 
                                                                   
                                                                  ofs-f2000x-pl/tests contains all uvm tests and sequences. 
                                                                   
                                                                  Users can run the simulation under \"ofs-f2000x-pl/verification/scripts\" directory and the simulation result is outputted to a \"sim\" directory for Synopsys VCS or sim_msim for Questasim.
                                                                   
                                                                  The simulation result folder is named after the test name with increasing suffix number. If user runs the same test multiple times, the suffix is incremented by 1 each time.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#63-uvm-test-suite","title":"6.3 UVM Test Suite","text":"
                                                                  The UVM environment contains a variety of tests that have been developed to test out the FIM portion of OFS.
                                                                   
                                                                  The table below has four columns which describe the \"Test Name\", \"DUT Scope\", \"Test Scenario\" and the \"Checking Criteria\".
                                                                   
                                                                  Tests are located at ofs-f2000x-pl/ofs-common/verification/fpga_family/agilex/tests
                                                                   Test Name DUT Scope Test Scenario Checking Criteria afu_mmio_flr_pf0_test PF0 FLR Reset Apply FLR Reset for PF0 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF0 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf0_vf0_test PF0_VF0_FLR Reset Apply FLR Reset for PF0_VF0 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR       access are working fine afu_mmio_flr_pf0_vf1_test PF0_VF1_FLR Reset Apply FLR Reset for PF0_VF1 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf0_vf2_test PF0_VF2_FLR Reset Apply FLR Reset for PF0_VF2 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf1_vf0_test PF1_VF0_FLR Reset Apply FLR Reset for PF1_VF0 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf2_test PF2 FLR Reset Apply FLR Reset for PF2 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF2 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf3_test PF3 FLR Reset Apply FLR Reset for PF3 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF3 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf4_test PF4 FLR Reset Apply FLR Reset for PF4 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF4 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_stress_5bit_tag_test AFU-StressAFU-Stress To check the AFU Stress by sending traffic with 5bit tag from all PF/VF. i.e Send traffic on HE-LPK/HE-MEM and initiating MMIO access to other PF/VFs Data checking afu_stress_8bit_tag_test AFU-Stress To check the AFU Stress by sending traffic with 5bit tag from all PF/VF. i.e Send traffic on HE-LPK/HE-MEM and initiating MMIO access to other PF/VFs Data checking afu_stress_test AFU-Stress 1. Initiate transactions to all the supported PF/VF from PCIE VIP and ensure that traffic is sent to all blocks of the AFU.                 2. Ensure that CE/HE-LB/HE-MEM/HSSI/BPF/FME are seeing traffic.                 3. Ensure that HE-LB/HE-MEM/CE sends DMWR/DMRD requests to PCIE VIP.                 4. Ensure the Mux/DeMux blocks is able to handle the traffic based on the PF's/VF's and proper muxing/demuxing happens. Data checking bar_32b_test PCIe MMIO Path Set the BAR values to 32bit and test mmio access BAR address, Register Base Offset bar_64b_test PCIe MMIO Path Set the BAR values to 64bit and test mmio access BAR address, Register Base Offset dfh_walking_test DFH DFH walking offset checking, eol checking -> tb emif_csr_test EMIF CSR access data checking fme_csr_test FME CSR CSR accesses data checking fme_err_intr_test Interrupt FME Interrupt request using FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read fme_intr_test Interrupt FME interrupt request using RAS ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read fme_multi_err_intr_test Interrupt FME interrupt request using RAS ERROR and FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests generated from FME and RAS ERROR bits he_hssi_csr_test HE-HSSI CSR accesses data checking he_hssi_rx_lpbk_25G_10G_test HE-HSSI Sending back to back ethernet data traffic with 25G speed on HSSI RX Port0-7 lanes using Ethernet VIPs Enable the loopback mode in HE-HSSI and compare the pkts recived on HSSI TX Port(DATA CHECKING) he_hssi_tx_lpbk_P0_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port0  lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P1_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port1 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P2_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port2 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P3_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port3 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P4_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port4 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P5_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port5 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P6_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port6 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P7_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port7 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_lpbk_cont_test HE-LPBK Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking he_lpbk_csr_test HE-LPBK CSR accesses data checking he_lpbk_flr_rst_test HE-LPBK FLR RST is generated in HE_LPBK access data checking, counter checking he_lpbk_long_rst_test HE-LPBK Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_lpbk_long_test HE-LPBK Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_lpbk_multi_user_intr_test HE-LPBK generate user HE_LB interrupt interrupt checking he_lpbk_rd_cont_test HE-LPBK Read only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking he_lpbk_rd_test HE-LPBK Read only mode. Randomize num_lines, addresses, req_len counter checking he_lpbk_reqlen16_test HE-LPBK To check the behavior of HE_LPK block when req_length 16 and num_lines set to 1024 Cache lines in Loopback mode data checking, counter checking he_lpbk_reqlen1_test HE-LPBK Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_lpbk_reqlen2_test HE-LPBK Loopback mode. 128 CLs, req_len = 2CL, random addresses data checking, counter checking he_lpbk_reqlen4_test HE-LPBK Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_lpbk_reqlen8_test HE-LPBK Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_lpbk_rst_in_middle_test PCIe MMIO Path Set HE_LPK in  all the modes randomly and iterate the transactions in loop. At the end of every loop assert the Soft reset in the middle of the transactions Register Base Offset he_lpbk_test HE-LPBK Loopback mode.  Randomize num_lines, addresses, req_len data checking, counter checking he_lpbk_thruput_contmode_test HE-LPBK Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking he_lpbk_thruput_test HE-LPBK Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_lpbk_wr_cont_test HE-LPBK Write only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking he_lpbk_wr_test HE-LPBK Write only mode. Randomize num_lines, addresses, req_len counter checking he_mem_cont_test HE-MEM Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking, counter checking he_mem_csr_test HE-MEM CSR accesses data checking he_mem_lpbk_long_rst_test HE-MEM Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_mem_lpbk_long_test HE-MEM Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_mem_lpbk_reqlen16_test HE-MEM To check the behavior of HE_LPK block when req_length 16 and num_lines set to 1024 Cache lines in Loopback mode data checking, counter checking he_mem_lpbk_reqlen1_test HE-MEM Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_mem_lpbk_reqlen2_test HE-MEM Loopback mode. 128 CLs, req_len = 2CL, random addresses data checking, counter checking he_mem_lpbk_reqlen4_test HE-MEM Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_mem_lpbk_reqlen8_test HE-MEM Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_mem_lpbk_test HE-MEM Loopback mode.  Randomize num_lines, addresses, req_len data checking, counter checking he_mem_multi_user_intr_test Interrupt Back to back multiple User interrupt request from HE MEM Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read for multiple back to back request he_mem_rd_cont_test HE-MEM Read only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_rd_test HE-MEM Read only mode. Randomize num_lines, addresses, req_len counter checking he_mem_thruput_contmode_directed_test HE-MEM Set HE_MEM in Thrput Continuous mode and test data checking, counter checking he_mem_thruput_contmode_test HE-MEM Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_thruput_test HE-MEM Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_mem_user_intr_test Interrupt FME interrupt request using RAS ERROR and FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests generated from FME and RAS ERROR bits he_mem_wr_cont_test HE-MEM Write only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_wr_test HE-MEM Write only mode. Randomize num_lines, addresses, req_len counter checkingt he_random_test All HEs Enable all HEs and randomize modes data checking if in lpbk mode, counter checking helb_rd_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Read only mode Measure the performance helb_rd_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Read only mode Measure the performance helb_rd_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Read only mode Measure the performance helb_thruput_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Thruput mode Measure the performance helb_thruput_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Thruput mode Measure the performance helb_thruput_4cl_5bit_tag_test Performance Set HE_LPK in thruput mode and send traffic with req len 4 and num_lines set to 1024. Measure the Read/Write performance Measure the performance helb_thruput_4cl_8bit_tag_test Performance Set HE_LPK in thruput mode and send traffic with req len 4 and num_lines set to 1024. Measure the Read/Write performance Measure the performance helb_thruput_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Thruput mode Measure the performance helb_wr_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Write only mode Measure the performance helb_wr_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Write only mode Measure the performance helb_wr_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Write only mode Measure the performance hssi_ss_test HSSI CSR accesses data checking malformedtlp_pcie_rst_test Protocol Checker generate mllformed TLP protocol error and initiate pcie reset to clear the error Check for error malformedtlp_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MaxTagError_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transactions are completing. mem_tg_csr_test MEM-TG CSR access data checking mem_tg_traffic_gen_test MEM-TG This test checks the MEM_TG traffic generator flow  data checking mini_smoke_test All HEs shorter simpler version of random test for turn-in sanity check data checking if in lpbk mode, counter checking mix_intr_test Interrupt Mix interrupt testcase to send multiple FME and User interrupt request simultaneously Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests through different sources - FME and HE-MEM modules mmio_pcie_mrrs_128B_mps_128B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_128B_mps_256B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_256B_mps_128B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_256B_mps_256B_test PCIe - Max Payload/Max Read Req Size Random length mmio Read Checking valid possible combination of MPS & MRRS mmio_stress_nonblocking_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux with non-blocking MMIO reads data checking mmio_stress_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux data checking mmio_test PCIe MMIO Path MMIO targeting PF0(ST2MM, FME, PMCI, QSFP, HSSI SS), PF1, PF2,PF3, PF4, PF1.VF1, PF1.VF2 data checking mmio_unimp_test PCIe MMIO Path MMIO acccess to unimplemented addresses data checking MMIODataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MMIOInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MMIOTimedout_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. pcie_csr_test PCIESS Earlier msix registers were in fme block but now it has moved from fme to pciess. Hence coded a seperate test for msix data checking pcie_pmci_mctp_multi_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM multiple packets received from PMCI controller over APF and BPF to PCIe HIA subsystem pcie_pmci_mctp_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM (Vendor Defined Message) single packet received from PCIe HIA subsystem to PMCI controller over APF and BPF via AXI4-lite memory write request pmci_csr_test PMCI CSR CSR accesses data checking pmci_fme_csr_test PMCI FME CSR CSR accesses data checking pmci_pcie_mctp_multi_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM multiple packets received from PMCI controller over APF and BPF to PCIe HIA subsystem pmci_pcie_mctp_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM single packet received from PMCI controller over APF and BPF to PCIe HIA subsystem pmci_pciess_csr_test PMCI PCIESS CSR CSR accesses data checking pmci_qsfp_csr_test PMCI QSFP CSR CSR accesses data checking port_gasket_csr_test PORT GASKET CSR accesses data checking qsfp_csr_test QSFP CSR accesses data checking TxMWrDataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. TxMWrInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. uart_intr_test UART Checking Generates UART interrupt Check interrupt UnexpMMIORspErr_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. vdm_err_vid_test Vendor ID check generate a packet with undefined Vendor-ID from Host to PMCI_SS ID check 
                                                                   
                                                                  The next section describes how to compile and build the UVM environment prior to running each UVM test and analyinsg the results in the log files 
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#64-ip-compile","title":"6.4 IP Compile","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-vcs","title":"Synopsys VCS","text":"
                                                                  To compile all IPs for the Synopsys VCS simulater:
                                                                   
                                                                      cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk cmplib_adp\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-tbd","title":"Questasim (TBD)","text":"
                                                                  To compile all IPs for the Questasim simulater:
                                                                   
                                                                      cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk cmplib_adp\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#65-rtl-test-bench-compile","title":"6.5 RTL & Test Bench Compile","text":"
                                                                  The RTL file list for compilation is located here: verification/scripts/rtl_comb.f
                                                                   
                                                                  The TB file list for compilation is located here: verification/scripts/ver_list.f
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-vcs_1","title":"Synopsys VCS","text":"
                                                                  To compile RTL and Testbench for the Synopsys VCS simulater
                                                                   
                                                                      cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_adp DUMP=1\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-tbd_1","title":"Questasim (TBD)","text":"
                                                                  To compile RTL and Testbench for the Questasim simulater
                                                                   
                                                                      cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_adp DUMP=1\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#66-ip-and-rtl-test-bench-compile","title":"6.6 IP and RTL & Test Bench Compile","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-vcs_2","title":"Synopsys VCS","text":"
                                                                  If the user wants to compile all IPs and RTL Testbench in one command for Synopsys VCS then follow the procedure below
                                                                   
                                                                      cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_all DUMP=1\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-tbd_2","title":"Questasim (TBD)","text":"
                                                                  If the user wants to compile all IPs and RTL Testbench in one command for Questasim then follow the procedure below
                                                                   
                                                                      cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_all DUMP=1\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-vcs_3","title":"Synopsys VCS","text":"
                                                                  To run a simulation for Synopsys VCS:
                                                                   
                                                                      cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk run TESTNAME=ofs_mmio_test DUMP=1\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-tbd_3","title":"Questasim (TBD)","text":"
                                                                  To run a simulation for Questasim:
                                                                   
                                                                      cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=ofs_mmio_test DUMP=1\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-vcs_4","title":"Synopsys VCS","text":"
                                                                  To dump the waveform, \"DUMP=1\" must be added into the command line for Synopsys VCS build and simulation. 
                                                                   
                                                                      gmake -f Makefile_VCS.mk build_adp DUMP=1\n\n    gmake -f Makefile_VCS.mk run TESTNAME=<test_case_name> DUMP=1\n
                                                                   
                                                                  Or
                                                                   
                                                                      gmake -f Makefile_VCS.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-tbd_4","title":"Questasim (TBD)","text":"
                                                                  To dump the waveform, \"DUMP=1\" must be added into the command line for Questasim build and simulation. 
                                                                   
                                                                      gmake -f Makefile_MSIM.mk build_adp DUMP=1\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=<test_case_name> DUMP=1\n
                                                                   
                                                                  Or
                                                                   
                                                                      gmake -f Makefile_MSIM.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                                                                   
                                                                  There are some optimizations in the Table below for convenience if you want to bypass some commands for both Synopsys VCS and Questasim:
                                                                   Command (Synopsys VCS) Command (Questasim) Details gmake -f Makefile_VCS.mk build_all DUMP=1 gmake -f Makefile_MSIM.mk build_all DUMP=1 compile IP + compile RTL gmake -f Makefile_VCS.mk build_run TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk build_run TESTNAME= DUMP=1 compile RTL + run test gmake -f Makefile_VCS.mk do_it_all TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk do_it_all TESTNAME= DUMP=1 compile IP, RTL and run test gmake -f Makefile_VCS.mk rundb TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk rundb TESTNAME= DUMP=1 run test in sim dir + over-writes content"},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#67-uvm-regression-test","title":"6.7 UVM Regression Test","text":"
                                                                  cd $VERDIR/scripts\n\npython uvm_regress.py -l -n 8 -p adp -k top_pkg -s vcs -c none\n\nFor Regression in VCS with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -p adp -k top_pkg -s vcs -c none\n\nResults are created in a sim directory ($VERDIR/sim) with individual testcase log dir\n\nFor Regression in MSIM with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -p adp -k top_pkg -s msim -c none\n
                                                                   
                                                                  Results are created in a sim directory ($VERDIR/sim_msim) with individual testcase log dir
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#68-uvm-waveform-and-transcript-analysis","title":"6.8 UVM Waveform and Transcript Analysis","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#synopsys-vcs_5","title":"Synopsys VCS","text":"
                                                                  Running Synopsys VCS UVM tests will generate a ofs-f2000x-pl/verification/sim directory
                                                                   
                                                                  \u2022 All build time logs are located at ofs-f2000x-pl/verification/sim\n\n\u2022 Each testcase will have separate directory inside sim ofs-f2000x-pl/verification/sim/<test_case_name>\n
                                                                   
                                                                  There are two tracker or log files that are available: runsim.log and trans.log. 
                                                                   
                                                                  runsim.log is the simulation log file generated from Synopsys VCS. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 5
                                                                   
                                                                   
                                                                  Figure 5 runsim.log
                                                                   
                                                                  trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 6
                                                                   
                                                                   
                                                                  Figure 6 trans.log
                                                                   
                                                                  The waveform generated is named as \"inter.vpd\". To open the waveform, go to simulation result directory and run 
                                                                   
                                                                      dve -full64 -vpd inter.vpd &\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#questasim-tbd_5","title":"Questasim (TBD)","text":"
                                                                  Running Questasim UVM tests will generate a ofs-f2000x-pl/verification/sim_msim directory
                                                                   
                                                                  \u2022 All build time logs are at ofs-f2000x-pl/verification/sim_msim\n\n\u2022 Each testcase will have separate directory inside sim ofs-f2000x-pl/verification/sim_msim/<test_case_name>\n
                                                                   
                                                                  There are two tracker or log files that are available: runsim.log and trans.log. 
                                                                   
                                                                  runsim.log is the simulation log file generated from Questasim. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 7
                                                                   
                                                                   
                                                                  Figure 7 runsim.log
                                                                   
                                                                  trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 8
                                                                   
                                                                   
                                                                  Figure 8 trans.log
                                                                   
                                                                  The waveform generated is named as \"vsim.wlf\". To open the waveform, go to simulation result directory and run 
                                                                   
                                                                      vsim -view vsim.wlf &\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#69-uvm-coverage-analysis","title":"6.9 UVM Coverage Analysis","text":"
                                                                  The following command allows to run a single testcase with coverage enabled
                                                                   
                                                                      gmake -f Makefile_VCS.mk cmplib_adp && gmake -f Makefile_VCS.mk build_adp DUMP=1 DEBUG=1 COV_FUNCTIONAL=1&& gmake -f Makefile_VCS.mk run TESTNAME=<TESTCASE-NAME> DUMP=1 DEBUG=1 COV_FUNCTIONAL=1 &\n
                                                                   
                                                                  The following command shows how to merge and generate the coverage report
                                                                   
                                                                      urg -dir <$VERDIR/sim/simv.vdb> <$VERDIR/sim/regression.vdb> -format both -dbname <regression_database_name>\n
                                                                   
                                                                  This will generate both urgreport directory and .vdb file Multiple regression.vdb from different regressions can be merged with the same command. 
                                                                      e.g \"urg -dir <path1_till_simv.vdb> <path1_till_regression.vdb> <path2_till_regression.vdb> -report <dir> -format both -dbname <dirname>\"\n
                                                                   
                                                                  The following commands shows how to launch DVE and check the coverage reports
                                                                   
                                                                  To open DVE of a single regression or testcase, execute:\n\n    dve -full64 -cov -covdir simv.vdb regression.vdb &\n\nTo open DVE of a merged regression test, execute:\n\n    dve -full64 -cov -covdir <dirname.vdb> &\n
                                                                   
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#7-csr-verification-using-uvm-ral","title":"7 CSR Verification using UVM RAL","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#71-overview","title":"7.1 Overview","text":"
                                                                  The UVM Register Layer provides a standard base class library that enable users to implement the object-oriented model to access the DUT registers and memories. The UVM Register Layer is also referred to as UVM Register Abstraction Layer (UVM RAL). Design registers can be accessed independently of the physical bus interface. i.e. by calling read/write methods. This is shown in Figure 9 below.
                                                                   
                                                                   
                                                                  Figure 9 RAL UVM Testbench
                                                                   
                                                                  The RAL register models for different CSR's mimics the design registers. All RAL files are located here.
                                                                   
                                                                      ofs-f2000x-pl/verification/testbench/ral\n
                                                                   
                                                                  The RAL model is generated through the Synopsys RALGEN tool and is used for CSR verification.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#72-ral-integration","title":"7.2 RAL Integration","text":"
                                                                  For UVM RAL model integration to the environment, adapters for each CSR is implemented and integrated into the Testbench Environment. It is used to convert the PCIe bus sequence items into uvm_reg_bus_op and vice versa. The CSR test cases pick up all the registers from the respective CSR blocks and perform a default value, wr/rd check on them. 
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#73-ral-model-generation","title":"7.3 RAL Model Generation","text":"
                                                                  Steps for RAL model generation
                                                                   
                                                                  Excel(xls) file containing the registers is required. Make sure there are separate xls files for each CSR and the worksheet name must contain the name \"reg_fields\".  
                                                                   
                                                                  Excel sheet snapshot example below for EMIF_CSR.xls located at /ipss/mem/rtl/adp
                                                                   
                                                                   
                                                                  \u2022 Navigate to ofs-f2000x-pl/ofs-common/verification/common/scripts/ral\n\u2022 Copy the excel file (xls) to the above area\n\u2022 In the bash terminal run mk_ral.sh <Excel sheet name without extension > <output *.sv file name without ral_  prepended >\n\u2022 The above steps generate two ral *.sv files. File with _cov suffix is a coverage enabled ral model. \n\u2022 Copy *.sv files to ofs-f2000x-pl/verification/testbench/ral\n
                                                                   
                                                                   
                                                                  \u2022 As an example to generate ral_ac_ce.sv from AC_CE_CSR.xls file the command is\n\n    mk_ral.sh AC_CE_CSR ac_ce\n
                                                                   
                                                                  This generates two ral models (ral_ac_ce.sv and ral_ac_ce_cov.sv)
                                                                   
                                                                  To add new registers
                                                                   
                                                                  \u2022 To create new registers, copy existing ones and modify the cells in the xls. Make sure the last line is also a copied blank line\n\u2022 Follow all the steps of RAL model generation\n
                                                                   
                                                                  To Generate a RAL model when a new xls sheet is created for a new component
                                                                   
                                                                  \u2022 Copy the relevant xls sheet to ofs-f2000x-pl/ofs-common/verification/common/scripts/ral\n\u2022 Follow all the steps of RAL model generation\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#74-top-level-verification-architecture-for-csr-testing","title":"7.4 Top Level Verification Architecture for CSR testing","text":""},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#741-testbench-components","title":"7.4.1 Testbench components","text":"
                                                                  The testbench components for RAL are defined below
                                                                   
                                                                  \u2022 ral_reg_iofs_* (uvm_reg) generated by the steps as mentioned in section 5.3\n
                                                                   
                                                                  The uvm register class is written by extending the uvm_reg. A register represents a set of fields that are accessible as a single entity Each register contains any number of fields, which mirror the values of the corresponding elements in hardware
                                                                   
                                                                  \u2022 ral_block_iofs_* (uvm_block) generated in the same register file\n
                                                                   
                                                                  A register model is an instance of a register block, which may contain any number of registers, register files, memories, and other blocks
                                                                   
                                                                  \u2022 ral_block_ofs (uvm_block) \u2013 Contains all the CSR block instantiations\n\u2022 Reg2vip_*_adapter (uvm_reg_adapter) \u2013 This class defines an interface for converting between uvm_reg_bus_op and a specific bus transaction. For each CSR a respective adapter is present\n
                                                                   
                                                                  All the components are defined in ofs-f2000x-pl/ofs-common/verification/testbench
                                                                   
                                                                  Integration of components in testbench
                                                                   
                                                                  \u2022 The RAL blocks and adapters for each CSR is instantiated in tb_env\n\u2022 The bar range for each CSR is also set in the tb_env\n
                                                                   
                                                                  Sample Environment Integration snippets 
                                                                   
                                                                   
                                                                   
                                                                   
                                                                   
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#8-modifying-uvm-testbench","title":"8 Modifying UVM Testbench","text":"
                                                                  The next sections describe what needs to be considered when modifying the UVM, adding a new interface to the testbench and creating a new UVM test for a customised OFS Accelerator platform.
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#81-modifying-uvm-environment-for-new-shell-variant","title":"8.1 Modifying UVM environment for new Shell Variant","text":"
                                                                  OFS f2000x comprises a shell based on PCIe Gen4x16 and is named base_x16
                                                                   
                                                                  This base_x16 shell is described by an RTL file list, IP File lists and setup scripts to complete the build flow
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#82-modifying-uvm-environment-and-setting-up-compile-and-run-flow","title":"8.2 Modifying UVM environment and setting up Compile and Run flow","text":"
                                                                  All the variants can mostly reuse the existing UVM infrastructure to setup the build and run flow
                                                                   
                                                                  \u2022 Create directory under $OFS_BUILD_ROOT new variant e.g ofs-n9000\n\u2022 Change directory to $OFS_BUILD_ROOT/ofs-n9000/verification/scripts\n\u2022 modify Makefile it to point to the new RTL, IP and script files.\n
                                                                   
                                                                  Following these three steps above will enable the build and sim flow to run the existing UVM TB and tests with new IOFS n9000 variant.
                                                                   
                                                                  Adding a new interface requires signal connections in the testbench. An additional BFM or verification IP is needed to drive the new interface. The main testbench file tb_top.sv is found at the following location
                                                                   
                                                                      ofs-f2000x-pl/verification/testbench\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#83-adding-a-new-ral-directory","title":"8.3 Adding a new RAL directory","text":"
                                                                  In case the design has many register changes and the user decides to generate all the new RAL models instead of reusing from existing base RAL models, the following steps will help to create and integrate a new RALDIR in the UVM environment.
                                                                   
                                                                  \u2022 Generate the new RAL files in desired directory. Preferably under the \"ofs-f2000x-pl/verification/common/testbench\" \n\u2022 By default, the Makefile points to base FIM RAL so set the RALDIR path in the Makefile to the new generated RAL file directory\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#84-modifying-tbtest-files-for-new-variant","title":"8.4 Modifying TB/Test files for new Variant","text":"
                                                                  Create a define for the new variant. e.g 'define FIM_NEW. If you are modifying common files then add new logic under this define so other projects will not get affected with variant specific change.
                                                                   
                                                                  If there are more changes, please create separate \"testbench\" and \"test\" folders under this new variant. 
                                                                   
                                                                  \u2022 Extend variant specific env file from base env file to add variant specific changes.\n\u2022 Create new test/seq lib files in \"tests\" folder.\n\u2022 Create new variant package, add new TB/Tests/Sequence lib files and also import the base package files.\n
                                                                   
                                                                  If you are adding new files then make sure it's included in Makefile for the build+run flow. 
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#85-uvm-pcie-drivers","title":"8.5 UVM PCIe Drivers","text":"
                                                                  The \"svt_pcie_driver_app_transaction_base_sequence\" is part of Synopsys PCIe VIP library. You can find the sequence definition in the following two directories
                                                                   
                                                                  \u2022 Navigate to \"$DESIGNWARE_HOME/vip/svt/pcie_svt/Q-2020.03/sverilog/src/vcs/svt_pcie_driver_app_transaction_sequence_collection.svp\" file. All the base and PCIe sequences are defined in this file.\n\n\u2022 When the OFS UVM build command is executed, it creates \"vip\" directory under \"$OFS_BUILD_ROOT/ofs-f2000x-pl/verification\". You can also find the same sequence file at \"$IOFS_BUILD_ROOT/ofs-f2000x/verification/vip/pcie_vip/src/sverilog/vcs/svt_pcie_driver_app_transaction_sequence_collection.sv\"\n
                                                                  "},{"location":"hw/f2000x/user_guides/ug_sim_ofs/ug_sim_ofs/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                  Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                   
                                                                  OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                  "},{"location":"hw/ftile_devkit/","title":"F-Series (2xF-tile) Development Kit Collateral for OFS","text":"
                                                                  This folder contains applicable collateral for OFS PCIe Attach reference shell targeting the F-Series (2xF-tile) Development Kit DK-DEV-AGF027F1.
                                                                  "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/","title":"FPGA Interface Manager Developer Guide for Open FPGA Stack: Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) PCIe Attach","text":"
                                                                  Last updated: February 03, 2024 
                                                                  "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1-introduction","title":"1. Introduction","text":""},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#11-about-this-document","title":"1.1. About This Document","text":"
                                                                  This document serves as a guide for OFS Agilex PCIe Attach developers targeting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile). The following topics are covered in this guide:
                                                                   
                                                                     
                                                                  • Compiling the OFS Agilex PCIe Attach FIM design
                                                                    •  
                                                                  • Simulating the OFS Agilex PCIe Attach design
                                                                    •  
                                                                  • Customizing the OFS Agilex PCIe Attach FIM design
                                                                    •  
                                                                  • Configuring the FPGA with an OFS Agilex PCIe Attach FIM design
                                                                    •  
                                                                   
                                                                  The FIM Development Walkthroughs Table lists all of the walkthroughs provided in this guide. These walkthroughs provide step-by-step instructions for performing different FIM Development tasks.
                                                                   
                                                                  Table: FIM Development Walkthroughs
                                                                   Walkthrough Name Category Install Quartus Prime Pro Software Setup Install Git Large File Storage Extension Setup Clone FIM Repository Setup Set Development Environment Variables Setup Set Up Development Environment Setup Compile OFS FIM Compilation Manually Generate OFS Out-Of-Tree PR FIM Compilation Change the Compilation Seed Compilation Run Individual Unit Level Simulation Simulation Run Regression Unit Level Simulation Simulation Add a new module to the OFS FIM Customization Modify and run unit tests for a FIM that has a new module Customization Hardware test a FIM that has a new module Customization Debug the FIM with Signal Tap Customization Compile the FIM in preparation for designing your AFU Customization Resize the Partial Reconfiguration Region Customization Modify PCIe Configuration Using OFSS Customization Modify PCIe Configuration Using IP Presets Customization Create a Minimal FIM Customization Migrate to a Different Agilex Device Number Customization Modify the Ethernet Sub-System to 2x4x10GbE Customization Modify the Ethernet Sub-System to 3x4x10GbE Customization Remove the HPS Customization Set up JTAG Configuration Program the FPGA via JTAG Configuration"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#111-knowledge-pre-requisites","title":"1.1.1 Knowledge Pre-Requisites","text":"
                                                                  It is recommended that you have the following knowledge and skills before using this developer guide.
                                                                   
                                                                     
                                                                  • Basic understanding of OFS and the difference between OFS designs. Refer to the OFS Welcome Page.
                                                                    •  
                                                                  • Review the release notes for the Intel Agilex 7 PCIe Attach Reference Shells, with careful consideration of the Known Issues.
                                                                    •  
                                                                  • Review of Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)).
                                                                    •  
                                                                  • FPGA compilation flows using Intel\u00ae Quartus\u00ae Prime Pro Edition.
                                                                    •  
                                                                  • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
                                                                    •  
                                                                  • RTL (System Verilog) and coding practices to create synthesized logic.
                                                                    •  
                                                                  • RTL simulation tools.
                                                                    •  
                                                                  • Intel\u00ae Quartus\u00ae Prime Pro Edition Signal Tap Logic Analyzer tool software.
                                                                    •  
                                                                  "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#12-fim-development-theory","title":"1.2. FIM Development Theory","text":"
                                                                  This section will help you understand how the OFS Agilex PCIe Attach FIM can be developed to fit your design goals.
                                                                   
                                                                  The Default FIM Features section provides general information about the default features of the OFS Agilex PCIe Attach FIM so you can become familiar with the default design. For more detailed information about the FIM architecture, refer to the OFS Agilex PCIe Attach FIM Technical Reference Manual.
                                                                   
                                                                  The Customization Options section then gives suggestions of how this default design can be customized. Step-by-step walkthroughs for many of the suggested customizations are later described in the FIM Customization section.
                                                                   
                                                                  FIM development for a new acceleration card generally consists of the following steps:
                                                                   
                                                                     
                                                                  1. Install OFS and familiarize yourself with provided scripts and source code
                                                                    2.  
                                                                  2. Develop high level design with your specific functionality
                                                                       
                                                                    1. Determine requirements and key performance metrics
                                                                      2.  
                                                                    2. Select IP cores
                                                                      3.  
                                                                    3. Select FPGA device
                                                                      4.  
                                                                    4. Develop software memory map
                                                                      5.  
                                                                     
                                                                    3.  
                                                                  3. Select and implement FIM Physical interfaces including:
                                                                       
                                                                    1. External clock sources and creation of internal PLL clocks
                                                                      2.  
                                                                    2. General I/O
                                                                      3.  
                                                                    3. Ethernet modules
                                                                      4.  
                                                                    4. External memories
                                                                      5.  
                                                                    5. FPGA programming methodology
                                                                      6.  
                                                                     
                                                                    4.  
                                                                  4. Develop device physical implementation
                                                                       
                                                                    1. FPGA device pin assignment
                                                                      2.  
                                                                    2. Create logic lock regions
                                                                      3.  
                                                                    3. Create of timing constraints
                                                                      4.  
                                                                    4. Create Intel Quartus Prime Pro FIM test project and validate:
                                                                         
                                                                      1. Placement
                                                                        2.  
                                                                      2. Timing constraints
                                                                        3.  
                                                                      3. Build script process
                                                                        4.  
                                                                      4. Review test FIM FPGA resource usage
                                                                        5.  
                                                                       
                                                                      5.  
                                                                     
                                                                    5.  
                                                                  5. Select FIM to AFU interfaces and development of PIM
                                                                    6.  
                                                                  6. Implement FIM design
                                                                       
                                                                    1. Develop RTL
                                                                      2.  
                                                                    2. Instantiate IPs
                                                                      3.  
                                                                    3. Develop test AFU to validate FIM
                                                                      4.  
                                                                    4. Develop unit and device level simulation
                                                                      5.  
                                                                    5. Develop timing constraints and build scripts
                                                                      6.  
                                                                    6. Perform timing closure and build validation
                                                                      7.  
                                                                     
                                                                    7.  
                                                                  7. Create FIM documentation to support AFU development and synthesis
                                                                    8.  
                                                                  8. Software Device Feature discovery
                                                                    9.  
                                                                  9. Integrate, validate, and debug hardware/software
                                                                    10.  
                                                                  10. Prepare for high volume production
                                                                    11.  
                                                                  "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#121-default-fim-features","title":"1.2.1 Default FIM Features","text":""},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1211-top-level","title":"1.2.1.1 Top Level","text":"
                                                                  Figure: OFS Agilex PCIe Attach fseries-dk FIM Top-Level Diagram
                                                                   
                                                                  "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1212-interfaces","title":"1.2.1.2 Interfaces","text":"
                                                                  The key interfaces in the OFS Agilex PCIe Attach design are listed in the Release Capabilities Table. It describes the capabilities of the fseries-dk hardware as well as the capabilities of the default OFS Agilex PCIe Attach design targeting the fseries-dk.
                                                                   
                                                                  Table: Release Capabilities
                                                                   Interface fseries-dk Hardware Capabilities OFS Agilex PCIe Attach Default Design Implementation Host Interface[1] PCIe Gen4x8 PCIe Gen4x16 Network Interface 1 QSFP-56 cage1 QSFPDD-56 Cage 2x4x25GbE External Memory 3xDDR4 DIMMs sockets - 72-bits (1 available for HPS) 2xDDR4 - 2400MHz - 8Gb (1Gbx8) - 64-bits - No ECC1xDDR4 - 2400MHz - 16Gb (2Gbx8) - 40-bits - With ECC - For HPS 
                                                                  [1] F-Tile ES version development kit has a form factor of PCIe Gen4x16, however it only supports PCIe Gen4x8. The OFS Agilex PCIe Attach design implements PCIe Gen4x16 with the assumption that it will train down to PCIe Gen4x8.
                                                                  "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1213-subsystems","title":"1.2.1.3 Subsystems","text":"
                                                                  The FIM Subsystems Table  describes the Platform Designer IP subsystems used in the OFS Agilex PCIe Attach fseries-dk FIM.
                                                                   
                                                                  Table: FIM Subsystems
                                                                   Subsystem User Guide Document ID PCIe Subsystem PCIe Subsystem Intel FPGA IP User Guide for Intel Agilex OFS N/A Memory Subsystem Memory Subsystem Intel FPGA IP User Guide for Intel Agilex OFS 686148[1] Ethernet Subsystem Ethernet Subsystem Intel FPGA IP User Guide 773413[1] 
                                                                  [1] You must log in to myIntel and request entitled access.
                                                                  "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1214-host-exercisers","title":"1.2.1.4 Host Exercisers","text":"
                                                                  The default AFU workload in the OFS Agilex PCIe Attach fseries-dk FIM contains several modules called Host Exercisers which are used to exercise the interfaces on the board. The Host Exerciser Descriptions Table describes these modules.
                                                                   
                                                                  Table: Host Exerciser Descriptions
                                                                   Name Acronym Description OPAE Command Host Exerciser Loopback HE-LB Used to exercise and characterize host to FPGA data transfer. host_exerciser Host Exerciser Memory HE_MEM Used to exercise and characterize host to Memory data transfer. host_exerciser Host Exerciser Memory Traffic Generator HE_MEM_TG Used to exercise and test available memory channels with a configurable traffic pattern. mem_tg Host Exerciser High Speed Serial Interface HE-HSSI Used to exercise and characterize HSSI interfaces. hssi 
                                                                  The host exercisers can be removed from the design at compile-time using command line arguments for the build script.
                                                                  "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1215-module-access-via-apfbpf","title":"1.2.1.5 Module Access via APF/BPF","text":"
                                                                  The OFS Agilex PCIe Attach fseries-dk FIM uses AXI4-Lite interconnect logic named the AFU Peripheral Fabric (APF) and Board Peripheral Fabric (BPF) to access the registers of the various modules in the design. The APF/BPF modules define master/slave interactions, namely between the host software and AFU and board peripherals. The APF Address Map Table describes the address mapping of the APF, followed by the BPF Address Map Table which describes the address mapping of the BPF.
                                                                   
                                                                  Table: APF Address Map
                                                                   Address Size (Bytes) Feature 0x00000\u20130x3FFFF 256K Board Peripherals (See BPF Address Map table) 0x40000 \u2013 0x4FFFF 64K ST2MM 0x50000 \u2013 0x5FFFF 64K Reserved 0x60000 \u2013 0x60FFF 4K UART 0x61000 \u2013 0x6FFFF 4K Reserved 0x70000 \u2013 0x7FFFF 56K PR Gasket:4K= PR Gasket DFH, control and status4K= Port DFH4K=User Clock52K=Remote STP 0x80000 \u2013 0x80FFF 4K AFU Error Reporting 
                                                                  Table: BPF Address Mapping
                                                                   Address Size (Bytes) Feature 0x00000 - 0x0FFFF 64K FME 0x10000 - 0x10FFF 4K PCIe 0x11000 - 0x11FFF 4K Reserved 0x12000 - 0x12FFF 4K QSFP0 0x13000 - 0x13FFF 4K QSFP1 0x14000 - 0x14FFF 4K HSSI 0x15000 - 0x15FFF 4K EMIF"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#122-customization-options","title":"1.2.2 Customization Options","text":"
                                                                  OFS is designed to be easily customizable to meet your design needs. The OFS FIM Customizations Table lists the general user flows for OFS Agilex PCIe Attach fseries-dk FIM development, along with example customizations for each user flow, plus links to step-by-step walkthroughs where available.
                                                                   
                                                                  Table: OFS FIM Customizations
                                                                   Customization Walkthrough Name Add a new module to the OFS FIM Modify and run unit tests for a FIM that has a new module Hardware test a FIM that has a new module Debug the FIM with Signal Tap Compile the FIM in preparation for designing your AFU Resize the Partial Reconfiguration Region Modify PCIe Configuration Using OFSS Modify PCIe Configuration Using IP Presets Create a Minimal FIM Migrate to a Different Agilex Device Number Modify the Ethernet Sub-System to 2x4x10GbE Modify the Ethernet Sub-System to 3x4x10GbE Remove the HPS"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#13-development-environment","title":"1.3 Development Environment","text":"
                                                                  This section describes the components required for OFS FIM development, and provides a walkthrough for setting up the environment on your development machine.
                                                                   
                                                                  Note that your development machine may be different than your deployment machine where the FPGA acceleration card is installed. FPGA development work and deployment work can be performed either on the same machine, or on different machines as desired. Please see the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up the environment for deployment machines.
                                                                  "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#131-development-tools","title":"1.3.1 Development Tools","text":"
                                                                  The Development Environment Table describes the Best Known Configuration (BKC) for the tools that are required for OFS FIM development.
                                                                   
                                                                  Table: Development Environment BKC
                                                                   Component Version Installation Walkthrough Operating System RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6 N/A Intel Quartus Prime Software Quartus Prime Pro Version 23.3 for Linux + Patches 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe) Section 1.3.1.1 Python 3.6.8 or later N/A GCC 7.4.0 or later N/A cmake 3.15 or later N/A git with git-lfs 1.8.3.1 or later Section 1.3.1.2 FIM Source Files ofs-2023.3-2 Section 1.3.2.1"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1311-walkthrough-install-quartus-prime-pro-software","title":"1.3.1.1 Walkthrough: Install Quartus Prime Pro Software","text":"
                                                                  Intel Quartus Prime Pro Version 23.3 is verified to work with the latest OFS release ofs-2023.3-2.  However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.
                                                                   
                                                                  Use RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6 for compatibility with your development flow and also testing your FIM design in your platform. 
                                                                   
                                                                  Prior to installing Quartus:
                                                                   
                                                                     
                                                                  1.  
                                                                    Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.
                                                                     
                                                                       
                                                                    • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                                                                      •  
                                                                    • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.
                                                                      •  
                                                                     
                                                                    2.  
                                                                  2.  
                                                                    Perform the following steps to satisfy the required dependencies.
                                                                     
                                                                    $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                                                                     
                                                                    Apply the following configurations.
                                                                     
                                                                    $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n
                                                                     
                                                                    3.  
                                                                  3.  
                                                                    Create the default installation path: /intelFPGA_pro/, where  is the default path of the Linux workstation, or as set by the system administrator and  is your Quartus version number. 
                                                                    The installation path must satisfy the following requirements:
                                                                     
                                                                       
                                                                    • Contain only alphanumeric characters
                                                                      •  
                                                                    • No special characters or symbols, such as !$%@^&*<>,
                                                                      •  
                                                                    • Only English characters
                                                                      •  
                                                                    • No spaces
                                                                      •  
                                                                     
                                                                  4.  
                                                                    Download your required Quartus Prime Pro Linux version here.
                                                                     
                                                                    5.  
                                                                  5.  
                                                                    Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe).
                                                                     
                                                                    6.  
                                                                  6.  
                                                                    After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                                                                     
                                                                    export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                                                                     
                                                                    For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:
                                                                     
                                                                    export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n
                                                                     
                                                                    7.  
                                                                  7.  
                                                                    Verify, Quartus is discoverable by opening a new shell:
                                                                     
                                                                    $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                                                                     
                                                                    8. "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1312-walkthrough-install-git-large-file-storage-extension","title":"1.3.1.2 Walkthrough: Install Git Large File Storage Extension","text":"
                                                                    To install the Git Large File Storage (LFS) extension, execute the following commands:
                                                                     
                                                                       
                                                                    1. Obtain Git LFS package     
                                                                      curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\n
                                                                      2.  
                                                                    2. Install Git LFS package     
                                                                      sudo dnf install git-lfs\n
                                                                      3.  
                                                                    3. Install Git LFS     
                                                                      git lfs install\n
                                                                      4.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#132-fim-source-files","title":"1.3.2 FIM Source Files","text":"
                                                                    The source files for the OFS Agilex PCIe Attach FIM are provided in the following repository: https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2
                                                                     
                                                                    Some essential directories in the repository are described as follows:
                                                                     
                                                                    ofs-agx7-pcie-attach\n|  syn                      // Contains files related to synthesis\n|  |  board                     // Contains synthesis files for several cards, including the fseries-dk \n|  |  |  fseries-dk                 // Contains synthesis files for fseries-dk\n|  |  |  |  setup                       // Contains setup files, including pin constraints and location constraints\n|  |  |  |  syn_top                     // Contains Quartus project files\n|  verification             // Contains files for UVM testing\n|  ipss                     // Contains files for IP Sub-Systems\n|  |  qsfp                      // Contains source files for QSFP Sub-System\n|  |  hssi                      // Contains source files for HSSI Sub-System\n|  |  pmci                      // Contains source files for PMCI Sub-System (not used in F-Tile FIM)\n|  |  pcie                      // Contains source files for PCIe Sub-System\n|  |  mem                       // Contains source files for Memory Sub-System\n|  sim                      // Contains simulation files\n|  |  unit_test                 // Contains files for all unit tests\n|  |  |  scripts                    // Contains script to run regression unit tests\n|  license                  // Contains Quartus patch\n|  ofs-common               // Contains files which are common across OFS platforms\n|  |  verification              // Contains common UVM files\n|  |  scripts                   // Contains common scripts\n|  |  |  common\n|  |  |  |  syn                         // Contains common scripts for synthesis, including build script\n|  |  |  |  sim                         // Contains common scripts for simulation\n|  |  tools                     // Contains common tools files\n|  |  |  mk_csr_module              // Contains common files for CSR modules\n|  |  |  fabric_generation          // Contains common files for APF/BPF fabric generation\n|  |  |  ofss_config                // Contains common files for OFSS configuration tool\n|  |  |  |  ip_params                   // Contains default IP parameters for certain Sub-Systems when using OFSS\n|  |  src                       // Contains common source files, including host exercisers\n|  tools                    //\n|  |  ofss_config               // Contains top level OFSS files for each pre-made board configuration\n|  |  |  hssi                       // Contains OFSS files for Ethernet-SS configuraiton\n|  |  |  memory                     // Contains OFSS files for Memory-SS configuration\n|  |  |  pcie                       // Contains OFSS files for PCIe-SS configuration\n|  |  |  iopll                      // Contains OFSS files for IOPLL configuration\n|  src                      // Contains source files for Agilex PCIe Attach FIM\n|  |  pd_qsys                   // Contains source files related to APF/BPF fabric\n|  |  includes                  // Contains source file header files\n|  |  top                       // Contains top-level source files, including design top module\n|  |  afu_top                   // Contains top-level source files for AFU\n
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1321-walkthrough-clone-fim-repository","title":"1.3.2.1 Walkthrough: Clone FIM Repository","text":"
                                                                    Perform the following steps to clone the OFS Agilex PCIe Attach FIM Repository:
                                                                     
                                                                       
                                                                    1. Create a new directory to use as a clean starting point to store the retrieved files.     
                                                                      mkdir OFS_BUILD_ROOT\ncd OFS_BUILD_ROOT\nexport OFS_BUILD_ROOT=$PWD\n
                                                                      2.  
                                                                    2. Clone GitHub repository using the HTTPS git method     
                                                                      git clone --recurse-submodules https://github.com/OFS/ofs-agx7-pcie-attach.git\n
                                                                      3.  
                                                                    3. Check out the correct tag of the repository     
                                                                      cd ofs-agx7-pcie-attach\ngit checkout --recurse-submodules tags/ofs-2023.3-2\n
                                                                      4.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#133-environment-variables","title":"1.3.3 Environment Variables","text":"
                                                                    The OFS FIM compilation and simulation scripts require certain environment variables be set prior to execution.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#1331-walkthrough-set-development-environment-variables","title":"1.3.3.1 Walkthrough: Set Development Environment Variables","text":"
                                                                    Perform the following steps to set the required environment variables. These environment variables must be set prior to simulation or compilation tasks so it is recommended that you create a script to set these variables.
                                                                     
                                                                       
                                                                    1.  
                                                                      Navigate to the top level directory of the cloned OFS FIM repository.
                                                                       
                                                                      cd ofs-agx7-pcie-attach\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set project variables   
                                                                      # Set OFS Root Directory - e.g. this is the top level directory of the cloned OFS FIM repository\nexport OFS_ROOTDIR=$PWD\n
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Set variables based on your development environment   
                                                                      # Set proxies if required for your server\nexport http_proxy=<YOUR_HTTP_PROXY>\nexport https_proxy=<YOUR_HTTPS_PROXY>\nexport ftp_proxy=<YOUR_FTP_PROXY>\nexport socks_proxy=<YOUR_SOCKS_PROXY>\nexport no_proxy=<YOUR_NO_PROXY>\n\n# Set Quartus license path\nexport LM_LICENSE_FILE=<YOUR_LM_LICENSE_FILE>\n\n# Set Synopsys License path (if using Synopsys for simulation)\nexport DW_LICENSE_FILE=<YOUR_DW_LICENSE_FILE>\nexport SNPSLMD_LICENSE_FILE=<YOUR_SNPSLMD_LICENSE_FILE>\n\n# Set Quartus Installation Directory - e.g. $QUARTUS_ROOTDIR/bin contains Quartus executables\nexport QUARTUS_ROOTDIR=<YOUR_QUARTUS_INSTALLATION_DIRECTORY>\n\n# Set the Tools Directory - e.g. $TOOLS_LOCATION contains the 'synopsys' directory if you are using Synopsys. Refer to the $VCS_HOME variable for an example.\nexport TOOLS_LOCATION=<YOUR_TOOLS_LOCATION>\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Set generic environment variables
                                                                       
                                                                      # Set Work directory \nexport WORKDIR=$OFS_ROOTDIR\n\n# Set Quartus Tools variables\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport QUARTUS_VER_AC=$QUARTUS_ROOTDIR\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport IMPORT_IP_ROOTDIR=$IP_ROOTDIR\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\n\n# Set Verification Tools variables (if running simulations)\nexport DESIGNWARE_HOME=$TOOLS_LOCATION/synopsys/vip_common/vip_Q-2020.03A\nexport UVM_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport MTI_HOME=$QUARTUS_ROOTDIR/../questa_fse\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n\n# Set PATH to include compilation and simulation tools\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/../qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/bin:$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$DESIGNWARE_HOME/bin:$VCS_HOME/bin:$PATH\n
                                                                       
                                                                      5.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#134-walkthrough-set-up-development-environment","title":"1.3.4 Walkthrough: Set Up Development Environment","text":"
                                                                    This walkthrough guides you through the process of setting up your development environment in preparation for FIM development. This flow only needs to be done once on your development machine.
                                                                     
                                                                       
                                                                    1.  
                                                                      Ensure that Quartus Prime Pro Version 23.3 for Linux with Intel Agilex FPGA device support is installed on your development machine. Refer to the Install Quartus Prime Pro Software section for step-by-step installation instructions.
                                                                       
                                                                         
                                                                      1. Verify version number
                                                                        quartus_sh --version\n
                                                                         
                                                                        Example Output:
                                                                         
                                                                        Quartus Prime Shell\nVersion 23.3 Build 94 06/14/2023 SC Pro Edition\nCopyright (C) 2023  Intel Corporation. All rights reserved.\n
                                                                         
                                                                        2.  
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Ensure that all support tools are installed on your development machine, and that they meet the version requirements.
                                                                       
                                                                         
                                                                      1.  
                                                                        Python 3.6.8 or later
                                                                         
                                                                           
                                                                        1.  
                                                                          Verify version number
                                                                           
                                                                          python --version\n
                                                                           
                                                                          Example Output:
                                                                           
                                                                          Python 3.6.8\n
                                                                           
                                                                          2.  
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        GCC 7.4.0 or later
                                                                         
                                                                           
                                                                        1.  
                                                                          Verify version number
                                                                           
                                                                          gcc --version\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          gcc (GCC) 7.4.0\n
                                                                           
                                                                          2.  
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        cmake 3.15 or later
                                                                         
                                                                           
                                                                        1.  
                                                                          Verify version number
                                                                           
                                                                          cmake --version\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          cmake version 3.15\n
                                                                           
                                                                          2.  
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        git with git-lfs 1.8.3.1 or later. Refer to the Install Git Large File Storage Extension section for step-by-step instructions on installing the Git Large File Storage (LFS) extension.
                                                                         
                                                                           
                                                                        1.  
                                                                          Verify version number
                                                                           
                                                                          git --version\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          git version 1.8.3.1\n
                                                                           
                                                                          2.  
                                                                         
                                                                        5.  
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Clone the ofs-agx7-pcie-attach repository. Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Install UART IP license patch .02.
                                                                       
                                                                         
                                                                      1.  
                                                                        Navigate to the license directory
                                                                         
                                                                        cd $IOFS_BUILD_ROOT/license\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Install Patch 0.02
                                                                         
                                                                        sudo ./quartus-0.0-0.02iofs-linux.run\n
                                                                         
                                                                        3.  
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Install Quartus Patches 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe). All required patches are provided in the Assets of the OFS FIM Release Tag: https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Verify that patches have been installed correctly. They should be listed in the output of the following command.
                                                                       
                                                                      quartus_sh --version\n
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Set required environment variables. Refer to the [Set Environment Variables] section for step-by-step instructions.
                                                                       
                                                                      8.  
                                                                     
                                                                    This concludes the walkthrough for setting up your development environment. At this point you are ready to begin FIM development.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2-fim-compilation","title":"2. FIM Compilation","text":"
                                                                    This section describes the process of compiling OFS FIM designs using the provided build scripts. It contains two main sections:
                                                                     
                                                                       
                                                                    • Compilation Theory - Describes the theory behind FIM compilation
                                                                      •  
                                                                    • Compilation Flows - Describes the process of compiling a FIM
                                                                      •  
                                                                     
                                                                    The walkthroughs provided in this section are:
                                                                     
                                                                       
                                                                    • Compile OFS FIM
                                                                      •  
                                                                    • Manually Generate OFS Out-Of-Tree PR FIM
                                                                      •  
                                                                    • Change the Compilation Seed
                                                                      •  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#21-compilation-theory","title":"2.1 Compilation Theory","text":"
                                                                    This sections describes the theory behind FIM compilation.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#211-fim-build-script","title":"2.1.1 FIM Build Script","text":"
                                                                    The OFS Common Repository contains a script named build_top.sh which is used to build OFS FIM designs and generate output files that can be programmed to the board. After cloning the OFS FIM repository (with the ofs-common repository included), the build script can be found in the following location:
                                                                     
                                                                    $OFS_ROOTDIR/ofs-common/scripts/common/syn/build_top.sh\n
                                                                     
                                                                    The usage of the build_top.sh script is as follows:
                                                                     
                                                                    build_top.sh [-k] [-p] [-e] [--stage=<action>] [--ofss=<ip_config>] <build_target>[:<fim_options>] [<work_dir_name>]\n
                                                                     Field Options Description Requirement -k None Keep. Preserves and rebuilds within an existing work tree instead of overwriting it. Optional -p None When set, and if the FIM supports partial reconfiguration, a PR template tree is generated at the end of the FIM build. The PR template tree is located in the top of the work directory but is relocatable and uses only relative paths. See $OFS_ROOTDIR/syn/common/scripts generate_pr_release.sh for details. Optional -e None Run only Quartus analysis and elaboration. It completes the setup stage, passes -end synthesis to the Quartus compilation flow and exits without running the finish stage. Optional --stage all | setup | compile | finish Controls which portion of the OFS build is run.\u00a0\u00a0- all: Run all build stages (default)\u00a0\u00a0- setup: Initialize a project in the work directory\u00a0\u00a0- compile: Run the Quartus compilation flow on a project that was already initialized with setup\u00a0\u00a0- finish: Complete OFS post-compilation tasks, such as generating flash images and, if -p is set, generating a release. Optional --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. This parameter is consumed during the setup stage and IP is updated only inside the work tree. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. Optional <build_target> fseries-dk Specifies which board is being targeted. Required <fim_options> flat | null_he_lb | null_he_hssi | null_he_mem | null_he_mem_tg | no_hssi Used to change how the FIM is built.\u00a0\u00a0\u2022 flat - Compiles a flat design (no PR assignments). This is useful for bringing up the design on a new board without dealing with PR complexity.\u00a0\u00a0\u2022 null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null.\u00a0\u00a0\u2022 null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null.\u00a0\u00a0\u2022 null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null.\u00a0\u00a0\u2022 null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null.\u00a0\u00a0\u2022 no_hssi - Removes the HSSI-SS from the design. More than one FIM option may be passed included in the <fim_options> list by concatenating them separated by commas. For example: <build_target>:flat,null_he_lb,null_he_hssi Optional <work_dir_name> String Specifies the name of the work directory in which the FIM will be built. If not specified, the default target is $OFS_ROOTDIR/work Optional 
                                                                    Note: The he_null is a minimal block with CSRs that responds to PCIe MMIO requests in order to keep PCIe alive.
                                                                     
                                                                    The build script copies source files from the existing cloned repository into the specified work directory, which are then used for compilation. As such, any changes made in the base source files will be included in all subsequent builds, unless the -k option is used, in which case an existing work directories files are used as-is. Likewise, any changes made in a work directory is only applied to that work directory, and will not be updated in the base repository by default.
                                                                     
                                                                    Refer to Compile OFS FIM which provides step-by-step instructions for running the build_top.sh script with some of the different available options.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#212-ofss-file-usage","title":"2.1.2 OFSS File Usage","text":"
                                                                    The OFS FIM build script can use OFSS files to easily customize design IP prior to compilation using preset configurations. The OFSS files specify certain parameters for different IPs. Using OFSS is provided as a convenience feature for building FIMs. The Provided OFSS Files table below describes the pre-made OFSS files for the fseries-dk that can be found in the $OFS_ROOTDIR/tools/ofss_config directory. Only the OFSS files listed in this table are compatible with the fseries-dk In order to compile an fseries-dk FIM, you must supply OFSS files corresponding to each IP that is present in your design.
                                                                     
                                                                    Table: Provided OFSS Files
                                                                     OFSS File Name Location Type Description fseries-dk.ofss $OFS_ROOTDIR/tools/ofss_config Top Includes the following OFSS files [1]: \u00a0\u00a0\u2022 fseries-dk_base.ofss \u00a0\u00a0\u2022 pcie_host.ofss \u00a0\u00a0\u2022 iopll.ofss \u00a0\u00a0\u2022 memory_ftile.ofss fseries-dk_base.ofss $OFS_ROOTDIR/tools/ofss_config ofs Defines certain attributes of the design, including the platform name, device family, fim type, part number, and device ID. pcie_host.ofss $OFS_ROOTDIR/tools/ofss_config/pcie pcie Defines the PCIe Subsystem with the following configuration:\u00a0\u00a0\u2022 PF0 (3 VFs)\u00a0\u00a0\u2022 PF1 (0 VFs)\u00a0\u00a0\u2022 PF2 (0 VFs)\u00a0\u00a0\u2022 PF3 (0 VFs)\u00a0\u00a0\u2022 PF4 (0 VFs) pcie_1pf_1vf.ofss $OFS_ROOTDIR/tools/ofss_config/pcie pcie Defines the PCIe Subsystem with the following configuration:\u00a0\u00a0\u2022 PF0 (1 VF) iopll.ofss $OFS_ROOTDIR/tools/ofss_config/iopll iopll Sets the IOPLL frequency to 470 MHz memory_ftile.ofss $OFS_ROOTDIR/tools/ofss_config/memory memory Defines the memory IP preset file to be used during the build as fseries-dk hssi_8x25_ftile.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP preset file to be used during the build as fseries-dk 
                                                                    [1] The fseries-dk.ofss file does not include an HSSI OFSS file by default. If you are using the fseries-dk.ofss file when building the FIM, you must also specify an HSSI OFSS file for F-tile. Refer to the Compile OFS FIM Section for examples of this flow.
                                                                     
                                                                    Note: Using OFSS is required for FIM builds targeting the F-tile Development Kit.
                                                                     
                                                                    There can typically be three sections contained within an OFSS file.
                                                                     
                                                                       
                                                                    •  
                                                                      [include]
                                                                       
                                                                         
                                                                      • This section of an OFSS file contains elements separated by a newline, where each element is the path to an OFSS file that is to be included for configuration by the OFSS Configuration Tool. Ensure that any environment variables (e.g. $OFS_ROOTDIR) are set correctly. The OFSS Config tool uses breadth first search to include all of the specified OFSS files; the ordering of OFSS files does not matter
                                                                        •  
                                                                       
                                                                      •  
                                                                     
                                                                       
                                                                    •  
                                                                      [ip]
                                                                       
                                                                         
                                                                      • This section of an OFSS file contains a key value pair that allows the OFSS Config tool to determine which IP configuration is being passed in. The currently supported values of IP are ofs, iopll, pcie, memory, and hssi.
                                                                        •  
                                                                       
                                                                      •  
                                                                     
                                                                       
                                                                    •  
                                                                      [settings]
                                                                       
                                                                         
                                                                      • This section of an OFSS file contains IP specific settings. Refer to existing IP OFSS file of different types to see what IP settings are set. For the IP type ofss, the settings will be information of the OFS device (platform, family, fim, part #, device_id)
                                                                        •  
                                                                       
                                                                      •  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2121-platform-ofss-file","title":"2.1.2.1 Platform OFSS File","text":"
                                                                    The <platform>.ofss file (e.g. $OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss) is the platform level OFSS wrapper file. This is typically the OFSS file that is provided to the build script. It only contains an include section which lists all other OFSS files that are to be used when the <platform>.ofss file is passed to the build script.
                                                                     
                                                                    An example structure of a <platform>.ofss file is as follows:
                                                                     
                                                                    [include]\n<PATH_TO_PLATFORM_BASE_OFSS_FILE>\n<PATH_TO_PCIE_OFSS_FILE>\n<PATH_TO_IOPLL_OFSS_FILE>\n<PATH_TO_MEMORY_OFSS_FILE>\n<PATH_TO_HSSI_OFSS_FILE>\n
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2122-ofs-ip-ofss-file","title":"2.1.2.2 OFS IP OFSS File","text":"
                                                                    An OFSS file with IP type ofs (e.g. $OFS_ROOTDIR/tools/ofss_config/fseries-dk_base.ofss) contains board specific information for the target board.
                                                                     
                                                                    Currently supported configuration options for an OFSS file with IP type ofs are described in the OFS IP OFSS File Options table.
                                                                     
                                                                    Table: OFS IP OFSS File Options
                                                                     Section Parameter fseries-dk Default Value [ip] type ofs [settings] platform N6001 family agilex fim base_x16 part AGFB027R24C2E2VR2 device_id 6001"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2123-pcie-ip-ofss-file","title":"2.1.2.3 PCIe IP OFSS File","text":"
                                                                    An OFSS file with IP type pcie (e.g. $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss) is used to configure the PCIe-SS in the FIM.
                                                                     
                                                                    The PCIe OFSS file has a special section type ([pf*]) which is used to define physical functions (PFs) in the FIM. Each PF has a dedicated section, where the * character is replaced with the PF number. For example, [pf0], [pf1], etc. For reference FIM configurations, 0 virtual functions (VFs) on PF0 is not supported. This is because the PR region cannot be left unconnected. A NULL AFU may need to be instantiated in this special case. PFs must be consecutive. The PFVF Limitations table describes the supported number of PFs and VFs.
                                                                     
                                                                    Table: PF/VF Limitations
                                                                     Parameter Value Min # of PFs 1 (on PF0) Max # of PFs 8 Min # of VFs 1 on PF0 Max # of VFs 2000 distributed across all PFs 
                                                                    Currently supported configuration options for an OFSS file with IP type pcie are described in the PCIe IP OFSS File Options table.
                                                                     
                                                                    Table: PCIe IP OFSS File Options
                                                                     Section Parameter Options Default Description [ip] type pcie N/A Specifies that this OFSS file configures the PCIe-SS [settings] output_name pcie_ss N/A Specifies the output name of the PCIe-SS IP preset String N/A OPTIONAL - Specifies the name of a PCIe-SS IP presets file to use when building the FIM. When used, a presets file will take priority over any other parameters set in this OFSS file. [pf*] num_vfs Integer 0 Specifies the number of Virtual Functions in the current PF bar0_address_width Integer 12 bar4_address_width Integer 14 vf_bar0_address_width Integer 12 ats_cap_enable 0 | 1 0 vf_ats_cap_enable 0 | 1 0 prs_ext_cap_enable 0 | 1 0 pasid_cap_enable 0 | 1 0 pci_type0_vendor_id 32'h Value 0x00008086 pci_type0_device_id 32'h Value 0x0000bcce revision_id 32'h Value 0x00000001 class_code 32'h Value 0x00120000 subsys_vendor_id 32'h Value 0x00008086 subsys_dev_id 32'h Value 0x00001771 sriov_vf_device_id 32'h Value 0x0000bccf exvf_subsysid 32'h Value 0x00001771 
                                                                    The default values for all PCIe-SS parameters (that are not defined in the PCIe IP OFSS file) are defined in $OFS_ROOTDIR/ofs-common/tools/ofss_config/ip_params/pcie_ss_component_parameters.py. When using a PCIe IP OFSS file during compilation, the PCIe-SS IP that is used will be defined based on the values in the PCIe IP OFSS file plus the parameters defined in pcie_ss_component_parameters.py.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2124-iopll-ip-ofss-file","title":"2.1.2.4 IOPLL IP OFSS File","text":"
                                                                    An OFSS file with IP type iopll (e.g. $OFS_ROOTDIR/tools/ofss_config/iopll/iopll.ofss) is used to configure the IOPLL in the FIM.
                                                                     
                                                                    The IOPLL OFSS file has a special section type ([p_clk]) which is used to define the IOPLL clock frequency.
                                                                     
                                                                    Currently supported configuration options for an OFSS file with IP type iopll are described in the IOPLL OFSS File Options table.
                                                                     
                                                                    Table: IOPLL OFSS File Options
                                                                     Section Parameter Options Description [ip] type iopll Specifies that this OFSS file configures the IOPLL [settings] output_name sys_pll Specifies the output name of the IOPLL. instance_name iopll_0 Specifies the instance name of the IOPLL. [p_clk] freq Integer: 250 - 470 Specifies the IOPLL clock frequency in MHz. 
                                                                    Note: The following frequencies have been tested on reference boards: 350MHz, 400MHz, 470MHz.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2125-memory-ip-ofss-file","title":"2.1.2.5 Memory IP OFSS File","text":"
                                                                    An OFSS file with IP type memory (e.g. $OFS_ROOTDIR/tools/ofss_config/memory/memory_ftile.ofss) is used to configure the Memory-SS in the FIM.
                                                                     
                                                                    The Memory OFSS file specifies a preset value, which selects a presets file (.qprs) to configure the Memory-SS.
                                                                     
                                                                    Currently supported configuration options for an OFSS file with IP type memory are described in the Memory OFSS File Options table.
                                                                     
                                                                    Table: Memory OFSS File Options
                                                                     Section Parameter Options Description [ip] type memory Specifies that this OFSS file configures the Memory-SS [settings] output_name mem_ss_fm Specifies the output name of the Memory-SS. preset fseries-dk | String[1] Specifies the name of the .qprs presets file that will be used to build the Memory-SS. 
                                                                    [1] You may generate your own .qprs presets file with a unique name using Quartus. 
                                                                     
                                                                    Pre-provided Memory-SS presets files are located in the $OFS_ROOTDIR/ipss/mem/qip/presets directory.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2126-hssi-ip-ofss-file","title":"2.1.2.6 HSSI IP OFSS File","text":"
                                                                    An OFSS file with IP type hssi (e.g. $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_8x25_ftile.ofss) is used to configure the Ethernet-SS in the FIM.
                                                                     
                                                                    Currently supported configuration options for an OFSS file with IP type hssi are described in the HSSI OFSS File Options table.
                                                                     
                                                                    Table: HSSI OFSS File Options
                                                                     Section Parameter Options Description [ip] type hssi Specifies that this OFSS file configures the Ethernet-SS [settings] output_name hssi_ss Specifies the output name of the Ethernet-SS num_channels Integer Specifies the number of channels. data_rate 10GbE | 25GbE Specifies the data rate[1] preset fseries-dk | String[2] Specifies the name of the .qprs preset file that will be used to build the Ethernet-SS. This will overwrite the other settings in this OFSS file. 
                                                                    [1] The presets file will take priority over the data_rate parameter, so this value will not take effect when using a presets file.
                                                                     
                                                                    [2] You may generate your own .qprs presets file with a unique name using Quartus. 
                                                                     
                                                                    Pre-provided Ethernet-SS presets are located in the  $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss/presets directory.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#213-ofs-build-script-outputs","title":"2.1.3 OFS Build Script Outputs","text":"
                                                                    The output files resulting from running the the OFS FIM build_top.sh build script are copied to a single directory during the finish stage of the build script. The path for this directory is: $OFS_ROOTDIR/<WORK_DIRECTORY>/syn/board/fseries-dk/syn_top/output_files.
                                                                     
                                                                    The output files include programmable images and compilation reports. The OFS Build Script Output Descriptions table describes the images that are generated by the build script.
                                                                     
                                                                    Table: OFS Build Script Output Descriptions
                                                                     File Name Description ofs_top.sof The FIM design SRAM Object File; a binary file of the compiled FIM image. ofs_top_hps.sof The build assembly process combines the FPGA ofs_top.sof programming file with u-boot-spl-dtb.hex to produce this file."},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#22-compilation-flows","title":"2.2 Compilation Flows","text":"
                                                                    This section provides information for using the build script to generate different FIM types. Walkthroughs are provided for each compilation flow. These walkthroughs require that the development environment has been set up as described in the Set Up Development Environment section.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#221-flat-fim","title":"2.2.1 Flat FIM","text":"
                                                                    A flat FIM is compiled such that there is no partial reconfiguration region, and the entire design is built as a flat design. This is useful for compiling new designs without worrying about the complexity introduced by partial reconfiguration. The flat compile removes the PR region and PR IP; thus, you cannot use the -p build flag when using the flat compile setting. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#222-in-tree-pr-fim","title":"2.2.2 In-Tree PR FIM","text":"
                                                                    An In-Tree PR FIM is the default compilation if no compile flags or compile settings are used. This flow will compile the design with the partial reconfiguration region, but it will not create a relocatable PR directory tree to aid in AFU development. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#223-out-of-tree-pr-fim","title":"2.2.3 Out-of-Tree PR FIM","text":"
                                                                    An Out-of-Tree PR FIM will compile the design with the partial reconfiguration region, and will create a relocatable PR directory tree to aid in AFU workload development. This is especially useful if you are developing a FIM to be used by another team developing AFU workloads. This is the recommended build flow in most cases. There are two ways to create the relocatable PR directory tree:
                                                                     
                                                                       
                                                                    • Run the FIM build script with the -p option. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                                                                      •  
                                                                    • Run the generate_pr_release.sh script after running the FIM build script. Refer to the Manually Generate OFS Out-Of-Tree PR FIM Section step-by-step instructions for this flow.
                                                                      •  
                                                                     
                                                                    In both cases, the generate_pr_release.sh is run to create the relocatable build tree. This script is located at $OFS_ROOTDIR/ofs-common/scripts/common/syn/generate_pr_release.sh. Usage for this script is as follows:
                                                                     
                                                                    generate_pr_release.sh -t <PATH_OF_RELOCATABLE_PR_TREE> <BOARD_TARGET> <WORK_DIRECTORY>\n
                                                                     
                                                                    The Generate PR Release Script Options table describes the options for the generate_pr_release.sh script.
                                                                     
                                                                    Table: Generate PR Release Script Options
                                                                     Parameter Options Description <PATH_OF_RELOCATABLE_PR_TREE> String Specifies the location of the relocatable PR directory tree to be created. <BOARD_TARGET> fseries-dk Specifies the name of the board target. <WORK_DIRECTORY> String Specifies the existing work directory from which the relocatable PR directory tree will be created from. 
                                                                    After generating the relocatable build tree, it is located in the $OFS_ROOTDIR/<WORK_DIRECTORY>/pr_build_template directory (or the directory you specified if generated separately). The contents of this directory have the following structure:
                                                                     
                                                                    \u251c\u2500\u2500 bin\n\u251c\u2500\u2500 \u251c\u2500\u2500 afu_synth\n\u251c\u2500\u2500 \u251c\u2500\u2500 qar_gen\n\u251c\u2500\u2500 \u251c\u2500\u2500 update_pim\n\u251c\u2500\u2500 \u251c\u2500\u2500 run.sh\n\u251c\u2500\u2500 \u251c\u2500\u2500 build_env_config\n\u251c\u2500\u2500 README\n\u251c\u2500\u2500 hw\n\u251c\u2500\u2500 \u251c\u2500\u2500 lib\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 build\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 fme-ifc-id.txt\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 platform\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 fme-platform-class.txt\n\u251c\u2500\u2500 \u251c\u2500\u2500 blue_bits\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 ofs_top_hps.sof\n\u2514\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 ofs_top.sof\n
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#223-he_null-fim","title":"2.2.3 HE_NULL FIM","text":"
                                                                    An HE_NULL FIM refers to a design with one, some, or all of the Host Exercisers replaced by he_null blocks. The he_null is a minimal block with CSRs that responds to PCIe MMIO requests in order to keep PCIe alive. You may use any of the build flows (flat, in-tree, out-of-tree) with the HE_NULL compile options. The HE_NULL compile options are as follows:
                                                                     
                                                                       
                                                                    • null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null
                                                                      •  
                                                                    • null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null
                                                                      •  
                                                                    • null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null
                                                                      •  
                                                                    • null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null
                                                                      •  
                                                                     
                                                                    The Compile OFS FIM section gives step-by-step instructions for this flow.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#225-walkthrough-compile-ofs-fim","title":"2.2.5 Walkthrough: Compile OFS FIM","text":"
                                                                    Perform the following steps to compile the OFS Agilex PCIe Attach FIM for fseries-dk:
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Navigate to the root directory.
                                                                       
                                                                      cd $OFS_ROOTDIR\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Run the build_top.sh script with the desired compile options using the fseries-dk OFSS presets. In this case you must use the fseries-dk.ofss file, and then specify the 8x25G HSSI configuration using the hssi_8x25_ftile.ofss (this is not necessary if using the no_hssi compile option).
                                                                       
                                                                         
                                                                      •  
                                                                        Flat FIM
                                                                         
                                                                        ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk:flat work_fseries-dk_flat\n
                                                                         
                                                                        •  
                                                                       
                                                                         
                                                                      •  
                                                                        In-Tree PR FIM
                                                                         
                                                                        ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_in_tree_pr\n
                                                                         
                                                                        •  
                                                                       
                                                                         
                                                                      •  
                                                                        Out-of-Tree PR FIM
                                                                         
                                                                        ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_oot_pr\n
                                                                         
                                                                        •  
                                                                       
                                                                         
                                                                      •  
                                                                        HE_NULL Flat FIM
                                                                         
                                                                        ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/fseries-dk.ofss fseries-dk:flat,null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_fseries-dk_flat\n
                                                                         
                                                                        •  
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Once the build script is complete, the build summary should report that the build is complete and passes timing. For example:
                                                                       
                                                                      ***********************************\n***\n***        OFS_PROJECT: ofs-agx7-pcie-attach\n***        OFS_BOARD: fseries-dk\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 6\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                                                       
                                                                      6.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#226-walkthrough-manually-generate-ofs-out-of-tree-pr-fim","title":"2.2.6 Walkthrough: Manually Generate OFS Out-Of-Tree PR FIM","text":"
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Navigate to the root directory.
                                                                       
                                                                      cd $OFS_ROOTDIR\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Run the build_top.sh script with the desired compile options using the fseries-dk OFSS presets. In order to create the relocatable PR tree, you may not compile with the flat option. For example:
                                                                       
                                                                      ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk\n
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Run the generate_pr_release.sh script to create the relocatable PR tree.
                                                                       
                                                                      ./ofs-common/scripts/common/syn/generate_pr_release.sh -t work_fseries-dk/pr_build_template fseries-dk work_fseries-dk\n
                                                                       
                                                                      6.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#227-compilation-seed","title":"2.2.7 Compilation Seed","text":"
                                                                    You may change the seed which is used by the build script during Quartus compilation to change the starting point of the fitter. Trying different seeds is useful when your design is failing timing by a small amount.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#2271-walkthrough-change-the-compilation-seed","title":"2.2.7.1 Walkthrough: Change the Compilation Seed","text":"
                                                                    Perform the following steps to change the compilation seed for the FIM build.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Edit the SEED assignment in the $OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_top.qsf file to your desired seed value. The value can be any non-negative integer value.
                                                                       
                                                                      set_global_assignment -name SEED 1\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Build the FIM. Refer to the Compile OFS FIM section for instructions.
                                                                       
                                                                      5.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#3-fim-simulation","title":"3. FIM Simulation","text":"
                                                                    Unit level simulation of key components in the FIM is provided for verification of the following areas:
                                                                     
                                                                       
                                                                    • Ethernet
                                                                      •  
                                                                    • PCIe
                                                                      •  
                                                                    • External Memory
                                                                      •  
                                                                    • Core FIM
                                                                      •  
                                                                     
                                                                    The Unit Level simulations work with Synopsys VCS/VCSMX or Mentor Graphics Questasim simulators. The scripts to run each unit level simulation are located in $OFS_ROOTDIR/sim/unit_test. Each unit test directory contains a README which describes the test in detail.  Refer to the Supported Unit Tests table for a list of the supported unit tests.
                                                                     
                                                                    Note: The OFS Agilex PCIe Attach FIM for the F-Tile Development Kit does not support all of the unit tests that are provided in the unit_test directory.
                                                                     
                                                                    Table: Supported Unit Tests
                                                                     Test Name Description bfm_test This is the unit test for PCIe BFM. The test uses HE-LB to perform memory loopback between FIM and the host. csr_test This is the unit test for FIM CSR access and AFU memory write/read dfh_walker This is the unit test for FME DFH walking flr This is the unit test for PCIe PF/VF FLR fme_csr_access This is the a unit test for the register access logic for $OFS_ROOTDIR/ofs-common/src/common/fme/fme_csr.sv fme_csr_directed This is the unit test for $OFS_ROOTDIR/ofs-common/src/common/fme/fme_csr.sv he_lb_test This is the unit test for HE_LPBK. The test uses HE-LB to perform memory loopback between FIM and the host. he_null_test This is the unit test for HE-NULL Exerciser. The test issues basic mmio Rd/Wr requests targetting HE-NULL CSRs. indirect_csr This is the unit test for axi4lite_indirect_csr_if module. pcie_csr_test This is the unit test for PCIE CSR access. pf_vf_access_test This is the unit test for PCIe PF/VF MMIO. Each function has a feature GUID at offset 0x8 with an associated register map. port_gasket_test This is the unit test for pg_csr block and it's connectivity to fabric. The test issues mmio Rd/Wr requests targetting the csrs in port_gasket. This test does not do any functional testing of partial reconfiguration, user clock or remote stp. remote_stp_test This is the unit test for remote stp. It covers mmio read access to remote_stp registers. uart_csr This is the unit test for UART CSR accesses."},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#31-simulation-file-generation","title":"3.1 Simulation File Generation","text":"
                                                                    The simulation files must be generated prior to running unit level simulations. The script to generate simulation files is in the following location:
                                                                     
                                                                    $OFS_ROOTDIR/ofs-common/scripts/common/sim/gen_sim_files.sh\n
                                                                     
                                                                    The usage of the gen_sim_files.sh script is as follows:
                                                                     
                                                                    gen_sim_files.sh [--ofss=<ip_config>] <build_target>[:<fim_options>] [<device>] [<family>]\n
                                                                     
                                                                    The Gen Sim Files Script Options table describes the options for the gen_sim_files.sh script.
                                                                     
                                                                    Table: Gen Sim Files Script Options
                                                                     Field Options Description Requirement --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. Platform Dependent[1] <build_target> fseries-dk | n6001 Specifies which board is being targeted. Required <fim_options> null_he_lb | null_he_hssi | null_he_mem | null_he_mem_tg Used to change how the FIM is built.\u00a0\u00a0- null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null.\u00a0\u00a0- null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null.\u00a0\u00a0- null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null.\u00a0\u00a0- null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null. More than one FIM option may be passed included in the <fim_options> list by concatenating them separated by commas. For example: <build_target>:null_he_lb,null_he_hssi Optional <device> string Specifies the device ID for the target FPGA. If not specified, the default device is parsed from the QSF file for the project. Optional <family> string Specifies the family for the target FPGA. If not specified, the default family is parsed from the QSF file for the project. Optional 
                                                                    [1] Using OFSS is required for the F-Tile Development Kit.
                                                                     
                                                                    Refer to the Run Individual Unit Level Simulation section for an example of the simulation files generation flow.
                                                                     
                                                                    When running regression tests, you may use the -g command line argument to generate simulation files; refer to the Run Regression Unit Level Simulation section for step-by-step instructions.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#32-individual-unit-tests","title":"3.2 Individual Unit Tests","text":"
                                                                    Each unit test may be run individually using the run_sim.sh script located in the following directory:
                                                                     
                                                                    $OFS_ROOTDIR/ofs-common/scripts/common/sim/run_sim.sh\n
                                                                     
                                                                    The usage for the run_sim.sh script is as follows:
                                                                     
                                                                    sh run_sim.sh TEST=<test> [VCSMX=<0|1> | MSIM=<0|1>]\n
                                                                     
                                                                    The Run Sim Script Options table describes the options for the run_sim.sh script.
                                                                     
                                                                    Table: Run Sim Script Options
                                                                     Field Options Description TEST String Specify the name of the test to run, e.g. dfh_walker VCSMX 0 | 1 When set, the VCSMX simulator will be used MSIM 0 | 1 When set, the QuestaSim simulator will be used 
                                                                    Note: The default simulator is VCS if neither VCSMX nor MSIM are set.
                                                                     
                                                                    The log for a unit test is stored in a transcript file in the simulation directory of the test that was run.
                                                                     
                                                                    $OFS_ROOTDIR/sim/unit_test/<TEST_NAME>/<SIMULATOR>/transcript\n
                                                                     
                                                                    For example, the log for the DFH walker test using VCSMX would be found at:
                                                                     
                                                                    $OFS_ROOTDIR/sim/unit_test/dfh_walker/sim_vcsmx/transcript\n
                                                                     
                                                                    The simulation waveform database is saved as vcdplus.vpd for post simulation review.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#321-walkthrough-run-individual-unit-level-simulation","title":"3.2.1 Walkthrough: Run Individual Unit Level Simulation","text":"
                                                                    Perform the following steps to run an individual unit test.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Generate the simulation files for the fseries-dk
                                                                       
                                                                      cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Navigate to the common simulation directory   
                                                                      cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Run the desired unit test using your desired simulator
                                                                       
                                                                         
                                                                      •  
                                                                        Using VCS
                                                                         
                                                                        sh run_sim.sh TEST=<test_name>\n
                                                                         
                                                                        •  
                                                                       
                                                                         
                                                                      •  
                                                                        Using VCSMX
                                                                         
                                                                        sh run_sim.sh TEST=<test_name> VCSMX=1\n
                                                                         
                                                                        •  
                                                                       
                                                                         
                                                                      •  
                                                                        Using QuestaSim
                                                                         
                                                                        sh run_sim.sh TEST=<test_name> MSIM=1\n
                                                                         
                                                                        •  
                                                                       
                                                                         
                                                                      •  
                                                                        For example, to run the DFH walker test using VCSMX:
                                                                         
                                                                        sh run_sim.sh TEST=dfh_walker VCSMX=1\n
                                                                         
                                                                        •  
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Once the test is complete, check the output for the simulation results. Review the log for detailed test results.
                                                                       
                                                                      Test status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/ofs-agx7-pcie-attach/sim/unit_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time         356233750000\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 356233750000 fs\nCPU Time:     57.730 seconds;       Data structure size:  47.2Mb\nTue Sep  5 09:44:19 2023\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                                                                       
                                                                      7.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#33-regression-unit-tests","title":"3.3 Regression Unit Tests","text":"
                                                                    You may use the regression script regress_run.py to run some or all of the unit tests available with a single command. The regression script is in the following location:
                                                                     
                                                                    $OFS_ROOTDIR/sim/unit_test/scripts/regress_run.py\n
                                                                     
                                                                    The usage of the regression script is as follows:
                                                                     
                                                                    regress_run.py [-h] [-l] [-n <num_procs>] [-k <test_package>] [-s <simulator>] [-g] [--ofss <ip_config>] [-b <board_name>] [-e]\n
                                                                     
                                                                    The Regression Unit Test Script Options table describes the options for the regress_run.py script.
                                                                     
                                                                    Table: Regression Unit Test Script Options
                                                                     Field Options Description -h | --help N/A Show the help message and exit -l | --local N/A Run regression locally -n | --n_procs Integer Maximum number of processes/tests to run in parallel when run locally. This has no effect on farm runs. -k | --pack all | fme | he | hssi | list | mem | pmci Test package to run during regression. The \"list\" option will look for a text file named \"list.txt\" in the \"unit_test\" directory for a text list of tests to run (top directory names). The default test package is all. -s | --sim vcs | vcsmx | msim Specifies the simulator used for the regression tests. The default simulator is vcs -g | --gen_sim_files N/A Generate simulation files. This only needs to be done once per repo update. This is the equivalent of running the gen_sim_files.sh script. -o | --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. -b | --board_name n6000 | n6001 | fseries-dk Specifies the board target -e | --email_list String Specifies email list to send results to multiple recipients 
                                                                    The log for each unit test that is run by the regression script is stored in a transcript file in the simulation directory of the test that was run.
                                                                     
                                                                    $OFS_ROOTDIR/sim/unit_test/<TEST_NAME>/<SIMULATOR>/transcript\n
                                                                     
                                                                    For example, the log for the DFH walker test using VCSMX would be found at:
                                                                     
                                                                    $OFS_ROOTDIR/sim/unit_test/dfh_walker/sim_vcsmx/transcript\n
                                                                     
                                                                    The simulation waveform database is saved as vcdplus.vpd for post simulation review.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#331-walkthrough-run-regression-unit-level-simulation","title":"3.3.1 Walkthrough: Run Regression Unit Level Simulation","text":"
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Create a test list file to only run the unit level simulations that are supported for the fseries-dk FIM.
                                                                       
                                                                      touch $OFS_ROOTDIR/sim/unit_test/list.txt\n
                                                                       
                                                                      Copy the following list into the new file. You may remove tests from this list as desired.
                                                                       
                                                                      ./bfm_test/set_params.sh\n./csr_test/set_params.sh\n./dfh_walker/set_params.sh\n./flr/set_params.sh\n./fme_csr_access/set_params.sh\n./fme_csr_directed/set_params.sh\n./he_lb_test/set_params.sh\n./he_null_test/set_params.sh\n./hssi_csr_test/set_params.sh\n./hssi_kpi_test/set_params.sh\n./hssi_test/set_params.sh\n./indirect_csr/set_params.sh\n./pcie_csr_test/set_params.sh\n./pf_vf_access_test/set_params.sh\n./port_gasket_test/set_params.sh\n./qsfp_test/set_params.sh\n./remote_stp_test/set_params.sh\n./uart_csr/set_params.sh\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Run regression test with the your desired options. For example, to simulate with the options to generate simulation files, run locally, use 8 processes, run the specified test list, use VCSMX simulator, and target the fseries-dk:
                                                                       
                                                                      cd $OFS_ROOTDIR/sim/unit_test/scripts\n\npython regress_run.py --ofss $OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss -g -l -n 8 -k list -s vcsmx -b fseries-dk\n
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Once all tests are complete, check that the tests have passed.
                                                                       
                                                                      2023-08-30 15:00:50,256: Passing Unit Tests:13/13 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-08-30 15:00:50,256:    bfm_test:......... PASS -- Time Elapsed:0:22:23.940917\n2023-08-30 15:00:50,256:    csr_test:......... PASS -- Time Elapsed:0:22:46.262916\n2023-08-30 15:00:50,256:    dfh_walker:....... PASS -- Time Elapsed:0:22:16.544732\n2023-08-30 15:00:50,256:    flr:.............. PASS -- Time Elapsed:0:22:21.332386\n2023-08-30 15:00:50,256:    fme_csr_access:... PASS -- Time Elapsed:0:17:12.454034\n2023-08-30 15:00:50,256:    fme_csr_directed:. PASS -- Time Elapsed:0:17:22.947134\n2023-08-30 15:00:50,256:    he_lb_test:....... PASS -- Time Elapsed:0:28:38.962424\n2023-08-30 15:00:50,256:    indirect_csr:..... PASS -- Time Elapsed:0:21:15.387478\n2023-08-30 15:00:50,256:    pcie_csr_test:.... PASS -- Time Elapsed:0:22:33.838949\n2023-08-30 15:00:50,256:    pf_vf_access_test: PASS -- Time Elapsed:0:22:28.704149\n2023-08-30 15:00:50,256:    port_gasket_test:. PASS -- Time Elapsed:0:22:32.592301\n2023-08-30 15:00:50,256:    remote_stp_test:.. PASS -- Time Elapsed:0:22:01.485914\n2023-08-30 15:00:50,256:    uart_csr:......... PASS -- Time Elapsed:0:22:31.848882\n2023-08-30 15:00:50,256: Failing Unit Tests: 0/13 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-08-30 15:00:50,256: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-08-30 15:00:50,256:       Number of Unit test results captured: 13\n2023-08-30 15:00:50,256:       Number of Unit test results passing.: 13\n2023-08-30 15:00:50,256:       Number of Unit test results failing.:  0\n2023-08-30 15:00:50,256:     End Unit regression running at date/time................: 2023-08-30 15:00:50.256725\n2023-08-30 15:00:50,256:     Elapsed time for Unit regression run....................: 0:54:48.172625\n
                                                                       
                                                                      6.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#4-fim-customization","title":"4. FIM Customization","text":"
                                                                    This section describes how to perform specific customizations of the FIM, and provides step-by-step walkthroughs for these customizations. Each walkthrough can be done independently. These walkthroughs require a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment. The FIM Customization Walkthroughs table lists the walkthroughs that are provided in this section. Some walkthroughs include steps for testing on hardware. Testing on hardware requires that you have a deployment environment set up. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                                                                     
                                                                    Table: FIM Customization Walkthroughs
                                                                     Customization Walkthrough Name Add a new module to the OFS FIM Modify and run unit tests for a FIM that has a new module Hardware test a FIM that has a new module Debug the FIM with Signal Tap Compile the FIM in preparation for designing your AFU Resize the Partial Reconfiguration Region Modify PCIe Configuration Using OFSS Modify PCIe Configuration Using IP Presets Create a Minimal FIM Migrate to a Different Agilex Device Number Modify the Ethernet Sub-System to 2x4x10GbE Modify the Ethernet Sub-System to 3x4x10GbE Remove the HPS"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#41-adding-a-new-module-to-the-fim","title":"4.1 Adding a new module to the FIM","text":"
                                                                    This section provides a information for adding a custom module to the FIM, simulating the new design, compiling the new design, implementing and testing the new design on hardware, and debugging the new design on hardware.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#411-hello-fim-theory-of-operation","title":"4.1.1 Hello FIM Theory of Operation","text":"
                                                                    If you intend to add a new module to the FIM area, then you will need to inform the host software of the new module. The FIM exposes its functionalities to host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). This set of CSR registers and their operation is described in FIM MMIO Regions.
                                                                     
                                                                    See FPGA Device Feature List (DFL) Framework Overview for a description of the software process to read and process the linked list of Device Feature Header (DFH) CSRs within a FPGA.
                                                                     
                                                                    This example will add a hello_fim module to the design. The Hello FIM example adds a simple DFH register and 64bit scratchpad register connected to the Board Peripheral Fabric (BPF) that can be accessed by the Host. You can use this example as the basis for adding a new feature to your FIM.
                                                                     
                                                                    The Hello FIM design can be verified by Unit Level simulation, Universal Verification Methodology (UVM) simulation, and running in hardware on the fseries-dk card. The process for these are described in this section.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#4111-hello-fim-board-peripheral-fabric-bpf","title":"4.1.1.1 Hello FIM Board Peripheral Fabric (BPF)","text":"
                                                                    The Hello FIM module will be connected to the Board Peripheral Fabric (BPF), and will be connected such that it can be mastered by the Host. The BPF is an interconnect generated by Platform Designer. The Hello FIM BPF Interface Diagram figure shows the APF/BPF Master/Slave interactions, as well as the added Hello FIM module.
                                                                     
                                                                    Figure: Hello FIM BPF Interface Diagram
                                                                     
                                                                     
                                                                    The BPF fabric is defined in $OFS_ROOTDIR/src/pd_qsys/fabric/bpf.txt.
                                                                     
                                                                    We will add the Hello FIM module to an un-used address space in the MMIO region. The Hello FIM MMIO Address Layout table below shows the MMIO region for the Host with the Hello FIM module added at base address 0x16000.
                                                                     
                                                                    Table: Hello FIM MMIO Address Layout
                                                                     Offset Feature CSR set 0x00000 FME AFU 0x10000 PCIe Interface 0x12000 QSFP Controller 0 0x13000 QSFP Controller 1 0x14000 HSSI Interface 0x15000 EMIF Interface 0x16000 Hello FIM"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#4112-hello-fim-csr","title":"4.1.1.2 Hello FIM CSR","text":"
                                                                    The Hello FIM CSR will consist of the three registers shown in the Hello FIM CSR table below. The DFH and Hello FIM ID registers are read-only. The Scratchpad register supports read and write accesses.
                                                                     
                                                                    Table: Hello FIM CSR
                                                                     Offset Attribute Description Default Value 0x016000 RO DFH(Device Feature Headers) register 0x30000006a0000100 0x016030 RW Scrachpad register 0x0 0x016038 RO Hello FIM ID register 0x6626070150000034"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#412-walkthrough-add-a-new-module-to-the-ofs-fim","title":"4.1.2 Walkthrough: Add a new module to the OFS FIM","text":"
                                                                    Perform the following steps to add a new module to the OFS FIM that can be accessed by the Host.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Make hello_fim source directory
                                                                       
                                                                      mkdir $OFS_ROOTDIR/src/hello_fim\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Create hello_fim_top.sv file.
                                                                       
                                                                      touch $OFS_ROOTDIR/src/hello_fim/hello_fim_top.sv\n
                                                                       
                                                                      Copy the following code into hello_fim_top.sv:
                                                                       
                                                                      // ***************************************************************************\n//                               INTEL CONFIDENTIAL\n//\n//        Copyright (C) 2023 Intel Corporation All Rights Reserved.\n//\n// The source code contained or described herein and all  documents related to\n// the  source  code  (\"Material\")  are  owned  by  Intel  Corporation  or its\n// suppliers  or  licensors.    Title  to  the  Material  remains  with  Intel\n// Corporation or  its suppliers  and licensors.  The Material  contains trade\n// secrets  and  proprietary  and  confidential  information  of  Intel or its\n// suppliers and licensors.  The Material is protected  by worldwide copyright\n// and trade secret laws and treaty provisions. No part of the Material may be\n// used,   copied,   reproduced,   modified,   published,   uploaded,  posted,\n// transmitted,  distributed,  or  disclosed  in any way without Intel's prior\n// express written permission.\n//\n// No license under any patent,  copyright, trade secret or other intellectual\n// property  right  is  granted  to  or  conferred  upon  you by disclosure or\n// delivery  of  the  Materials, either expressly, by implication, inducement,\n// estoppel or otherwise.  Any license under such intellectual property rights\n// must be express and approved by Intel in writing.\n//\n// You will not, and will not allow any third party to modify, adapt, enhance, \n// disassemble, decompile, reverse engineer, change or create derivative works \n// from the Software except and only to the extent as specifically required by \n// mandatory applicable laws or any applicable third party license terms \n// accompanying the Software.\n//\n// -----------------------------------------------------------------------------\n// Engineer     : \n// Create Date  : September 2023\n// Module Name  : hello_fim_top.sv\n// Project      : OFS\n// -----------------------------------------------------------------------------\n//\n// Description: \n// This is a simple module that implements DFH registers and \n// AVMM address decoding logic.\n\nmodule hello_fim_top  #(\n   parameter ADDR_WIDTH  = 12, \n   parameter DATA_WIDTH = 64, \n   parameter bit [11:0] FEAT_ID = 12'h001,\n   parameter bit [3:0]  FEAT_VER = 4'h1,\n   parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n   parameter bit END_OF_LIST = 1'b0\n)(\n   input  logic    clk,\n   input  logic    reset,\n// -----------------------------------------------------------\n//  AXI4LITE Interface\n// -----------------------------------------------------------\n   ofs_fim_axi_lite_if.slave   csr_lite_if\n);\n\nimport ofs_fim_cfg_pkg::*;\nimport ofs_csr_pkg::*;\n\n//-------------------------------------\n// Signals\n//-------------------------------------\n   logic [ADDR_WIDTH-1:0]              csr_waddr;\n   logic [DATA_WIDTH-1:0]              csr_wdata;\n   logic [DATA_WIDTH/8-1:0]            csr_wstrb;\n   logic                               csr_write;\n   logic                               csr_slv_wready;\n   csr_access_type_t                   csr_write_type;\n\n   logic [ADDR_WIDTH-1:0]              csr_raddr;\n   logic                               csr_read;\n   logic                               csr_read_32b;\n   logic [DATA_WIDTH-1:0]              csr_readdata;\n   logic                               csr_readdata_valid;\n   logic [ADDR_WIDTH-1:0]              csr_addr;\n\n   logic [63:0]                        com_csr_writedata;\n   logic                               com_csr_read;\n   logic                               com_csr_write;\n   logic [63:0]                        com_csr_readdata;\n   logic                               com_csr_readdatavalid;\n   logic [5:0]                         com_csr_address;\n\n// AXI-M CSR interfaces\nofs_fim_axi_mmio_if #(\n   .AWID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .AWADDR_WIDTH (ADDR_WIDTH),\n   .WDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH),\n   .ARID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .ARADDR_WIDTH (ADDR_WIDTH),\n   .RDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH)\n) csr_if();\n\n// AXI4-lite to AXI-M adapter\naxi_lite2mmio axi_lite2mmio (\n   .clk       (clk),\n   .rst_n     (~reset),\n   .lite_if   (csr_lite_if),\n   .mmio_if   (csr_if)\n);\n\n//---------------------------------\n// Map AXI write/read request to CSR write/read,\n// and send the write/read response back\n//---------------------------------\nofs_fim_axi_csr_slave #(\n   .ADDR_WIDTH (ADDR_WIDTH),\n   .USE_SLV_READY (1'b1)\n\n   ) csr_slave (\n   .csr_if             (csr_if),\n\n   .csr_write          (csr_write),\n   .csr_waddr          (csr_waddr),\n   .csr_write_type     (csr_write_type),\n   .csr_wdata          (csr_wdata),\n   .csr_wstrb          (csr_wstrb),\n   .csr_slv_wready     (csr_slv_wready),\n   .csr_read           (csr_read),\n   .csr_raddr          (csr_raddr),\n   .csr_read_32b       (csr_read_32b),\n   .csr_readdata       (csr_readdata),\n   .csr_readdata_valid (csr_readdata_valid)\n);\n\n// Address mapping\nassign csr_addr             = csr_write ? csr_waddr : csr_raddr;\nassign com_csr_address      = csr_addr[5:0];  // byte address\nassign csr_slv_wready       = 1'b1 ;\n// Write data mapping\nassign com_csr_writedata    = csr_wdata;\n\n// Read-Write mapping\nalways_comb\nbegin\n   com_csr_read             = 1'b0;\n   com_csr_write            = 1'b0;\n   casez (csr_addr[11:6])\n      6'h00 : begin // Common CSR\n         com_csr_read       = csr_read;\n         com_csr_write      = csr_write;\n      end   \n      default: begin\n         com_csr_read       = 1'b0;\n         com_csr_write      = 1'b0;\n      end\n   endcase\nend\n\n// Read data mapping\nalways_comb begin\n   if (com_csr_readdatavalid) begin\n      csr_readdata       = com_csr_readdata;\n      csr_readdata_valid = 1'b1;\n   end\n   else begin\n      csr_readdata       = '0;\n      csr_readdata_valid = 1'b0;\n   end\nend\n\nhello_fim_com  #(\n   .FEAT_ID          (FEAT_ID),\n   .FEAT_VER         (FEAT_VER),\n   .NEXT_DFH_OFFSET  (NEXT_DFH_OFFSET),\n   .END_OF_LIST      (END_OF_LIST)\n) hello_fim_com_inst (\n   .clk                   (clk                     ),\n   .reset                 (reset                   ),\n   .writedata             (com_csr_writedata       ),\n   .read                  (com_csr_read            ),\n   .write                 (com_csr_write           ),\n   .byteenable            (4'hF                    ),\n   .readdata              (com_csr_readdata        ),\n   .readdatavalid         (com_csr_readdatavalid   ),\n   .address               (com_csr_address         )\n   );\nendmodule\n
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Create hello_fim_com.sv file.
                                                                       
                                                                      touch $OFS_ROOTDIR/src/hello_fim/hello_fim_com.sv\n
                                                                       
                                                                      Copy the following code to hello_fim_com.sv:
                                                                       
                                                                      module hello_fim_com #(\n  parameter bit [11:0] FEAT_ID = 12'h001,\n  parameter bit [3:0]  FEAT_VER = 4'h1,\n  parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n  parameter bit END_OF_LIST = 1'b0\n)(\ninput clk,\ninput reset,\ninput [63:0] writedata,\ninput read,\ninput write,\ninput [3:0] byteenable,\noutput reg [63:0] readdata,\noutput reg readdatavalid,\ninput [5:0] address\n);\n\nwire reset_n = !reset;  \nreg [63:0] rdata_comb;\nreg [63:0] scratch_reg;\n\nalways @(negedge reset_n ,posedge clk)  \n   if (!reset_n) readdata[63:0] <= 64'h0; else readdata[63:0] <= rdata_comb[63:0];\n\nalways @(negedge reset_n , posedge clk)\n   if (!reset_n) readdatavalid <= 1'b0; else readdatavalid <= read;\n\nwire wr = write;\nwire re = read;\nwire [5:0] addr = address[5:0];\nwire [63:0] din  = writedata [63:0];\nwire wr_scratch_reg = wr & (addr[5:0]  == 6'h30)? byteenable[0]:1'b0;\n\n// 64 bit scratch register\nalways @( negedge  reset_n,  posedge clk)\n   if (!reset_n)  begin\n      scratch_reg <= 64'h0;\n   end\n   else begin\n   if (wr_scratch_reg) begin \n      scratch_reg <=  din;  \n   end\nend\n\nalways @ (*)\nbegin\nrdata_comb = 64'h0000000000000000;\n   if(re) begin\n      case (addr)  \n        6'h00 : begin\n                rdata_comb [11:0]   = FEAT_ID ;  // dfh_feature_id  is reserved or a constant value, a read access gives the reset value\n                rdata_comb [15:12]  = FEAT_VER ;  // dfh_feature_rev    is reserved or a constant value, a read access gives the reset value\n                rdata_comb [39:16]  = NEXT_DFH_OFFSET ;  // dfh_dfh_ofst is reserved or a constant value, a read access gives the reset value\n                rdata_comb [40]     = END_OF_LIST ;        //dfh_end_of_list\n                rdata_comb [59:40]  = 20'h00000 ;  // dfh_rsvd1     is reserved or a constant value, a read access gives the reset value\n                rdata_comb [63:60]  = 4'h3 ;  // dfh_feat_type  is reserved or a constant value, a read access gives the reset value\n        end\n        6'h30 : begin\n                rdata_comb [63:0]   = scratch_reg; \n        end\n        6'h38 : begin\n                rdata_comb [63:0]       = 64'h6626_0701_5000_0034;\n        end\n        default : begin\n                rdata_comb = 64'h0000000000000000;\n        end\n      endcase\n   end\nend\n\nendmodule\n
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Create hello_fim_design_files.tcl file.
                                                                       
                                                                      touch $OFS_ROOTDIR/src/hello_fim/hello_fim_design_files.tcl\n
                                                                       
                                                                      Copy the following code into hello_fim_design_files.tcl
                                                                       
                                                                      # Copyright 2023 Intel Corporation.\n#\n# THIS SOFTWARE MAY CONTAIN PREPRODUCTION CODE AND IS PROVIDED BY THE\n# COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED\n# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE\n# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR\n# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\n# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR\n# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,\n# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE\n# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,\n# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# Hello FIM Files\n#--------------------\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_com.sv\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_top.sv\n
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Modify $OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_top.qsf to include Hello FIM module
                                                                       
                                                                      ######################################################\n# Verilog Macros\n######################################################\n.....\nset_global_assignment -name VERILOG_MACRO \"INCLUDE_HELLO_FIM\"     # Includes Hello FIM\n
                                                                       
                                                                      8.  
                                                                    8.  
                                                                      Modify $OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_top_sources.tcl to include Hello FIM design files
                                                                       
                                                                      ############################################\n# Design Files\n############################################\n...\n# Subsystems\n...\nset_global_assignment -name SOURCE_TCL_SCRIPT_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_design_files.tcl\n
                                                                       
                                                                      9.  
                                                                    9.  
                                                                      Modify $OFS_ROOTDIR/src/pd_qsys/fabric/fabric_design_files.tcl to include BPF Hello FIM Slave IP.
                                                                       
                                                                      #--------------------\n# BPF\n#--------------------\n...\nset_global_assignment -name IP_FILE   $::env(BUILD_ROOT_REL)/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip\n
                                                                       
                                                                      10.  
                                                                    10.  
                                                                      Modify $OFS_ROOTDIR/src/includes/fabric_width_pkg.sv to add Hello FIM slave information and update EMIF slave next offset.
                                                                       
                                                                      localparam bpf_hello_fim_slv_baseaddress = 'h16000;    // New\nlocalparam bpf_hello_fim_slv_address_width = 12;       // New\nlocalparam bpf_emif_slv_next_dfh_offset = 'h1000;      // Old value: 'hB000\nlocalparam bpf_hello_fim_slv_next_dfh_offset = 'hA000; // New\nlocalparam bpf_hello_fim_slv_eol = 'b0;                // New\n
                                                                       
                                                                      11.  
                                                                    11.  
                                                                      Modify $OFS_ROOTDIR/src/top/top.sv
                                                                       
                                                                         
                                                                      1.  
                                                                        Add bpf_hello_fim_slv_if to AXI interfaces
                                                                         
                                                                        // AXI4-lite interfaces\n...\nofs_fim_axi_lite_if #(.AWADDR_WIDTH(fabric_width_pkg::bpf_hello_fim_slv_address_width), .ARADDR_WIDTH(fabric_width_pkg::bpf_hello_fim_slv_address_width)) bpf_hello_fim_slv_if();\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Add Hello FIM instantiation
                                                                         
                                                                        //*******************************\n// Hello FIM Subsystem\n//*******************************\n\n`ifdef INCLUDE_HELLO_FIM\nhello_fim_top #(\n   .ADDR_WIDTH       (fabric_width_pkg::bpf_hello_fim_slv_address_width),\n   .DATA_WIDTH       (64),\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset),\n   .END_OF_LIST      (fabric_width_pkg::bpf_hello_fim_slv_eol)\n) hello_fim_top_inst (\n    .clk (clk_csr),\n    .reset(~rst_n_csr),\n    .csr_lite_if    (bpf_hello_fim_slv_if)\n);\n`else\ndummy_csr #(\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset),\n   .END_OF_LIST      (fabric_width_pkg::bpf_hello_fim_slv_eol)\n) hello_fim_dummy (\n   .clk         (clk_csr),\n   .rst_n       (rst_n_csr),\n   .csr_lite_if (bpf_hello_fim_slv_if)\n);\n\n`endif\n
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Add interfaces for Hello FIM slv to bpf instantiation
                                                                         
                                                                        bpf bpf (\n...\n  .bpf_hello_fim_slv_awaddr   (bpf_hello_fim_slv_if.awaddr  ),\n  .bpf_hello_fim_slv_awprot   (bpf_hello_fim_slv_if.awprot  ),\n  .bpf_hello_fim_slv_awvalid  (bpf_hello_fim_slv_if.awvalid ),\n  .bpf_hello_fim_slv_awready  (bpf_hello_fim_slv_if.awready ),\n  .bpf_hello_fim_slv_wdata    (bpf_hello_fim_slv_if.wdata   ),\n  .bpf_hello_fim_slv_wstrb    (bpf_hello_fim_slv_if.wstrb   ),\n  .bpf_hello_fim_slv_wvalid   (bpf_hello_fim_slv_if.wvalid  ),\n  .bpf_hello_fim_slv_wready   (bpf_hello_fim_slv_if.wready  ),\n  .bpf_hello_fim_slv_bresp    (bpf_hello_fim_slv_if.bresp   ),\n  .bpf_hello_fim_slv_bvalid   (bpf_hello_fim_slv_if.bvalid  ),\n  .bpf_hello_fim_slv_bready   (bpf_hello_fim_slv_if.bready  ),\n  .bpf_hello_fim_slv_araddr   (bpf_hello_fim_slv_if.araddr  ),\n  .bpf_hello_fim_slv_arprot   (bpf_hello_fim_slv_if.arprot  ),\n  .bpf_hello_fim_slv_arvalid  (bpf_hello_fim_slv_if.arvalid ),\n  .bpf_hello_fim_slv_arready  (bpf_hello_fim_slv_if.arready ),\n  .bpf_hello_fim_slv_rdata    (bpf_hello_fim_slv_if.rdata   ),\n  .bpf_hello_fim_slv_rresp    (bpf_hello_fim_slv_if.rresp   ),\n  .bpf_hello_fim_slv_rvalid   (bpf_hello_fim_slv_if.rvalid  ),\n  .bpf_hello_fim_slv_rready   (bpf_hello_fim_slv_if.rready  ),\n...\n);\n
                                                                         
                                                                        4.  
                                                                       
                                                                      12.  
                                                                    12.  
                                                                      Modify $OFS_ROOTDIR/src/pd_qsys/fabric/bpf.txt to add the hello_fim module as a slave to the apf.
                                                                       
                                                                      # NAME   FABRIC      BASEADDRESS    ADDRESS_WIDTH SLAVES\napf         mst     n/a             18            fme,pcie,pmci,qsfp0,qsfp1,emif,hssi,hello_fim\n...\nhello_fim   slv     0x16000         12            n/a\n
                                                                       
                                                                      13.  
                                                                    13.  
                                                                      Execute helper script to generate BPF design files
                                                                       
                                                                      cd $OFS_ROOTDIR/src/pd_qsys/fabric/\n\nsh gen_fabrics.sh\n
                                                                       
                                                                      14.  
                                                                    14.  
                                                                      Once the script completes, the following new IP is created: $OFS_ROOTDIR/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip.
                                                                       
                                                                      15.  
                                                                    15.  
                                                                      [OPTIONAL] You may verify the BPF changes have been made correctly by opening bpf.qsys to analyze the BPF.
                                                                       
                                                                      cd $OFS_ROOTDIR/src/pd_qsys/fabric\n\nqsys-edit bpf.qsys --quartus-project=$OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_top.qpf\n
                                                                       
                                                                      Find the bpf_hello_fim_slv instance:
                                                                       
                                                                       
                                                                      16.  
                                                                    16.  
                                                                      Compile the Hello FIM design
                                                                       
                                                                      cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_hello_fim\n
                                                                       
                                                                      17.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#413-walkthrough-modify-and-run-unit-tests-for-a-fim-that-has-a-new-module","title":"4.1.3 Walkthrough: Modify and run unit tests for a FIM that has a new module","text":"
                                                                    Perform the following steps to modify the unit test files to support a FIM that has had a new module added to it.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design in order to simulate.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Modify $OFS_ROOTDIR/sim/unit_test/dfh_walker/test_csr_defs.sv
                                                                       
                                                                         
                                                                      1.  
                                                                        Add HELLO_FIM_IDX entry to t_dfh_idx enumeration.
                                                                         
                                                                        ...\ntypedef enum {\n    FME_DFH_IDX,\n    THERM_MNGM_DFH_IDX,\n    GLBL_PERF_DFH_IDX,\n    GLBL_ERROR_DFH_IDX,\n    QSFP0_DFH_IDX,\n    QSFP1_DFH_IDX,\n    HSSI_DFH_IDX,\n    EMIF_DFH_IDX,\n    HELLO_FIM_DFH_IDX,  // New\n    PMCI_DFH_IDX,\n    ST2MM_DFH_IDX,\n    VUART_DFH_IDX,\n    PG_PR_DFH_IDX,\n    PG_PORT_DFH_IDX,\n    PG_USER_CLK_DFH_IDX,\n    PG_REMOTE_STP_DFH_IDX,\n    AFU_ERR_DFH_IDX,\n    MAX_DFH_IDX\n} t_dfh_idx;\n...\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Add HELLO_FIM_DFH to get_dfh_names function.
                                                                         
                                                                        ...\nfunction automatic dfh_name[MAX_DFH_IDX-1:0] get_dfh_names();\n...\n  dfh_names[PMCI_DFH_IDX]        = \"PMCI_DFH\";\n  dfh_names[HELLO_FIM_DFH_IDX]   = \"HELLO_FIM_DFH\";  // New\n  dfh_names[ST2MM_DFH_IDX]       = \"ST2MM_DFH\";\n...\nreturn dfh_names;\n...\n
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Add expected DFH value for Hello FIM to the get_dfh_values function.
                                                                         
                                                                        ...\nfunction automatic [MAX_DFH_IDX-1:0][63:0] get_dfh_values();\n...\n  dfh_values[PMCI_DFH_IDX]       = 64'h3_00000_xxxxxx_1012;\n  dfh_values[PMCI_DFH_IDX][39:16] = fabric_width_pkg::bpf_pmci_slv_next_dfh_offset;\n\n  dfh_values[HELLO_FIM_DFH_IDX]  = 64'h3_00000_xxxxxx_0100;  // New\n  dfh_values[HELLO_FIM_DFH_IDX][39:16] = fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset; // New\n\n  dfh_values[ST2MM_DFH_IDX]      = 64'h3_00000_xxxxxx_0014;\n  dfh_values[ST2MM_DFH_IDX][39:16] = fabric_width_pkg::apf_st2mm_slv_next_dfh_offset;\n...\nreturn dfh_values;\n...\n
                                                                         
                                                                        4.  
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Generate simulation files
                                                                       
                                                                      cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Run DFH Walker Simulation   
                                                                      cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\nsh run_sim.sh TEST=dfh_walker\n
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Verify that the test passes, and that the output shows the Hello FIM in the DFH sequence
                                                                       
                                                                      ********************************************\n Running TEST(0) : test_dfh_walking\n********************************************\n...\n\nREAD64: address=0x00015000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x3000000010001009\n\nEMIF_DFH\n   Address   (0x15000)\n   DFH value (0x3000000010001009)\n\nREAD64: address=0x00016000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x30000000a0000100\n\nHELLO_FIM_DFH\n   Address   (0x16000)\n   DFH value (0x30000000a0000100)\n\nREAD64: address=0x00020000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x3000000200001012\n\nPMCI_DFH\n   Address   (0x20000)\n   DFH value (0x3000000200001012)\n\n...\n\nTest status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/ofs-agx7-pcie-attach/sim/unit_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time         356791250000\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 356791250000 fs\nCPU Time:     61.560 seconds;       Data structure size:  47.4Mb\nTue Aug 15 16:29:45 2023\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                                                                       
                                                                      6.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#414-walkthrough-hardware-test-a-fim-that-has-a-new-module","title":"4.1.4 Walkthrough: Hardware test a FIM that has a new module","text":"
                                                                    Perform the following steps to program and hardware test a FIM that has had a new module added to it.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough uses a FIM design that has been generated with a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for generating a Hello FIM design.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      [OPTIONAL] In the work directory where the FIM was compiled, determine the PR Interface ID of your design. You can use this value at the end of the walkthrough to verify that the design has been configured to the FPGA.
                                                                       
                                                                      cd $OFS_ROOTDIR/<work_directory>/syn/board/fseries-dk/syn_top/\n\ncat fme-ifc-id.txt\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      5bcd682f-5093-5fc7-8cd2-ae8073e19452\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Switch to your deployment environment.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Program the FPGA with the Hello FIM image. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Run fpgainfo to determine the PCIe B:D.F of your board, and to verify the PR Interface ID matches the ID you found in Step 1.
                                                                       
                                                                      fpgainfo fme\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 360571655976424377 \nBitstream Version                : 5.0.1\nPr Interface Id                  : 5bcd682f-5093-5fc7-8cd2-ae8073e19452\nBoot Page                        : N/A\n
                                                                       
                                                                      Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Initialize opae.io
                                                                       
                                                                      sudo opae.io init -d <B:D.F>\n
                                                                       
                                                                      For example:
                                                                       
                                                                      sudo opae.io init -d b1:00.0\n
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Run DFH walker. Note the value read back from offset 0x16000 indicates the DFH ID is 0x100 which matches the Hello FIM module.
                                                                       
                                                                      sudo opae.io walk -d <B:D.F>\n
                                                                       
                                                                      For example:
                                                                       
                                                                      sudo opae.io walk -d b1:00.0\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      ...\noffset: 0x15000, value: 0x3000000010001009\n    dfh: id = 0x9, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x16000, value: 0x30000000a0000100\n    dfh: id = 0x100, rev = 0x0, next = 0xa000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x20000, value: 0x3000000200001012\n    dfh: id = 0x12, rev = 0x1, next = 0x20000, eol = 0x0, reserved = 0x0, feature_type = 0x3\n...\n
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Read all of the registers in the Hello FIM module
                                                                       
                                                                         
                                                                      1.  
                                                                        Read the DFH Register
                                                                         
                                                                        opae.io -d b1:00.0 -r 0 peek 0x16000\n
                                                                         
                                                                        Example Output:
                                                                         
                                                                        0x30000006a0000100\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Read the Scratchpad Register
                                                                         
                                                                        opae.io -d b1:00.0 -r 0 peek 0x16030\n
                                                                         
                                                                        Example Output:
                                                                         
                                                                        0x0\n
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Read the ID Register
                                                                         
                                                                        opae.io -d b1:00.0 -r 0 peek 0x16038\n
                                                                         
                                                                        Example Output:
                                                                         
                                                                        0x6626070150000034\n
                                                                         
                                                                        4.  
                                                                       
                                                                      8.  
                                                                    8.  
                                                                      Verify the scratchpad register at 0x16030 by writing and reading back from it.
                                                                       
                                                                         
                                                                      1.  
                                                                        Write to Scratchpad register
                                                                         
                                                                        opae.io -d b1:00.0 -r 0 poke 0x16030 0x123456789abcdef\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Read from Scratchpad register
                                                                         
                                                                        opae.io -d b1:00.0 -r 0 peek 0x16030\n
                                                                         
                                                                        Expected output:
                                                                         
                                                                        0x123456789abcdef\n
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Write to Scratchpad register
                                                                         
                                                                        opae.io -d b1:00.0 -r 0 poke 0x16030 0xfedcba9876543210\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Read from Scratchpad register
                                                                         
                                                                        opae.io -d b1:00.0 -r 0 peek 0x16030\n
                                                                         
                                                                        Expected output:
                                                                         
                                                                        0xfedcba9876543210\n
                                                                         
                                                                        5.  
                                                                       
                                                                      9.  
                                                                    9.  
                                                                      Release the opae.io tool
                                                                       
                                                                      opae.io release -d b1:00.0\n
                                                                       
                                                                      10.  
                                                                    10.  
                                                                      Confirm the driver has been set back to dfl-pci
                                                                       
                                                                      opae.io ls\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      [0000:b1:00.0] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                                                                       
                                                                      11.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#415-walkthrough-debug-the-fim-with-signal-tap","title":"4.1.5 Walkthrough: Debug the FIM with Signal Tap","text":"
                                                                    The following steps guide you through the process of adding a Signal Tap instance to your design. The added Signal Tap instance provides hardware to capture the desired internal signals and connect the stored trace information via JTAG. Please be aware that the added Signal Tap hardware will consume FPGA resources and may require additional floorplanning steps to accommodate these resources. Some areas of the FIM use logic lock regions and these regions may need to be re-sized.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design.
                                                                      •  
                                                                     
                                                                    Perform the following steps in your development environment:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Synthesize the design using the -e build script option. You may skip this step if you are using a pre-synthesized design.
                                                                       
                                                                      cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -e --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_hello_fim_with_stp\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Open the design in Quartus. The Compilation Dashboard should show that the Analysis & Synthesis step has completed.
                                                                       
                                                                      quartus $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/fseries-dk/syn_top/ofs_top.qpf\n
                                                                       
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Open Tools -> Signal Tap Logic Analyzer
                                                                       
                                                                       
                                                                         
                                                                      1.  
                                                                        Select the Default template and click Create
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Assign the clock for sampling the Signal Tap instrumented signals of interest. Note that the clock selected should correspond to the signals you want to view for best trace fidelity. Different clocks can be used, however, there maybe issues with trace inaccuracy due to sampling time differences. This example instruments the hello_fim_top module previously intetegrated into the FIM. If unfamiliar with code, it is helpful to use the Quartus Project Navigator to find the block of interest and open the design instance for review.
                                                                         
                                                                           
                                                                        1.  
                                                                          In the Signal Configuration -> Clock box of the Signal Tap Logic Analyzer window, click the \"...\" button
                                                                           
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          In the Node Finder tool that opens, type hello_fim_top_inst|clock into the Named field, then click Search. Select the clk signal from the Matching Nodes box and click the \">\" button to move it to the Nodes Found box. Click OK to close the Node Finder dialog.
                                                                           
                                                                           
                                                                          3.  
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Update the sample depth and other Signal Tap settings as needed for your debugging criteria.
                                                                         
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        In the Signal Tap GUI add the nodes to be instrumented by double-clicking on the \"Double-click to add nodes\" legend.
                                                                         
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        This brings up the Node Finder to add the signals to be traced. In this example we will monitor the memory mapped interface to the Hello FIM. Select the signals that appear from the  search patterns hello_fim_top_inst|reset and hello_fim_top_inst|csr_lite_if\\*. Click Insert and close the Node Finder dialog.
                                                                         
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        To provide a unique name for your Signal Tap instance, select \"auto_signaltap_0\", right-click, and select Rename Instance (F2). Provide a descriptive name for your instance, for example, stp_for_hello_fim.
                                                                         
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Save the newly created Signal Tap file, in the $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/fseries-dk/syn_top/ directory, and give it the same name as the instance. Ensure that the Add file to current project checkbox is ticked.
                                                                         
                                                                         
                                                                        8.  
                                                                      8.  
                                                                        In the dialog that pops up, click \"Yes\" to add the newly created Signal Tap file to the project settings files.
                                                                         
                                                                         
                                                                        This will aurtomatically add the following lines to $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/fseries-dk/syn_top/ofs_top.qsf:
                                                                         
                                                                        set_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE stp_for_hello_fim.stp\nset_global_assignment -name SIGNALTAP_FILE stp_for_hello_fim.stp\n
                                                                         
                                                                        9.  
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Close all Quartus GUIs.
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Compile the project with the Signal Tap file added to the project. Use the -k switch to perform the compilation using the files in a specified working directory and not the original ones from the cloned repository. 
                                                                       
                                                                      ./ofs-common/scripts/common/syn/build_top.sh -p -k --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_hello_fim_with_stp\n
                                                                       
                                                                      Alternatively, you can copy the ofs_top.qsf and stp_for_hello_fim.stp files from the Hello FIM with STP work directory to replace the original files in the cloned OFS repository. In this scenario, all further FIM compilation projects will include the Signal Tap instance integrated into the design. Execute the following commands for this alternative flow:
                                                                       
                                                                      Copy the modified file work_hello_fim_with_stp/syn/board/fseries-dk/syn_top/ofs_top.qsf to the source OFS repository, into syn/board/fseries-dk/syn_top/.
                                                                       
                                                                      cd $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/fseries-dk/syn_top\n\ncp ofs_top.qsf $OFS_ROOTDIR/syn/board/fseries-dk/syn_top\n\ncp stp_for_hello_fim.stp $OFS_ROOTDIR/syn/board/fseries-dk/syn_top\n
                                                                       
                                                                      Compile the FIM to create a new work directory.
                                                                       
                                                                      cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_hello_fim_with_stp_src_repo\n
                                                                       
                                                                      8.  
                                                                    8.  
                                                                      Ensure that the compile completes successfully and meets timing:
                                                                       
                                                                      ***********************************\n***\n***        OFS_PROJECT: n6001\n***        OFS_BOARD: fseries-dk\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 6\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                                                       
                                                                      9.  
                                                                    9.  
                                                                      Set up a JTAG connection to the fseries-dk. Refer to the Set up JTAG section for step-by-step instructions.
                                                                       
                                                                      10.  
                                                                    10.  
                                                                      Copy the ofs_top.sof and stp_for_hello_fim.stp files to the machine which is connected to the fseries-dk via JTAG.
                                                                       
                                                                      11.  
                                                                    11.  
                                                                      From the JTAG connected machine, program the $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/fseries-dk/syn_top/output_files/ofs_top.sof image to the fseries-dk FPGA. Refer to the Program the FPGA via JTAG section for step-by-step programming instructions.
                                                                       
                                                                      12.  
                                                                    12.  
                                                                      Open the Quartus Signal Tap GUI
                                                                       
                                                                      $QUARTUS_ROOTDIR/bin/quartus_stpw\n
                                                                       
                                                                      13.  
                                                                    13.  
                                                                      In the Signal Tap Logic Analyzer window, select File -> Open, and choose the stp_for_hello_fim.stp file.
                                                                       
                                                                       
                                                                      14.  
                                                                    14.  
                                                                      In the right pane of the Signal Tap GUI, in the Hardware: selection box select the cable for the fseries-dk. In the Device: selection box select the Agilex device.
                                                                       
                                                                       
                                                                      15.  
                                                                    15.  
                                                                      If the Agilex Device is not displayed in the Device: list, click the 'Scan Chain' button to re-scan the JTAG device chain.
                                                                       
                                                                      16.  
                                                                    16.  
                                                                      Create the trigger conditions. In this example, we will capture data on a rising edge of the Read Address Valid signal.
                                                                       
                                                                       
                                                                      17.  
                                                                    17.  
                                                                      Start analysis by selecting the 'stp_for_hello_fim' instance and pressing 'F5' or clicking the Run Analysis icon in the toolbar. You should see a green message indicating the Acquisition is in progress. Then, move to the Data Tab to observe the signals captured.
                                                                       
                                                                       
                                                                      18.  
                                                                    18.  
                                                                      To generate traffic in the csr_lite_if signals of the Hello FIM module, walk the DFH list or peek/poke the Hello FIM registers.
                                                                       
                                                                      opae.io init -d 0000:b1:00.0\nopae.io walk -d 0000:b1:00.0\nopae.io release -d 0000:b1:00.0\n
                                                                       
                                                                      The signals should be captured on the rising edge of arvalid in this example. Zoom in to get a better view of the signals.
                                                                       
                                                                       
                                                                      19.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#42-preparing-fim-for-afu-development","title":"4.2 Preparing FIM for AFU Development","text":"
                                                                    To save area, the default Host Excercisers in the FIM can be replaced by a \"he_null\" blocks. There are a few things to note:
                                                                     
                                                                       
                                                                    • \"he_null\" is a minimal block with registers that responds to PCIe MMIO request. MMIO responses are required to keep PCIe alive (end points enabled in PCIe-SS needs service downstream requests).
                                                                      •  
                                                                    • If an exerciser with other I/O connections such as \"he_mem\" or \"he_hssi\" is replaced, then then those I/O ports are simply tied off.
                                                                      •  
                                                                    • The options supported are null_he_lb, null_he_hssi, null_he_mem and null_he_mem_tg. Any combination, order or all can be enabled at the same time. 
                                                                      •  
                                                                    • Finer grain control is provided for you to, for example, turn off only the exercisers in the Static Region in order to save area.
                                                                      •  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#421-walkthrough-compile-the-fim-in-preparation-for-designing-your-afu","title":"4.2.1 Walkthrough: Compile the FIM in preparation for designing your AFU","text":"
                                                                    Perform the following steps to compile a FIM for where the exercisers are removed and replaced with an he_null module while keeping the PF/VF multiplexor connections.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the [Walkthrough: Set Up Development Environment] Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the FIM repository (or use an existing cloned repository). Refer to the [Walkthrough: Clone FIM Repository] section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the [Walkthrough: Set Development Environment Variables] section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Compile the FIM with the HE_NULL compile options
                                                                       
                                                                      cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss fseries-dk:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_fseries-dk\n
                                                                       
                                                                      4.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#43-partial-reconfiguration-region","title":"4.3 Partial Reconfiguration Region","text":"
                                                                    To take advantage of the available resources in the Intel\u00ae Agilex\u00ae 7 FPGA for an AFU design, you can adjust the size of the AFU PR partition. An example reason for the changing the size of PR region is if you add more logic to the FIM region, then you may need to adjust the PR region to fit the additional logic into the static region.  Similarly, if you reduce logic in the FIM region, then you can adjust the PR region to provide more logic resources for the AFU.
                                                                     
                                                                    After the compilation of the FIM, the resources usage broken down by partitions as reported in the following file:
                                                                     
                                                                    $OFS_ROOTDIR/<WORDK_DIRECTORY>/syn/board/fseries-dk/syn_top/output_files/ofs_top.fit.rpt\n
                                                                     
                                                                    An example report of the resources usage by partitions defined for the FIM is shown as follows:
                                                                     
                                                                    +------------------------------------------------------------------------------------------+\n; Logic Lock Region Constraints                                                            ;\n+--------------------------------------+-------------------------+-------------------------+\n; Name                                 ; Place Region Constraint ; Route Region Constraint ;\n+--------------------------------------+-------------------------+-------------------------+\n; afu_top|port_gasket|pr_slot|afu_main ; (90, 40) to (350, 220)  ; (0, 0) to (385, 329)    ;\n+--------------------------------------+-------------------------+-------------------------+\n\n\n+----------------------------------------------------------------------------------------------+\n; Logic Lock Region Usage Summary                                                              ;\n+-------------------------------------------------------+--------------------------------------+\n; Statistic                                             ; afu_top|port_gasket|pr_slot|afu_main ;\n+-------------------------------------------------------+--------------------------------------+\n; ALMs needed [=A-B+C]                                  ; 48011.2 / 351140 ( 13 % )            ;\n;     [A] ALMs used in final placement                  ; 53324.4 / 351140 ( 15 % )            ;\n;     [B] Estimate of ALMs recoverable by dense packing ; 5452.3 / 351140 ( 1 % )              ;\n;     [C] Estimate of ALMs unavailable                  ; 139.0 / 351140 ( < 1 % )             ;\n; ALMs used for memory                                  ; 450.0                                ;\n; Combinational ALUTs                                   ; 67166                                ;\n; Dedicated Logic Registers                             ; 87533 / 1404560 ( 6 % )              ;\n; I/O Registers                                         ; 0                                    ;\n; Block Memory Bits                                     ; 1737568                              ;\n; M20Ks                                                 ; 137 / 5049 ( 2 % )                   ;\n; DSP Blocks needed [=A-B]                              ; 0 / 3439 ( 0 % )                     ;\n;     [A] DSP Blocks used in final placement            ; 0 / 3439 ( 0 % )                     ;\n;     [B] Estimate of DSPs recoverable by dense merging ; 0 / 3439 ( 0 % )                     ;\n; Pins                                                  ; 0                                    ;\n; IOPLLs                                                ; 0                                    ;\n;                                                       ;                                      ;\n; Region Placement                                      ; (90, 40) to (350, 220)               ;\n+-------------------------------------------------------+--------------------------------------+\n
                                                                     
                                                                    In this case, the default size for the afu_top|port_gasket|pr_slot|afu_main PR partition is large enough to hold the logic of the default AFU, which is mainly occupied by the Host Exercisers. However, larger designs might require additional resources.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#431-walkthrough-resize-the-partial-reconfiguration-region","title":"4.3.1 Walkthrough: Resize the Partial Reconfiguration Region","text":"
                                                                    Perform the following steps to first analyze the PR logic lock regions in a default FIM design, then resize the PR region:
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      The OFS flow provides the TCL file $OFS_ROOTDIR/syn/board/fseries-dk/setup/pr_assignments.tcl which defines the PR partition where the AFU is allocated. Each region is a rectangle defined by the origin coordinate (X0, Y0) and the top right corner coordinate (X1, Y1).
                                                                       
                                                                      #####################################################\n# Main PR Partition -- green_region\n#####################################################\nset_instance_assignment -name PARTITION green_region -to afu_top|port_gasket|pr_slot|afu_main\nset_instance_assignment -name CORE_ONLY_PLACE_REGION ON -to afu_top|port_gasket|pr_slot|afu_main\nset_instance_assignment -name RESERVE_PLACE_REGION ON -to afu_top|port_gasket|pr_slot|afu_main\nset_instance_assignment -name PARTIAL_RECONFIGURATION_PARTITION ON -to afu_top|port_gasket|pr_slot|afu_main\n\n\nset_instance_assignment -name PLACE_REGION \"X90 Y40 X350 Y220\" -to afu_top|port_gasket|pr_slot|afu_main\nset_instance_assignment -name ROUTE_REGION \"X0 Y0 X385 Y329\" -to afu_top|port_gasket|pr_slot|afu_main\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      [OPTIONAL] Use Quartus Chip Planner to visualize the default PR region allocation.
                                                                       
                                                                         
                                                                      1.  
                                                                        Run the setup stage of the build script to create a work directory.
                                                                         
                                                                        cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_resize_pr\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Open the project in the Quartus GUI
                                                                         
                                                                        quartus $OFS_ROOTDIR/work_fseries-dk_resize_pr/syn/board/fseries-dk/syn_top/ofs_top.qpf\n
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        In the Compilation Dashboard, click the arrow next to Compile Design to start the compilation. 
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Once compilation is complete, click Tools -> Chip Planner to open the Chip Planner.
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Analyze the regions shown. Note that the regions are made of rectangles described by an origin coordinate, region height, and region width. If you are modifying the regions, you will need to identify the coordinates of your desired region.
                                                                         
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Close the Quartus GUI.
                                                                         
                                                                        7.  
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Make changes to the partial reconfiguraton region in the $OFS_ROOTDIR/syn/board/fseries-dk/setup/pr_assignments.tcl file. You can modify the size and location of existing lock regions or add new ones and assign them to the AFU PR partition.
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Recompile your FIM and create the PR relocatable build tree using the following commands.
                                                                       
                                                                      cd $OFS_ROOTDIR    \n\nofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_resize_pr\n
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Analyze the resource utilization report $OFS_ROOTDIR/work_fseries-dk/syn/board/fseries-dk/syn_top/output_files/ofs_top.fit.rpt produced after recompiling the FIM.
                                                                       
                                                                      8.  
                                                                    8.  
                                                                      Perform further modification to the PR regions until the results are satisfactory. Make sure timing constraints are met.
                                                                       
                                                                      9.  
                                                                     
                                                                    For more information on how to optimize the floor plan of your Partial Reconfiguration design refer to the following online documentation.
                                                                     
                                                                       
                                                                    • Analyzing and Optimizing the Design Floorplan
                                                                      •  
                                                                     
                                                                       
                                                                    • Partial Reconfiguration Design Flow - Step 3 - Floorplan the Design
                                                                      •  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#44-pcie-configuration","title":"4.4 PCIe Configuration","text":"
                                                                    The PCIe sub-system IP and PF/VF MUX can be modified either using the OFSS flow or the IP Presets flow. The OFSS flow supports a subset of all available PCIe Sub-system settings, while the IP Preset flow can make any available PCIe Sub-system settings change. With PCIe-SS modifcations related to the PFs and VFs, the PF/VF MUX logic is automatically configured based on the PCIe-SS configuration. The sections below describe each flow.
                                                                     
                                                                       
                                                                    • PCIe Configuration Using OFSS
                                                                      •  
                                                                    • PCIe Configuration Using IP Presets
                                                                      •  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#441-pfvf-mux-configuration","title":"4.4.1 PF/VF MUX Configuration","text":"
                                                                    The default PF/VF MUX configuration for OFS PCIe Attach FIM for the fseries-dk can support up to 8 PFs and 2000 VFs distributed accross all PFs.
                                                                     
                                                                    For reference FIM configurations, 0 VFs on PF0 is not supported. This is because the PR region cannot be left unconnected. A NULL AFU may need to be instantiated in this special case. PFs must be consecutive. the PF/VF Limitations table describes the supported number of PFs and VFs.
                                                                     
                                                                    Table: PF/VF Limitations
                                                                     Parameter Value Min # of PFs 1 (on PF0) Max # of PFs 8 Min # of VFs 1 on PF0 Max # of VFs 2000 distributed across all PFs 
                                                                    New PF or VF instances will automatically have a null_afu module instantiated and connected to the new PF or VF.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#442-pcie-ss-configuration-registers","title":"4.4.2 PCIe-SS Configuration Registers","text":"
                                                                    The PCIe-SS configuration registers contain the Vendor, Device and Subsystem Vendor ID registers which are used in PCIe add-in cards to uniquely identify the card for assignment to software drivers. OFS has these registers set with Intel values for out of the box usage. If you are using OFS for a PCIe add in card that your company manufactures, then update the PCIe Subsytem Subsystem ID and Vendor ID registers as described below and change OPAE provided software code to properly operate with your company's register values.
                                                                     
                                                                    The Vendor ID is assigned to your company by the PCI-SIG (Special Interest Group). The PCI-SIG is the only body officially licensed to give out IDs. You must be a member of PCI-SIG to request your own ID. Information about joining PCI-SIG is available here: PCI-SIG. You select the Subsystem Device ID for your PCIe add in card.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#443-pcie-configuration-using-ofss","title":"4.4.3 PCIe Configuration Using OFSS","text":"
                                                                    The general flow for using OFSS to modify the PCIe Sub-system and PF/VF MUX is as follows:
                                                                     
                                                                       
                                                                    1. Create or modify a PCIe OFSS file with the desired PCIe configuration. 
                                                                      2.  
                                                                    2. Call this PCIe OFSS file when running the FIM build script.
                                                                      3.  
                                                                     
                                                                    The PCIe IP OFSS File Options table lists all of the configuration options supported by the OFSS flow. Any other customizations to the PCIe sub-system must be done with the IP Presets Flow.
                                                                     
                                                                    Table: PCIe IP OFSS File Options
                                                                     Section Parameter Options Default Description [ip] type pcie N/A Specifies that this OFSS file configures the PCIe-SS [settings] output_name pcie_ss N/A Specifies the output name of the PCIe-SS IP preset String N/A OPTIONAL - Specifies the name of a PCIe-SS IP presets file to use when building the FIM. When used, a presets file will take priority over any other parameters set in this OFSS file. [pf*] num_vfs Integer 0 Specifies the number of Virtual Functions in the current PF bar0_address_width Integer 12 bar4_address_width Integer 14 vf_bar0_address_width Integer 12 ats_cap_enable 0 | 1 0 vf_ats_cap_enable 0 | 1 0 prs_ext_cap_enable 0 | 1 0 pasid_cap_enable 0 | 1 0 pci_type0_vendor_id 32'h Value 0x00008086 pci_type0_device_id 32'h Value 0x0000bcce revision_id 32'h Value 0x00000001 class_code 32'h Value 0x00120000 subsys_vendor_id 32'h Value 0x00008086 subsys_dev_id 32'h Value 0x00001771 sriov_vf_device_id 32'h Value 0x0000bccf exvf_subsysid 32'h Value 0x00001771"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#4431-walkthrough-modify-pcie-configuration-using-ofss","title":"4.4.3.1 Walkthrough: Modify PCIe Configuration Using OFSS","text":"
                                                                    Perform the following steps to use OFSS files to configure the PCIe Sub-system and PF/VF MUX. In this example, we will add a PF with 2 VFs to the default configuration.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment to build the FIM. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment to test the design. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      View the default OFS PCIe Attach FIM for the fseries-dk PF/VF configuration in the the $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss file.
                                                                       
                                                                      [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n\n[pf0]\nnum_vfs = 3\nbar0_address_width = 20\nvf_bar0_address_width = 20\n\n[pf1]\n\n[pf2]\nbar0_address_width = 18\n\n[pf3]\n\n[pf4]\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Create a new PCIe OFSS file from the existing pcie_host.ofss file
                                                                       
                                                                      cp $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_pfvf_mod.ofss\n
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Configure the new pcie_pfvf_mod.ofss with your desired PCIe settings. In this example we will add PF5 with 2 VFs.
                                                                       
                                                                      [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n\n[pf0]\nnum_vfs = 3\nbar0_address_width = 20\nvf_bar0_address_width = 20\n\n[pf1]\n\n[pf2]\nbar0_address_width = 18\n\n[pf3]\n\n[pf4]\n\n[pf5]\nnum_vfs = 2\n
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Edit the $OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss file to use the new PCIe configuration file pcie_pfvf_mod.ofss
                                                                       
                                                                      [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/fseries-dk_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_pfvf_mod.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory_ftile.ofss\n
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Compile the FIM. 
                                                                       
                                                                      cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_pfvf_mod\n
                                                                       
                                                                      8.  
                                                                    8.  
                                                                      Copy the resulting $OFS_ROOTDIR/work_fseries-dk_pfvf_mod/syn/board/fseries-dk/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                                                                       
                                                                      9.  
                                                                    9.  
                                                                      Switch to your deployment environment.
                                                                       
                                                                      10.  
                                                                    10.  
                                                                      Program the ofs_top.sof image to the fseries-dk FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                                                                       
                                                                      11.  
                                                                    11.  
                                                                      Verify the number of VFs on the newly added PF5. In this example, we defined 2 VFs on PF5 in Step 5.
                                                                       
                                                                      sudo lspci -vvv -s b1:00.5 | grep VF\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      Initial VFs: 2, Total VFs: 2, Number of VFs: 0, Function Dependency Link: 05\nVF offset: 4, stride: 1, Device ID: bccf\nVF Migration: offset: 00000000, BIR: 0\n
                                                                       
                                                                      12.  
                                                                    12.  
                                                                      Verify communication with the newly added PF5. New PF/VF are seamlessly connected to their own CSR stub, which can be read at DFH Offset 0x0. You can bind to the function and perform opae.io peek commands to read from the stub CSR. Similarly, perform opae.io poke commands to write into the stub CSRs. Use this mechanism to verify that the new PF/VF Mux configuration allows to write and read back values from the stub CSRs. 
                                                                       
                                                                      The GUID for every new PF/VF CSR stub is the same.
                                                                       
                                                                      * NULL_GUID_L           = 64'haa31f54a3e403501\n* NULL_GUID_H           = 64'h3e7b60a0df2d4850\n
                                                                       
                                                                         
                                                                      1.  
                                                                        Initialize the driver on PF5
                                                                         
                                                                        sudo opae.io init -d b1:00.5\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Read the GUID for the PF5 CSR stub.
                                                                         
                                                                        sudo opae.io -d b1:00.5 -r 0 peek 0x8\nsudo opae.io -d b1:00.5 -r 0 peek 0x10\n
                                                                         
                                                                        Example output:
                                                                         
                                                                        0xaa31f54a3e403501\n0x3e7b60a0df2d4850\n
                                                                         
                                                                        3.  
                                                                       
                                                                      13.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#444-pcie-configuration-using-ip-presets","title":"4.4.4 PCIe Configuration Using IP Presets","text":"
                                                                    The general flow for using IP Presets to modify he PCIe Sub-system is as follows:
                                                                     
                                                                       
                                                                    1. [OPTIONAL] Create a work directory using OFSS files if you wish to use OFSS configuration settings.
                                                                      2.  
                                                                    2. Open the PCIe-SS IP and make desired modifications.
                                                                      3.  
                                                                    3. Create an IP Presets file.
                                                                      4.  
                                                                    4. Create an PCIe OFSS file that uses the IP Presets file.
                                                                      5.  
                                                                    5. Build the FIM with the PCIe OFSS file from Step 4.
                                                                      6.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#4441-walkthrough-modify-pcie-configuration-using-ip-presets","title":"4.4.4.1 Walkthrough: Modify PCIe Configuration Using IP Presets","text":"
                                                                    Perform the following steps to use an IP Preset file to configure the PCIe Sub-system and PF/VF MUX. In this example, we will change the Revision ID of PF0. While this modification can be done with the OFSS flow, this walkthrough is intended to show the procedure for making any PCIe configuration change using IP presets.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment to build the FIM. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment to test the design. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Run the setup stage of the build script using your desired OFSS configration to create a working directory for the fseries-dk design.
                                                                       
                                                                      ./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Open the PCIe-SS in the work directory using Quartus Parameter Editor.
                                                                       
                                                                      qsys-edit $OFS_ROOTDIR/work_fseries-dk/ipss/pcie/qip/pcie_ss.ip\n
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      In the IP Parameter Editor window, scroll down and select the PCIe Interfaces Port Settings -> Port 0 -> PCIe0 Device Identification Registers -> PCIe0 PF0 IDs tab. Modify the settings as desired. In this case, we are changing the Revision ID to 0x00000002. You may make any desired modifications.
                                                                       
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Once you are satisfied with your modifcations, create a new IP Preset file.
                                                                       
                                                                         
                                                                      1.  
                                                                        click New... in the Presets window.
                                                                         
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        In the New Preset window, set a unique Name for the preset; for example, fseries-dk-rev2.
                                                                         
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Click the ... button to set the save location for the IP Preset file to $OFS_ROOTDIR/ipss/pcie/presets. Set the File Name to match the name selected in Step 9. Click OK.
                                                                         
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        In the New Preset window, click Save. Click No when prompted to add the file to the IP search path.
                                                                         
                                                                         
                                                                        5.  
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Close IP Parameter Editor without saving or generating HDL.
                                                                       
                                                                      8.  
                                                                    8.  
                                                                      Create a new PCIe OFSS file in the $OFS_ROOTDIR/tools/ofss_config/pcie directory. For example:
                                                                       
                                                                      touch $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host_mod_preset.ofss\n
                                                                       
                                                                      Insert the following into the OFSS file to specify the IP Preset file created in Step 6. 
                                                                       
                                                                      [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\npreset = fseries-dk-rev2\n
                                                                       
                                                                      9.  
                                                                    9.  
                                                                      Edit the $OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss file to call new OFSS file created in the previous step.
                                                                       
                                                                      [include]  \"$OFS_ROOTDIR\"/tools/ofss_config/fseries-dk_base.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host_mod_preset.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory_ftile.ofss
                                                                       
                                                                      10.  
                                                                    10.  
                                                                      Compile the design with the modified fseries-dk.ofss file.
                                                                       
                                                                      cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_pcie_mod\n
                                                                       
                                                                      11.  
                                                                    11.  
                                                                      Copy the resulting $OFS_ROOTDIR/work_fseries-dk_pcie_mod/syn/board/fseries-dk/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                                                                       
                                                                      12.  
                                                                    12.  
                                                                      Switch to your deployment environment.
                                                                       
                                                                      13.  
                                                                    13.  
                                                                      Program the ofs_top.sof image to the fseries-dk FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                                                                       
                                                                      14.  
                                                                    14.  
                                                                      Use lspci to verify that the PCIe changes have been implemented. In this example, the Rev for PF0 is 02.
                                                                       
                                                                      lspci -nvmms 98:00.0\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      Slot:   98:00.0\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nPhySlot:        1-1\nRev:    02\nNUMANode:       1\nIOMMUGroup:     8\n
                                                                       
                                                                      15.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#45-minimal-fim","title":"4.5 Minimal FIM","text":"
                                                                    In a minimal FIM, the exercisers are removed. This minimal FIM is useful for HDL applications.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#451-walkthrough-create-a-minimal-fim","title":"4.5.1 Walkthrough: Create a Minimal FIM","text":"
                                                                    Perform the following steps to create a Minimal FIM.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Create a new platform OFSS file that will use a 1PF/1VF PCIe configuration.
                                                                       
                                                                      cp $OFS_ROOTDIR/tools/ofss_config/fseries-dk.ofss $OFS_ROOTDIR/tools/ofss_config/fseries-dk_1pf_1vf.ofss\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Edit the $OFS_ROOTDIR/tools/ofss_config/fseries-dk_1pf_1vf.ofss file to use the 1PF/1VF PCIe configuration file pcie_1pf_1vf.ofss
                                                                       
                                                                      [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/fseries-dk_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_1pf_1vf.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory_ftile.ofss\n
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Compile the FIM with Null HEs compile option, the No HSSI compile option, and 1PF/1VF configuration OFSS file.
                                                                       
                                                                      cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk_1pf_1vf.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg,no_hssi work_fseries-dk_minimal_fim\n
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Review the $OFS_ROOTDIR/work_fseries-dk_minimal_fim/syn/board/fseries-dk/syn_top/output_files/ofs_top.fit.rpt utilization report to see the utilization statistics for the Minimal fim. Refer to [Appendix A] Table A-4 for the expected utilization for this Minimal FIM.
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Copy the resulting $OFS_ROOTDIR/work_fseries-dk_minimal_fim/syn/board/fseries-dk/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                                                                       
                                                                      8.  
                                                                    8.  
                                                                      Switch to your deployment environment, if different than your development environment.
                                                                       
                                                                      9.  
                                                                    9.  
                                                                      Program the ofs_top.sof image to the fseries-dk FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                                                                       
                                                                      10.  
                                                                    10.  
                                                                      Use lspci to verify that the PCIe changes have been implemented.
                                                                       
                                                                      sudo lspci -vvv -s b1:00.0 | grep VF\n
                                                                       
                                                                      Example output: 
                                                                      Initial VFs: 1, Total VFs: 1, Number of VFs: 0, Function Dependency Link: 00\nVF offset: 1, stride: 1, Device ID: bcce\nVF Migration: offset: 00000000, BIR: 0\n
                                                                       
                                                                      11.  
                                                                    11.  
                                                                      You may wish to adjust the PR logic lock regions to maximize the resources available for the AFU. Refer to the Resize the Partial Reconfiguration Region section for instructions.
                                                                       
                                                                      12.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#46-migrating-to-a-different-agilex-device-number","title":"4.6 Migrating to a Different Agilex Device Number","text":"
                                                                    The following instructions enable a user to change the device part number of the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile). Please be aware that this release works with Intel\u00ae Agilex\u00ae 7 FPGA devices that have F-tile for PCIe and Ethernet.  Other tiles will take further work.
                                                                     
                                                                    You may wish to change the device part number for the following reasons
                                                                     
                                                                       
                                                                    1. Migrate to same device package but with a different density
                                                                      2.  
                                                                    2. Migrate to a different package and with a different or same density
                                                                      3.  
                                                                     
                                                                    The default device for the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) is AGFB027R24C2E2VR2
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#461-walkthrough-migrate-to-a-different-agilex-device-number","title":"4.6.1 Walkthrough: Migrate to a Different Agilex Device Number","text":"
                                                                    Perform the following steps to migrate to a different Agilex Device. In this example, we will migrate from the default AGFB027R24C2E2VR2 device to AGFB027R31C2E2V. The package will change from R24C to R31C.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Modify the part field in the $OFS_ROOTDIR/tools/ofss_config/fseries-dk_base.ofss file to use AGFB027R31C2E2V.
                                                                       
                                                                      [ip]\ntype = ofs\n\n[settings]\nplatform = n6001\nfamily = agilex\nfim = base_x16\npart = AGFB027R31C2E2V \ndevice_id = 6001\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Modify the DEVICE field in the $OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_top.qsf file to use AGFB027R31C2E2V.
                                                                       
                                                                      ############################################################################################\n# FPGA Device\n############################################################################################\n\nset_global_assignment -name FAMILY \"Agilex 7\"\nset_global_assignment -name DEVICE AGFB027R31C2E2V \n
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Modify the DEVICE field in the $OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_pr_afu.qsf file to use AGFB027R31C2E2V.
                                                                       
                                                                      ############################################################################################\n# FPGA Device\n############################################################################################\n\nset_global_assignment -name FAMILY Agilex 7\nset_global_assignment -name DEVICE AGFB027R31C2E2V \n
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      If the device you are migrating to uses the same package and pinout, you do not need to modify the pinout constraints. In this example, because we are migrating from package R24C to R31C, we need to modify the pinout to match the new device. If you would like Quartus to attempt to pin out the design automatically, you may remove all pin assignments. Typically, you will still need to set certain pins manually in order to guide Quartus for a successful compile (e.g. transceiver reference clocks).
                                                                       
                                                                         
                                                                      1.  
                                                                        Commment out all pin assignments in the following files:     * $OFS_ROOTDIR/syn/board/fseries-dk/setup/emif_loc.tcl     * $OFS_ROOTDIR/syn/board/fseries-dk/setup/hps_loc.tcl     * $OFS_ROOTDIR/syn/board/fseries-dk/setup/top_loc.tcl
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Identify the pins in the design that will be constrained. In this example we will manually constrain the QSFP reference clock and the PCIe reference clock to help guide the fitter. The Device Migration Pinout Table shows th pin assignments that will be set, along with the pin number for both the old R24C package and the new R31C package.
                                                                         
                                                                        3.  
                                                                       Net Name Pin Name AGF 027 R24C Pin # AGF 027 R31C Pin # \"qsfp_ref_clk(n)\" REFCLK_FGTL12C_Q1_RX_CH2n AV48 BD57 qsfp_ref_clk REFCLK_FGTL12C_Q1_RX_CH2p AW49 BB57 \"PCIE_REFCLK0(n)\" REFCLK_FGTR13A_Q1_RX_CH2n BU7 BP13 PCIE_REFCLK0 REFCLK_FGTR13A_Q1_RX_CH2p BR7 BN14 
                                                                         
                                                                      1.  
                                                                        Constrain the pins identified in Step 6.B in the $OFS_ROOTDIR/syn/board/fseries-dk/setup/top_loc.tcl file for the new pinout for the AGF 027 R31C package. 
                                                                         
                                                                        set_location_assignment PIN_BB57 -to qsfp_ref_clk\nset_location_assignment PIN_BD57 -to \"qsfp_ref_clk(n)\"\n\nset_location_assignment PIN_BP13 -to \"PCIE_REFCLK0(n)\"\nset_location_assignment PIN_BN14 -to PCIE_REFCLK0\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Uncomment the instance assignments related to he QSFP reference clock in the $OFS_ROOTDIR/syn/board/fseries-dk/setup/top_loc.tcl file.
                                                                         
                                                                        set_instance_assignment -name IO_STANDARD \"CURRENT MODE LOGIC (CML)\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=disable_3p3v_tol\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=156250000\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=TRUE\" -to qsfp_ref_clk\n
                                                                         
                                                                        3.  
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Compile a flat design. It is recommended to compile a flat design first before incorporating a PR region in the design.
                                                                       
                                                                      cd $OFS_ROOTDIR\n./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk:flat work_fseries-dk\n
                                                                       
                                                                      8.  
                                                                    8.  
                                                                      Verify that the build completes successfuly. If there are timing violation, try building with a different seed. Refer to the Change the Compilation Seed section for instructions on changing the build seed.
                                                                       
                                                                      9.  
                                                                    9.  
                                                                      When you are satisfied with the pinout, preserve it by hard-coding the desired pinout to followig files:
                                                                       
                                                                         
                                                                      • $OFS_ROOTDIR/syn/board/fseries-dk/setup/emif_loc.tcl
                                                                        •  
                                                                      • $OFS_ROOTDIR/syn/board/fseries-dk/setup/hps_loc.tcl
                                                                        •  
                                                                      • $OFS_ROOTDIR/syn/board/fseries-dk/setup/top_loc.tcl
                                                                        •  
                                                                       
                                                                      10.  
                                                                    10.  
                                                                      When you are ready to re-incorporate PR into the design, modify the PR region to be compatible with the new device. Refer to the Resize the Partial Reconfiguration Region section for instructions.
                                                                       
                                                                      11.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#47-modify-the-ethernet-sub-system","title":"4.7 Modify the Ethernet Sub-System","text":"
                                                                    This section describes the flows for modifying the Ethernet Sub-System.
                                                                     
                                                                    Note: The default HSSI-SS configuration for the fseries-dk is 2x4x25GbE.
                                                                     
                                                                    Note: The fseries-dk does not support 2x200GbE or 1x400GbE configurations.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#471-walkthrough-modify-the-ethernet-sub-system-to-2x4x10gbe","title":"4.7.1 Walkthrough: Modify the Ethernet Sub-System to 2x4x10GbE","text":"
                                                                    Perform the following steps to modify the Ethernet Sub-System to 2x4x10GbE. In this walkthrough, we will create a new IP presets file that will be used during the build flow.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Run the setup stage of the build script to create a work directry for the fseries-dk design.
                                                                       
                                                                      ./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Open the HSSI-SS IP in the work directory using qsys-edit
                                                                       
                                                                      qsys-edit $OFS_ROOTDIR/work_fseries-dk/ipss/hssi/qip/hssi_ss/hssi_ss.ip\n
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Change the PORT_PROFILE parameter from 25GbE to 10GbE for Ports 0 through 7.
                                                                       
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Create an IP Presets file for this Ethernet Subsystem IP configuration.
                                                                       
                                                                         
                                                                      1.  
                                                                        Click New in the bottom right corner of the Presets window.
                                                                         
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        In the New Preset window, set the Preset name. In this example, we will name it 2x4x10g-fseries-dk.
                                                                         
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Click the ... button to select the save location of the Preset file.
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        In the Save As window, navigate to the $OFS_ROODIR/ipss/hssi/qip/hssi_ss/presets directory. Set the File Name to 2x4x10g-fseries-dk.qprs. Then click OK.
                                                                         
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        In the New Preset window, click Save. Click No when prompted to add the file to the IP search path.
                                                                         
                                                                        6.  
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Close out of the IP Parameter Editor GUI. Do not save or generate the IP.
                                                                       
                                                                      8.  
                                                                    8.  
                                                                      Create a new HSSI OFSS file named $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_2x4x10_ftile.ofss with the following contents. This will call the IP Presets file generated in Step 6.
                                                                       
                                                                      [ip]\n  type = hssi\n\n  [settings]\n  output_name = hssi_ss\n  num_channels = 8\n  data_rate = 10GbE\n  preset = 2x4x10g-fseries-dk\n
                                                                       
                                                                      9.  
                                                                    9.  
                                                                      Run the FIM build script with the new HSSI OFSS file created in Step 8.
                                                                       
                                                                      ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_2x4x10_ftile.ofss fseries-dk work_fseries-dk_eth_2x4x10\n
                                                                       
                                                                      10.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#472-walkthrough-modify-the-ethernet-sub-system-to-3x4x10gbe","title":"4.7.2 Walkthrough: Modify the Ethernet Sub-System to 3x4x10GbE","text":"
                                                                    Perform the following steps to modify the Ethernet Sub-System to 3x4x10GbE. In this walkthrough, we will create a new IP presets file that will be used during the build flow.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Run the setup stage of the build script to create a work directry for the fseries-dk design.
                                                                       
                                                                      ./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Open the HSSI-SS IP in the work directory using qsys-edit
                                                                       
                                                                      qsys-edit $OFS_ROOTDIR/work_fseries-dk/ipss/hssi/qip/hssi_ss/hssi_ss.ip\n
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Change the NUM_ENABLED_PORTS parameter from 8 to 12.
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Change the PORT_PROFILE parameter from 25GbE to 10GbE for Ports 0 through 11.
                                                                       
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Create an IP Presets file for this Ethernet Subsystem IP configuration.
                                                                       
                                                                         
                                                                      1.  
                                                                        Click New in the bottom right corner of the Presets window.
                                                                         
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        In the New Preset window, set the Preset name. In this example, we will name it 3x4x10g-fseries-dk.
                                                                         
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Click the ... button to select the save location of the Preset file.
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        In the Save As window, navigate to the $OFS_ROODIR/ipss/hssi/qip/hssi_ss/presets directory. Set the File Name to 3x4x10g-fseries-dk.qprs. Then click OK.
                                                                         
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        In the New Preset window, click Save. Click No when prompted to add the file to the IP search path.
                                                                         
                                                                        6.  
                                                                       
                                                                      8.  
                                                                    8.  
                                                                      Close out of the IP Parameter Editor GUI. Do not save or generate the IP.
                                                                       
                                                                      9.  
                                                                    9.  
                                                                      Create a new HSSI OFSS file named $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_3x4x10_ftile.ofss with the following contents. This will call the IP Presets file generated in Step 6.
                                                                       
                                                                      [ip]\n  type = hssi\n\n  [settings]\n  output_name = hssi_ss\n  num_channels = 12\n  data_rate = 10GbE\n  preset = 3x4x10g-fseries-dk\n
                                                                       
                                                                      10.  
                                                                    10.  
                                                                      Change the number of QSFP ports from 2 to 3 in the $OFS_ROOTDIR/ipss/hssi/rtl/inc/ofs_fim_eth_plat_if_pkg.sv file.
                                                                       
                                                                      localparam NUM_QSFP_PORTS = 3; // QSFP cage on board\n
                                                                       
                                                                      11.  
                                                                    11.  
                                                                      Add the pin location assignments for the new QSFP channels in the $OFS_ROOTDIR/syn/board/fseries-dk/setup/top_loc.tcl file. In this example we will use channels 4 through 7 of the QSFPDD-56.
                                                                       
                                                                      set_location_assignment PIN_AC55 -to qsfp_serial[2].rx_p[0]\nset_location_assignment PIN_R55  -to qsfp_serial[2].rx_p[1]\nset_location_assignment PIN_AG55 -to qsfp_serial[2].rx_p[2]\nset_location_assignment PIN_W55  -to qsfp_serial[2].rx_p[3]\n\nset_location_assignment PIN_AD54 -to qsfp_serial[2].rx_n[0]\nset_location_assignment PIN_T54  -to qsfp_serial[2].rx_n[1]\nset_location_assignment PIN_AH54 -to qsfp_serial[2].rx_n[2]\nset_location_assignment PIN_Y54  -to qsfp_serial[2].rx_n[3]\n\nset_location_assignment PIN_AF52 -to qsfp_serial[2].tx_p[0]\nset_location_assignment PIN_W49  -to qsfp_serial[2].tx_p[1]\nset_location_assignment PIN_AK52 -to qsfp_serial[2].tx_p[2]\nset_location_assignment PIN_AB52  -to qsfp_serial[2].tx_p[3]\n\nset_location_assignment PIN_AE51 -to qsfp_serial[2].tx_n[0]\nset_location_assignment PIN_Y48  -to qsfp_serial[2].tx_n[1]\nset_location_assignment PIN_AJ51 -to qsfp_serial[2].tx_n[2]\nset_location_assignment PIN_AA51  -to qsfp_serial[2].tx_n[3]\n
                                                                       
                                                                      12.  
                                                                    12.  
                                                                      Build the FIM. 
                                                                       
                                                                      ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_3x4x10_ftile.ofss fseries-dk work_fseries-dk_eth_12x10g\n
                                                                       
                                                                      13.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#48-modifying-the-hps","title":"4.8 Modifying the HPS","text":"
                                                                    This section describes ways to modify the HPS.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#481-walkthrough-remove-the-hps","title":"4.8.1 Walkthrough: Remove the HPS","text":"
                                                                    Perform the following steps to remove the HPS from the FIM design.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Run the setup stage of the build script to create a work directry for the fseries-dk design.
                                                                       
                                                                      ./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Create a Memory Sub-system IP presets file with the connection to the HPS removed.
                                                                       
                                                                         
                                                                      1.  
                                                                        In the work directory created in Step 3, open the mem_ss.ip
                                                                         
                                                                        qsys-edit $OFS_ROOTDIR/work_fseries-dk/ipss/mem/qip/mem_ss/mem_ss.ip\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        In the IP Parameter Editor window that opens, remove the entries corresponding to the HPS (row #2) in the Memory Interfaces and Application Interfaces sections. To do this, click the row to be removed, then click the minus (-) button.
                                                                         
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        In the Presets pane, click New... to create a new IP presets file.
                                                                         
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        In the New Preset window, create a unique preset name. For example, fseries-dk-mem-no-hps.
                                                                         
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Click the ... button to select the save location of the IP presets file. In the Save As window, set the Look In field to the memory IP presets directory $OFS_ROOTDIR/ipss/mem/qip/presets. Set the File Name field to match the name selected in Step 4. Click OK.
                                                                         
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Click Save in the New Preset window. Click No when prompted to add the file to the IP search path.
                                                                         
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Close IP Parameter Editor without saving or generating HDL.
                                                                         
                                                                        8.  
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Edit the Memory OFSS file $OFS_ROOTDIR/tools/ofss_config/memory/memory_ftile.ofss to use the IP presets file generated in Step 4.
                                                                       
                                                                      [ip]\ntype = memory\n\n[settings]\noutput_name = mem_ss\npreset = fseries-dk-mem-no-hps\n
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Edit $OFS_ROOTDIR/syn/board/fseries-dk/syn_top/ofs_top.qsf to comment out the INCLUDE_HPS macro definition.
                                                                       
                                                                      #set_global_assignment -name VERILOG_MACRO \"INCLUDE_HPS\"\n
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Build the FIM.
                                                                       
                                                                      ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/fseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss fseries-dk work_fseries-dk_no_hps\n
                                                                       
                                                                      8.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#5-fpga-configuration","title":"5. FPGA Configuration","text":"
                                                                    Configuring the Agilex FPGA on the fseries-dk is done by programming a SOF image to the FPGA via JTAG using Quartus Programer.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#51-walkthrough-set-up-jtag","title":"5.1 Walkthrough: Set up JTAG","text":"
                                                                    The Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) has an on-board FPGA Download Cable II module which is used to program the FPGA via JTAG.
                                                                     
                                                                    Perform the following steps to establish a JTAG connection with the fseries-dk.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires a workstation with Quartus Prime Pro Version 23.3 tools installed, specifically the jtagconfig tool.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Refer to the following figure for Steps 2 and 3.
                                                                       
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Locate Single DIP Switch SW2 and 4-position DIP switch SW3 on the fseries-dk. These switches control the JTAG setup for the board. Ensure that both SW2 and SW3.3 are set to ON.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Locate the J10 Micro-USB port on the fseries-dk. Connect a Micro-USB to USB-A cable between the J10 port and the workstation that has Quartus Prime Pro tools installed.
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Use the jtagconfig tool to check that the JTAG chain contains the AGFB027R24C2E2VR2 device.
                                                                       
                                                                      <QUARTUS_INSTALL_DIR>/23.3/quartus/bin/jtagconfig\n
                                                                       
                                                                      Example expected output:
                                                                       
                                                                      1) Agilex F-Series FPGA Dev Kit [1-6]\n0343B0DD   AGFB027R24C(.|R2|R0)/..\n020D10DD   VTAP10\n
                                                                       
                                                                      5.  
                                                                     
                                                                    This concludes the walkthrough for establishing a JTAG connection on the fseries-dk.
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#52-walkthrough-program-the-fpga-via-jtag","title":"5.2 Walkthrough: Program the FPGA via JTAG","text":"
                                                                    This walkthrough describes the steps to program the Agilex FPGA on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) with a SOF image via JTAG.
                                                                     
                                                                    Pre-Requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires a SOF image which will be programmed to the Agilex FPGA. Refer to the Compile OFS FIM Section for step-by-step instructions for generating a SOF image.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires a JTAG connection to the fseries-dk. Refer to the Set up JTAG section for step-by-step instructions.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) is connected via JTAG.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Start in your deployment environment.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      If the card is already programmed with an OFS enabled design, determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                                                                       
                                                                      fpgainfo fme\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010202A8769764\nBitstream Version                : 5.0.1\nPr Interface Id                  : b541eb7c-3c7e-5678-a660-a54f71594b34\nBoot Page                        : N/A\n
                                                                       
                                                                      Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Remove the card from PCIe prior to programming. This will disable AER for the PCIe root port to prevent a surprise link-down event during programming.
                                                                       
                                                                      sudo pci_device b1:00.0 unplug\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Switch to the machine with JTAG connection to the fseries-dk, if different than your deployment machine.
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Open the Quartus programmer GUI
                                                                       
                                                                      quartus_pgmw\n
                                                                       
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Click Hardware Setup to open the Hardware Setup dialog window.
                                                                       
                                                                         
                                                                      1.  
                                                                        In the Currently selected hardware field select the fseries-dk.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        In the Hardware frequency field enter 16000000 Hz
                                                                         
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Click Close
                                                                         
                                                                        4.  
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      In the Quartus Prime Programmer window, click Auto Detect.
                                                                       
                                                                      8.  
                                                                    8.  
                                                                      If prompted, select the AGFB027R24C2E2VR2 device. The JTAG chain should show the device.
                                                                       
                                                                       
                                                                      9.  
                                                                    9.  
                                                                      Right click the AGFB027R24C2E2VR2 row and selct Change File.
                                                                       
                                                                       
                                                                      10.  
                                                                    10.  
                                                                      In the Select New Programming File window that opens, select the .sof image you wish to program and click Open.
                                                                       
                                                                      11.  
                                                                    11.  
                                                                      Check the Program/Configure box for the AGFB027R24C2E2VR2 row, then click Start. Wait for the Progress bar to show 100% (Success).
                                                                       
                                                                       
                                                                      12.  
                                                                    12.  
                                                                      Close the Quartus Programmer GUI.
                                                                       
                                                                      13.  
                                                                    13.  
                                                                      Switch to the deployment environment, if different than the JTAG connected machine.
                                                                       
                                                                      14.  
                                                                    14.  
                                                                      Replug the board PCIe
                                                                       
                                                                      sudo pci_device b1:00.0 plug\n
                                                                       
                                                                      15.  
                                                                    15.  
                                                                      Run fpgainfo fme to verify communication with the board, and to check the PR Interface ID.
                                                                       
                                                                      Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x501020241BF165B\nBitstream Version                : 5.0.1\nPr Interface Id                  : e7f69412-951f-5d1a-8cb7-8c778ac02055\nBoot Page                        : N/A\n
                                                                       
                                                                      Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                                                                       
                                                                      16.  
                                                                    "},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#appendix","title":"Appendix","text":""},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#appendix-a-resource-utilization-tables","title":"Appendix A: Resource Utilization Tables","text":"
                                                                    Table A-1 Default Flat FIM Resource Utilization
                                                                     Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 153105.5 16.77 624 4.7 afu_top 83365.9 9.13 274 2.06 auto_fab_0 1339.6 0.15 9 0.07 bpf 780.1 0.09 0 0.0 fme_top 658.5 0.07 6 0.05 hps_ss 0.0 0.0 0 0.0 hssi_wrapper 11681.9 1.28 81 0.61 mem_ss_top 5712.9 0.63 38 0.29 ofs_top_auto_tiles 8180.5 0.9 20 0.15 pcie_wrapper 39444.9 4.32 188 1.42 pmci_dummy_csr 668.2 0.07 0 0.0 qsfp_0 627.1 0.07 4 0.03 qsfp_1 624.6 0.07 4 0.03 rst_ctrl 17.1 0.0 0 0.0 sys_pll 0.4 0.0 0 0.0 
                                                                    Table A-2 Default In-Tree PR FIM Resource Utilization
                                                                     Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 158371.6 17.35 624 4.7 afu_top 88125.9 9.65 274 2.06 auto_fab_0 1305.6 0.14 9 0.07 bpf 782.5 0.09 0 0.0 fme_top 650.5 0.07 6 0.05 hps_ss 0.0 0.0 0 0.0 hssi_wrapper 11757.4 1.29 81 0.61 mem_ss_top 6155.1 0.67 38 0.29 ofs_top_auto_tiles 8263.6 0.91 20 0.15 pcie_wrapper 39393.5 4.32 188 1.42 pmci_dummy_csr 658.9 0.07 0 0.0 qsfp_0 625.0 0.07 4 0.03 qsfp_1 632.8 0.07 4 0.03 rst_ctrl 16.3 0.0 0 0.0 sys_pll 0.4 0.0 0 0.0 
                                                                    Table A-3 Default Out-of-Tree FIM Resource Utilization
                                                                     Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 158287.4 17.34 624 4.7 afu_top 88046.1 9.65 274 2.06 auto_fab_0 1313.2 0.14 9 0.07 bpf 784.7 0.09 0 0.0 fme_top 656.1 0.07 6 0.05 hps_ss 0.0 0.0 0 0.0 hssi_wrapper 11781.0 1.29 81 0.61 mem_ss_top 6164.6 0.68 38 0.29 ofs_top_auto_tiles 8224.2 0.9 20 0.15 pcie_wrapper 39365.6 4.31 188 1.42 pmci_dummy_csr 665.1 0.07 0 0.0 qsfp_0 635.1 0.07 4 0.03 qsfp_1 631.7 0.07 4 0.03 rst_ctrl 15.1 0.0 0 0.0 sys_pll 0.5 0.0 0 0.0 
                                                                    Table A-4 Minimal FIM Resource Utilization
                                                                     Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 101487.5 11.12 453 3.41 afu_top 31185.9 3.42 105 0.79 auto_fab_0 1309.6 0.14 9 0.07 bpf 797.6 0.09 0 0.0 fme_top 656.8 0.07 6 0.05 hps_ss 0.0 0.0 0 0.0 hssi_wrapper 11649.8 1.28 81 0.61 mem_ss_top 6447.4 0.71 38 0.29 ofs_top_auto_tiles 8288.5 0.91 20 0.15 pcie_wrapper 39208.5 4.3 186 1.4 pmci_dummy_csr 670.4 0.07 0 0.0 qsfp_0 623.6 0.07 4 0.03 qsfp_1 627.8 0.07 4 0.03 rst_ctrl 16.2 0.0 0 0.0 sys_pll 0.4 0.0 0 0.0"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#appendix-b-glossary","title":"Appendix B: Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/ftile_devkit/dev_guides/fim_dev/ug_ofs_ftile_dk_fim_dev/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                    Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                     
                                                                    OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                    "},{"location":"hw/ftile_devkit/doc_modules/ftile_wt_program_fpga_via_jtag/","title":"Ftile wt program fpga via jtag","text":"
                                                                    This walkthrough describes the steps to program the Agilex FPGA on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) with a SOF image via JTAG.
                                                                     
                                                                    Pre-Requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 SoC Attach FPGAs (Intel Agilx 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires a SOF image which will be programmed to the Agilex FPGA. Refer to the [Walkthrough: Compile OFS FIM] Section for step-by-step instructions for generating a SOF image.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires a JTAG connection to the fseries-dk. Refer to the [Walkthrough: Set up JTAG] section for step-by-step instructions.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) is connected via JTAG.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Start in your deployment environment.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      If the card is already programmed with an OFS enabled design, determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                                                                       
                                                                      sudo fpgainfo fme\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010202A8769764\nBitstream Version                : 5.0.1\nPr Interface Id                  : b541eb7c-3c7e-5678-a660-a54f71594b34\nBoot Page                        : N/A\n
                                                                       
                                                                      Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Remove the card from PCIe prior to programming. This will disable AER for the PCIe root port to prevent a surprise link-down event during programming.
                                                                       
                                                                      sudo pci_device b1:00.0 unplug\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Switch to the machine with JTAG connection to the fseries-dk, if different than your deployment machine.
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Open the Quartus programmer GUI
                                                                       
                                                                      quartus_pgmw\n
                                                                       
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Click Hardware Setup to open the Hardware Setup dialog window.
                                                                       
                                                                         
                                                                      1.  
                                                                        In the Currently selected hardware field select the fseries-dk.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        In the Hardware frequency field enter 16000000 Hz
                                                                         
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Click Close
                                                                         
                                                                        4.  
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      In the Quartus Prime Programmer window, click Auto Detect.
                                                                       
                                                                      8.  
                                                                    8.  
                                                                      If prompted, select the AGFB027R24C2E2VR2 device. The JTAG chain should show the device.
                                                                       
                                                                       
                                                                      9.  
                                                                    9.  
                                                                      Right click the AGFB027R24C2E2VR2 row and selct Change File.
                                                                       
                                                                       
                                                                      10.  
                                                                    10.  
                                                                      In the Select New Programming File window that opens, select  ofs_top_hps.sof and click Open.
                                                                       
                                                                      11.  
                                                                    11.  
                                                                      Check the Program/Configure box for the AGFB027R24C2E2VR2 row, then click Start. Wait for the Progress bar to show 100% (Success).
                                                                       
                                                                       
                                                                      12.  
                                                                    12.  
                                                                      Close the Quartus Programmer GUI.
                                                                       
                                                                      13.  
                                                                    13.  
                                                                      Switch to the deployment environment, if different than the JTAG connected machine.
                                                                       
                                                                      14.  
                                                                    14.  
                                                                      Replug the board PCIe
                                                                       
                                                                      sudo pci_device b1:00.0 plug\n
                                                                       
                                                                      15.  
                                                                    15.  
                                                                      Run fpgainfo fme to verify communication with the board, and to check the PR Interface ID.
                                                                       
                                                                      Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 360571655976424377\nBitstream Version                : 5.0.1\nPr Interface Id                  : d8fd88a7-8683-57ba-8be6-a1e058b7d4ed\nBoot Page                        : N/A\n
                                                                       
                                                                      Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                                                                       
                                                                      16.  
                                                                    "},{"location":"hw/ftile_devkit/doc_modules/ftile_wt_set_up_jtag/","title":"Ftile wt set up jtag","text":"
                                                                    The Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) has an on-board FPGA Download Cable II module which is used to program the FPGA via JTAG.
                                                                     
                                                                    Perform the following steps to establish a JTAG connection with the fseries-dk.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the [OFS Agilex PCIe Attach Getting Started Guide] for instructions on setting up a deployment environment.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires a workstation with Quartus Prime Pro Version 23.3 tools installed, specifically the jtagconfig tool.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Refer to the following figure for Steps 2 and 3.
                                                                       
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Locate Single DIP Switch SW2 and 4-position DIP switch SW3 on the fseries-dk. These switches control the JTAG setup for the board. Ensure that both SW2 and SW3.3 are set to ON.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Locate the J10 Micro-USB port on the fseries-dk. Connect a Micro-USB to USB-A cable between the J10 port and the workstation that has Quartus Prime Pro tools installed.
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Use the jtagconfig tool to check that the JTAG chain contains the AGFB027R24C2E2VR2 device.
                                                                       
                                                                      <QUARTUS_INSTALL_DIR>/23.3/quartus/bin/jtagconfig\n
                                                                       
                                                                      Example expected output:
                                                                       
                                                                      1) Agilex F-Series FPGA Dev Kit [1-6]\n0343B0DD   AGFB027R24C(.|R2|R0)/..\n020D10DD   VTAP10\n
                                                                       
                                                                      5.  
                                                                     
                                                                    This concludes the walkthrough for establishing a JTAG connection on the fseries-dk.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/","title":"Getting Started Guide: Open FPGA Stack for Intel Agilex 7 FPGAs Targeting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile)","text":"
                                                                    Last updated: February 03, 2024 
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#10-about-this-document","title":"1.0 About This Document","text":"
                                                                    The purpose of this document is to help users get started in evaluating the 2023.3 version of the PCIe Attach release targeting the F-tile Development Kit. After reviewing this document, a user shall be able to:
                                                                     
                                                                       
                                                                    • Set up a server environment according to the Best Known Configuration (BKC)
                                                                      •  
                                                                    • Load and verify firmware targeting the FIM and AFU regions of the AGFB027R24C2E2VR2 FPGA
                                                                      •  
                                                                    • Verify full stack functionality offered by the PCIe Attach OFS solution
                                                                      •  
                                                                    • Learn where to find additional information on other PCIe Attach ingredients
                                                                      •  
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#11-audience","title":"1.1 Audience","text":"
                                                                    The information in this document is intended for customers evaluating the PCIe Attach shell targeting the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile). This platform is a Development Kit  intended to be used as a starting point for evaluation and development of the Intel Agilex 7 FPGA F-Series with two F-Tiles.
                                                                     
                                                                    Note: Code command blocks are used throughout the document. Commands that are intended for you to run are preceded with the symbol '$', and comments with '#'. Full command output may not be shown.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-1-terminology","title":"Table 1: Terminology","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-2-software-and-component-version-summary-for-ofs-pcie-attach-targeting-the-f-tile-development-kit","title":"Table 2: Software and Component Version Summary for OFS PCIe Attach targeting the F-tile Development Kit","text":"
                                                                    The OFS 2023.3 PCIe Attach release targeting the F-tile Development Kit is built upon tightly coupled software and Operating System version(s). The repositories listed below are used to manually build the Shell and the AFU portion of any potential workloads. Use this section as a general reference for the versions which compose this release. Specific instructions on building the FIM or AFU are discussed in their respective documents.
                                                                     Component Version Download Link Quartus Quartus Prime Pro Version 23.3 https://www.intel.com/content/www/us/en/software-kit/782411/intel-quartus-prime-pro-edition-design-software-version-23-3-for-linux.html, patches: 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe) Host Operating System RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6 https://access.redhat.com/downloads/content/479/ver=/rhel---8/8.6/x86_64/product-software OneAPI-ASP ofs-2023.3-2 https://github.com/OFS/oneapi-asp/releases/tag/ofs-2023.3-2, patches: 0.02 OFS Platform AFU BBB ofs-2023.3-2 https://github.com/OFS/ofs-platform-afu-bbb/releases/tag/ofs-2023.3-2 OFS FIM Common Resources 2023.3 https://github.com/OFS/ofs-fim-common/releases/tag/ofs-2023.3-2 AFU Examples tag: ofs-2023.3-2 https://github.com/OFS/examples-afu/releases/tag/ofs-2023.3-2 OPAE-SIM tag: 2.10.0-1 https://github.com/OPAE/opae-sim"},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-3-programmable-firmware-version-summary-for-ofs-pcie-attach-targeting-the-f-tile-development-kit","title":"Table 3: Programmable Firmware Version Summary for OFS PCIe Attach targeting the F-tile Development Kit","text":"
                                                                    OFS releases include pre-built binaries for the FPGA, OPAE SDK and Linux DFL which can be programmed out-of-box (OOB) and include known identifiers shown below. Installation of artifacts provided with this release will be discussed in their relevant sections.
                                                                     Component Version Link FIM (shell) Pr Interface ID: 5bcd682f-5093-5fc7-8cd2-ae8073e19452 https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2 Host OPAE SDK https://github.com/OPAE/opae-sdk, tag: 2.10.0-1 https://github.com/OFS/opae-sdk/releases/tag/2.10.0-1 Host Linux DFL Drivers https://github.com/OPAE/linux-dfl, tag: ofs-2023.3-6.1-3 https://github.com/OFS/linux-dfl/releases/tag/ofs-2023.3-6.1-3"},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-4-hardware-bkc-for-ofs-pcie-attach-targeting-the-f-tile-development-kit","title":"Table 4: Hardware BKC for OFS PCIe Attach targeting the F-tile Development Kit","text":"
                                                                    The following table highlights the hardware which composes the Best Known Configuation (BKC) for the OFS 2023.3 PCIe Attach release targeting F-tile Development Kit.
                                                                     
                                                                    Note: The Dell R750 server product line is known not to work with this release.
                                                                     Component Link Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) https://www.intel.com/content/www/us/en/products/details/fpga/development-kits/agilex/agf027-and-agf023.html Intel FPGA Download Cable II https://www.intel.com/content/www/us/en/products/sku/215664/intel-fpga-download-cable-ii/specifications.html SuperMicro SYS-220HE-FTNR https://www.supermicro.com/en/products/system/hyper/2u/sys-220he-ftnr"},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#12-server-requirements","title":"1.2 Server Requirements","text":""},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#121-host-server-specifications","title":"1.2.1 Host Server Specifications","text":"
                                                                    The host server must meet the following specifications:
                                                                     
                                                                       
                                                                    • The server platform must contain 64 GB of RAM to run certain demos, and to compile FIM Images
                                                                      •  
                                                                    • The server platform must be able to fit, power, and cool an Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) as described by the product page
                                                                      •  
                                                                    • The server should be able to run PCIe at Gen 4 speeds to properly test designs and demos
                                                                      •  
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#122-host-bios","title":"1.2.2 Host BIOS","text":"
                                                                    These are the host BIOS settings known to work with the F-tile Development Kit. Information about the server's currently loaded firmware and BIOS settings can be found through its remote access controller, or by manually entering the BIOS by hitting a specific key during power on. Your specific server platform will include instructions on proper BIOS configuration and should be followed when altering settings.
                                                                     
                                                                       
                                                                    • PCIe slot width must be set to x16
                                                                      •  
                                                                    • PCIe slot speed must be set to 4
                                                                      •  
                                                                    • PCIe slot must have iommu enabled
                                                                      •  
                                                                    • Intel VT for Directed I/O (VT-d) must be enabled
                                                                      •  
                                                                     
                                                                    Specific BIOS paths are not listed here as they can differ between BIOS vendors and versions.
                                                                     
                                                                    In addition to BIOS settings required to support the operation of an OFS PCIe Attach solution targeting the F-tile Development Kit, server fan speed may need to be adjusted in the BIOS settings depending on local air temperature and air flow. The OFS PCIe Attach design does not automatically communicate cooling curve information with the on-board server management interface. Increasing air flow will mitigate possible thermal runaway and thermal throttling that may occur as a result.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#123-host-server-kernel-and-grub-configuration","title":"1.2.3 Host Server Kernel and GRUB Configuration","text":"
                                                                    While many host Linux kernel and OS distributions may work with this design, only the following configuration(s) have been tested:
                                                                     
                                                                    OS: RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6  Kernel: 6.1-lts
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#13-preparing-the-f-tile-development-kit-for-installation-into-a-server","title":"1.3 Preparing the F-tile Development Kit for Installation into a Server","text":"
                                                                       
                                                                    1.  
                                                                      The DK-DEV-AGF027F1ES (or it is called the F - tile Dev Kit, or FM86 Dev Kit) has LED light pipes on top of the QSFP cages.
                                                                       
                                                                       
                                                                      These light pipes interfere with the server PCIe slot faceplate.
                                                                       
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      The light pipes can be easily removed by prying them off using a small screwdriver for leverage, then pushing the light pipes back to remove the retaining clips from the QSFP cage.
                                                                       
                                                                       
                                                                       
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Board switch definitions can be found in the Intel Agilex\u00ae 7 F-Series FPGA (Two F-Tiles) Development Kit User Guide.
                                                                       
                                                                       
                                                                      See the image below for SW1, SW4 and SW3.
                                                                       
                                                                       
                                                                      Before inserting into a server, set SW5 to 'ON'.
                                                                       
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Below shows a card installed into a PCIe riser with the light pipes removed.
                                                                       
                                                                       
                                                                      5.  
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#14-f-tile-development-kit-jtag-setup","title":"1.4 F-tile Development Kit JTAG Setup","text":"
                                                                    The Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) has an on-board FPGA Download Cable II module which is used to program the FPGA via JTAG.
                                                                     
                                                                    Perform the following steps to establish a JTAG connection with the fseries-dk.
                                                                     
                                                                    Pre-requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the [OFS Agilex PCIe Attach Getting Started Guide] for instructions on setting up a deployment environment.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires a workstation with Quartus Prime Pro Version 23.3 tools installed, specifically the jtagconfig tool.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Refer to the following figure for Steps 2 and 3.
                                                                       
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Locate Single DIP Switch SW2 and 4-position DIP switch SW3 on the fseries-dk. These switches control the JTAG setup for the board. Ensure that both SW2 and SW3.3 are set to ON.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Locate the J10 Micro-USB port on the fseries-dk. Connect a Micro-USB to USB-A cable between the J10 port and the workstation that has Quartus Prime Pro tools installed.
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Use the jtagconfig tool to check that the JTAG chain contains the AGFB027R24C2E2VR2 device.
                                                                       
                                                                      <QUARTUS_INSTALL_DIR>/23.3/quartus/bin/jtagconfig\n
                                                                       
                                                                      Example expected output:
                                                                       
                                                                      1) Agilex F-Series FPGA Dev Kit [1-6]\n0343B0DD   AGFB027R24C(.|R2|R0)/..\n020D10DD   VTAP10\n
                                                                       
                                                                      5.  
                                                                     
                                                                    This concludes the walkthrough for establishing a JTAG connection on the fseries-dk.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#15-upgrading-the-f-tile-development-kit-fim-via-jtag","title":"1.5 Upgrading the F-tile Development Kit FIM via JTAG","text":"
                                                                    Intel provides a pre-built FIM that can be used out-of-box for platform bring-up. This shell design is available on the OFS 2023.3 Release Page. After programming the shell and installing both the OPAE SDK and Linux DFL kernel drivers (as shown in sections 3.0 OFS DFL Kernel Drivers and 4.0 OPAE Software Development Kit), you can confirm the correct FIM has been configured by checking the output of fpgainfo fme against the following table:
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-5-fim-version","title":"Table 5: FIM Version","text":"Identifier Value Pr Interface ID 5bcd682f-5093-5fc7-8cd2-ae8073e19452 Bitstream ID 360571655976424377 
                                                                    You will need to download and unpack the artifact images for this release before upgrading your device. The file ofs_top_hps.sof is the base OFS FIM file. This file is loaded into the FPGA using the development kit built in USB Blaster. Please be aware this FPGA is not loaded into non-volatile storage, therefore if the server is power cycled, you will need to reload the FPGA .sof file.
                                                                     
                                                                    wget https://github.com/OFS/ofs-agx7-pcie-attach/releases/download/ofs-2023.3-2/fseries-images_ofs-2023-3-2.tar.gz\ntar xf fseries-dk-images.tar.gz\ncd fseries-dk-images/\n
                                                                     
                                                                    This walkthrough describes the steps to program the Agilex FPGA on the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) with a SOF image via JTAG.
                                                                     
                                                                    Pre-Requisites:
                                                                     
                                                                       
                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 SoC Attach FPGAs (Intel Agilx 7 FPGA F-Series Development Kit (2xF-Tile)) for instructions on setting up a deployment environment.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires a SOF image which will be programmed to the Agilex FPGA. Refer to the [Walkthrough: Compile OFS FIM] Section for step-by-step instructions for generating a SOF image.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires a JTAG connection to the fseries-dk. Refer to the [Walkthrough: Set up JTAG] section for step-by-step instructions.
                                                                      •  
                                                                     
                                                                       
                                                                    • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel Agilex\u00ae 7 FPGA F-Series Development Kit (2x F-Tile) is connected via JTAG.
                                                                      •  
                                                                     
                                                                    Steps:
                                                                     
                                                                       
                                                                    1.  
                                                                      Start in your deployment environment.
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      If the card is already programmed with an OFS enabled design, determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                                                                       
                                                                      sudo fpgainfo fme\n
                                                                       
                                                                      Example output:
                                                                       
                                                                      Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010202A8769764\nBitstream Version                : 5.0.1\nPr Interface Id                  : b541eb7c-3c7e-5678-a660-a54f71594b34\nBoot Page                        : N/A\n
                                                                       
                                                                      Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Remove the card from PCIe prior to programming. This will disable AER for the PCIe root port to prevent a surprise link-down event during programming.
                                                                       
                                                                      sudo pci_device b1:00.0 unplug\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Switch to the machine with JTAG connection to the fseries-dk, if different than your deployment machine.
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Open the Quartus programmer GUI
                                                                       
                                                                      quartus_pgmw\n
                                                                       
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Click Hardware Setup to open the Hardware Setup dialog window.
                                                                       
                                                                         
                                                                      1.  
                                                                        In the Currently selected hardware field select the fseries-dk.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        In the Hardware frequency field enter 16000000 Hz
                                                                         
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Click Close
                                                                         
                                                                        4.  
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      In the Quartus Prime Programmer window, click Auto Detect.
                                                                       
                                                                      8.  
                                                                    8.  
                                                                      If prompted, select the AGFB027R24C2E2VR2 device. The JTAG chain should show the device.
                                                                       
                                                                       
                                                                      9.  
                                                                    9.  
                                                                      Right click the AGFB027R24C2E2VR2 row and selct Change File.
                                                                       
                                                                       
                                                                      10.  
                                                                    10.  
                                                                      In the Select New Programming File window that opens, select  ofs_top_hps.sof and click Open.
                                                                       
                                                                      11.  
                                                                    11.  
                                                                      Check the Program/Configure box for the AGFB027R24C2E2VR2 row, then click Start. Wait for the Progress bar to show 100% (Success).
                                                                       
                                                                       
                                                                      12.  
                                                                    12.  
                                                                      Close the Quartus Programmer GUI.
                                                                       
                                                                      13.  
                                                                    13.  
                                                                      Switch to the deployment environment, if different than the JTAG connected machine.
                                                                       
                                                                      14.  
                                                                    14.  
                                                                      Replug the board PCIe
                                                                       
                                                                      sudo pci_device b1:00.0 plug\n
                                                                       
                                                                      15.  
                                                                    15.  
                                                                      Run fpgainfo fme to verify communication with the board, and to check the PR Interface ID.
                                                                       
                                                                      Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 360571655976424377\nBitstream Version                : 5.0.1\nPr Interface Id                  : d8fd88a7-8683-57ba-8be6-a1e058b7d4ed\nBoot Page                        : N/A\n
                                                                       
                                                                      Note: The errors related to the BMC are the result of the OFS BMC not being present on the fseries-dk design. These will be removed in a future release.
                                                                       
                                                                      16.  
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#16-f-tile-development-kit-installation-procedure","title":"1.6 F-tile Development Kit Installation Procedure","text":"
                                                                    The following instructions will help ensure safe installation of the F-tile Development Kit into a supported server platform. Safety and Regulatory information can be found under the product page for this development kit. It is assumed you have previously removed the light pipes mounted above the platform's QSFP cages before attempting to slot into a server mounted riser.
                                                                     
                                                                       
                                                                    1. Position the board over the selected connector on the motherboard
                                                                      2.  
                                                                    2. Press down gently and firmly seat the card in a PCIe slot. Depending on the server model being used, you may need to secure a retention screw or rotate retention clips over the development kit's faceplate.
                                                                      3.  
                                                                    3. Do not bend the card while inserting in a slot. Do not apply too much pressure while inserting.
                                                                      4.  
                                                                    4. Plug a standard 2x4 auxiliary power cord available from the server's ATX power supply or from the riser itself to the respective matching power connected on the board (J11). Both the PCIe slot and auxiliary PCIe power cable are required to power the entire board.
                                                                      5.  
                                                                    5. If you haven't already, follow the instructions in section 1.5 F-tile Development Kit JTAG Setup and connect a USB Blaster II Cable from the board to the server housing it.
                                                                      6.  
                                                                     
                                                                    The EMIF subsystem and base FIM in this release do not work with the RDIMM memory installed in the Agilex 7 FPGA F-Series Development Kit. The RDIMM memory must be removed and replaced. The following RDIMM memory module(s) are known to work with this platform:
                                                                     Amount Type Link 2 DDR4-2400 https://memory.net/product/mta9asf1g72az-2g3-micron-1x-8gb-ddr4-2400-udimm-pc4-19200t-e-single-rank-x8-module/"},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#20-ofs-stack-architecture-overview-for-reference-platform","title":"2.0 OFS Stack Architecture Overview for Reference Platform","text":""},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#21-hardware-components","title":"2.1 Hardware Components","text":"
                                                                    The OFS hardware architecture decomposes all designs into a standard set of modules, interfaces, and capabilities. Although the OFS infrastructure provides a standard set of functionality and capability, the user is responsible for making the customizations to their specific design in compliance with the specifications outlined in the FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA.
                                                                     
                                                                    OFS is a hardware and software infrastructure that provides an efficient approach to developing a customer FPGA-based platform or workload using an Intel, 3rd party, or custom board.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#211-fpga-interface-manager","title":"2.1.1 FPGA Interface Manager","text":"
                                                                    The FPGA Interface Manager (FIM), or shell of the FPGA provides platform management functionality, clocks, resets, and interface access to the host and peripheral features on the acceleration platform. The OFS architecture for Intel Agilex 7 FPGA provides modularity, configurability, and scalability. The primary components of the FPGA Interface Manager or shell of the reference design are:
                                                                     
                                                                       
                                                                    • PCIe Subsystem - a hierarchical design that targets the P-tile PCIe hard IP and is configured to support Gen4 speeds and Arm AXI4-Stream Data Mover functional mode.
                                                                      •  
                                                                    • Ethernet Subsystem - provides portability to different Ethernet configurations across platforms and generations and reusability of the hardware framework and software stack.
                                                                      •  
                                                                    • Memory Subsystem - composed of 5 DDR4 channels; two HPS DDR4 banks, x40 (x32 Data and x8 ECC), 1200 MHz, 1GB each, and four Fabric DDR4 banks, x32 (no ECC), 1200 MHz, 4GB
                                                                      •  
                                                                    • Hard Processor System - 64-bit quad core ARM\u00ae Cortex*-A53 MPCore with integrated peripherals.
                                                                      •  
                                                                    • Reset Controller
                                                                      •  
                                                                    • FPGA Management Engine - Provides a way to manage the platform and enable acceleration functions on the platform.
                                                                      •  
                                                                    • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                                                                      •  
                                                                    • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                                                                      •  
                                                                    • Platform Management Controller Interface (PMCI) to the board management controller
                                                                      •  
                                                                     
                                                                    The FPGA Management Engine (FME) provides management features for the platform and the loading/unloading of accelerators through partial reconfiguration. Each feature of the FME exposes itself to the kernel-level OFS drivers on the host through a Device Feature Header (DFH) register that is placed at the beginning of Control Status Register (CSR) space. Only one PCIe link can access the FME register space in a multi-host channel design architecture at a time.
                                                                     
                                                                    Note: For more information on the FIM and its external connections, refer to the FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#212-afu","title":"2.1.2 AFU","text":"
                                                                    An AFU is an acceleration workload that interfaces to the FIM. The AFU boundary in this reference design comprises both static and partial reconfiguration (PR) regions. You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region. A port gasket within the design provides all the PR specific modules and logic required to support partial reconfiguration. Only one partial reconfiguration region is supported in this design.
                                                                     
                                                                    Like the FME, the port gasket exposes its capability to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space.
                                                                     
                                                                    You can compile your design in one of the following ways:
                                                                     
                                                                       
                                                                    • Your entire AFU resides in a partial reconfiguration region of the FPGA.
                                                                      •  
                                                                    • The AFU is part of the static region and is compiled as a flat design.
                                                                      •  
                                                                    • Your AFU contains both static and PR regions.
                                                                      •  
                                                                     
                                                                    In this design, the AFU region is comprised of:
                                                                     
                                                                       
                                                                    • AFU Interface handler to verify transactions coming from AFU region.
                                                                      •  
                                                                    • PF/VF Mux to route transactions to and from corresponding AFU components: ST2MM module, Virtio LB stub, PCIe loopback host exerciser (HE-LB), HSSI host exerciser (HE-HSSI), Memory Host Exerciser (HE-MEM), Traffic Generator to memory (HE-MEM-TG), Port Gasket (PRG) and HPS Copy Engine.
                                                                      •  
                                                                    • AXI4 Streaming to Memory Map (ST2MM) Module that routes MMIO CSR accesses to FME and board peripherals.
                                                                      •  
                                                                    • Host exercisers to test PCIe, memory and HSSI interfaces (these can be removed from the AFU region after your FIM design is complete to provide more resource area for workloads)
                                                                      •  
                                                                    • Basic HPS Copy Engine to copy second-stage bootloader and Linux OS image from Host DDR to HPS DDR.
                                                                      •  
                                                                    • Port gasket and partial reconfiguration support.
                                                                      •  
                                                                    • Component for handling PLDM over MCTP over PCIe Vendor Defined Messages (VDM)
                                                                      •  
                                                                     
                                                                    The AFU has the option to consume native packets from the host or interface channels or to instantiate a shim provided by the Platform Interface Manager (PIM) to translate between protocols.
                                                                     
                                                                    Note: For more information on the Platform Interface Manager and AFU development and testing, refer to the AFU Development Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#22-ofs-software-overview","title":"2.2 OFS Software Overview","text":"
                                                                    The responsibility of the OFS kernel drivers is to act as the lowest software layer in the FPGA software stack, providing a minimalist driver implementation between the host software and functionality that has been implemented on the development platform. This leaves the implementation of IP-specific software in user-land, not the kernel. The OFS software stack also provides a mechanism for interface and feature discovery of FPGA platforms.
                                                                     
                                                                    The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space, and can be found on the OPAE SDK Github.
                                                                     
                                                                    The OFS drivers decompose implemented functionality, including external FIM features such as HSSI, EMIF and SPI, into sets of individual Device Features. Each Device Feature has its associated Device Feature Header (DFH), which enables a uniform discovery mechanism by software. A set of Device Features are exposed through the host interface in a Device Feature List (DFL). The OFS drivers discover and \"walk\" the Device Features in a Device Feature List and associate each Device Feature with its matching kernel driver.
                                                                     
                                                                    In this way the OFS software provides a clean and extensible framework for the creation and integration of additional functionalities and their features.
                                                                     
                                                                    Note: A deeper dive on available SW APIs and programming model is available in the Software Reference Manual: Intel\u00ae Open FPGA Stack, on kernel.org, and through the Linux DFL wiki pages.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#30-ofs-dfl-kernel-drivers","title":"3.0 OFS DFL Kernel Drivers","text":"
                                                                    OFS DFL driver software provides the bottom-most API to FPGA platforms. Libraries such as OPAE and frameworks like DPDK are consumers of the APIs provided by OFS. Applications may be built on top of these frameworks and libraries. The OFS software does not cover any out-of-band management interfaces. OFS driver software is designed to be extendable, flexible, and provide for bare-metal and virtualized functionality. An in depth look at the various aspects of the driver architecture such as the API, an explanation of the DFL framework, and instructions on how to port DFL driver patches to other kernel distributions can be found on https://github.com/OPAE/linux-dfl/wiki.
                                                                     
                                                                    An in-depth review of the Linux device driver architecture can be found on opae.github.io.
                                                                     
                                                                    The DFL driver suite can be automatically installed using a supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3 Release Page.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#31-ofs-dfl-kernel-driver-installation-environment-setup","title":"3.1 OFS DFL Kernel Driver Installation Environment Setup","text":"
                                                                    All OFS DFL kernel driver primary release code resides in the Linux DFL GitHub repository. This repository is open source and should not require any permissions to access. It includes a snapshot of the Linux kernel with the OFS driver included in /drivers/fpga/*. Downloading, configuration, and compilation will be discussed in this section. Refer back to section 1.2.3 Host Server Kernel and GRUB Configuration for a list of supported Operating System(s).
                                                                     
                                                                    You can choose to install the DFL kernel drivers by either using pre-built binaries created for the BKC OS, or by building them on your local server. If you decide to use the pre-built packages available on the OFS 2023.3 Release Page, skip to section 3.3 Installing the OFS DFL Kernel Drivers from Pre-Built Packages. Regardless of your choice you will need to follow the steps in this section to prepare your server for installation.
                                                                     
                                                                    This installation process assumes the user has access to an internet connection to clone specific GitHub repositories, and to satisfy package dependencies.
                                                                     
                                                                       
                                                                    1.  
                                                                      It is recommended you lock your Red Hat release version to 8.6 to prevent accidental upgrades. Update installed system packages to their latest versions.
                                                                       
                                                                      subscription-manager release --set=8.6\nsudo dnf update\nsubscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms\nsudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Install the following package dependencies if building and installing drivers from source. If you do not require the use of a proxy to pull in downloads using dnf, you can safely remove those parameters from the following commands:
                                                                       
                                                                      If you require the use of a proxy, add it to DNF using by editing the following file\nsudo nano /etc/dnf/dnf.conf\n# Include your proxy by adding the following line, replacing the URL with your proxy's URL\n# proxy=http://proxy.server.com:port\n\nsudo dnf install  python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel python3-pybind11 numactl-devel\n\npython3 -m pip install --user jsonschema virtualenv pudb pyyaml setuptools pybind11\n\n# If setuptools and pybind11 were already installed\n\npython3 -m pip upgrade pybind11 setuptools\n
                                                                       
                                                                      3.  
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#32-building-and-installing-the-ofs-dfl-kernel-drivers-from-source","title":"3.2 Building and Installing the OFS DFL Kernel Drivers from Source","text":"
                                                                    It is recommended you create an empty top level directory for your OFS related repositories to keep the working environment clean. All steps in this installation will use a generic top-level directory at /home/OFS/. If you have created a different top-level directory, replace this path with your custom path.
                                                                     
                                                                    1. Initialize an empty git repository and clone the DFL driver source code:
                                                                     
                                                                    ```bash\nmkdir /home/OFS/\ncd /home/OFS/\ngit init\ngit clone https://github.com/OFS/linux-dfl\ncd /home/OFS/linux-dfl\ngit checkout tags/ofs-2023.3-6.1-3\n```\n\n*Note: The linux-dfl repository is roughly 5 GB in size.*\n
                                                                     
                                                                    2. Verify that the correct tag/branch have been checked out.
                                                                     
                                                                    ```bash\ngit describe --tags\nofs-2023.3-6.1-3\n```\n\n*Note: If two different tagged releases are tied to the same commit, running git describe tags may report the other release's tag. This is why the match is made explicit.*\n
                                                                     
                                                                    3. Copy an existing kernel configuration file from /boot and apply the minimal required settings changes.
                                                                     
                                                                    ```bash\ncd /home/OFS/linux-dfl\ncp /boot/config-`uname -r` .config\ncat configs/dfl-config >> .config\necho 'CONFIG_LOCALVERSION=\"-dfl\"' >> .config\necho 'CONFIG_LOCALVERSION_AUTO=y' >> .config\nsed -i -r 's/CONFIG_SYSTEM_TRUSTED_KEYS=.*/CONFIG_SYSTEM_TRUSTED_KEYS=\"\"/' .config\nsed -i '/^CONFIG_DEBUG_INFO_BTF/ s/./#&/' .config\necho 'CONFIG_DEBUG_ATOMIC_SLEEP=y' >> .config\nexport LOCALVERSION=\nmake olddefconfig\n```\n
                                                                     
                                                                    4. The above command may report errors resembling symbol value 'm' invalid for CHELSIO_IPSEC_INLINE. These errors indicate that the nature of the config has changed between the currently executing kernel and the kernel being built. The option \"m\" for a particular kernel module is no longer a valid option, and the default behavior is to simply turn the option off. However, the option can likely be turned back on by setting it to 'y'. If the user wants to turn the option back on, change it to 'y' and re-run \"make olddefconfig\":
                                                                     
                                                                    ```bash\ncd /home/OFS/linux-dfl\necho 'CONFIG_CHELSIO_IPSEC_INLINE=y' >> .config\nmake olddefconfig\n```\n\n*Note: To use the built-in GUI menu for editing kernel configuration parameters, you can opt to run `make menuconfig`.*\n
                                                                     
                                                                    5. Linux kernel builds take advantage of multiple processors to parallelize the build process. Display how many processors are available with the nproc command, and then specify how many make threads to utilize with the -j option. Note that number of threads can exceed the number of processors. In this case, the number of threads is set to the number of processors in the system.
                                                                     
                                                                    ```bash\ncd /home/OFS/linux-dfl\nmake -j $(nproc)\n```\n
                                                                     
                                                                    6. You have two options to build the source:
                                                                     
                                                                    - Using the built-in install option from the kernel Makefile.\n- Locally building a set of RPM/DEP packages.\n\n\nThis first flow will directly install the kernel and kernel module files without the need to create a package first:\n\n```bash\ncd /home/OFS/linux-dfl\nsudo make modules_install -j $(nproc)\nsudo make install\n```\n\nIn this second flow, the OFS Makefile contains a few options for package creation:\n\n- rpm-pkg: Build both source and binary RPM kernel packages\n- binrpm-pkg: Build only the binary kernel RPM package\n- deb-pkg: Build both source and binary deb kernel packages\n- bindeb-pkg: Build only the binary kernel deb package\n\nIf you are concerned about the size of the resulting package and binaries, they can significantly reduce the size of the package and object files by using the make variable INSTALL_MOD_STRIP. If this is not a concern, feel free to skip this step. The below instructions will build a set of binary RPM packages:\n\n```bash\ncd /home/OFS/linux-dfl\nmake INSTALL_MOD_STRIP=1 binrpm-pkg\n```\n\nBy default, a directory is created in your home directory called `rpmbuild`. This directory will house all the kernel packages which have been built. You need to navigate to the newly built kernel packages and install them. The following files were generated using the build command executed in the previous step:\n\n```bash\ncd ~/rpmbuild/RPMS/x86_64\nls\nkernel-6.1.41_dfl.x86_64.rpm  kernel-headers-6.1.41_dfl.x86_64.rpm\nsudo dnf localinstall kernel*.rpm\n```\n
                                                                     
                                                                    7. The system will need to be rebooted in order for changes to take effect. After a reboot, select the newly built kernel as the boot target. This can be done pre-boot using the command grub2-reboot, which removes the requirement for user intervention. After boot, verify that the currently running kernel matches expectation.
                                                                     
                                                                    ```bash\nuname -r\n6.1.41-dfl\n```\n
                                                                     
                                                                    8. Verify the DFL drivers have been successfully installed by reading version information directly from /lib/modules. Recall that the name of the kernel built as a part of this section is 6.1.41-dfl. If the user set a different name for their kernel, change this path as needed:
                                                                     
                                                                    ```bash\ncd /usr/lib/modules/6.1.41-dfl/kernel/drivers/fpga\nls\ndfl-afu.ko     dfl-fme.ko      dfl-fme-region.ko  dfl.ko             dfl-pci.ko      fpga-mgr.ko     intel-m10-bmc-sec-update.ko\ndfl-fme-br.ko  dfl-fme-mgr.ko  dfl-hssi.ko        dfl-n3000-nios.ko  fpga-bridge.ko  fpga-region.ko\n```\n\nIf an OFS device that is compatible with these drivers is installed on the server, you can double check the driver versions by listing the currently loaded kernel modules with `lsmod`:\n\n\n```bash\nlsmod | grep dfl\nuio_dfl                20480  0\ndfl_emif               16384  0\nuio                    20480  1 uio_dfl\nptp_dfl_tod            16384  0\ndfl_intel_s10_iopll    20480  0\n8250_dfl               20480  0\ndfl_fme_region         20480  0\ndfl_fme_br             16384  0\ndfl_fme_mgr            20480  2\ndfl_fme                49152  0\ndfl_afu                36864  0\ndfl_pci                20480  0\ndfl                    40960  11 dfl_pci,uio_dfl,dfl_fme,intel_m10_bmc_pmci,dfl_fme_br,8250_dfl,qsfp_mem,ptp_dfl_tod,dfl_afu,dfl_intel_s10_iopll,dfl_emif\nfpga_region            20480  3 dfl_fme_region,dfl_fme,dfl\nfpga_bridge            20480  4 dfl_fme_region,fpga_region,dfl_fme,dfl_fme_br\nfpga_mgr               20480  4 dfl_fme_region,fpga_region,dfl_fme_mgr,dfl_fme\n```\n
                                                                     
                                                                    9. Two kernel parameters must be added to the boot command line for the newly installed kernel. First, open the file grub:
                                                                     
                                                                    ```bash\nsudo vim /etc/default/grub\n```\n
                                                                     
                                                                    10. In the variable GRUB_CMDLINE_LINUX add the following parameters in bold: GRUB_CMDLINE_LINUX=\"crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\".
                                                                     
                                                                    *Note: If you wish to instead set hugepages on a per session basis, you can perform the following steps. These settings will be lost on reboot.*\n\n```bash\nmkdir -p /mnt/huge \nmount -t hugetlbfs nodev /mnt/huge \necho 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages \necho 2048 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages \n```\n
                                                                     
                                                                    11. Save your edits, then apply them to the GRUB2 configuration file.
                                                                     
                                                                    ```bash\nsudo grub2-mkconfig\n```\n
                                                                     
                                                                    12. Warm reboot. Your kernel parameter changes should have taken affect.
                                                                     
                                                                    ```bash\ncat /proc/cmdline\nBOOT_IMAGE=(hd1,gpt2)/vmlinuz-6.1.41-dfl root=/dev/mapper/cl-root ro crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200 rhgb quiet\n```\n\nA list of all DFL drivers and their purpose is maintained on the [DFL Wiki](https://github.com/OFS/linux-dfl/wiki/FPGA-DFL-Driver-Modules#fpga-driver-modules).\n
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#33-installing-the-ofs-dfl-kernel-drivers-from-pre-built-packages","title":"3.3 Installing the OFS DFL Kernel Drivers from Pre-Built Packages","text":"
                                                                    To use the pre-built Linux DFL packages, you first need to download the files from the OFS 2023.3 Release Page. You can choose to either install using the SRC RPMs, or to use the pre-built RPM packages targeting the official supported release platform.
                                                                     
                                                                    tar xf kernel-6.1.41_dfl-1.x86_64-<<version>>.tar.gz\n\nsudo dnf localinstall kernel-6.1.41_dfl_<<version>>.x86_64.rpm \\\nkernel-devel-6.1.41_dfl_<<version>>.x86_64.rpm \\\nkernel-headers-6.1.41_dfl_<<version>>.x86_64.rpm\n\n### OR\n\nsudo dnf localinstall kernel-6.1.41_dfl_<<version>>.src.rpm\n
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#40-opae-software-development-kit","title":"4.0 OPAE Software Development Kit","text":"
                                                                    The OPAE SDK software stack sits in user space on top of the OFS kernel drivers. It is a common software infrastructure layer that simplifies and streamlines integration of programmable accelerators such as FPGAs into software applications and environments. OPAE consists of a set of drivers, user-space libraries, and tools to discover, enumerate, share, query, access, manipulate, and reconfigure programmable accelerators. OPAE is designed to support a layered, common programming model across different platforms and devices. To learn more about OPAE, its documentation, code samples, an explanation of the available tools, and an overview of the software architecture, visit opae.github.io.
                                                                     
                                                                    The OPAE SDK source code is contained within a single GitHub repository hosted at the OPAE Github. This repository is open source and does not require any permissions to access.
                                                                     
                                                                    You can choose to install the OPAE SDK by either using pre-built binaries created for the BKC OS, or by building them on your local server. If you decide to use the pre-built packages available on the OFS 2023.3 Release Page, skip to section 4.3 Installing the OPAE SDK with Pre-built Packages. Regardless of your choice you will need to follow the steps in this section to prepare your server for installation.
                                                                     
                                                                    You may also choose to use the supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3 Release Page.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#41-opae-sdk-installation-environment-setup","title":"4.1 OPAE SDK Installation Environment Setup","text":"
                                                                    This installation process assumes you have access to an internet connection in order to pull specific GitHub repositories, and to satisfy package dependencies.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-6-opae-package-description","title":"Table 6: OPAE Package Description","text":"Package Name Description opae OPAE SDK is a collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. It provides a library implementing the OPAE C API for presenting a streamlined and easy-to-use interface for software applications to discover, access, and manage FPGA devices and accelerators using the OPAE software stack. opae-debuginfo This package provides debug information for package opae. Debug information is useful when developing applications that use this package or when debugging this package. opae-debugsource This package provides debug sources for package opae. Debug sources are useful when developing applications that use this package or when debugging this package. opae-devel OPAE headers, tools, sample source, and documentation opae-devel-debuginfo This package provides debug information for package opae-devel. Debug information is useful when developing applications that use this package or when debugging this package. opae-tools This package contains OPAE base tools binaries opae-extra-tools Additional OPAE tools opae-extra-tools-debuginfo This package provides debug information for package opae-extra-tools. Debug information is useful when developing applications that use this package or when debugging this package. 
                                                                       
                                                                    1.  
                                                                      Remove any currently installed OPAE packages.
                                                                       
                                                                      sudo dnf remove opae*\n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Initialize an empty git repository and clone the tagged OPAE SDK source code.
                                                                       
                                                                      cd /home/OFS/\ngit init\ngit clone https://github.com/OFS/opae-sdk opae-sdk\ncd /home/OFS/opae-sdk\ngit checkout tags/2.10.0-1\n
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Verify that the correct tag/branch have been checkout out.
                                                                       
                                                                      git describe --tags\n2.10.0-1\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Set up a temporary podman container to build OPAE, which will allow you to customize the python installation without affecting system packages.
                                                                       
                                                                      cd /home/OFS\npodman pull registry.access.redhat.com/ubi8:8.6\npodman run -ti -v \"$PWD\":/src:Z -w /src registry.access.redhat.com/ubi8:8.6\n\n# Everything after runs within container:\n\n# Enable EPEL\ndnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n\ndnf install --enablerepo=codeready-builder-for-rhel-8-x86_64-rpms -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel make\n\npip3 install --upgrade --prefix=/usr pip setuptools pybind11\n\n./opae-sdk/packaging/opae/rpm/create unrestricted\n\nexit\n
                                                                       
                                                                      The following packages will be built in the same directory as create:
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Install the packages you just created.
                                                                       
                                                                      cd /home/OFS/opae-sdk/packaging/opae/rpm\nrm -rf opae-2.10.0-1.el8.src.rpm \nsudo dnf localinstall -y opae*.rpm\n
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Check that all packages have been installed and match expectation:
                                                                       
                                                                      rpm -qa | grep opae\nopae-2.8.0-1.el8.x86_64.rpm\nopae-debuginfo-2.8.0-1.el8.x86_64.rpm\nopae-debugsource-2.8.0-1.el8.x86_64.rpm\nopae-devel-2.8.0-1.el8.x86_64.rpm\nopae-devel-debuginfo-2.8.0-1.el8.x86_64.rpm\nopae-extra-tools-2.8.0-1.el8.x86_64.rpm\nopae-extra-tools-debuginfo-2.8.0-1.el8.x86_64.rpm\n
                                                                       
                                                                      7.  
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#42-installing-the-opae-sdk-with-pre-built-packages","title":"4.2 Installing the OPAE SDK with Pre-Built Packages","text":"
                                                                    You can skip the entire build process and use a set of pre-built binaries supplied by Intel. Visit the OFS 2023.3 Release Page and navigate to the bottom of the page, under the Assets tab you will see a file named opae-2.10.0-1.x86_64-<>_<>.tar.gz. Download this package and extract its contents: 
                                                                    tar xf opae-2.10.0-1.x86_64-<<date>>_<<build>>.tar.gz\n
                                                                     
                                                                    For a fast installation you can delete the source RPM as it isn't necessary, and install all remaining OPAE RPMs:
                                                                     
                                                                    rm opae-*.src.rpm\nsudo dnf localinstall opae*.rpm\n
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#43-fpga-device-access-permissions","title":"4.3 FPGA Device Access Permissions","text":"
                                                                    Access to FPGA accelerators and devices is controlled using file access permissions on the Intel\u00ae FPGA device files, /dev/dfl-fme.* and /dev/dfl-port.*, as well as to the files reachable through /sys/class/fpga_region/.
                                                                     
                                                                    In order to allow regular (non-root) users to access accelerators, you need to grant them read and write permissions on /dev/dfl-port.* (with * denoting the respective socket, i.e. 0 or 1). E.g.:
                                                                     
                                                                    sudo chmod a+rw /dev/dfl-port.0\n
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#44-memlock-limit","title":"4.4 Memlock limit","text":"
                                                                    Depending on the requirements of your application, you may also want to increase the maximum amount of memory a user process is allowed to lock. The exact way to do this depends on your Linux distribution.
                                                                     
                                                                    You can check the current memlock limit using
                                                                     
                                                                    ulimit -l\n
                                                                     
                                                                    A way to permanently remove the limit for locked memory for a regular user is to add the following lines to your /etc/security/limits.conf:
                                                                     
                                                                    user1    hard   memlock           unlimited\nuser1    soft   memlock           unlimited\n
                                                                     
                                                                    This removes the limit on locked memory for user user1. To remove it for all users, you can replace user1 with *:
                                                                     
                                                                    *    hard   memlock           unlimited\n*    soft   memlock           unlimited\n
                                                                     
                                                                    Note that settings in the /etc/security/limits.conf file don't apply to services. To increase the locked memory limit for a service you need to modify the application's systemd service file and add the line:
                                                                     
                                                                    [Service]\nLimitMEMLOCK=infinity\n
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#45-opae-tools-overview","title":"4.5 OPAE Tools Overview","text":"
                                                                    The following section offers a brief introduction including expected output values for the utilities included with OPAE. A full explanation of each command with a description of its syntax is available in the opae-sdk GitHub repo.
                                                                     
                                                                    A list of all tools included in the OPAE SDK release can be found on the OPAE FPGA Tools tab of ofs.github.io.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#451-board-management-with-fpgainfo","title":"4.5.1 Board Management with fpgainfo","text":"
                                                                    The fpgainfo utility displays FPGA information derived from sysfs files.
                                                                     
                                                                    Displays FPGA information derived from sysfs files. The command argument is one of the following: errors, power, temp, port, fme, bmc, phy or mac, security. Some commands may also have other arguments or options that control their behavior.
                                                                     
                                                                    For systems with multiple FPGA devices, you can specify the BDF to limit the output to the FPGA resource with the corresponding PCIe configuration. If not specified, information displays for all resources for the given command.
                                                                     
                                                                    Note: Your Bitstream ID and PR Interface Id may not match the below examples.
                                                                     
                                                                    The following examples walk through sample outputs generated by fpgainfo. As the F-tile Development Kit does not contain a traditional BMC as used by other OFS products, those lines in fpgainfo's output will not return valid objects. The subcommand fpgainfo bmc will likewise fail to report telemetry data.
                                                                     
                                                                    Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x501020241BF165B\nBitstream Version                : 5.0.1\nPr Interface Id                  : b4eda250-cdb7-5891-a06e-13d28d09bc32\nBoot Page                        : N/A\n
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#452-updating-with-fpgasupdate","title":"4.5.2 Updating with fpgasupdate","text":"
                                                                    The fpgasupdate tool is used to program AFU workloads into an open slot in a FIM. The fpgasupdate tool only accepts images that have been formatted using PACsign.
                                                                     
                                                                    As the F-tile Development Kit does not contain a traditional BMC, you do not have access to a factory, user1, and user2 programmed image for both the FIM and BMC FW and RTL. Only the programming of a GBS workload is supported for this release.
                                                                     
                                                                    The process of programming a SOF with a new FIM version is shown in section 1.5 F-tile Development Kit JTAG Setup
                                                                     
                                                                    sudo fpgasupdate ofs_pr_afu.gbs   <PCI ADDRESS>\n[2022-04-14 16:42:31.58] [WARNING ] Update starting. Please do not interrupt.                                           \n[2022-04-14 16:42:31.58] [INFO    ] updating from file ofs_pr_afu.gbs with size 19928064               \n[2022-04-14 16:42:31.60] [INFO    ] waiting for idle                                                                 \n[2022-04-14 16:42:31.60] [INFO    ] preparing image file                                                                \n[2022-04-14 16:42:38.61] [INFO    ] writing image file                                                                 \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [19928064/19928064 bytes][Elapsed Time: 0:00:16.01]                                       \n[2022-04-14 16:42:54.63] [INFO    ] programming image file                                                              \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588][Elapsed Time: 0:06:16.40]                                                                 \n[2022-04-14 16:49:11.03] [INFO    ] update of 0000:b1:00.0 complete                                                     \n[2022-04-14 16:49:11.03] [INFO    ] Secure update OK                                                                   \n[2022-04-14 16:49:11.03] [INFO    ] Total time: 0:06:39.45\n
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#453-verify-fme-interrupts-with-hello_events","title":"4.5.3 Verify FME Interrupts with hello_events","text":"
                                                                    The hello_events utility is used to verify FME interrupts. This tool injects FME errors and waits for error interrupts, then clears the errors.
                                                                     
                                                                    Sample output from sudo hello_events.
                                                                     
                                                                    sudo hello_events\nWaiting for interrupts now...\ninjecting error\nFME Interrupt occurred\nSuccessfully tested Register/Unregister for FME events!\nclearing error\n
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#454-host-exercisor-modules","title":"4.5.4 Host Exercisor Modules","text":"
                                                                    The reference FIM and unchanged FIM compilations contain Host Exerciser Modules (HEMs). These are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. There are three HEMs present in the Intel OFS Reference FIM - HE-LPBK, HE-MEM, and HE-HSSI. These exercisers are tied to three different VFs that must be enabled before they can be used. Execution of these exercisers requires you bind specific VF endpoint to vfio-pci. The host-side software looks for these endpoints to grab the correct FPGA resource.
                                                                     
                                                                    Refer to the Intel FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA for a full description of these modules.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-7-module-pfvf-mappings","title":"Table 7: Module PF/VF Mappings","text":"Module PF/VF ST2MM PF0 HE-MEM PF0-VF0 HE-HSSI PF0-VF1 HE-MEM_TG PF0-VF2 HE-LB Stub PF1-VF0 HE-LB PF2 VirtIO LB Stub PF3 HPS Copy Engine PF4"},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#4541-he-mem-he-lb","title":"4.5.4.1 HE-MEM / HE-LB","text":"
                                                                    The host exerciser used to exercise and characterize the various host-FPGA interactions eg. MMIO, Data transfer from host to FPGA , PR, host to FPGA memory etc. The Host Exerciser Loopback (HE-LBK) AFU can move data between host memory and FPGA.
                                                                     
                                                                    HE-LBK supports: - Latency (AFU to Host memory read) - MMIO latency (Write+Read) - MMIO BW (64B MMIO writes) - BW (Read/Write, Read only, Wr only)
                                                                     
                                                                    The Host Exerciser Loopback Memory (HE-MEM) AFU is used to exercise use of FPGA connected DDR, data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host.
                                                                     
                                                                    HE-LB is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth. HE-MEM is used to exercise use of FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host. HE-MEM uses external DDR memory (i.e. EMIF) to store data. It has a customized version of the AVMM interface to communicate with the EMIF memory controller. Both exercisers rely on the user-space tool host_exerciser. When using the F-tile Development Kit SmartNIC Platform, optimal performance requires the exercisers be run at 400 MHz.
                                                                     
                                                                    Execution of these exercisers requires you to bind specific VF endpoint to vfio-pci. The following commands will bind the correct endpoint for a device with B/D/F 0000:b1:00.0 and run through a basic loopback test.
                                                                     
                                                                    Note: While running the opae.io init command listed below, the command has failed if no output is present after completion. Double check that Intel VT-D and IOMMU have been enabled in the kernel as discussed in section 3.0 OFS DFL Kernel Drivers.
                                                                     
                                                                    sudo pci_device  0000:b1:00.0 vf 3\n\nsudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci                                                             \nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci \niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188                                                                  \nAssigning /dev/vfio/188 to user                                                                 \nChanging permissions for /dev/vfio/188 to rw-rw----\n\n\nsudo host_exerciser --clock-mhz 400 lpbk\n    starting test run, count of 1\nAPI version: 1\nAFU clock from command line: 400 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Number of clocks: 5224\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1022\n    Bandwidth: 5.018 GB/s\n    Test lpbk(1): PASS\n
                                                                     
                                                                    The following example will run a loopback throughput test using one cache line per request.
                                                                     
                                                                    sudo pci_device  0000:b1:00.0 vf 3\n\nsudo opae.io init -d 0000:b1:00.2 user:user\n\nsudo host_exerciser --clock-mhz 400 --mode trput --cls cl_1 lpbk\n    starting test run, count of 1\nAPI version: 1\nAFU clock from command line: 400 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 512\n    Host Exerciser numWrites: 513\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Number of clocks: 3517\n    Total number of Reads sent: 512\n    Total number of Writes sent: 512\n    Bandwidth: 7.454 GB/s\n    Test lpbk(1): PASS\n
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#4542-traffic-generator-afu-test-application","title":"4.5.4.2 Traffic Generator AFU Test Application","text":"
                                                                    Beginning in OPAE version 2.0.11-1+ the TG AFU has an OPAE application to access & exercise traffic, targeting a specific bank. The supported arguments for test configuration are:
                                                                     
                                                                       
                                                                    • Number of test loops: --loops
                                                                      •  
                                                                    • Number of read transfers per test loop: -r,--read
                                                                      •  
                                                                    • Number of write transfers per test loop: -w,--write
                                                                      •  
                                                                    • Burst size of each transfer: -b,--bls
                                                                      •  
                                                                    • Address stride between each transfer: --stride
                                                                      •  
                                                                    • Target memory TG: -m,--mem-channel
                                                                      •  
                                                                     
                                                                    Below are some example commands for how to execute the test application. To run the preconfigured write/read traffic test on channel 0:
                                                                     
                                                                    mem_tg tg_test\n
                                                                     
                                                                    Target channel 1 with a 1MB single-word write only test for 1000 iterations
                                                                     
                                                                    mem_tg --loops 1000 -r 0 -w 2000 -m 1 tg_test\n
                                                                     
                                                                    Target channel 2 with 4MB write/read test of max burst length for 10 iterations
                                                                     
                                                                    mem_tg --loops 10 -r 8 -w 8 --bls 255 -m 2 tg_test\n
                                                                     
                                                                    sudo mem_tg --loops 1000 -r 2000 -w 2000 --stride 2 --bls 2  -m 1 tg_test\n[2022-07-15 00:13:16.349] [tg_test] [info] starting test run, count of 1\nMemory channel clock frequency unknown. Assuming 300 MHz.\nTG PASS\nMem Clock Cycles: 17565035\nWrite BW: 4.37232 GB/s\nRead BW: 4.37232 GB/s\n
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#4543-he-hssi","title":"4.5.4.3 HE-HSSI","text":"
                                                                    HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic.
                                                                     
                                                                    The hssi application provides a means of interacting with the 10G and with the 100G HSSI AFUs. In both 10G and 100G operating modes, the application initializes the AFU, completes the desired transfer as described by the mode- specific options, and displays the ethernet statistics by invoking ethtool --statistics INTERFACE.
                                                                     
                                                                    Due to Ethernet differential pair routing on the ES version of the Intel Agilex\u00ae 7 F-Series FPGA (Two F-Tiles) Development Kit, some differential pairs were swapped to improve signal routing.  To account for the pair swap, there is a requirement to run a script to invert the differential traces.  If you run the command \u201cfpgainfo phy B:d.f\u201d when the Ethernet ports are connected to known good sources and observe the following three ports are down as shown below:
                                                                     
                                                                    sudo fpgainfo phy b1:00.0\nIntel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** PHY ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x501020241BF165B\nBitstream Version                : 5.0.1\nPr Interface Id                  : b4eda250-cdb7-5891-a06e-13d28d09bc32\nQSFP0                            : Connected\nQSFP1                            : Connected\n//****** HSSI information ******//\nHSSI version                     : 2.0\nNumber of ports                  : 8\nPort0                            :25GbE        UP\nPort1                            :25GbE        UP\nPort2                            :25GbE        UP\nPort3                            :25GbE        DOWN\nPort4                            :25GbE        UP\nPort5                            :25GbE        UP\nPort6                            :25GbE        DOWN\nPort7                            :25GbE        DOWN\n
                                                                     
                                                                    Create the following script called \u201cset_tx_inverse_polarity.sh\u201d to set make Transceiver PAM register settings:
                                                                     
                                                                    #!/bin/sh\n\n# Port 3\nbase_addr=$(printf \"%08d\" \"0x500000\")\naddress=`expr $base_addr + 589884`\n#address=`expr $base_addr + 589884`\noffset=`expr $address/4`\n\nhex_number=$(printf \"0x%06x\" \"$(($offset))\")\necho $hex_number\ncmd_sts=$(printf \"%32x\" \"$(($offset2))\")\ncsraddr=\"${hex_number}0500000002\"\ncsraddr1=\"${hex_number}0600000001\"\ndata=1a040\necho $csraddr\nsudo opae.io poke  0x140b0 0x0001a26500000000\nsleep 1\nsudo opae.io poke  0x140a8 $csraddr\nsleep 1\nsudo opae.io peek  0x140a8\nsleep 1\n\nsudo opae.io poke  0x140b0 0x0001226500000000\nsleep 1\nsudo opae.io poke  0x140a8 $csraddr\nsleep 1\nsudo opae.io peek  0x140a8\n# Port 6\n\nbase_addr=$(printf \"%08d\" \"0xb00000\")\naddress=`expr $base_addr + 589884`\n#address=`expr $base_addr + 589884`\noffset=`expr $address/4`\n\nhex_number=$(printf \"0x%06x\" \"$(($offset))\")\necho $hex_number\ncmd_sts=$(printf \"%32x\" \"$(($offset2))\")\ncsraddr=\"${hex_number}0500000002\"\ncsraddr1=\"${hex_number}0600000001\"\ndata=1a040\necho $csraddr\nsudo opae.io poke  0x140b0 0x0001a16500000000\nsleep 1\nsudo opae.io poke  0x140a8 $csraddr\nsleep 1\nsudo opae.io peek  0x140a8\nsleep 1\n\nsudo opae.io poke  0x140b0 0x0001216500000000\nsleep 1\nsudo opae.io poke  0x140a8 $csraddr\nsleep 1\nsudo opae.io peek  0x140a8\n# Port 7\n\nbase_addr=$(printf \"%08d\" \"0x1100000\")\naddress=`expr $base_addr + 589884`\n#address=`expr $base_addr + 589884`\noffset=`expr $address/4`\n\nhex_number=$(printf \"0x%06x\" \"$(($offset))\")\necho $hex_number\ncmd_sts=$(printf \"%32x\" \"$(($offset2))\")\ncsraddr=\"${hex_number}0500000002\"\ncsraddr1=\"${hex_number}0600000001\"\ndata=1a040\necho $csraddr\nsudo opae.io poke  0x140b0 0x0001a26500000000\nsleep 1\nsudo opae.io poke  0x140a8 $csraddr\nsleep 1\nsudo opae.io peek  0x140a8\nsleep 1\n\nsudo opae.io poke  0x140b0 0x0001226500000000\nsleep 1\nsudo opae.io poke  0x140a8 $csraddr\nsleep 1\nsudo opae.io peek  0x140a8\n
                                                                     
                                                                    The script set_tx_inverse_polarity.sh requires the VFIO driver on PF0 to access the Transceiver registers.  You will use the opae.io command prior to running set_tx_inverse_polarity.sh to bind the VFIO driver.  Once the script completes, release the VFIO driver with opae.io release.  
                                                                     
                                                                    The listing below shows the script being run:
                                                                     
                                                                    sudo opae.io init -d 0000:b1:00.0 $USER\nUnbinding (0x8086,0xbcce) at 0000:b1:00.0 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.0 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.0 is 8\n\nsh set_tx_inverse_polarity.sh\n0x16400f\n0x16400f0500000002\n0x16400f0500000006\n0x16400f0500000006\n0x2e400f\n0x2e400f0500000002\n0x2e400f0500000006\n0x2e400f0500000006\n0x46400f\n0x46400f0500000002\n0x46400f0500000006\n0x46400f0500000006\n\nsudo opae.io release -d 0000:b1:00.0\nReleasing (0x8086,0xbcce) at 0000:b1:00.0 from vfio-pci\nRebinding (0x8086,0xbcce) at 0000:b1:00.0 to dfl-pci\n\nsudo fpgainfo phy b1:00.0\nIntel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** PHY ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x501020241BF165B\nBitstream Version                : 5.0.1\nPr Interface Id                  : 767712e5-b1d0-5777-aea9-592572a6817f\nQSFP0                            : Connected\nQSFP1                            : Connected\n//****** HSSI information ******//\nHSSI version                     : 2.0\nNumber of ports                  : 8\nPort0                            :25GbE        UP\nPort1                            :25GbE        UP\nPort2                            :25GbE        UP\nPort3                            :25GbE        UP\nPort4                            :25GbE        UP\nPort5                            :25GbE        UP\nPort6                            :25GbE        UP\nPort7                            :25GbE        UP\n
                                                                     
                                                                    The following example walks through the process of binding the VF corresponding with the HE-HSSI exerciser to vfio-pci, sending traffic, and verifying that traffic was received.
                                                                    "},{"location":"hw/ftile_devkit/user_guides/ug_qs_ofs_ftile/ug_qs_ofs_ftile/#table-8-accelerator-pfvf-and-guid-mappings","title":"Table 8: Accelerator PF/VF and GUID Mappings","text":"Component VF Accelerator GUID F Series Dev Kit base PF XXXX:XX:XX.0 N/A VirtIO Stub XXXX:XX:XX.1 3e7b60a0-df2d-4850-aa31-f54a3e403501 HE-MEM Stub XXXX:XX:XX.2 56e203e9-864f-49a7-b94b-12284c31e02b Copy Engine XXXX:XX:XX.4 44bfc10d-b42a-44e5-bd42-57dc93ea7f91 HE-MEM XXXX:XX:XX.5 8568ab4e-6ba5-4616-bb65-2a578330a8eb HE-HSSI XXXX:XX:XX.6 823c334c-98bf-11ea-bb37-0242ac130002 MEM-TG XXXX:XX:XX.7 4dadea34-2c78-48cb-a3dc-5b831f5cecbb 
                                                                       
                                                                    1.  
                                                                      Create 3 VFs in the PR region.
                                                                       
                                                                      sudo pci_device b1:00.0 vf 3 \n
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Verify all 3 VFs were created.
                                                                       
                                                                      lspci -s b1:00 \nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01) \nb1:00.1 Processing accelerators: Intel Corporation Device bcce \nb1:00.2 Processing accelerators: Intel Corporation Device bcce \nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device \nb1:00.4 Processing accelerators: Intel Corporation Device bcce \nb1:00.5 Processing accelerators: Intel Corporation Device bccf \nb1:00.6 Processing accelerators: Intel Corporation Device bccf \nb1:00.7 Processing accelerators: Intel Corporation Device bccf \n
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Bind all the PF/VF endpoints to the vfio-pci driver.
                                                                       
                                                                      sudo opae.io init -d 0000:b1:00.1 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 187\nAssigning /dev/vfio/187 to user\nChanging permissions for /dev/vfio/187 to rw-rw----\n\nsudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188\nAssigning /dev/vfio/188 to user\nChanging permissions for /dev/vfio/188 to rw-rw----\n\n...\n\nsudo opae.io init -d 0000:b1:00.7 user:user\nBinding (0x8086,0xbccf) at 0000:b1:00.7 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.7 is 319\nAssigning /dev/vfio/319 to user\nChanging permissions for /dev/vfio/319 to rw-rw----\n
                                                                       
                                                                      4.  
                                                                    4.  
                                                                      Check that the accelerators are present using fpgainfo. Note your port configuration may differ from the below.
                                                                       
                                                                      sudo fpgainfo port \n//****** PORT ******//\nObject Id                        : 0xEC00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0xE0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.7\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 4dadea34-2c78-48cb-a3dc-5b831f5cecbb\n//****** PORT ******//\nObject Id                        : 0xC0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.6\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 823c334c-98bf-11ea-bb37-0242ac130002\n//****** PORT ******//\nObject Id                        : 0xA0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.5\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n//****** PORT ******//\nObject Id                        : 0x80B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.4\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 44bfc10d-b42a-44e5-bd42-57dc93ea7f91\n//****** PORT ******//\nObject Id                        : 0x40B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n//****** PORT ******//\nObject Id                        : 0x20B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 3e7b60a0-df2d-4850-aa31-f54a3e403501\n
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Check Ethernet PHY settings with fpgainfo.
                                                                       
                                                                      sudo fpgainfo phy -B 0xb1 \nIIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\n//****** HSSI information ******//\nHSSI version                     : 1.0\nNumber of ports                  : 8\nPort0                            :25GbE        DOWN\nPort1                            :25GbE        DOWN\nPort2                            :25GbE        DOWN\nPort3                            :25GbE        DOWN\nPort4                            :25GbE        DOWN\nPort5                            :25GbE        DOWN\nPort6                            :25GbE        DOWN\nPort7                            :25GbE        DOWN\n
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      Set loopback mode.
                                                                       
                                                                      sudo hssiloopback --loopback enable  --pcie-address 0000:b1:00.0 \nargs Namespace(loopback='enable', pcie_address='0000:b1:00.0', port=0)\nsbdf: 0000:b1:00.0\nFPGA dev: {'segment': 0, 'bus': 177, 'dev': 0, 'func': 0, 'path': '/sys/class/fpga_region/region0', 'pcie_address': '0000:b1:00.0'}\nargs.hssi_grps{0: ['dfl_dev.6', ['/sys/bus/pci/devices/0000:b1:00.0/fpga_region/region0/dfl-fme.0/dfl_dev.6/uio/uio0']]}\nfpga uio dev:dfl_dev.6\n\n--------HSSI INFO START-------\nDFH                     :0x3000000010002015\nHSSI ID                 :0x15\nDFHv                    :0.5\nguidl                   :0x99a078ad18418b9d\nguidh                   :0x4118a7cbd9db4a9b\nHSSI version            :1.0\nFirmware Version        :1\nHSSI num ports          :8\nPort0                   :25GbE\nPort1                   :25GbE\nPort2                   :25GbE\nPort3                   :25GbE\nPort4                   :25GbE\nPort5                   :25GbE\nPort6                   :25GbE\nPort7                   :25GbE\n--------HSSI INFO END-------\n\nhssi loopback enabled to port0\n
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Send traffic through the 10G AFU.
                                                                       
                                                                      sudo hssi --pci-address b1:00.6 hssi_10g --num-packets 100       \n10G loopback test\n  port: 0\n  eth_loopback: on\n  he_loopback: none\n  num_packets: 100\n  packet_length: 64\n  src_address: 11:22:33:44:55:66\n    (bits): 0x665544332211\n  dest_address: 77:88:99:aa:bb:cc\n    (bits): 0xccbbaa998877\n  random_length: fixed\n  random_payload: incremental\n  rnd_seed0: 5eed0000\n  rnd_seed1: 5eed0001\n  rnd_seed2: 25eed\n  eth:\n\nNo eth interface, so not honoring --eth-loopback.\n0x40000           ETH_AFU_DFH: 0x1000010000001000\n0x40008          ETH_AFU_ID_L: 0xbb370242ac130002\n0x40010          ETH_AFU_ID_H: 0x823c334c98bf11ea\n0x40030      TRAFFIC_CTRL_CMD: 0x0000000000000000\n0x40038     TRAFFIC_CTRL_DATA: 0x0000000100000000\n0x40040 TRAFFIC_CTRL_PORT_SEL: 0x0000000000000000\n0x40048        AFU_SCRATCHPAD: 0x0000000045324511\n\n0x3c00         number_packets: 0x00000064\n0x3c01          random_length: 0x00000000\n0x3c02         random_payload: 0x00000000\n0x3c03                  start: 0x00000000\n0x3c04                   stop: 0x00000000\n0x3c05           source_addr0: 0x44332211\n0x3c06           source_addr1: 0x00006655\n0x3c07             dest_addr0: 0xaa998877\n0x3c08             dest_addr1: 0x0000ccbb\n0x3c09        packet_tx_count: 0x00000064\n0x3c0a              rnd_seed0: 0x5eed0000\n0x3c0b              rnd_seed1: 0x5eed0001\n0x3c0c              rnd_seed2: 0x00025eed\n0x3c0d             pkt_length: 0x00000040\n0x3cf4          tx_end_tstamp: 0x000003d2\n0x3d00                num_pkt: 0xffffffff\n0x3d01               pkt_good: 0x00000064\n0x3d02                pkt_bad: 0x00000000\n0x3d07            avst_rx_err: 0x00000000\n0x3d0b          rx_sta_tstamp: 0x00000103\n0x3d0c          rx_end_tstamp: 0x0000053b\n0x3e00               mac_loop: 0x00000000\n\nHSSI performance:\n        Selected clock frequency : 402.832 MHz\n        Latency minimum : 642.948 ns\n        Latency maximum : 896.155 ns\n        Achieved Tx throughput : 18.4528 GB/s\n        Achieved Rx throughput : 16.7101 GB/s\n\nNo eth interface, so not showing stats.\n
                                                                       
                                                                      8.  
                                                                     
                                                                    The hssi_loopback utility works in conjunction with a packet generator accelerator function unit (AFU) to test high-speed serial interface (HSSI) cards. hssi_loopback tests both external and internal loopbacks.
                                                                     
                                                                    The hssistats tool provides the MAC statistics.
                                                                    "},{"location":"hw/iseries_devkit/","title":"I-Series (2xR-tile and 1xF-Tile) Development Kit Collateral for OFS","text":"
                                                                    This folder contains applicable collateral for OFS PCIe Attach reference shell targeting the I-Series (2xR-tile and 1xF-Tile) Development Kit DK-DEV-AGI027RBES.
                                                                    "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/","title":"FPGA Interface Manager Developer Guide for Open FPGA Stack: Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) PCIe Attach","text":"
                                                                    Last updated: February 03, 2024 
                                                                    "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1-introduction","title":"1. Introduction","text":""},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#11-about-this-document","title":"1.1. About This Document","text":"
                                                                    This document serves as a guide for OFS Agilex PCIe Attach developers targeting the Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile). The following topics are covered in this guide:
                                                                     
                                                                       
                                                                    • Compiling the OFS Agilex PCIe Attach FIM design
                                                                      •  
                                                                    • Simulating the OFS Agilex PCIe Attach design
                                                                      •  
                                                                    • Customizing the OFS Agilex PCIe Attach FIM design
                                                                      •  
                                                                    • Configuring the FPGA with an OFS Agilex PCIe Attach FIM design
                                                                      •  
                                                                     
                                                                    The FIM Development Walkthroughs Table lists all of the walkthroughs provided in this guide. These walkthroughs provide step-by-step instructions for performing different FIM Development tasks.
                                                                     
                                                                    Table: FIM Development Walkthroughs
                                                                     Walkthrough Name Category Install Quartus Prime Pro Software Setup Install Git Large File Storage Extension Setup Clone FIM Repository Setup Set Development Environment Variables Setup Set Up Development Environment Setup Compile OFS FIM Compilation Manually Generate OFS Out-Of-Tree PR FIM Compilation Change the Compilation Seed Compilation Running Individual Unit Level Simulation Simulation Running Regression Unit Level Simulation Simulation Add a new module to the OFS FIM Customization Modify and run unit tests for a FIM that has a new module Customization Hardware test a FIM that has a new module Customization Debug the FIM with Signal Tap Customization Compile the FIM in preparation for designing your AFU Customization Resize the Partial Reconfiguration Region Customization Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS Customization Modify PCIe Configuration Using IP Presets Customization Migrate to a Different Agilex Device Number Customization Modify the Ethernet Sub-System to 1x400GbE Customization Set up JTAG Configuration Program the FPGA via JTAG Configuration"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#111-knowledge-pre-requisites","title":"1.1.1 Knowledge Pre-Requisites","text":"
                                                                    It is recommended that you have the following knowledge and skills before using this developer guide.
                                                                     
                                                                       
                                                                    • Basic understanding of OFS and the difference between OFS designs. Refer to the OFS Welcome Page.
                                                                      •  
                                                                    • Review the release notes for the Intel Agilex 7 PCIe Attach Reference Shells, with careful consideration of the Known Issues.
                                                                      •  
                                                                    • Review of Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile))
                                                                      •  
                                                                    • FPGA compilation flows using Intel\u00ae Quartus\u00ae Prime Pro Edition.
                                                                      •  
                                                                    • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
                                                                      •  
                                                                    • RTL (System Verilog) and coding practices to create synthesized logic.
                                                                      •  
                                                                    • RTL simulation tools.
                                                                      •  
                                                                    • Intel\u00ae Quartus\u00ae Prime Pro Edition Signal Tap Logic Analyzer tool software.
                                                                      •  
                                                                    "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#12-fim-development-theory","title":"1.2. FIM Development Theory","text":"
                                                                    This section will help you understand how the OFS Agilex PCIe Attach FIM can be developed to fit your design goals.
                                                                     
                                                                    The Default FIM Features section provides general information about the default features of the OFS Agilex PCIe Attach FIM so you can become familiar with the default design. For more detailed information about the FIM architecture, refer to the OFS Agilex PCIe Attach FIM Technical Reference Manual.
                                                                     
                                                                    The Customization Options section then gives suggestions of how this default design can be customized. Step-by-step walkthroughs for many of the suggested customizations are later described in the FIM Customization section.
                                                                     
                                                                    FIM development for a new acceleration card generally consists of the following steps:
                                                                     
                                                                       
                                                                    1. Install OFS and familiarize yourself with provided scripts and source code
                                                                      2.  
                                                                    2. Develop high level design with your specific functionality
                                                                         
                                                                      1. Determine requirements and key performance metrics
                                                                        2.  
                                                                      2. Select IP cores
                                                                        3.  
                                                                      3. Select FPGA device
                                                                        4.  
                                                                      4. Develop software memory map
                                                                        5.  
                                                                       
                                                                      3.  
                                                                    3. Select and implement FIM Physical interfaces including:
                                                                         
                                                                      1. External clock sources and creation of internal PLL clocks
                                                                        2.  
                                                                      2. General I/O
                                                                        3.  
                                                                      3. Ethernet modules
                                                                        4.  
                                                                      4. External memories
                                                                        5.  
                                                                      5. FPGA programming methodology
                                                                        6.  
                                                                       
                                                                      4.  
                                                                    4. Develop device physical implementation
                                                                         
                                                                      1. FPGA device pin assignment
                                                                        2.  
                                                                      2. Create logic lock regions
                                                                        3.  
                                                                      3. Create of timing constraints
                                                                        4.  
                                                                      4. Create Intel Quartus Prime Pro FIM test project and validate:
                                                                           
                                                                        1. Placement
                                                                          2.  
                                                                        2. Timing constraints
                                                                          3.  
                                                                        3. Build script process
                                                                          4.  
                                                                        4. Review test FIM FPGA resource usage
                                                                          5.  
                                                                         
                                                                        5.  
                                                                       
                                                                      5.  
                                                                    5. Select FIM to AFU interfaces and development of PIM
                                                                      6.  
                                                                    6. Implement FIM design
                                                                         
                                                                      1. Develop RTL
                                                                        2.  
                                                                      2. Instantiate IPs
                                                                        3.  
                                                                      3. Develop test AFU to validate FIM
                                                                        4.  
                                                                      4. Develop unit and device level simulation
                                                                        5.  
                                                                      5. Develop timing constraints and build scripts
                                                                        6.  
                                                                      6. Perform timing closure and build validation
                                                                        7.  
                                                                       
                                                                      7.  
                                                                    7. Create FIM documentation to support AFU development and synthesis
                                                                      8.  
                                                                    8. Software Device Feature discovery
                                                                      9.  
                                                                    9. Integrate, validate, and debug hardware/software
                                                                      10.  
                                                                    10. Prepare for high volume production
                                                                      11.  
                                                                    "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#121-default-fim-features","title":"1.2.1 Default FIM Features","text":""},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1211-top-level","title":"1.2.1.1 Top Level","text":"
                                                                    Figure: OFS Agilex PCIe Attach iseries-dk FIM Top-Level Diagram
                                                                     
                                                                    "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1212-interfaces","title":"1.2.1.2 Interfaces","text":"
                                                                    The key interfaces in the OFS Agilex PCIe Attach design are listed in the Release Capabilities Table. It describes the capabilities of the iseries-dk hardware as well as the capabilities of the default OFS Agilex PCIe Attach design targeting the iseries-dk.
                                                                     
                                                                    Table: Release Capabilities
                                                                     Interface iseries-dk Hardware Capabilities OFS Agilex PCIe Attach Default Design Implementation Host Interface[1] PCIe Gen5x16 Bifurcated PCIe Gen5x8 (Only 1 PCIe interface implemented) Network Interface 2 - QSFP-DD 3 Build Options: 1. QSFP 1,0 = 25 GbE2. QSFP 1,0 = 200 GbE  3. QSFP 0 = 400 GbE External Memory 2 - board mounted independent single rank DDR4-2666 8GB (1 Gb x 64 + 8b ECC)Two DIMM sockets where each socket is single memory channels or independent channels (Check Dev Kit OPN for support option).DIMM socket supports DDR4 x72 (ECC) and can operate up to DDR-3200 (depending on the speed of the FPGA used) 2xDDR4-2666 - 8GB (1Gb x 64 bits) - ECC not implemented 
                                                                    [1] The I-Series development kit has a form factor of PCIe x16, however in this release the PCIe Subsystem only supports bifurcated PCIe Gen5 x 8.  The FIM only connects 1 of the 2 PCIe links.  Future releases will support Gen5 x 16.
                                                                    "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1213-subsystems","title":"1.2.1.3 Subsystems","text":"
                                                                    The FIM Subsystems Table  describes the Platform Designer IP subsystems used in the OFS Agilex PCIe Attach iseries-dk FIM.
                                                                     
                                                                    Table: FIM Subsystems
                                                                     Subsystem User Guide Document ID PCIe Subsystem PCIe Subsystem Intel FPGA IP User Guide for Intel Agilex OFS N/A Memory Subsystem Memory Subsystem Intel FPGA IP User Guide for Intel Agilex OFS 686148[1] Ethernet Subsystem Ethernet Subsystem Intel FPGA IP User Guide 773413[1] 
                                                                    [1] You must log in to myIntel and request entitled access.
                                                                    "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1214-host-exercisers","title":"1.2.1.4 Host Exercisers","text":"
                                                                    The default AFU workload in the OFS Agilex PCIe Attach iseries-dk FIM contains several modules called Host Exercisers which are used to exercise the interfaces on the board. The Host Exerciser Descriptions Table describes these modules.
                                                                     
                                                                    Table: Host Exerciser Descriptions
                                                                     Name Acronym Description OPAE Command Host Exerciser Loopback HE-LB Used to exercise and characterize host to FPGA data transfer. host_exerciser Host Exerciser Memory HE_MEM Used to exercise and characterize host to Memory data transfer. host_exerciser Host Exerciser Memory Traffic Generator HE_MEM_TG Used to exercise and test available memory channels with a configurable traffic pattern. mem_tg Host Exerciser High Speed Serial Interface HE-HSSI Used to exercise and characterize HSSI interfaces. hssi 
                                                                    The host exercisers can be removed from the design at compile-time using command line arguments for the build script.
                                                                    "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1215-module-access-via-apfbpf","title":"1.2.1.5 Module Access via APF/BPF","text":"
                                                                    The OFS Agilex PCIe Attach iseries-dk FIM uses AXI4-Lite interconnect logic named the AFU Peripheral Fabric (APF) and Board Peripheral Fabric (BPF) to access the registers of the various modules in the design. The APF/BPF modules define master/slave interactions, namely between the host software and AFU and board peripherals. The APF Address Map Table describes the address mapping of the APF, followed by the BPF Address Map Table which describes the address mapping of the BPF.
                                                                     
                                                                    Table: APF Address Map
                                                                     Address Size (Bytes) Feature 0x00000\u20130x3FFFF 256K Board Peripherals (See BPF Address Map table) 0x40000 \u2013 0x4FFFF 64K ST2MM 0x50000 \u2013 0x5FFFF 64K Reserved 0x60000 \u2013 0x60FFF 4K UART (not used) 0x61000 \u2013 0x6FFFF 4K Reserved 0x70000 \u2013 0x7FFFF 56K PR Gasket:4K= PR Gasket DFH, control and status4K= Port DFH4K=User Clock52K=Remote STP 0x80000 \u2013 0x80FFF 4K AFU Error Reporting 
                                                                    Table: BPF Address Mapping
                                                                     Address Size (Bytes) Feature 0x00000 - 0x0FFFF 64K FME 0x10000 - 0x10FFF 4K PCIe 0x11000 - 0x11FFF 4K Reserved 0x12000 - 0x12FFF 4K QSFP0 0x13000 - 0x13FFF 4K QSFP1 0x14000 - 0x14FFF 4K HSSI 0x15000 - 0x15FFF 4K EMIF 0x20000 - 0x3FFFF 128K PMCI (note, PMCI is not implemented)"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#122-customization-options","title":"1.2.2 Customization Options","text":"
                                                                    OFS is designed to be easily customizable to meet your design needs. The OFS FIM Customizations Table lists the general user flows for OFS Agilex PCIe Attach iseries-dk FIM development, along with example customizations for each user flow, plus links to step-by-step walkthroughs where available.
                                                                     
                                                                    Table: OFS FIM Customizations
                                                                     Customization Walkthrough Name Add a new module to the OFS FIM Modify and run unit tests for a FIM that has a new module Hardware test a FIM that has a new module Debug the FIM with Signal Tap Compile the FIM in preparation for designing your AFU Resize the Partial Reconfiguration Region Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS Modify PCIe Configuration Using IP Presets Migrate to a Different Agilex Device Number Modify the Ethernet Sub-System to 1x400GbE"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#13-development-environment","title":"1.3 Development Environment","text":"
                                                                    This section describes the components required for OFS FIM development, and provides a walkthrough for setting up the environment on your development machine.
                                                                     
                                                                    Note that your development machine may be different than your deployment machine where the FPGA acceleration card is installed. FPGA development work and deployment work can be performed either on the same machine, or on different machines as desired. Please see the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up the environment for deployment machines.
                                                                    "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#131-development-tools","title":"1.3.1 Development Tools","text":"
                                                                    The Development Environment Table describes the Best Known Configuration (BKC) for the tools that are required for OFS FIM development.
                                                                     
                                                                    Table: Development Environment BKC
                                                                     Component Version Installation Walkthrough Operating System RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6 N/A Intel Quartus Prime Software Quartus Prime Pro Version 23.3 for Linux + Patches 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe) Section 1.3.1.1 Python 3.6.8 or later N/A GCC 7.4.0 or later N/A cmake 3.15 or later N/A git with git-lfs 1.8.3.1 or later Section 1.3.1.2 FIM Source Files ofs-2023.3-2 Section 1.3.2.1"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1311-walkthrough-install-quartus-prime-pro-software","title":"1.3.1.1 Walkthrough: Install Quartus Prime Pro Software","text":"
                                                                    Intel Quartus Prime Pro Version 23.3 is verified to work with the latest OFS release ofs-2023.3-2.  However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.
                                                                     
                                                                    Use RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6 for compatibility with your development flow and also testing your FIM design in your platform. 
                                                                     
                                                                    Prior to installing Quartus:
                                                                     
                                                                       
                                                                    1.  
                                                                      Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.
                                                                       
                                                                         
                                                                      • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                                                                        •  
                                                                      • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.
                                                                        •  
                                                                       
                                                                      2.  
                                                                    2.  
                                                                      Perform the following steps to satisfy the required dependencies.
                                                                       
                                                                      $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                                                                       
                                                                      Apply the following configurations.
                                                                       
                                                                      $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n
                                                                       
                                                                      3.  
                                                                    3.  
                                                                      Create the default installation path: /intelFPGA_pro/, where  is the default path of the Linux workstation, or as set by the system administrator and  is your Quartus version number. 
                                                                      The installation path must satisfy the following requirements:
                                                                       
                                                                         
                                                                      • Contain only alphanumeric characters
                                                                        •  
                                                                      • No special characters or symbols, such as !$%@^&*<>,
                                                                        •  
                                                                      • Only English characters
                                                                        •  
                                                                      • No spaces
                                                                        •  
                                                                       
                                                                    4.  
                                                                      Download your required Quartus Prime Pro Linux version here.
                                                                       
                                                                      5.  
                                                                    5.  
                                                                      Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe).
                                                                       
                                                                      6.  
                                                                    6.  
                                                                      After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                                                                       
                                                                      export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                                                                       
                                                                      For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:
                                                                       
                                                                      export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n
                                                                       
                                                                      7.  
                                                                    7.  
                                                                      Verify, Quartus is discoverable by opening a new shell:
                                                                       
                                                                      $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                                                                       
                                                                      8. "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1312-walkthrough-install-git-large-file-storage-extension","title":"1.3.1.2 Walkthrough: Install Git Large File Storage Extension","text":"
                                                                      To install the Git Large File Storage (LFS) extension, execute the following commands:
                                                                       
                                                                         
                                                                      1. Obtain Git LFS package     
                                                                        curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\n
                                                                        2.  
                                                                      2. Install Git LFS package     
                                                                        sudo dnf install git-lfs\n
                                                                        3.  
                                                                      3. Install Git LFS     
                                                                        git lfs install\n
                                                                        4.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#132-fim-source-files","title":"1.3.2 FIM Source Files","text":"
                                                                      The source files for the OFS Agilex PCIe Attach FIM are provided in the following repository: https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2
                                                                       
                                                                      Some essential directories in the repository are described as follows:
                                                                       
                                                                      ofs-agx7-pcie-attach\n|  syn                      // Contains files related to synthesis\n|  |  board                     // Contains synthesis files for several cards, including the iseries-dk \n|  |  |  iseries-dk                 // Contains synthesis files for iseries-dk\n|  |  |  |  setup                       // Contains setup files, including pin constraints and location constraints\n|  |  |  |  syn_top                     // Contains Quartus project files\n|  verification             // Contains files for UVM testing\n|  ipss                     // Contains files for IP Sub-Systems\n|  |  qsfp                      // Contains source files for QSFP Sub-System\n|  |  hssi                      // Contains source files for HSSI Sub-System\n|  |  pmci                      // Contains source files for PMCI Sub-System (not used in F-Tile FIM)\n|  |  pcie                      // Contains source files for PCIe Sub-System\n|  |  mem                       // Contains source files for Memory Sub-System\n|  sim                      // Contains simulation files\n|  |  unit_test                 // Contains files for all unit tests\n|  |  |  scripts                    // Contains script to run regression unit tests\n|  license                  // Contains Quartus patch\n|  ofs-common               // Contains files which are common across OFS platforms\n|  |  verification              // Contains common UVM files\n|  |  scripts                   // Contains common scripts\n|  |  |  common\n|  |  |  |  syn                         // Contains common scripts for synthesis, including build script\n|  |  |  |  sim                         // Contains common scripts for simulation\n|  |  tools                     // Contains common tools files\n|  |  |  mk_csr_module              // Contains common files for CSR modules\n|  |  |  fabric_generation          // Contains common files for APF/BPF fabric generation\n|  |  |  ofss_config                // Contains common files for OFSS configuration tool\n|  |  |  |  ip_params                   // Contains default IP parameters for certain Sub-Systems when using OFSS\n|  |  src                       // Contains common source files, including host exercisers\n|  tools                    //\n|  |  ofss_config               // Contains top level OFSS files for each pre-made board configuration\n|  |  |  hssi                       // Contains OFSS files for Ethernet-SS configuraiton\n|  |  |  memory                     // Contains OFSS files for Memory-SS configuration\n|  |  |  pcie                       // Contains OFSS files for PCIe-SS configuration\n|  |  |  iopll                      // Contains OFSS files for IOPLL configuration\n|  src                      // Contains source files for Agilex PCIe Attach FIM\n|  |  pd_qsys                   // Contains source files related to APF/BPF fabric\n|  |  includes                  // Contains source file header files\n|  |  top                       // Contains top-level source files, including design top module\n|  |  afu_top                   // Contains top-level source files for AFU\n
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1321-walkthrough-clone-fim-repository","title":"1.3.2.1 Walkthrough: Clone FIM Repository","text":"
                                                                      Perform the following steps to clone the OFS Agilex PCIe Attach FIM Repository:
                                                                       
                                                                         
                                                                      1. Create a new directory to use as a clean starting point to store the retrieved files.     
                                                                        mkdir OFS_BUILD_ROOT\ncd OFS_BUILD_ROOT\nexport OFS_BUILD_ROOT=$PWD\n
                                                                        2.  
                                                                      2. Clone GitHub repository using the HTTPS git method     
                                                                        git clone --recurse-submodules https://github.com/OFS/ofs-agx7-pcie-attach.git\n
                                                                        3.  
                                                                      3. Check out the correct tag of the repository     
                                                                        cd ofs-agx7-pcie-attach\ngit checkout --recurse-submodules tags/ofs-2023.3-2\n
                                                                        4.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#133-environment-variables","title":"1.3.3 Environment Variables","text":"
                                                                      The OFS FIM compilation and simulation scripts require certain environment variables be set prior to execution.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#1331-walkthrough-set-development-environment-variables","title":"1.3.3.1 Walkthrough: Set Development Environment Variables","text":"
                                                                      Perform the following steps to set the required environment variables. These environment variables must be set prior to simulation or compilation tasks so it is recommended that you create a script to set these variables.
                                                                       
                                                                         
                                                                      1.  
                                                                        Navigate to the top level directory of the cloned OFS FIM repository.
                                                                         
                                                                        cd ofs-agx7-pcie-attach\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set project variables   
                                                                        # Set OFS Root Directory - e.g. this is the top level directory of the cloned OFS FIM repository\nexport OFS_ROOTDIR=$PWD\n
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Set variables based on your development environment   
                                                                        # Set proxies if required for your server\nexport http_proxy=<YOUR_HTTP_PROXY>\nexport https_proxy=<YOUR_HTTPS_PROXY>\nexport ftp_proxy=<YOUR_FTP_PROXY>\nexport socks_proxy=<YOUR_SOCKS_PROXY>\nexport no_proxy=<YOUR_NO_PROXY>\n\n# Set Quartus license path\nexport LM_LICENSE_FILE=<YOUR_LM_LICENSE_FILE>\n\n# Set Synopsys License path (if using Synopsys for simulation)\nexport DW_LICENSE_FILE=<YOUR_DW_LICENSE_FILE>\nexport SNPSLMD_LICENSE_FILE=<YOUR_SNPSLMD_LICENSE_FILE>\n\n# Set Quartus Installation Directory - e.g. $QUARTUS_ROOTDIR/bin contains Quartus executables\nexport QUARTUS_ROOTDIR=<YOUR_QUARTUS_INSTALLATION_DIRECTORY>\n\n# Set the Tools Directory - e.g. $TOOLS_LOCATION contains the 'synopsys' directory if you are using Synopsys. Refer to the $VCS_HOME variable for an example.\nexport TOOLS_LOCATION=<YOUR_TOOLS_LOCATION>\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Set generic environment variables
                                                                         
                                                                        # Set Work directory \nexport WORKDIR=$OFS_ROOTDIR\n\n# Set Quartus Tools variables\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport QUARTUS_VER_AC=$QUARTUS_ROOTDIR\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport IMPORT_IP_ROOTDIR=$IP_ROOTDIR\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\n\n# Set Verification Tools variables (if running simulations)\nexport DESIGNWARE_HOME=$TOOLS_LOCATION/synopsys/vip_common/vip_Q-2020.03A\nexport UVM_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport MTI_HOME=$QUARTUS_ROOTDIR/../questa_fse\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n\n# Set PATH to include compilation and simulation tools\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/../qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/bin:$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$DESIGNWARE_HOME/bin:$VCS_HOME/bin:$PATH\n
                                                                         
                                                                        5.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#134-walkthrough-set-up-development-environment","title":"1.3.4 Walkthrough: Set Up Development Environment","text":"
                                                                      This walkthrough guides you through the process of setting up your development environment in preparation for FIM development. This flow only needs to be done once on your development machine.
                                                                       
                                                                         
                                                                      1.  
                                                                        Ensure that Quartus Prime Pro Version 23.3 for Linux with Intel Agilex FPGA device support is installed on your development machine. Refer to the Install Quartus Prime Pro Software section for step-by-step installation instructions.
                                                                         
                                                                           
                                                                        1. Verify version number
                                                                          quartus_sh --version\n
                                                                           
                                                                          Example Output:
                                                                           
                                                                          Quartus Prime Shell\nVersion 23.3 Build 94 06/14/2023 SC Pro Edition\nCopyright (C) 2023  Intel Corporation. All rights reserved.\n
                                                                           
                                                                          2.  
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Ensure that all support tools are installed on your development machine, and that they meet the version requirements.
                                                                         
                                                                           
                                                                        1.  
                                                                          Python 3.6.8 or later
                                                                           
                                                                             
                                                                          1.  
                                                                            Verify version number
                                                                             
                                                                            python --version\n
                                                                             
                                                                            Example Output:
                                                                             
                                                                            Python 3.6.8\n
                                                                             
                                                                            2.  
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          GCC 7.4.0 or later
                                                                           
                                                                             
                                                                          1.  
                                                                            Verify version number
                                                                             
                                                                            gcc --version\n
                                                                             
                                                                            Example output:
                                                                             
                                                                            gcc (GCC) 7.4.0\n
                                                                             
                                                                            2.  
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          cmake 3.15 or later
                                                                           
                                                                             
                                                                          1.  
                                                                            Verify version number
                                                                             
                                                                            cmake --version\n
                                                                             
                                                                            Example output:
                                                                             
                                                                            cmake version 3.15\n
                                                                             
                                                                            2.  
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          git with git-lfs 1.8.3.1 or later. Refer to the Install Git Large File Storage Extension section for step-by-step instructions on installing the Git Large File Storage (LFS) extension.
                                                                           
                                                                             
                                                                          1.  
                                                                            Verify version number
                                                                             
                                                                            git --version\n
                                                                             
                                                                            Example output:
                                                                             
                                                                            git version 1.8.3.1\n
                                                                             
                                                                            2.  
                                                                           
                                                                          5.  
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Clone the ofs-agx7-pcie-attach repository. Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Install UART IP license patch .02.
                                                                         
                                                                           
                                                                        1.  
                                                                          Navigate to the license directory
                                                                           
                                                                          cd $IOFS_BUILD_ROOT/license\n
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Install Patch 0.02
                                                                           
                                                                          sudo ./quartus-0.0-0.02iofs-linux.run\n
                                                                           
                                                                          3.  
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Install Quartus Patches 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe). All required patches are provided in the Assets of the OFS FIM Release Tag: https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Verify that patches have been installed correctly. They should be listed in the output of the following command.
                                                                         
                                                                        quartus_sh --version\n
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Set required environment variables. Refer to the [Set Environment Variables] section for step-by-step instructions.
                                                                         
                                                                        8.  
                                                                       
                                                                      This concludes the walkthrough for setting up your development environment. At this point you are ready to begin FIM development.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2-fim-compilation","title":"2. FIM Compilation","text":"
                                                                      This section describes the process of compiling OFS FIM designs using the provided build scripts. It contains two main sections:
                                                                       
                                                                         
                                                                      • Compilation Theory - Describes the theory behind FIM compilation
                                                                        •  
                                                                      • Compilation Flows - Describes the process of compiling a FIM
                                                                        •  
                                                                       
                                                                      The walkthroughs provided in this section are:
                                                                       
                                                                         
                                                                      • Compile OFS FIM
                                                                        •  
                                                                      • Manually Generate OFS Out-Of-Tree PR FIM
                                                                        •  
                                                                      • Change the Compilation Seed
                                                                        •  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#21-compilation-theory","title":"2.1 Compilation Theory","text":"
                                                                      This section describes the theory behind FIM compilation.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#211-fim-build-script","title":"2.1.1 FIM Build Script","text":"
                                                                      The OFS Common Repository contains a script named build_top.sh which is used to build OFS FIM designs and generate output files that can be programmed to the board. After cloning the OFS FIM repository (with the ofs-common repository included), the build script can be found in the following location:
                                                                       
                                                                      $OFS_ROOTDIR/ofs-common/scripts/common/syn/build_top.sh\n
                                                                       
                                                                      The usage of the build_top.sh script is as follows:
                                                                       
                                                                      build_top.sh [-k] [-p] [-e] [--stage=<action>] [--ofss=<ip_config>] <build_target>[:<fim_options>] [<work_dir_name>]\n
                                                                       Field Options Description Requirement -k None Keep. Preserves and rebuilds within an existing work tree instead of overwriting it. Optional -p None When set, and if the FIM supports partial reconfiguration, a PR template tree is generated at the end of the FIM build. The PR template tree is located in the top of the work directory but is relocatable and uses only relative paths. See $OFS_ROOTDIR/syn/common/scripts generate_pr_release.sh for details. Optional -e None Run only Quartus analysis and elaboration. It completes the setup stage, passes -end synthesis to the Quartus compilation flow and exits without running the finish stage. Optional --stage all | setup | compile | finish Controls which portion of the OFS build is run.\u00a0\u00a0- all: Run all build stages (default)\u00a0\u00a0- setup: Initialize a project in the work directory\u00a0\u00a0- compile: Run the Quartus compilation flow on a project that was already initialized with setup\u00a0\u00a0- finish: Complete OFS post-compilation tasks, such as generating flash images and, if -p is set, generating a release. Optional --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. This parameter is consumed during the setup stage and IP is updated only inside the work tree. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. Optional <build_target> iseries-dk Specifies which board is being targeted. Required <fim_options> flat | null_he_lb | null_he_hssi | null_he_mem | null_he_mem_tg | no_hssi Used to change how the FIM is built.\u00a0\u00a0\u2022 flat - Compiles a flat design (no PR assignments). This is useful for bringing up the design on a new board without dealing with PR complexity.\u00a0\u00a0\u2022 null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null.\u00a0\u00a0\u2022 null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null.\u00a0\u00a0\u2022 null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null.\u00a0\u00a0\u2022 null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null.\u00a0\u00a0\u2022 no_hssi - Removes the HSSI-SS from the design. More than one FIM option may be passed included in the <fim_options> list by concatenating them separated by commas. For example: <build_target>:flat,null_he_lb,null_he_hssi Optional <work_dir_name> String Specifies the name of the work directory in which the FIM will be built. If not specified, the default target is $OFS_ROOTDIR/work Optional 
                                                                      Note: The he_null is a minimal block with CSRs that responds to PCIe MMIO requests in order to keep PCIe alive.
                                                                       
                                                                      The build script copies source files from the existing cloned repository into the specified work directory, which are then used for compilation. As such, any changes made in the base source files will be included in all subsequent builds, unless the -k option is used, in which case an existing work directories files are used as-is. Likewise, any changes made in a work directory is only applied to that work directory, and will not be updated in the base repository by default.
                                                                       
                                                                      Refer to Compile OFS FIM which provides step-by-step instructions for running the build_top.sh script with some of the different available options.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#212-ofss-file-usage","title":"2.1.2 OFSS File Usage","text":"
                                                                      The OFS FIM build script can use OFSS files to easily customize design IP prior to compilation using preset configurations. The OFSS files specify certain parameters for different IPs. Pre-made OFSS files can be found in the $OFS_ROOTDIR/tools/ofss_config directory. Using OFSS is provided as a convenience tool for building pre-defined FIMs.
                                                                       
                                                                      Table: Provided OFSS Files
                                                                       OFSS File Name Location Type Description iseries-dk.ofss $OFS_ROOTDIR/tools/ofss_config Top Includes the following OFSS files [1]: \u00a0\u00a0\u2022 iseries-dk_base.ofss \u00a0\u00a0\u2022 pcie_host.ofss \u00a0\u00a0\u2022 iopll.ofss \u00a0\u00a0\u2022 memory_ftile.ofss iseries-dk_base.ofss $OFS_ROOTDIR/tools/ofss_config ofs Defines certain attributes of the design, including the platform name, device family, fim type, part number, and device ID. pcie_host_rtile.ofss $OFS_ROOTDIR/tools/ofss_config/pcie pcie Defines the PCIe Subsystem with the following configuration:\u00a0\u00a0\u2022 PF0 (3 VFs)\u00a0\u00a0\u2022 PF1 (0 VFs)\u00a0\u00a0\u2022 PF2 (0 VFs)\u00a0\u00a0\u2022 PF3 (0 VFs)\u00a0\u00a0\u2022 PF4 (0 VFs) iopll.ofss $OFS_ROOTDIR/tools/ofss_config/iopll iopll Sets the IOPLL frequency to 470 MHz memory_rtile.ofss $OFS_ROOTDIR/tools/ofss_config/memory memory Defines the memory IP preset file to be used during the build as iseries-dk hssi_8x25_ftile.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP preset file to be used during the build as fseries-dk hssi_2x200_ftile.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP preset file to be used during the build as 200g-fseries-dk hssi_1x400_ftile.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP preset file to be used during the build as 400g-fseries-dk 
                                                                      [1] The iseries-dk.ofss file does not include an HSSI OFSS file by default. If you are using the iseries-dk.ofss file when building the FIM, you must also specify an HSSI OFSS file for F-tile. Refer to the Compile OFS FIM Section for examples of this flow.
                                                                       
                                                                      Note: Using OFSS is required for FIM builds targeting the I-Series Development Kit.
                                                                       
                                                                      There are typically three sections contained within an OFSS file.
                                                                       
                                                                         
                                                                      •  
                                                                        [include]
                                                                         
                                                                           
                                                                        • This section of an OFSS file contains elements separated by a newline, where each element is the path to an OFSS file that is to be included for configuration by the OFSS Configuration Tool. Ensure that any environment variables (e.g. $OFS_ROOTDIR) is set correctly. The OFSS Config tool uses breadth first search to include all of the specified OFSS files; the ordering of OFSS files does not matter
                                                                          •  
                                                                         
                                                                        •  
                                                                       
                                                                         
                                                                      •  
                                                                        [ip]
                                                                         
                                                                           
                                                                        • This section of an OFSS file contains a key value pair that allows the OFSS Config tool to determine which IP configuration is being passed in. The currently supported values of IP are ofs, iopll, pcie, memory, and hssi.
                                                                          •  
                                                                         
                                                                        •  
                                                                       
                                                                         
                                                                      •  
                                                                        [settings]
                                                                         
                                                                           
                                                                        • This section of an OFSS file contains IP specific settings. Refer to an existing IP OFSS file to see what IP settings are set. For the IP type `ofss``, the settings will be information of the OFS device (platform, family, fim, part #, device_id)
                                                                          •  
                                                                         
                                                                        •  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2121-platform-ofss-file","title":"2.1.2.1 Platform OFSS File","text":"
                                                                      The <platform>.ofss file (e.g. $OFS_ROOTDIR/tools/ofss_config/iseries-dk.ofss) is the platform level OFSS wrapper file. This is typically the OFSS file that is provided to the build script. It only contains an include section which lists all other OFSS files that are to be used when the <platform>.ofss file is passed to the build script.
                                                                       
                                                                      The generic structure of a <platform>.ofss file is as follows:
                                                                       
                                                                      [include]\n<PATH_TO_PLATFORM_BASE_OFSS_FILE>\n<PATH_TO_PCIE_OFSS_FILE>\n<PATH_TO_IOPLL_OFSS_FILE>\n<PATH_TO_MEMORY_OFSS_FILE>\n<PATH_TO_HSSI_OFSS_FILE>\n
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2122-ofs-ip-ofss-file","title":"2.1.2.2 OFS IP OFSS File","text":"
                                                                      An OFSS file with IP type ofs (e.g. $OFS_ROOTDIR/tools/ofss_config/iseries-dk_base.ofss) contains board specific information for the target board.
                                                                       
                                                                      Currently supported configuration options for an OFSS file with IP type ofs are described in the OFS IP OFSS File Options table.
                                                                       
                                                                      Table: OFS IP OFSS File Options
                                                                       Section Parameter iseries-dk Default Value [ip] type ofs [settings] platform N6001 family agilex fim base_x16 part AGIB027R29A1E2VR3 device_id 6001"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2123-pcie-ip-ofss-file","title":"2.1.2.3 PCIe IP OFSS File","text":"
                                                                      An OFSS file with IP type pcie (e.g. $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss) is used to configure the PCIe-SS in the FIM.
                                                                       
                                                                      The PCIe OFSS file has a special section type ([pf*]) which is used to define physical functions (PFs) in the FIM. Each PF has a dedicated section, where the * character is replaced with the PF number. For example, [pf0], [pf1], etc. For reference FIM configurations, 0 virtual functions (VFs) on PF0 is not supported. This is because the PR region cannot be left unconnected. A NULL AFU may need to be instantiated in this special case. PFs must be consecutive. The PFVF Limitations table describes the supported number of PFs and VFs.
                                                                       
                                                                      Table: PF/VF Limitations
                                                                       Parameter Value Min # of PFs 1 (on PF0) Max # of PFs 8 Min # of VFs 1 on PF0 Max # of VFs 2000 distributed across all PFs 
                                                                      Currently supported configuration options for an OFSS file with IP type pcie are described in the PCIe IP OFSS File Options table.
                                                                       
                                                                      Table: PCIe IP OFSS File Options
                                                                       Section Parameter Options Default Description [ip] type pcie N/A Specifies that this OFSS file configures the PCIe-SS [settings] output_name pcie_ss N/A Specifies the output name of the PCIe-SS IP preset String N/A OPTIONAL - Specifies the name of a PCIe-SS IP presets file to use when building the FIM. When used, a presets file will take priority over any other parameters set in this OFSS file. [pf*] num_vfs Integer 0 Specifies the number of Virtual Functions in the current PF bar0_address_width Integer 12 bar4_address_width Integer 14 vf_bar0_address_width Integer 12 ats_cap_enable 0 | 1 0 vf_ats_cap_enable 0 | 1 0 prs_ext_cap_enable 0 | 1 0 pasid_cap_enable 0 | 1 0 pci_type0_vendor_id 32'h Value 0x00008086 pci_type0_device_id 32'h Value 0x0000bcce revision_id 32'h Value 0x00000001 class_code 32'h Value 0x00120000 subsys_vendor_id 32'h Value 0x00008086 subsys_dev_id 32'h Value 0x00001771 sriov_vf_device_id 32'h Value 0x0000bccf exvf_subsysid 32'h Value 0x00001771 
                                                                      The default values for all PCIe-SS parameters (that are not defined in the PCIe IP OFSS file) are defined in $OFS_ROOTDIR/ofs-common/tools/ofss_config/ip_params/pcie_ss_component_parameters.py. When using a PCIe IP OFSS file during compilation, the PCIe-SS IP that is used will be defined based on the values in the PCIe IP OFSS file plus the parameters defined in pcie_ss_component_parameters.py.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2124-iopll-ip-ofss-file","title":"2.1.2.4 IOPLL IP OFSS File","text":"
                                                                      An OFSS file with IP type iopll (e.g. $OFS_ROOTDIR/tools/ofss_config/iopll/iopll.ofss) is used to configure the IOPLL in the FIM.
                                                                       
                                                                      The IOPLL OFSS file has a special section type ([p_clk]) which is used to define the IOPLL clock frequency.
                                                                       
                                                                      Currently supported configuration options for an OFSS file with IP type iopll are described in the IOPLL OFSS File Options table.
                                                                       
                                                                      Table: IOPLL OFSS File Options
                                                                       Section Parameter Options Description [ip] type iopll Specifies that this OFSS file configures the IOPLL [settings] output_name sys_pll Specifies the output name of the IOPLL. instance_name iopll_0 Specifies the instance name of the IOPLL. [p_clk] freq Integer: 250 - 470 Specifies the IOPLL clock frequency in MHz. 
                                                                      Note: The following frequencies have been tested on reference boards: 350MHz, 400MHz, 470MHz.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2125-memory-ip-ofss-file","title":"2.1.2.5 Memory IP OFSS File","text":"
                                                                      An OFSS file with IP type memory (e.g. $OFS_ROOTDIR/tools/ofss_config/memory/memory_iseries.ofss) is used to configure the Memory-SS in the FIM.
                                                                       
                                                                      The Memory OFSS file specifies a preset value, which selects a presets file (.qprs) to configure the Memory-SS.
                                                                       
                                                                      Currently supported configuration options for an OFSS file with IP type memory are described in the Memory OFSS File Options table.
                                                                       
                                                                      Table: Memory OFSS File Options
                                                                       Section Parameter Options Description [ip] type memory Specifies that this OFSS file configures the Memory-SS [settings] output_name mem_ss_fm Specifies the output name of the Memory-SS. preset iseries-dk | String[1] Specifies the name of the .qprs presets file that will be used to build the Memory-SS. 
                                                                      [1] You may generate your own .qprs presets file with a unique name using Quartus. 
                                                                       
                                                                      Pre-provided Memory-SS presets files are located in the $OFS_ROOTDIR/ipss/mem/qip/presets directory.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2126-hssi-ip-ofss-file","title":"2.1.2.6 HSSI IP OFSS File","text":"
                                                                      An OFSS file with IP type hssi (e.g. $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_8x25_ftile.ofss) is used to configure the Ethernet-SS in the FIM.
                                                                       
                                                                      Currently supported configuration options for an OFSS file with IP type hssi are described in the HSSI OFSS File Options table.
                                                                       
                                                                      Table: HSSI OFSS File Options
                                                                       Section Parameter Options Description [ip] type hssi Specifies that this OFSS file configures the Ethernet-SS [settings] output_name hssi_ss Specifies the output name of the Ethernet-SS num_channels Integer Specifies the number of channels. data_rate 10GbE | 25GbE | 200GAUI-4 | 400GAUI-8 Specifies the data rate[1] preset fseries-dk | 200g-fseries-dk | 400g-fseries-dk | String[2] Specifies the name of the .qprs preset file that will be used to build the Ethernet-SS. This will overwrite the other settings in this OFSS file. 
                                                                      [1] The presets file will take priority over the data_rate parameter, so this value will not take effect when using a presets file.
                                                                       
                                                                      [2] You may generate your own .qprs presets file with a unique name using Quartus. 
                                                                       
                                                                      Pre-provided Ethernet-SS presets are located in the  $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss/presets directory.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#213-ofs-build-script-outputs","title":"2.1.3 OFS Build Script Outputs","text":"
                                                                      The output files resulting from running the the OFS FIM build_top.sh build script are copied to a single directory during the finish stage of the build script. The path for this directory is: $OFS_ROOTDIR/<WORK_DIRECTORY>/syn/board/iseries-dk/syn_top/output_files.
                                                                       
                                                                      The output files include programmable images and compilation reports. The OFS Build Script Output Descriptions table describes the images that are generated by the build script.
                                                                       
                                                                      Table: OFS Build Script Output Descriptions
                                                                       File Name Description ofs_top.sof The FIM design SRAM Object File; a binary file of the compiled FIM image."},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#22-compilation-flows","title":"2.2 Compilation Flows","text":"
                                                                      This section provides information for using the build script to generate different FIM types. Walkthroughs are provided for each compilation flow. These walkthroughs require that the development environment has been set up as described in the Set Up Development Environment section.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#221-flat-fim","title":"2.2.1 Flat FIM","text":"
                                                                      A flat FIM is compiled such that there is no partial reconfiguration region, and the entire design is built as a flat design. This is useful for compiling new designs without worrying about the complexity introduced by partial reconfiguration. The flat compile removes the PR region and PR IP; thus, you cannot use the -p build flag when using the flat compile setting. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#222-in-tree-pr-fim","title":"2.2.2 In-Tree PR FIM","text":"
                                                                      An In-Tree PR FIM is the default compilation if no compile flags or compile settings are used. This flow will compile the design with the partial reconfiguration region, but it will not create a relocatable PR directory tree to aid in AFU development. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#223-out-of-tree-pr-fim","title":"2.2.3 Out-of-Tree PR FIM","text":"
                                                                      An Out-of-Tree PR FIM will compile the design with the partial reconfiguration region, and will create a relocatable PR directory tree to aid in AFU workload development. This is especially useful if you are developing a FIM to be used by another team developing AFU workloads. This is the recommended build flow in most cases. There are two ways to create the relocatable PR directory tree:
                                                                       
                                                                         
                                                                      • Run the FIM build script with the -p option. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                                                                        •  
                                                                      • Run the generate_pr_release.sh script after running the FIM build script. Refer to the Manually Generate OFS Out-Of-Tree PR FIM Section step-by-step instructions for this flow.
                                                                        •  
                                                                       
                                                                      In both cases, the generate_pr_release.sh is run to create the relocatable build tree. This script is located at $OFS_ROOTDIR/ofs-common/scripts/common/syn/generate_pr_release.sh. Usage for this script is as follows:
                                                                       
                                                                      generate_pr_release.sh -t <PATH_OF_RELOCATABLE_PR_TREE> <BOARD_TARGET> <WORK_DIRECTORY>\n
                                                                       
                                                                      The Generate PR Release Script Options table describes the options for the generate_pr_release.sh script.
                                                                       
                                                                      Table: Generate PR Release Script Options
                                                                       Parameter Options Description <PATH_OF_RELOCATABLE_PR_TREE> String Specifies the location of the relocatable PR directory tree to be created. <BOARD_TARGET> iseries-dk Specifies the name of the board target. <WORK_DIRECTORY> String Specifies the existing work directory from which the relocatable PR directory tree will be created from. 
                                                                      After generating the relocatable build tree, it is located in the $OFS_ROOTDIR/<WORK_DIRECTORY>/pr_build_template directory (or the directory you specified if generated separately). The contents of this directory have the following structure:
                                                                       
                                                                      \u251c\u2500\u2500 bin\n\u251c\u2500\u2500 \u251c\u2500\u2500 afu_synth\n\u251c\u2500\u2500 \u251c\u2500\u2500 qar_gen\n\u251c\u2500\u2500 \u251c\u2500\u2500 update_pim\n\u251c\u2500\u2500 \u251c\u2500\u2500 run.sh\n\u251c\u2500\u2500 \u251c\u2500\u2500 build_env_config\n\u251c\u2500\u2500 README\n\u251c\u2500\u2500 hw\n\u251c\u2500\u2500 \u251c\u2500\u2500 lib\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 build\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 fme-ifc-id.txt\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 platform\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 fme-platform-class.txt\n\u251c\u2500\u2500 \u251c\u2500\u2500 blue_bits\n\u2514\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 ofs_top.sof\n
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#224-he_null-fim","title":"2.2.4 HE_NULL FIM","text":"
                                                                      An HE_NULL FIM refers to a design with one, some, or all of the Host Exercisers replaced by he_null blocks. The he_null is a minimal block with CSRs that responds to PCIe MMIO requests in order to keep PCIe alive. You may use any of the build flows (flat, in-tree, out-of-tree) with the HE_NULL compile options. The HE_NULL compile options are as follows:
                                                                       
                                                                         
                                                                      • null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null
                                                                        •  
                                                                      • null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null
                                                                        •  
                                                                      • null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null
                                                                        •  
                                                                      • null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null
                                                                        •  
                                                                       
                                                                      The Compile OFS FIM section gives step-by-step instructions for this flow.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#225-walkthrough-compile-ofs-fim","title":"2.2.5 Walkthrough: Compile OFS FIM","text":"
                                                                      Perform the following steps to compile the OFS Agilex PCIe Attach FIM for iseries-dk:
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Navigate to the root directory.
                                                                         
                                                                        cd $OFS_ROOTDIR\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Run the build_top.sh script with the desired compile options using the iseries-dk OFSS presets. In the examples below, the iseries-dk.ofss file is used to call the OFS OFSS file, the IOPLL OFSS File, the PCIe OFSS file, and the Memory OFSS file. The HSSI OFSS file is specified in the command, in this case, the hssi_8x25_ftile.ofss file is used. This is not necessary if using the no_hssi compile option.
                                                                         
                                                                           
                                                                        •  
                                                                          Flat FIM
                                                                           
                                                                          ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk:flat work_iseries-dk_flat\n
                                                                           
                                                                          •  
                                                                         
                                                                           
                                                                        •  
                                                                          In-Tree PR FIM
                                                                           
                                                                          ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk_in_tree_pr\n
                                                                           
                                                                          •  
                                                                         
                                                                           
                                                                        •  
                                                                          Out-of-Tree PR FIM
                                                                           
                                                                          ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk_oot_pr\n
                                                                           
                                                                          •  
                                                                         
                                                                           
                                                                        •  
                                                                          HE_NULL Flat FIM
                                                                           
                                                                          ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk:flat,null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_iseries-dk_flat\n
                                                                           
                                                                          •  
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Once the build script is complete, the build summary should report that the build is complete and passes timing. For example:
                                                                         
                                                                        ***********************************\n***\n***        OFS_PROJECT: ofs-agx7-pcie-attach\n***        OFS_BOARD: iseries-dk\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 1\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                                                         
                                                                        6.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#226-walkthrough-manually-generate-ofs-out-of-tree-pr-fim","title":"2.2.6 Walkthrough: Manually Generate OFS Out-Of-Tree PR FIM","text":"
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Navigate to the root directory.
                                                                         
                                                                        cd $OFS_ROOTDIR\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Run the build_top.sh script with the desired compile options using the iseries-dk OFSS presets. In order to create the relocatable PR tree, you may not compile with the flat option. For example:
                                                                         
                                                                        ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk\n
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Run the generate_pr_release.sh script to create the relocatable PR tree.
                                                                         
                                                                        ./ofs-common/scripts/common/syn/generate_pr_release.sh -t work_iseries-dk/pr_build_template iseries-dk work_iseries-dk\n
                                                                         
                                                                        6.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#227-compilation-seed","title":"2.2.7 Compilation Seed","text":"
                                                                      You may change the seed which is used by the build script during Quartus compilation to change the starting point of the fitter. Trying different seeds is useful when your design is failing timing by a small amount.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#2271-walkthrough-change-the-compilation-seed","title":"2.2.7.1 Walkthrough: Change the Compilation Seed","text":"
                                                                      Perform the following steps to change the compilation seed for the FIM build.
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Edit the SEED assignment in the $OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_top.qsf file to your desired seed value. The value can be any non-negative integer value.
                                                                         
                                                                        set_global_assignment -name SEED 2\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Build the FIM. Refer to the Compile OFS FIM section for instructions.
                                                                         
                                                                        5.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#3-fim-simulation","title":"3. FIM Simulation","text":"
                                                                      Unit level simulation of key components in the FIM is provided for verification of the following areas:
                                                                       
                                                                         
                                                                      • Ethernet
                                                                        •  
                                                                      • PCIe
                                                                        •  
                                                                      • External Memory
                                                                        •  
                                                                      • Core FIM
                                                                        •  
                                                                       
                                                                      The Unit Level simulations work with Synopsys VCS/VCSMX or Mentor Graphics Questasim simulators. The scripts to run each unit level simulation are located in $OFS_ROOTDIR/sim/unit_test. Each unit test directory contains a README which describes the test in detail.  Refer to the Supported Unit Tests table for a list of the supported unit tests.
                                                                       
                                                                      Note: The OFS Agilex PCIe Attach FIM for the F-Tile Development Kit does not support all of the unit tests that are provided in the unit_test directory.
                                                                       
                                                                      Table: Supported Unit Tests
                                                                       Test Name Description bfm_test This is the unit test for PCIe BFM. The test uses HE-LB to perform memory loopback between FIM and the host. csr_test This is the unit test for FIM CSR access and AFU memory write/read dfh_walker This is the unit test for FME DFH walking flr This is the unit test for PCIe PF/VF FLR fme_csr_access This is the a unit test for the register access logic for $OFS_ROOTDIR/ofs-common/src/common/fme/fme_csr.sv fme_csr_directed This is the unit test for $OFS_ROOTDIR/ofs-common/src/common/fme/fme_csr.sv he_lb_test This is the unit test for HE_LPBK. The test uses HE-LB to perform memory loopback between FIM and the host. he_null_test This is the unit test for HE-NULL Exerciser. The test issues basic mmio Rd/Wr requests targetting HE-NULL CSRs. indirect_csr This is the unit test for axi4lite_indirect_csr_if module. pcie_csr_test This is the unit test for PCIE CSR access. pf_vf_access_test This is the unit test for PCIe PF/VF MMIO. Each function has a feature GUID at offset 0x8 with an associated register map. port_gasket_test This is the unit test for pg_csr block and it's connectivity to fabric. The test issues mmio Rd/Wr requests targetting the csrs in port_gasket. This test does not do any functional testing of partial reconfiguration, user clock or remote stp. remote_stp_test This is the unit test for remote stp. It covers mmio read access to remote_stp registers. uart_csr This is the unit test for UART CSR accesses."},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#31-simulation-file-generation","title":"3.1 Simulation File Generation","text":"
                                                                      The simulation files must be generated prior to running unit level simulations. The script to generate simulation files is in the following location:
                                                                       
                                                                      $OFS_ROOTDIR/ofs-common/scripts/common/sim/gen_sim_files.sh\n
                                                                       
                                                                      The usage of the gen_sim_files.sh script is as follows:
                                                                       
                                                                      gen_sim_files.sh [--ofss=<ip_config>] <build_target>[:<fim_options>] [<device>] [<family>]\n
                                                                       
                                                                      The Gen Sim Files Script Options table describes the options for the gen_sim_files.sh script.
                                                                       
                                                                      Table: Gen Sim Files Script Options
                                                                       Field Options Description Requirement --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. Platform Dependent[1] <build_target> iseries-dk Specifies which board is being targeted. Required <fim_options> null_he_lb | null_he_hssi | null_he_mem | null_he_mem_tg Used to change how the FIM is built.\u00a0\u00a0- null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null.\u00a0\u00a0- null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null.\u00a0\u00a0- null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null.\u00a0\u00a0- null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null. More than one FIM option may be passed included in the <fim_options> list by concatenating them separated by commas. For example: <build_target>:null_he_lb,null_he_hssi Optional <device> string Specifies the device ID for the target FPGA. If not specified, the default device is parsed from the QSF file for the project. Optional <family> string Specifies the family for the target FPGA. If not specified, the default family is parsed from the QSF file for the project. Optional 
                                                                      [1] Using OFSS is required for the F-Tile Development Kit.
                                                                       
                                                                      Refer to the Running Individual Unit Level Simulation section for an example of the simulation files generation flow.
                                                                       
                                                                      When running regression tests, you may use the -g command line argument to generate simulation files; refer to the Running Regression Unit Level Simulation section for step-by-step instructions.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#32-individual-unit-tests","title":"3.2 Individual Unit Tests","text":"
                                                                      Each unit test may be run individually using the run_sim.sh script located in the following directory:
                                                                       
                                                                      $OFS_ROOTDIR/ofs-common/scripts/common/sim/run_sim.sh\n
                                                                       
                                                                      The usage for the run_sim.sh script is as follows:
                                                                       
                                                                      sh run_sim.sh TEST=<test> [VCSMX=<0|1> | MSIM=<0|1>]\n
                                                                       
                                                                      The Run Sim Script Options table describes the options for the run_sim.sh script.
                                                                       
                                                                      Table: Run Sim Script Options
                                                                       Field Options Description TEST String Specify the name of the test to run, e.g. dfh_walker VCSMX 0 | 1 When set, the VCSMX simulator will be used MSIM 0 | 1 When set, the QuestaSim simulator will be used 
                                                                      Note: The default simulator is VCS if neither VCSMX nor MSIM are set.
                                                                       
                                                                      The log for a unit test is stored in a transcript file in the simulation directory of the test that was run.
                                                                       
                                                                      $OFS_ROOTDIR/sim/unit_test/<TEST_NAME>/<SIMULATOR>/transcript\n
                                                                       
                                                                      For example, the log for the DFH walker test using VCSMX would be found at:
                                                                       
                                                                      $OFS_ROOTDIR/sim/unit_test/dfh_walker/sim_vcsmx/transcript\n
                                                                       
                                                                      The simulation waveform database is saved as vcdplus.vpd for post simulation review.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#321-walkthrough-running-individual-unit-level-simulation","title":"3.2.1 Walkthrough: Running Individual Unit Level Simulation","text":"
                                                                      Perform the following steps to run an individual unit test.
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Generate the simulation files for the iseries-dk
                                                                         
                                                                        cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/iseries-dk.ofss iseries-dk\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Navigate to the common simulation directory   
                                                                        cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Run the desired unit test using your desired simulator
                                                                         
                                                                           
                                                                        •  
                                                                          Using VCS
                                                                           
                                                                          sh run_sim.sh TEST=<test_name>\n
                                                                           
                                                                          •  
                                                                         
                                                                           
                                                                        •  
                                                                          Using VCSMX
                                                                           
                                                                          sh run_sim.sh TEST=<test_name> VCSMX=1\n
                                                                           
                                                                          •  
                                                                         
                                                                           
                                                                        •  
                                                                          Using QuestaSim
                                                                           
                                                                          sh run_sim.sh TEST=<test_name> MSIM=1\n
                                                                           
                                                                          •  
                                                                         
                                                                           
                                                                        •  
                                                                          For example, to run the DFH walker test using VCSMX:
                                                                           
                                                                          sh run_sim.sh TEST=dfh_walker VCSMX=1\n
                                                                           
                                                                          •  
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Once the test is complete, check the output for the simulation results. Review the log for detailed test results.
                                                                         
                                                                        Test status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/ofs-agx7-pcie-attach/sim/unit_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time         356233750000\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 356233750000 fs\nCPU Time:     57.730 seconds;       Data structure size:  47.2Mb\nTue Sep  5 09:44:19 2023\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                                                                         
                                                                        7.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#33-regression-unit-tests","title":"3.3 Regression Unit Tests","text":"
                                                                      You may use the regression script regress_run.py to run some or all of the unit tests available with a single command. The regression script is in the following location:
                                                                       
                                                                      $OFS_ROOTDIR/sim/unit_test/scripts/regress_run.py\n
                                                                       
                                                                      The usage of the regression script is as follows:
                                                                       
                                                                      regress_run.py [-h] [-l] [-n <num_procs>] [-k <test_package>] [-s <simulator>] [-g] [--ofss <ip_config>] [-b <board_name>] [-e]\n
                                                                       
                                                                      The Regression Unit Test Script Options table describes the options for the regress_run.py script.
                                                                       
                                                                      Table: Regression Unit Test Script Options
                                                                       Field Options Description -h | --help N/A Show the help message and exit -l | --local N/A Run regression locally -n | --n_procs Integer Maximum number of processes/tests to run in parallel when run locally. This has no effect on farm runs. -k | --pack all | fme | he | hssi | list | mem | pmci Test package to run during regression. The \"list\" option will look for a text file named \"list.txt\" in the \"unit_test\" directory for a text list of tests to run (top directory names). The default test package is all. -s | --sim vcs | vcsmx | msim Specifies the simulator used for the regression tests. The default simulator is vcs -g | --gen_sim_files N/A Generate simulation files. This only needs to be done once per repo update. This is the equivalent of running the gen_sim_files.sh script. -o | --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. -b | --board_name iseries-dk Specifies the board target -e | --email_list String Specifies email list to send results to multiple recipients 
                                                                      The log for each unit test that is run by the regression script is stored in a transcript file in the simulation directory of the test that was run.
                                                                       
                                                                      $OFS_ROOTDIR/sim/unit_test/<TEST_NAME>/<SIMULATOR>/transcript\n
                                                                       
                                                                      For example, the log for the DFH walker test using VCSMX would be found at:
                                                                       
                                                                      $OFS_ROOTDIR/sim/unit_test/dfh_walker/sim_vcsmx/transcript\n
                                                                       
                                                                      The simulation waveform database is saved as vcdplus.vpd for post simulation review.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#331-walkthrough-running-regression-unit-level-simulation","title":"3.3.1 Walkthrough: Running Regression Unit Level Simulation","text":"
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Create a test list file to only run the unit level simulations that are supported for the iseries-dk FIM.
                                                                         
                                                                        touch $OFS_ROOTDIR/sim/unit_test/list.txt\n
                                                                         
                                                                        Copy the following list into the new file. You may remove tests from this list as desired.
                                                                         
                                                                        ./bfm_test/set_params.sh\n./csr_test/set_params.sh\n./dfh_walker/set_params.sh\n./flr/set_params.sh\n./fme_csr_access/set_params.sh\n./fme_csr_directed/set_params.sh\n./he_lb_test/set_params.sh\n./he_null_test/set_params.sh\n./hssi_csr_test/set_params.sh\n./hssi_kpi_test/set_params.sh\n./hssi_test/set_params.sh\n./indirect_csr/set_params.sh\n./pcie_csr_test/set_params.sh\n./pf_vf_access_test/set_params.sh\n./port_gasket_test/set_params.sh\n./qsfp_test/set_params.sh\n./remote_stp_test/set_params.sh\n./uart_csr/set_params.sh\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Run regression test with the your desired options. For example, to simulate with the options to generate simulation files, run locally, use 8 processes, run the specified test list, use VCSMX simulator, and target the iseries-dk:
                                                                         
                                                                        cd $OFS_ROOTDIR/sim/unit_test/scripts\n\npython regress_run.py --ofss $OFS_ROOTDIR/tools/ofss_config/iseries-dk.ofss -g -l -n 8 -k list -s vcsmx -b iseries-dk\n
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Once all tests are complete, check that the tests have passed.
                                                                         
                                                                        2023-08-30 15:00:50,256: Passing Unit Tests:13/13 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-08-30 15:00:50,256:    bfm_test:......... PASS -- Time Elapsed:0:22:23.940917\n2023-08-30 15:00:50,256:    csr_test:......... PASS -- Time Elapsed:0:22:46.262916\n2023-08-30 15:00:50,256:    dfh_walker:....... PASS -- Time Elapsed:0:22:16.544732\n2023-08-30 15:00:50,256:    flr:.............. PASS -- Time Elapsed:0:22:21.332386\n2023-08-30 15:00:50,256:    fme_csr_access:... PASS -- Time Elapsed:0:17:12.454034\n2023-08-30 15:00:50,256:    fme_csr_directed:. PASS -- Time Elapsed:0:17:22.947134\n2023-08-30 15:00:50,256:    he_lb_test:....... PASS -- Time Elapsed:0:28:38.962424\n2023-08-30 15:00:50,256:    indirect_csr:..... PASS -- Time Elapsed:0:21:15.387478\n2023-08-30 15:00:50,256:    pcie_csr_test:.... PASS -- Time Elapsed:0:22:33.838949\n2023-08-30 15:00:50,256:    pf_vf_access_test: PASS -- Time Elapsed:0:22:28.704149\n2023-08-30 15:00:50,256:    port_gasket_test:. PASS -- Time Elapsed:0:22:32.592301\n2023-08-30 15:00:50,256:    remote_stp_test:.. PASS -- Time Elapsed:0:22:01.485914\n2023-08-30 15:00:50,256:    uart_csr:......... PASS -- Time Elapsed:0:22:31.848882\n2023-08-30 15:00:50,256: Failing Unit Tests: 0/13 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-08-30 15:00:50,256: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n2023-08-30 15:00:50,256:       Number of Unit test results captured: 13\n2023-08-30 15:00:50,256:       Number of Unit test results passing.: 13\n2023-08-30 15:00:50,256:       Number of Unit test results failing.:  0\n2023-08-30 15:00:50,256:     End Unit regression running at date/time................: 2023-08-30 15:00:50.256725\n2023-08-30 15:00:50,256:     Elapsed time for Unit regression run....................: 0:54:48.172625\n
                                                                         
                                                                        6.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#4-fim-customization","title":"4. FIM Customization","text":"
                                                                      This section describes how to perform specific customizations of the FIM, and provides step-by-step walkthroughs for these customizations. Each walkthrough can be done independently. These walkthroughs require a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment. The FIM Customization Walkthroughs table lists the walkthroughs that are provided in this section. Some walkthroughs include steps for testing on hardware. Testing on hardware requires that you have a deployment environment set up. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                                                       
                                                                      Table: FIM Customization Walkthroughs
                                                                       Customization Walkthrough Name Add a new module to the OFS FIM Modify and run unit tests for a FIM that has a new module Hardware test a FIM that has a new module Debug the FIM with Signal Tap Compile the FIM in preparation for designing your AFU Resize the Partial Reconfiguration Region Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS Modify PCIe Configuration Using IP Presets Migrate to a Different Agilex Device Number Modify the Ethernet Sub-System to 1x400GbE"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#41-adding-a-new-module-to-the-fim","title":"4.1 Adding a new module to the FIM","text":"
                                                                      This section provides a information for adding a custom module to the FIM, simulating the new design, compiling the new design, implementing and testing the new design on hardware, and debugging the new design on hardware.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#411-hello-fim-theory-of-operation","title":"4.1.1 Hello FIM Theory of Operation","text":"
                                                                      If you intend to add a new module to the FIM area, then you will need to inform the host software of the new module. The FIM exposes its functionalities to host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). This set of CSR registers and their operation is described in FIM MMIO Regions.
                                                                       
                                                                      See FPGA Device Feature List (DFL) Framework Overview for a description of the software process to read and process the linked list of Device Feature Header (DFH) CSRs within a FPGA.
                                                                       
                                                                      This example will add a hello_fim module to the design. The Hello FIM example adds a simple DFH register and 64bit scratchpad register connected to the Board Peripheral Fabric (BPF) that can be accessed by the Host. You can use this example as the basis for adding a new feature to your FIM.
                                                                       
                                                                      The Hello FIM design can be verified by Unit Level simulation, Universal Verification Methodology (UVM) simulation, and running in hardware on the iseries-dk card. The process for these are described in this section.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#4111-hello-fim-board-peripheral-fabric-bpf","title":"4.1.1.1 Hello FIM Board Peripheral Fabric (BPF)","text":"
                                                                      The Hello FIM module will be connected to the Board Peripheral Fabric (BPF), and will be connected such that it can be mastered by the Host. The BPF is an interconnect generated by Platform Designer. The Hello FIM BPF Interface Diagram figure shows the APF/BPF Master/Slave interactions, as well as the added Hello FIM module.
                                                                       
                                                                      Figure: Hello FIM BPF Interface Diagram
                                                                       
                                                                       
                                                                      The BPF fabric is defined in $OFS_ROOTDIR/src/pd_qsys/fabric/bpf.txt.
                                                                       
                                                                      We will add the Hello FIM module to an un-used address space in the MMIO region. The Hello FIM MMIO Address Layout table below shows the MMIO region for the Host with the Hello FIM module added at base address 0x16000.
                                                                       
                                                                      Table: Hello FIM MMIO Address Layout
                                                                       Offset Feature CSR set 0x00000 FME AFU 0x10000 PCIe Interface 0x12000 QSFP Controller 0 0x13000 QSFP Controller 1 0x14000 HSSI Interface 0x15000 EMIF Interface 0x16000 Hello FIM"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#4112-hello-fim-csr","title":"4.1.1.2 Hello FIM CSR","text":"
                                                                      The Hello FIM CSR will consist of the three registers shown in the Hello FIM CSR table below. The DFH and Hello FIM ID registers are read-only. The Scratchpad register supports read and write accesses.
                                                                       
                                                                      Table: Hello FIM CSR
                                                                       Offset Attribute Description Default Value 0x016000 RO DFH(Device Feature Headers) register 0x30000006a0000100 0x016030 RW Scrachpad register 0x0 0x016038 RO Hello FIM ID register 0x6626070150000034"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#412-walkthrough-add-a-new-module-to-the-ofs-fim","title":"4.1.2 Walkthrough: Add a new module to the OFS FIM","text":"
                                                                      Perform the following steps to add a new module to the OFS FIM that can be accessed by the Host.
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Make hello_fim source directory
                                                                         
                                                                        mkdir $OFS_ROOTDIR/src/hello_fim\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Create hello_fim_top.sv file.
                                                                         
                                                                        touch $OFS_ROOTDIR/src/hello_fim/hello_fim_top.sv\n
                                                                         
                                                                        Copy the following code into hello_fim_top.sv:
                                                                         
                                                                        // ***************************************************************************\n//                               INTEL CONFIDENTIAL\n//\n//        Copyright (C) 2023 Intel Corporation All Rights Reserved.\n//\n// The source code contained or described herein and all  documents related to\n// the  source  code  (\"Material\")  are  owned  by  Intel  Corporation  or its\n// suppliers  or  licensors.    Title  to  the  Material  remains  with  Intel\n// Corporation or  its suppliers  and licensors.  The Material  contains trade\n// secrets  and  proprietary  and  confidential  information  of  Intel or its\n// suppliers and licensors.  The Material is protected  by worldwide copyright\n// and trade secret laws and treaty provisions. No part of the Material may be\n// used,   copied,   reproduced,   modified,   published,   uploaded,  posted,\n// transmitted,  distributed,  or  disclosed  in any way without Intel's prior\n// express written permission.\n//\n// No license under any patent,  copyright, trade secret or other intellectual\n// property  right  is  granted  to  or  conferred  upon  you by disclosure or\n// delivery  of  the  Materials, either expressly, by implication, inducement,\n// estoppel or otherwise.  Any license under such intellectual property rights\n// must be express and approved by Intel in writing.\n//\n// You will not, and will not allow any third party to modify, adapt, enhance, \n// disassemble, decompile, reverse engineer, change or create derivative works \n// from the Software except and only to the extent as specifically required by \n// mandatory applicable laws or any applicable third party license terms \n// accompanying the Software.\n//\n// -----------------------------------------------------------------------------\n// Engineer     : \n// Create Date  : September 2023\n// Module Name  : hello_fim_top.sv\n// Project      : OFS\n// -----------------------------------------------------------------------------\n//\n// Description: \n// This is a simple module that implements DFH registers and \n// AVMM address decoding logic.\n\nmodule hello_fim_top  #(\n   parameter ADDR_WIDTH  = 12, \n   parameter DATA_WIDTH = 64, \n   parameter bit [11:0] FEAT_ID = 12'h001,\n   parameter bit [3:0]  FEAT_VER = 4'h1,\n   parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n   parameter bit END_OF_LIST = 1'b0\n)(\n   input  logic    clk,\n   input  logic    reset,\n// -----------------------------------------------------------\n//  AXI4LITE Interface\n// -----------------------------------------------------------\n   ofs_fim_axi_lite_if.slave   csr_lite_if\n);\n\nimport ofs_fim_cfg_pkg::*;\nimport ofs_csr_pkg::*;\n\n//-------------------------------------\n// Signals\n//-------------------------------------\n   logic [ADDR_WIDTH-1:0]              csr_waddr;\n   logic [DATA_WIDTH-1:0]              csr_wdata;\n   logic [DATA_WIDTH/8-1:0]            csr_wstrb;\n   logic                               csr_write;\n   logic                               csr_slv_wready;\n   csr_access_type_t                   csr_write_type;\n\n   logic [ADDR_WIDTH-1:0]              csr_raddr;\n   logic                               csr_read;\n   logic                               csr_read_32b;\n   logic [DATA_WIDTH-1:0]              csr_readdata;\n   logic                               csr_readdata_valid;\n   logic [ADDR_WIDTH-1:0]              csr_addr;\n\n   logic [63:0]                        com_csr_writedata;\n   logic                               com_csr_read;\n   logic                               com_csr_write;\n   logic [63:0]                        com_csr_readdata;\n   logic                               com_csr_readdatavalid;\n   logic [5:0]                         com_csr_address;\n\n// AXI-M CSR interfaces\nofs_fim_axi_mmio_if #(\n   .AWID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .AWADDR_WIDTH (ADDR_WIDTH),\n   .WDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH),\n   .ARID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .ARADDR_WIDTH (ADDR_WIDTH),\n   .RDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH)\n) csr_if();\n\n// AXI4-lite to AXI-M adapter\naxi_lite2mmio axi_lite2mmio (\n   .clk       (clk),\n   .rst_n     (~reset),\n   .lite_if   (csr_lite_if),\n   .mmio_if   (csr_if)\n);\n\n//---------------------------------\n// Map AXI write/read request to CSR write/read,\n// and send the write/read response back\n//---------------------------------\nofs_fim_axi_csr_slave #(\n   .ADDR_WIDTH (ADDR_WIDTH),\n   .USE_SLV_READY (1'b1)\n\n   ) csr_slave (\n   .csr_if             (csr_if),\n\n   .csr_write          (csr_write),\n   .csr_waddr          (csr_waddr),\n   .csr_write_type     (csr_write_type),\n   .csr_wdata          (csr_wdata),\n   .csr_wstrb          (csr_wstrb),\n   .csr_slv_wready     (csr_slv_wready),\n   .csr_read           (csr_read),\n   .csr_raddr          (csr_raddr),\n   .csr_read_32b       (csr_read_32b),\n   .csr_readdata       (csr_readdata),\n   .csr_readdata_valid (csr_readdata_valid)\n);\n\n// Address mapping\nassign csr_addr             = csr_write ? csr_waddr : csr_raddr;\nassign com_csr_address      = csr_addr[5:0];  // byte address\nassign csr_slv_wready       = 1'b1 ;\n// Write data mapping\nassign com_csr_writedata    = csr_wdata;\n\n// Read-Write mapping\nalways_comb\nbegin\n   com_csr_read             = 1'b0;\n   com_csr_write            = 1'b0;\n   casez (csr_addr[11:6])\n      6'h00 : begin // Common CSR\n         com_csr_read       = csr_read;\n         com_csr_write      = csr_write;\n      end   \n      default: begin\n         com_csr_read       = 1'b0;\n         com_csr_write      = 1'b0;\n      end\n   endcase\nend\n\n// Read data mapping\nalways_comb begin\n   if (com_csr_readdatavalid) begin\n      csr_readdata       = com_csr_readdata;\n      csr_readdata_valid = 1'b1;\n   end\n   else begin\n      csr_readdata       = '0;\n      csr_readdata_valid = 1'b0;\n   end\nend\n\nhello_fim_com  #(\n   .FEAT_ID          (FEAT_ID),\n   .FEAT_VER         (FEAT_VER),\n   .NEXT_DFH_OFFSET  (NEXT_DFH_OFFSET),\n   .END_OF_LIST      (END_OF_LIST)\n) hello_fim_com_inst (\n   .clk                   (clk                     ),\n   .reset                 (reset                   ),\n   .writedata             (com_csr_writedata       ),\n   .read                  (com_csr_read            ),\n   .write                 (com_csr_write           ),\n   .byteenable            (4'hF                    ),\n   .readdata              (com_csr_readdata        ),\n   .readdatavalid         (com_csr_readdatavalid   ),\n   .address               (com_csr_address         )\n   );\nendmodule\n
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Create hello_fim_com.sv file.
                                                                         
                                                                        touch $OFS_ROOTDIR/src/hello_fim/hello_fim_com.sv\n
                                                                         
                                                                        Copy the following code to hello_fim_com.sv:
                                                                         
                                                                        module hello_fim_com #(\n  parameter bit [11:0] FEAT_ID = 12'h001,\n  parameter bit [3:0]  FEAT_VER = 4'h1,\n  parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n  parameter bit END_OF_LIST = 1'b0\n)(\ninput clk,\ninput reset,\ninput [63:0] writedata,\ninput read,\ninput write,\ninput [3:0] byteenable,\noutput reg [63:0] readdata,\noutput reg readdatavalid,\ninput [5:0] address\n);\n\nwire reset_n = !reset;  \nreg [63:0] rdata_comb;\nreg [63:0] scratch_reg;\n\nalways @(negedge reset_n ,posedge clk)  \n   if (!reset_n) readdata[63:0] <= 64'h0; else readdata[63:0] <= rdata_comb[63:0];\n\nalways @(negedge reset_n , posedge clk)\n   if (!reset_n) readdatavalid <= 1'b0; else readdatavalid <= read;\n\nwire wr = write;\nwire re = read;\nwire [5:0] addr = address[5:0];\nwire [63:0] din  = writedata [63:0];\nwire wr_scratch_reg = wr & (addr[5:0]  == 6'h30)? byteenable[0]:1'b0;\n\n// 64 bit scratch register\nalways @( negedge  reset_n,  posedge clk)\n   if (!reset_n)  begin\n      scratch_reg <= 64'h0;\n   end\n   else begin\n   if (wr_scratch_reg) begin \n      scratch_reg <=  din;  \n   end\nend\n\nalways @ (*)\nbegin\nrdata_comb = 64'h0000000000000000;\n   if(re) begin\n      case (addr)  \n        6'h00 : begin\n                rdata_comb [11:0]   = FEAT_ID ;  // dfh_feature_id  is reserved or a constant value, a read access gives the reset value\n                rdata_comb [15:12]  = FEAT_VER ;  // dfh_feature_rev    is reserved or a constant value, a read access gives the reset value\n                rdata_comb [39:16]  = NEXT_DFH_OFFSET ;  // dfh_dfh_ofst is reserved or a constant value, a read access gives the reset value\n                rdata_comb [40]     = END_OF_LIST ;        //dfh_end_of_list\n                rdata_comb [59:40]  = 20'h00000 ;  // dfh_rsvd1     is reserved or a constant value, a read access gives the reset value\n                rdata_comb [63:60]  = 4'h3 ;  // dfh_feat_type  is reserved or a constant value, a read access gives the reset value\n        end\n        6'h30 : begin\n                rdata_comb [63:0]   = scratch_reg; \n        end\n        6'h38 : begin\n                rdata_comb [63:0]       = 64'h6626_0701_5000_0034;\n        end\n        default : begin\n                rdata_comb = 64'h0000000000000000;\n        end\n      endcase\n   end\nend\n\nendmodule\n
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Create hello_fim_design_files.tcl file.
                                                                         
                                                                        touch $OFS_ROOTDIR/src/hello_fim/hello_fim_design_files.tcl\n
                                                                         
                                                                        Copy the following code into hello_fim_design_files.tcl
                                                                         
                                                                        # Copyright 2023 Intel Corporation.\n#\n# THIS SOFTWARE MAY CONTAIN PREPRODUCTION CODE AND IS PROVIDED BY THE\n# COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED\n# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE\n# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR\n# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\n# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR\n# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,\n# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE\n# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,\n# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# Hello FIM Files\n#--------------------\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_com.sv\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_top.sv\n
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Modify $OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_top.qsf to include Hello FIM module
                                                                         
                                                                        ######################################################\n# Verilog Macros\n######################################################\n.....\nset_global_assignment -name VERILOG_MACRO \"INCLUDE_HELLO_FIM\"     # Includes Hello FIM\n
                                                                         
                                                                        8.  
                                                                      8.  
                                                                        Modify $OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_top_sources.tcl to include Hello FIM design files
                                                                         
                                                                        ############################################\n# Design Files\n############################################\n...\n# Subsystems\n...\nset_global_assignment -name SOURCE_TCL_SCRIPT_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_design_files.tclset_global_assignment -name SOURCE_TCL_SCRIPT_FILE ../setup/hello_fim_design_files.tcl\n
                                                                         
                                                                        9.  
                                                                      9.  
                                                                        Modify $OFS_ROOTDIR/src/pd_qsys/fabric/fabric_design_files.tcl to include BPF Hello FIM Slave IP.
                                                                         
                                                                        #--------------------\n# BPF\n#--------------------\n...\nset_global_assignment -name IP_FILE   $::env(BUILD_ROOT_REL)/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip\n
                                                                         
                                                                        10.  
                                                                      10.  
                                                                        Modify $OFS_ROOTDIR/src/includes/fabric_width_pkg.sv to add Hello FIM slave information and update EMIF slave next offset.
                                                                         
                                                                        localparam bpf_hello_fim_slv_baseaddress = 'h16000;    // New\nlocalparam bpf_hello_fim_slv_address_width = 12;       // New\nlocalparam bpf_emif_slv_next_dfh_offset = 'h1000;      // Old value: 'hB000\nlocalparam bpf_hello_fim_slv_next_dfh_offset = 'hA000; // New\nlocalparam bpf_hello_fim_slv_eol = 'b0;                // New\n
                                                                         
                                                                        11.  
                                                                      11.  
                                                                        Modify $OFS_ROOTDIR/src/top/top.sv
                                                                         
                                                                           
                                                                        1.  
                                                                          Add bpf_hello_fim_slv_if to AXI interfaces
                                                                           
                                                                          // AXI4-lite interfaces\n...\nofs_fim_axi_lite_if #(.AWADDR_WIDTH(fabric_width_pkg::bpf_hello_fim_slv_address_width), .ARADDR_WIDTH(fabric_width_pkg::bpf_hello_fim_slv_address_width)) bpf_hello_fim_slv_if();\n
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Add Hello FIM instantiation
                                                                           
                                                                          //*******************************\n// Hello FIM Subsystem\n//*******************************\n\n`ifdef INCLUDE_HELLO_FIM\nhello_fim_top #(\n   .ADDR_WIDTH       (fabric_width_pkg::bpf_hello_fim_slv_address_width),\n   .DATA_WIDTH       (64),\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset),\n   .END_OF_LIST      (fabric_width_pkg::bpf_hello_fim_slv_eol)\n) hello_fim_top_inst (\n    .clk (clk_csr),\n    .reset(~rst_n_csr),\n    .csr_lite_if    (bpf_hello_fim_slv_if)\n);\n`else\ndummy_csr #(\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset),\n   .END_OF_LIST      (fabric_width_pkg::bpf_hello_fim_slv_eol)\n) hello_fim_dummy (\n   .clk         (clk_csr),\n   .rst_n       (rst_n_csr),\n   .csr_lite_if (bpf_hello_fim_slv_if)\n);\n\n`endif\n
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Add interfaces for Hello FIM slv to bpf instantiation
                                                                           
                                                                          bpf bpf (\n...\n  .bpf_hello_fim_slv_awaddr   (bpf_hello_fim_slv_if.awaddr  ),\n  .bpf_hello_fim_slv_awprot   (bpf_hello_fim_slv_if.awprot  ),\n  .bpf_hello_fim_slv_awvalid  (bpf_hello_fim_slv_if.awvalid ),\n  .bpf_hello_fim_slv_awready  (bpf_hello_fim_slv_if.awready ),\n  .bpf_hello_fim_slv_wdata    (bpf_hello_fim_slv_if.wdata   ),\n  .bpf_hello_fim_slv_wstrb    (bpf_hello_fim_slv_if.wstrb   ),\n  .bpf_hello_fim_slv_wvalid   (bpf_hello_fim_slv_if.wvalid  ),\n  .bpf_hello_fim_slv_wready   (bpf_hello_fim_slv_if.wready  ),\n  .bpf_hello_fim_slv_bresp    (bpf_hello_fim_slv_if.bresp   ),\n  .bpf_hello_fim_slv_bvalid   (bpf_hello_fim_slv_if.bvalid  ),\n  .bpf_hello_fim_slv_bready   (bpf_hello_fim_slv_if.bready  ),\n  .bpf_hello_fim_slv_araddr   (bpf_hello_fim_slv_if.araddr  ),\n  .bpf_hello_fim_slv_arprot   (bpf_hello_fim_slv_if.arprot  ),\n  .bpf_hello_fim_slv_arvalid  (bpf_hello_fim_slv_if.arvalid ),\n  .bpf_hello_fim_slv_arready  (bpf_hello_fim_slv_if.arready ),\n  .bpf_hello_fim_slv_rdata    (bpf_hello_fim_slv_if.rdata   ),\n  .bpf_hello_fim_slv_rresp    (bpf_hello_fim_slv_if.rresp   ),\n  .bpf_hello_fim_slv_rvalid   (bpf_hello_fim_slv_if.rvalid  ),\n  .bpf_hello_fim_slv_rready   (bpf_hello_fim_slv_if.rready  ),\n...\n);\n
                                                                           
                                                                          4.  
                                                                         
                                                                        12.  
                                                                      12.  
                                                                        Modify $OFS_ROOTDIR/src/pd_qsys/fabric/bpf.txt to add the hello_fim module as a slave to the apf.
                                                                         
                                                                        # NAME   FABRIC      BASEADDRESS    ADDRESS_WIDTH SLAVES\napf         mst     n/a             18            fme,pcie,pmci,qsfp0,qsfp1,emif,hssi,hello_fim\n...\nhello_fim   slv     0x16000         12            n/a\n
                                                                         
                                                                        13.  
                                                                      13.  
                                                                        Execute helper script to generate BPF design files
                                                                         
                                                                        cd $OFS_ROOTDIR/src/pd_qsys/fabric/\n\nsh gen_fabrics.sh\n
                                                                         
                                                                        14.  
                                                                      14.  
                                                                        Once the script completes, the following new IP is created: $OFS_ROOTDIR/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip.
                                                                         
                                                                        15.  
                                                                      15.  
                                                                        [OPTIONAL] You may verify the BPF changes have been made correctly by opening bpf.qsys to analyze the BPF.
                                                                         
                                                                        cd $OFS_ROOTDIR/src/pd_qsys/fabric\n\nqsys-edit bpf.qsys --quartus-project=$OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_top.qpf\n
                                                                         
                                                                        Find the bpf_hello_fim_slv instance:
                                                                         
                                                                         
                                                                        16.  
                                                                      16.  
                                                                        Compile the Hello FIM design
                                                                         
                                                                        cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk_hello_fim\n
                                                                         
                                                                        17.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#413-walkthrough-modify-and-run-unit-tests-for-a-fim-that-has-a-new-module","title":"4.1.3 Walkthrough: Modify and run unit tests for a FIM that has a new module","text":"
                                                                      Perform the following steps to modify the unit test files to support a FIM that has had a new module added to it.
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                         
                                                                      • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design in order to simulate.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Modify $OFS_ROOTDIR/sim/unit_test/dfh_walker/test_csr_defs.sv
                                                                         
                                                                           
                                                                        1.  
                                                                          Add HELLO_FIM_IDX entry to t_dfh_idx enumeration.
                                                                           
                                                                          ...\ntypedef enum {\n    FME_DFH_IDX,\n    THERM_MNGM_DFH_IDX,\n    GLBL_PERF_DFH_IDX,\n    GLBL_ERROR_DFH_IDX,\n    QSFP0_DFH_IDX,\n    QSFP1_DFH_IDX,\n    HSSI_DFH_IDX,\n    EMIF_DFH_IDX,\n    HELLO_FIM_DFH_IDX,  // New\n    PMCI_DFH_IDX,\n    ST2MM_DFH_IDX,\n    VUART_DFH_IDX,\n    PG_PR_DFH_IDX,\n    PG_PORT_DFH_IDX,\n    PG_USER_CLK_DFH_IDX,\n    PG_REMOTE_STP_DFH_IDX,\n    AFU_ERR_DFH_IDX,\n    MAX_DFH_IDX\n} t_dfh_idx;\n...\n
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Add HELLO_FIM_DFH to get_dfh_names function.
                                                                           
                                                                          ...\nfunction automatic dfh_name[MAX_DFH_IDX-1:0] get_dfh_names();\n...\n  dfh_names[PMCI_DFH_IDX]        = \"PMCI_DFH\";\n  dfh_names[HELLO_FIM_DFH_IDX]   = \"HELLO_FIM_DFH\";  // New\n  dfh_names[ST2MM_DFH_IDX]       = \"ST2MM_DFH\";\n...\nreturn dfh_names;\n...\n
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Add expected DFH value for Hello FIM to the get_dfh_values function.
                                                                           
                                                                          ...\nfunction automatic [MAX_DFH_IDX-1:0][63:0] get_dfh_values();\n...\n  dfh_values[PMCI_DFH_IDX]       = 64'h3_00000_xxxxxx_1012;\n  dfh_values[PMCI_DFH_IDX][39:16] = fabric_width_pkg::bpf_pmci_slv_next_dfh_offset;\n\n  dfh_values[HELLO_FIM_DFH_IDX]  = 64'h3_00000_xxxxxx_0100;  // New\n  dfh_values[HELLO_FIM_DFH_IDX][39:16] = fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset; // New\n\n  dfh_values[ST2MM_DFH_IDX]      = 64'h3_00000_xxxxxx_0014;\n  dfh_values[ST2MM_DFH_IDX][39:16] = fabric_width_pkg::apf_st2mm_slv_next_dfh_offset;\n...\nreturn dfh_values;\n...\n
                                                                           
                                                                          4.  
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Generate simulation files
                                                                         
                                                                        cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/iseries-dk.ofss iseries-dk\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Run DFH Walker Simulation   
                                                                        cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\nsh run_sim.sh TEST=dfh_walker\n
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Verify that the test passes, and that the output shows the Hello FIM in the DFH sequence
                                                                         
                                                                        ********************************************\n Running TEST(0) : test_dfh_walking\n********************************************\n\n...\n\nTX: Tag Search Comparison: CPLD Tag: 00a   Active Tag: 00a\nTX: PU Completion ADDED to transaction!\nTX: Match found for PU CPLD!\nRead64  Method Data Transaction: Address: 0000_0000_0001_5000   Data: 3000_0000_1000_1009\nEMIF_DFH\n   Address   (0x15000)\n   DFH value (0x3000000010001009)\n\nTX: Tag Search Comparison: CPLD Tag: 00b   Active Tag: 00b\nTX: PU Completion ADDED to transaction!\nTX: Match found for PU CPLD!\nRead64  Method Data Transaction: Address: 0000_0000_0001_6000   Data: 3000_0000_a000_0100\nHELLO_FIM_DFH\n   Address   (0x16000)\n   DFH value (0x30000000a0000100)\n\nTX: Tag Search Comparison: CPLD Tag: 00c   Active Tag: 00c\nTX: PU Completion ADDED to transaction!\nTX: Match found for PU CPLD!\nRead64  Method Data Transaction: Address: 0000_0000_0002_0000   Data: 3000_0002_0000_1012\nPMCI_DFH\n   Address   (0x20000)\n   DFH value (0x3000000200001012)\n\n...\n\nTest status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\n$finish called from file \"/home/applications.fpga.ofs.fim-n6001/sim/unit_test/dfh_walker/unit_test.sv\", line 236.\n$finish at simulation time 13720.141ns\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 13720141000 fs\nCPU Time:      3.290 seconds;       Data structure size:   5.8Mb\nMon Dec  4 09:27:50 2023\nTotal of 5 minutes elapsed for dfh_walker\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                                                                         
                                                                        6.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#414-walkthrough-hardware-test-a-fim-that-has-a-new-module","title":"4.1.4 Walkthrough: Hardware test a FIM that has a new module","text":"
                                                                      Perform the following steps to program and hardware test a FIM that has had a new module added to it.
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                                                        •  
                                                                       
                                                                         
                                                                      • This walkthrough uses a FIM design that has been generated with a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for generating a Hello FIM design.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        [OPTIONAL] In the work directory where the FIM was compiled, determine the PR Interface ID of your design. You can use this value at the end of the walkthrough to verify that the design has been configured to the FPGA.
                                                                         
                                                                        cd $OFS_ROOTDIR/<work_directory>/syn/board/iseries-dk/syn_top/\n\ncat fme-ifc-id.txt\n
                                                                         
                                                                        Example output:
                                                                         
                                                                        5bcd682f-5093-5fc7-8cd2-ae8073e19452\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Switch to your deployment environment.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Program the FPGA with the Hello FIM image. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Run fpgainfo to determine the PCIe B:D.F of your board, and to verify the PR Interface ID matches the ID you found in Step 1.
                                                                         
                                                                        fpgainfo fme\n
                                                                         
                                                                        Example output:
                                                                         
                                                                        Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:b1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : TBD \nBitstream Version                : 5.0.1\nPr Interface Id                  : 5bcd682f-5093-5fc7-8cd2-ae8073e19452\nBoot Page                        : N/A\n
                                                                         
                                                                        Note: The errors related to the BMC are the result of the OFS BMC not being present on the iseries-dk design. These will be removed in a future release.
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Initialize opae.io
                                                                         
                                                                        sudo opae.io init -d <B:D.F>\n
                                                                         
                                                                        For example:
                                                                         
                                                                        sudo opae.io init -d b1:00.0\n
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Run DFH walker. Note the value read back from offset 0x16000 indicates the DFH ID is 0x100 which matches the Hello FIM module.
                                                                         
                                                                        sudo opae.io walk -d <B:D.F>\n
                                                                         
                                                                        For example:
                                                                         
                                                                        sudo opae.io walk -d b1:00.0\n
                                                                         
                                                                        Example output:
                                                                         
                                                                        ...\noffset: 0x15000, value: 0x3000000010001009\n    dfh: id = 0x9, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x16000, value: 0x30000000a0000100\n    dfh: id = 0x100, rev = 0x0, next = 0xa000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x20000, value: 0x3000000200001012\n    dfh: id = 0x12, rev = 0x1, next = 0x20000, eol = 0x0, reserved = 0x0, feature_type = 0x3\n...\n
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Read all of the registers in the Hello FIM module
                                                                         
                                                                           
                                                                        1.  
                                                                          Read the DFH Register
                                                                           
                                                                          opae.io -d b1:00.0 -r 0 peek 0x16000\n
                                                                           
                                                                          Example Output:
                                                                           
                                                                          0x30000006a0000100\n
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Read the Scratchpad Register
                                                                           
                                                                          opae.io -d b1:00.0 -r 0 peek 0x16030\n
                                                                           
                                                                          Example Output:
                                                                           
                                                                          0x0\n
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Read the ID Register
                                                                           
                                                                          opae.io -d b1:00.0 -r 0 peek 0x16038\n
                                                                           
                                                                          Example Output:
                                                                           
                                                                          0x6626070150000034\n
                                                                           
                                                                          4.  
                                                                         
                                                                        8.  
                                                                      8.  
                                                                        Verify the scratchpad register at 0x16030 by writing and reading back from it.
                                                                         
                                                                           
                                                                        1.  
                                                                          Write to Scratchpad register
                                                                           
                                                                          opae.io -d b1:00.0 -r 0 poke 0x16030 0x123456789abcdef\n
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Read from Scratchpad register
                                                                           
                                                                          opae.io -d b1:00.0 -r 0 peek 0x16030\n
                                                                           
                                                                          Expected output:
                                                                           
                                                                          0x123456789abcdef\n
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Write to Scratchpad register
                                                                           
                                                                          opae.io -d b1:00.0 -r 0 poke 0x16030 0xfedcba9876543210\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Read from Scratchpad register
                                                                           
                                                                          opae.io -d b1:00.0 -r 0 peek 0x16030\n
                                                                           
                                                                          Expected output:
                                                                           
                                                                          0xfedcba9876543210\n
                                                                           
                                                                          5.  
                                                                         
                                                                        9.  
                                                                      9.  
                                                                        Release the opae.io tool
                                                                         
                                                                        opae.io release -d b1:00.0\n
                                                                         
                                                                        10.  
                                                                      10.  
                                                                        Confirm the driver has been set back to dfl-pci
                                                                         
                                                                        opae.io ls\n
                                                                         
                                                                        Example output:
                                                                         
                                                                        [0000:b1:00.0] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                                                                         
                                                                        11.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#415-walkthrough-debug-the-fim-with-signal-tap","title":"4.1.5 Walkthrough: Debug the FIM with Signal Tap","text":"
                                                                      The following steps guide you through the process of adding a Signal Tap instance to your design. The added Signal Tap instance provides hardware to capture the desired internal signals and connect the stored trace information via JTAG. Please be aware that the added Signal Tap hardware will consume FPGA resources and may require additional floorplanning steps to accommodate these resources. Some areas of the FIM use logic lock regions and these regions may need to be re-sized.
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                         
                                                                      • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                                                        •  
                                                                       
                                                                         
                                                                      • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design.
                                                                        •  
                                                                       
                                                                      Perform the following steps in your development environment:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Synthesize the design using the -e build script option. You may skip this step if you are using a pre-synthesized design.
                                                                         
                                                                        cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -e --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_hello_fim_with_stp\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Open the design in Quartus. The Compilation Dashboard should show that the Analysis & Synthesis step has completed.
                                                                         
                                                                        quartus $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/iseries-dk/syn_top/ofs_top.qpf\n
                                                                         
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Open Tools -> Signal Tap Logic Analyzer
                                                                         
                                                                         
                                                                           
                                                                        1.  
                                                                          Select the Default template and click Create
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Assign the clock for sampling the Signal Tap instrumented signals of interest. Note that the clock selected should correspond to the signals you want to view for best trace fidelity. Different clocks can be used, however, there maybe issues with trace inaccuracy due to sampling time differences. This example instruments the hello_fim_top module previously intetegrated into the FIM. If unfamiliar with code, it is helpful to use the Quartus Project Navigator to find the block of interest and open the design instance for review.
                                                                           
                                                                             
                                                                          1.  
                                                                            In the Signal Configuration -> Clock box of the Signal Tap Logic Analyzer window, click the \"...\" button
                                                                             
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            In the Node Finder tool that opens, select hello_fim_top_inst in the Look in: field. Type clk into the Named field, then click Search. Select the hello_fim_top_inst|clk signal from the Matching Nodes box and click the \">\" button to move it to the Nodes Found box. Click OK to close the Node Finder dialog.
                                                                             
                                                                             
                                                                            3.  
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Update the sample depth and other Signal Tap settings as needed for your debugging criteria.
                                                                           
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          In the Signal Tap GUI add the nodes to be instrumented by double-clicking on the \"Double-click to add nodes\" legend.
                                                                           
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          This brings up the Node Finder to add the signals to be traced. In this example we will monitor the memory mapped interface to the Hello FIM. Enter csr_lite_if* in the Named: field. Select hello_fim_top_inst in the Look in: field.
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Set the Object type: field to net_bus then click Search. Select the signals that appear in the Matching Nodes column, then click the > button.
                                                                           
                                                                           
                                                                          7.  
                                                                        7.  
                                                                          Set the Object type: field to net then click Search. Select the non-bus signals that appear in the Matching Nodes column, then click the > button. Click Insert and close the Node Finder dialog.
                                                                           
                                                                           
                                                                          8.  
                                                                        8.  
                                                                          To provide a unique name for your Signal Tap instance, select \"auto_signaltap_0\", right-click, and select Rename Instance (F2). Provide a descriptive name for your instance, for example, stp_for_hello_fim.
                                                                           
                                                                           
                                                                          9.  
                                                                        9.  
                                                                          Save the newly created Signal Tap file, in the $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/iseries-dk/syn_top/ directory, and give it the same name as the instance. Ensure that the Add file to current project checkbox is ticked.
                                                                           
                                                                           
                                                                          10.  
                                                                        10.  
                                                                          In the dialog that pops up, click \"Yes\" to add the newly created Signal Tap file to the project settings files.
                                                                           
                                                                           
                                                                          This will aurtomatically add the following lines to $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/iseries-dk/syn_top/ofs_top.qsf:
                                                                           
                                                                          set_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE STP_For_Hello_FIM.stp\nset_global_assignment -name SIGNALTAP_FILE STP_For_Hello_FIM.stp\n
                                                                           
                                                                          11.  
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Close all Quartus GUIs.
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Compile the project with the Signal Tap file added to the project. Use the -k switch to perform the compilation using the files in a specified working directory and not the original ones from the cloned repository. 
                                                                         
                                                                        ./ofs-common/scripts/common/syn/build_top.sh -p -k --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_hello_fim_with_stp\n
                                                                         
                                                                        8.  
                                                                      8.  
                                                                        Ensure that the compile completes successfully and meets timing:
                                                                         
                                                                        ***********************************\n***\n***        OFS_PROJECT: ofs-agx7-pcie-attach\n***        OFS_BOARD: iseries-dk\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 1\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                                                         
                                                                        9.  
                                                                      9.  
                                                                        Set up a JTAG connection to the iseries-dk. Refer to Set up JTAG section for step-by-step instructions.
                                                                         
                                                                        10.  
                                                                      10.  
                                                                        Copy the ofs_top.sof and stp_for_hello_fim.stp files to the machine which is connected to the iseries-dk via JTAG.
                                                                         
                                                                        11.  
                                                                      11.  
                                                                        From the JTAG connected machine, program the $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/iseries-dk/syn_top/output_files/ofs_top.sof image to the iseries-dk FPGA. Refer to the Program the FPGA via JTAG section for step-by-step programming instructions.
                                                                         
                                                                        12.  
                                                                      12.  
                                                                        Open the Quartus Signal Tap GUI
                                                                         
                                                                        $QUARTUS_ROOTDIR/bin/quartus_stpw\n
                                                                         
                                                                        13.  
                                                                      13.  
                                                                        In the Signal Tap Logic Analyzer window, select File -> Open, and choose the stp_for_hello_fim.stp file.
                                                                         
                                                                         
                                                                        14.  
                                                                      14.  
                                                                        In the right pane of the Signal Tap GUI, in the Hardware: selection box select the cable for the iseries-dk. In the Device: selection box select the Agilex device.
                                                                         
                                                                         
                                                                        15.  
                                                                      15.  
                                                                        If the Agilex Device is not displayed in the Device: list, click the 'Scan Chain' button to re-scan the JTAG device chain.
                                                                         
                                                                        16.  
                                                                      16.  
                                                                        Create the trigger conditions. In this example, we will capture data on a rising edge of the Read Address Valid signal.
                                                                         
                                                                         
                                                                        17.  
                                                                      17.  
                                                                        Start analysis by selecting the 'stp_for_hello_fim' instance and pressing 'F5' or clicking the Run Analysis icon in the toolbar. You should see a green message indicating the Acquisition is in progress. Then, move to the Data Tab to observe the signals captured.
                                                                         
                                                                         
                                                                        18.  
                                                                      18.  
                                                                        To generate traffic in the csr_lite_if signals of the Hello FIM module, walk the DFH list or peek/poke the Hello FIM registers.
                                                                         
                                                                        opae.io init -d 0000:b1:00.0\nopae.io walk -d 0000:b1:00.0\nopae.io release -d 0000:b1:00.0\n
                                                                         
                                                                        The signals should be captured on the rising edge of arvalid in this example. Zoom in to get a better view of the signals.
                                                                         
                                                                         
                                                                        19.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#42-preparing-fim-for-afu-development","title":"4.2 Preparing FIM for AFU Development","text":"
                                                                      To save area, the default Host Excercisers in the FIM can be replaced by a \"he_null\" blocks. There are a few things to note:
                                                                       
                                                                         
                                                                      • \"he_null\" is a minimal block with registers that responds to PCIe MMIO request. MMIO responses are required to keep PCIe alive (end points enabled in PCIe-SS needs service downstream requests).
                                                                        •  
                                                                      • If an exerciser with other I/O connections such as \"he_mem\" or \"he_hssi\" is replaced, then then those I/O ports are simply tied off.
                                                                        •  
                                                                      • The options supported are null_he_lb, null_he_hssi, null_he_mem and null_he_mem_tg. Any combination, order or all can be enabled at the same time. 
                                                                        •  
                                                                      • Finer grain control is provided for you to, for example, turn off only the exercisers in the Static Region in order to save area.
                                                                        •  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#421-walkthrough-compile-the-fim-in-preparation-for-designing-your-afu","title":"4.2.1 Walkthrough: Compile the FIM in preparation for designing your AFU","text":"
                                                                      Perform the following steps to compile a FIM for where the exercisers are removed and replaced with an he_null module while keeping the PF/VF multiplexor connections.
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the [Walkthrough: Set Up Development Environment] Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the FIM repository (or use an existing cloned repository). Refer to the [Walkthrough: Clone FIM Repository] section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the [Walkthrough: Set Development Environment Variables] section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Compile the FIM with the HE_NULL compile options
                                                                         
                                                                        cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss iseries-dk:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_iseries-dk\n
                                                                         
                                                                        4.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#43-partial-reconfiguration-region","title":"4.3 Partial Reconfiguration Region","text":"
                                                                      To take advantage of the available resources in the Intel\u00ae Agilex\u00ae 7 FPGA for an AFU design, you can adjust the size of the AFU PR partition. An example reason for the changing the size of PR region is if you add more logic to the FIM region, then you may need to adjust the PR region to fit the additional logic into the static region.  Similarly, if you reduce logic in the FIM region, then you can adjust the PR region to provide more logic resources for the AFU.
                                                                       
                                                                      After the compilation of the FIM, the resources usage broken down by partitions as reported in the following file:
                                                                       
                                                                      $OFS_ROOTDIR/<WORDK_DIRECTORY>/syn/board/iseries-dk/syn_top/output_files/ofs_top.fit.rpt\n
                                                                       
                                                                      An example report of the resources usage by partitions defined for the FIM is shown as follows:
                                                                       
                                                                      +------------------------------------------------------------------------------------------+\n; Logic Lock Region Constraints                                                            ;\n+--------------------------------------+-------------------------+-------------------------+\n; Name                                 ; Place Region Constraint ; Route Region Constraint ;\n+--------------------------------------+-------------------------+-------------------------+\n; afu_top|port_gasket|pr_slot|afu_main ; (90, 40) to (350, 220)  ; (0, 0) to (385, 329)    ;\n+--------------------------------------+-------------------------+-------------------------+\n\n\n+----------------------------------------------------------------------------------------------+\n; Logic Lock Region Usage Summary                                                              ;\n+-------------------------------------------------------+--------------------------------------+\n; Statistic                                             ; afu_top|port_gasket|pr_slot|afu_main ;\n+-------------------------------------------------------+--------------------------------------+\n; ALMs needed [=A-B+C]                                  ; 48011.2 / 351140 ( 13 % )            ;\n;     [A] ALMs used in final placement                  ; 53324.4 / 351140 ( 15 % )            ;\n;     [B] Estimate of ALMs recoverable by dense packing ; 5452.3 / 351140 ( 1 % )              ;\n;     [C] Estimate of ALMs unavailable                  ; 139.0 / 351140 ( < 1 % )             ;\n; ALMs used for memory                                  ; 450.0                                ;\n; Combinational ALUTs                                   ; 67166                                ;\n; Dedicated Logic Registers                             ; 87533 / 1404560 ( 6 % )              ;\n; I/O Registers                                         ; 0                                    ;\n; Block Memory Bits                                     ; 1737568                              ;\n; M20Ks                                                 ; 137 / 5049 ( 2 % )                   ;\n; DSP Blocks needed [=A-B]                              ; 0 / 3439 ( 0 % )                     ;\n;     [A] DSP Blocks used in final placement            ; 0 / 3439 ( 0 % )                     ;\n;     [B] Estimate of DSPs recoverable by dense merging ; 0 / 3439 ( 0 % )                     ;\n; Pins                                                  ; 0                                    ;\n; IOPLLs                                                ; 0                                    ;\n;                                                       ;                                      ;\n; Region Placement                                      ; (90, 40) to (350, 220)               ;\n+-------------------------------------------------------+--------------------------------------+\n
                                                                       
                                                                      In this case, the default size for the afu_top|port_gasket|pr_slot|afu_main PR partition is large enough to hold the logic of the default AFU, which is mainly occupied by the Host Exercisers. However, larger designs might require additional resources.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#431-walkthrough-resize-the-partial-reconfiguration-region","title":"4.3.1 Walkthrough: Resize the Partial Reconfiguration Region","text":"
                                                                      Perform the following steps to first analyze the PR logic lock regions in a default FIM design, then resize the PR region:
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        The OFS flow provides the TCL file $OFS_ROOTDIR/syn/board/iseries-dk/setup/pr_assignments.tcl which defines the PR partition where the AFU is allocated. Each region is a rectangle defined by the origin coordinate (X0, Y0) and the top right corner coordinate (X1, Y1).
                                                                         
                                                                        #####################################################\n# Main PR Partition -- green_region\n#####################################################\nset_instance_assignment -name PARTITION green_region -to afu_top|pg_afu.port_gasket|pr_slot|afu_main\nset_instance_assignment -name CORE_ONLY_PLACE_REGION ON -to afu_top|pg_afu.port_gasket|pr_slot|afu_main\nset_instance_assignment -name RESERVE_PLACE_REGION ON -to afu_top|pg_afu.port_gasket|pr_slot|afu_main\nset_instance_assignment -name PARTIAL_RECONFIGURATION_PARTITION ON -to afu_top|pg_afu.port_gasket|pr_slot|afu_main\n\n\nset_instance_assignment -name PLACE_REGION \"X90 Y40 X350 Y220\" -to afu_top|pg_afu.port_gasket|pr_slot|afu_main\nset_instance_assignment -name ROUTE_REGION \"X0 Y0 X385 Y329\" -to afu_top|pg_afu.port_gasket|pr_slot|afu_main\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        [OPTIONAL] Use Quartus Chip Planner to visualize the default PR region allocation.
                                                                         
                                                                           
                                                                        1.  
                                                                          Compile the design.
                                                                           
                                                                          ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk\n
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Once the design is compiled, open it in Quartus.
                                                                           
                                                                          quartus $OFS_ROOTDIR/work_iseries-dk/syn/board/iseries-dk/syn_top/ofs_top.qpf\n
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Switch to ofs_top.
                                                                           
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          click Tools -> Chip Planner to open the Chip Planner.
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Analyze the regions shown. Note that the regions are made of rectangles described by an origin coordinate, region height, and region width. If you are modifying the regions, you will need to identify the coordinates of your desired region.
                                                                           
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Close the Quartus GUI.
                                                                           
                                                                          7.  
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Make changes to the partial reconfiguraton region in the $OFS_ROOTDIR/syn/board/iseries-dk/setup/pr_assignments.tcl file. You can modify the size and location of existing lock regions or add new ones and assign them to the AFU PR partition.
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Recompile your FIM and create the PR relocatable build tree using the following commands.
                                                                         
                                                                        cd $OFS_ROOTDIR    \n\nofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk_resize_pr\n
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Analyze the resource utilization report $OFS_ROOTDIR/work_iseries-dk/syn/board/iseries-dk/syn_top/output_files/ofs_top.fit.rpt produced after recompiling the FIM.
                                                                         
                                                                        8.  
                                                                      8.  
                                                                        Perform further modification to the PR regions until the results are satisfactory. Make sure timing constraints are met.
                                                                         
                                                                        9.  
                                                                       
                                                                      For more information on how to optimize the floor plan of your Partial Reconfiguration design refer to the following online documentation.
                                                                       
                                                                         
                                                                      • Analyzing and Optimizing the Design Floorplan
                                                                        •  
                                                                       
                                                                         
                                                                      • Partial Reconfiguration Design Flow - Step 3 - Floorplan the Design
                                                                        •  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#44-pcie-configuration","title":"4.4 PCIe Configuration","text":"
                                                                      The PCIe Subsystem can be easily modified using OFS provided script and the PCIe subsystem IP core.  In this section both the PCIe SR-IOV configuration and PCIe configuration registers will be modified.  You can use this process for setting up your specific settings.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#441-pcie-ss-configuration-registers","title":"4.4.1 PCIe-SS Configuration Registers","text":"
                                                                      The PCIe-SS configuration registers contain the Vendor, Device and Subsystem Vendor ID registers which are used in PCIe add-in cards to uniquely identify the card for assignment to software drivers. OFS has these registers set with Intel values for out of the box usage. If you are using OFS for a PCIe add in card that your company manufactures, then update the PCIe Subsytem Subsystem ID and Vendor ID registers as described below and change OPAE provided software code to properly operate with your company's register values.
                                                                       
                                                                      The Vendor ID is assigned to your company by the PCI-SIG (Special Interest Group). The PCI-SIG is the only body officially licensed to give out IDs. You must be a member of PCI-SIG to request your own ID. Information about joining PCI-SIG is available here: PCI-SIG. You select the Subsystem Device ID for your PCIe add in card.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#442-pfvf-mux-configuration","title":"4.4.2 PF/VF MUX Configuration","text":"
                                                                      The default PF/VF MUX configuration for OFS PCIe Attach FIM for the iseries-dk can support up to 8 PFs and 2000 VFs distributed accross all PFs.
                                                                       
                                                                      For reference FIM configurations, 0 VFs on PF0 is not supported. This is because the PR region cannot be left unconnected. A NULL AFU may need to be instantiated in this special case. PFs must be consecutive. the PF/VF Limitations table describes the supported number of PFs and VFs.
                                                                       
                                                                      Table: PF/VF Limitations
                                                                       Parameter Value Min # of PFs 1 (on PF0) Max # of PFs 8 Min # of VFs 1 on PF0 Max # of VFs 2000 distributed across all PFs 
                                                                      Please be aware, as the number of VFs goes up, timing closure can become more difficult.
                                                                       
                                                                      The scripts provided in ${OFS_ROOTDIR}/ofs-common/tools/ofss_config allows you to easily reconfigure the number of PFs and VFs, bar addresses, vendor/device ID values and more.  The PCIe Subsystem IP parameters that can be modified can be seen by reviewing ${OFS_ROOTDIR}/ofs-common/tools/ofss_config/ip_params/pcie_ss_component_parameters.py 
                                                                       
                                                                      New PF or VF instances will automatically have a null_afu module instantiated and connected to the new PF or VF.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#443-pcie-configuration-using-ofss","title":"4.4.3 PCIe Configuration Using OFSS","text":"
                                                                      The general flow for using OFSS to modify the PCIe Sub-system and PF/VF MUX is as follows:
                                                                       
                                                                         
                                                                      1. Create or modify a PCIe OFSS file with the desired PCIe configuration. 
                                                                        2.  
                                                                      2. Call this PCIe OFSS file when running the FIM build script.
                                                                        3.  
                                                                       
                                                                      The PCIe IP OFSS File Options table lists all of the configuration options supported by the OFSS flow. Any other customizations to the PCIe sub-system must be done with the IP Presets Flow.
                                                                       
                                                                      Table: PCIe IP OFSS File Options
                                                                       Section Parameter Options Default Description [ip] type pcie N/A Specifies that this OFSS file configures the PCIe-SS [settings] output_name pcie_ss N/A Specifies the output name of the PCIe-SS IP preset String N/A OPTIONAL - Specifies the name of a PCIe-SS IP presets file to use when building the FIM. When used, a presets file will take priority over any other parameters set in this OFSS file. pcie_gen 4 | 5 [1] N/A Specifies the PCIe generation pcie_instances 1 | 2 [1] N/A Specifies the number of PCIe instances [pf*] num_vfs Integer 0 Specifies the number of Virtual Functions in the current PF bar0_address_width Integer 12 bar4_address_width Integer 14 vf_bar0_address_width Integer 12 ats_cap_enable 0 | 1 0 vf_ats_cap_enable 0 | 1 0 prs_ext_cap_enable 0 | 1 0 pasid_cap_enable 0 | 1 0 pci_type0_vendor_id 32'h Value 0x00008086 pci_type0_device_id 32'h Value 0x0000bcce revision_id 32'h Value 0x00000001 class_code 32'h Value 0x00120000 subsys_vendor_id 32'h Value 0x00008086 subsys_dev_id 32'h Value 0x00001771 sriov_vf_device_id 32'h Value 0x0000bccf exvf_subsysid 32'h Value 0x00001771 
                                                                      [1] For the iseries-dk Gen5 2x8 is supported. Gen5 1x16 is not supported. Gen4 1x16 has not been validated.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#4431-walkthrough-modify-the-pcie-sub-system-and-pfvf-mux-configuration-using-ofss","title":"4.4.3.1 Walkthrough: Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS","text":"
                                                                      Perform the following steps to use OFSS files to configure the PCIe Sub-system and PF/VF MUX. In this example both the PCIe SR-IOV configuration and PCIe configuration registers will be modified.
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                         
                                                                      • To demonstrate updated PCIe PF/VF in hardware, use an OFS Agilex I-Series PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        View the default OFS PCIe Attach FIM for the iseries-dk PF/VF configuration in the the $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host_rtile.ofss file.
                                                                         
                                                                        [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n# pcie_ss top_topology_hw.tcl parameter using the variables below\n# assuming x16 lane, following will map to  \"Gen<pcie_gen> <pcie_instances>x<16/pcie_instances>\"\n# i.e. for R-tile PCIe SS example \"Gen5 2x8\". For combinations supported.\n# Note:\n# - \"Gen5 2x8\" is supported at the moment\n# - \"Gen5 1x16\" is not supported in PCIe SS\n# - \"Gen4 1x16\" should work but not validated\npcie_gen = 5\npcie_instances = 2\n\n[pf0]\nnum_vfs = 3\nbar0_address_width = 20\nvf_bar0_address_width = 20\n\n[pf1]\n\n[pf2]\nbar0_address_width = 18\n\n[pf3]\n\n[pf4]\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Create a new PCIe OFSS file from the existing pcie_host_rtile.ofss file
                                                                         
                                                                        cp $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host_rtile.ofss $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host_rtile_pfvf_new.ofss\n
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Configure the new pcie_host_rtile_pfvf_new.ofss with updated PF/VF settings. In this example the following changes are made:
                                                                         
                                                                           
                                                                        1. PF0: 1 VF is added by changing num_vfs from 3 to 4.
                                                                          2.  
                                                                        2. PF1, PF2: Vendor, device, subsystem vendor and subsystem device IDs are changed.
                                                                          3.  
                                                                        3. PF3: 1 VF is added by inserting num_vfs = 1
                                                                          4.  
                                                                        4. PF5, PF6 and PF7 are added to use the maximum supported number (8) PFs
                                                                          5.  
                                                                         
                                                                        The block diagram of the updated afu_top is shown below:
                                                                         
                                                                         
                                                                        You can use this example as a template for creating PCI settings for your specific requirements.
                                                                         
                                                                        [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n# pcie_ss top_topology_hw.tcl parameter using the variables below\n# assuming x16 lane, following will map to  \"Gen<pcie_gen> <pcie_instances>x<16/pcie_instances>\"\n# i.e. for R-tile PCIe SS example \"Gen5 2x8\". For combinations supported.\n# Note:\n# - \"Gen5 2x8\" is supported at the moment\n# - \"Gen5 1x16\" is not supported in PCIe SS\n# - \"Gen4 1x16\" should work but not validated\npcie_gen = 5\npcie_instances = 2\n\n[pf0]\nnum_vfs = 4\nbar0_address_width = 20\nvf_bar0_address_width = 20\n\n[pf1]\npci_type0_vendor_id = \"0x00001172\"\npci_type0_device_id = \"0x0000c001\"\nsubsys_vendor_id = \"0x00001172\"\nsubsys_dev_id = \"0x0000bad1\"\n\n[pf2]\nbar0_address_width = 18\npci_type0_vendor_id = \"0x00001172\"\npci_type0_device_id = \"0x00001122\"\nsubsys_vendor_id = \"0x00001172\"\nsubsys_dev_id = \"0x0000ddcc\"\n\n[pf3]\nnum_vfs = 1\n[pf4]\n\n[pf5]\n\n[pf6]\n\n[pf7]\n
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Edit the $OFS_ROOTDIR/tools/ofss_config/tools/ofss_config/iseries-dk.ofss file to use the new PCIe configuration file pcie_pfvf_new.ofss
                                                                         
                                                                        [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/iseries-dk_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host_rtile_pfvf_new.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory_rtile.ofss\n
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Compile the FIM. 
                                                                         
                                                                        cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk_pf8\n
                                                                         
                                                                        8.  
                                                                      8.  
                                                                        If needed, copy the resulting $OFS_ROOTDIR/work_iseries-dk_pfvf_pf8/syn/board/iseries-dk/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                                                                         
                                                                        9.  
                                                                      9.  
                                                                        Switch to your deployment environment.
                                                                         
                                                                        10.  
                                                                      10.  
                                                                        Program the ofs_top.sof image to the iseries-dk FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                                                                         
                                                                        11.  
                                                                      11.  
                                                                        Verify the 8 PFs are present with proper device ID.
                                                                         
                                                                        $ lspci | grep bcce\nad:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)\nad:00.3 Processing accelerators: Intel Corporation Device bcce (rev 01)\nad:00.4 Processing accelerators: Intel Corporation Device bcce (rev 01)\nad:00.5 Processing accelerators: Intel Corporation Device bcce (rev 01)\nad:00.6 Processing accelerators: Intel Corporation Device bcce (rev 01)\nad:00.7 Processing accelerators: Intel Corporation Device bcce (rev 01)\n\n  $ lspci | grep c001\nad:00.1 Processing accelerators: Altera Corporation Device c001 (rev 01)\n\n $ lspci | grep 1122\nad:00.2 Processing accelerators: Altera Corporation Device 1122 (rev 01)\n
                                                                         
                                                                        12.  
                                                                      12.  
                                                                        The PCIe Subssytem is setup in bifurcation mode so the unused PCIe channel will enumerate.  You can see this enumeration:
                                                                         
                                                                        $ lspci | grep Altera\nac:00.0 Non-VGA unclassified device: Altera Corporation Device 0000 (rev 01)\nad:00.1 Processing accelerators: Altera Corporation Device c001 (rev 01)\nad:00.2 Processing accelerators: Altera Corporation Device 1122 (rev 01)\n\nNote, ac:00.0 is unused PCIe channel.\n
                                                                         
                                                                        13.  
                                                                      13.  
                                                                        Verify the new VFs can be added.  Use the OPAE SDK command pci_device to create VFs.  Verify PF 0 and PF 3 have proper number of VFs and have device ID of bccf.
                                                                         
                                                                        $ sudo pci_device ad:00.0 vf 4\n$ sudo pci_device ad:00.3 vf 1\n$ sudo lspci -vvv -s ad:00.0 | grep VF\n                Initial VFs: 4, Total VFs: 4, Number of VFs: 4, Function Dependency Link: 00\n                VF offset: 8, stride: 1, Device ID: bccf\n                VF Migration: offset: 00000000, BIR: 0\n$ sudo lspci -vvv -s ad:00.3 | grep VF\n                Initial VFs: 1, Total VFs: 1, Number of VFs: 1, Function Dependency Link: 03\n                VF offset: 9, stride: 1, Device ID: bccf\n                VF Migration: offset: 00000000, BIR: 0\n
                                                                         
                                                                        14.  
                                                                      14.  
                                                                        Verify communication with the newly added PF5. The OFSS script creates a Null AFU with a basic set of command/status (CSR) registers connected to new PF/VF instances.  This basic set of CSRs can be used as the starting point for your new function.  In this step, the PF/VF is bound to the VFIO driver and the OPAE SDK commands opae.io peek is used to read the CSR registers in the Null AFU instance coonected to PF5.  
                                                                         
                                                                        You can use this mechanism to verify access to your newly developed AFU.
                                                                         
                                                                        The GUID for every new PF/VF that has the automatically instantiated null_afu is:
                                                                         
                                                                        * NULL_GUID_L           = 64'haa31f54a3e403501\n* NULL_GUID_H           = 64'h3e7b60a0df2d4850\n
                                                                         
                                                                           
                                                                        1.  
                                                                          Initialize the driver on PF5
                                                                           
                                                                          sudo opae.io init -d ad:00.5 $USER\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          Unbinding (0x8086,0xbcce) at 0000:ad:00.5 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:ad:00.5 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:ad:00.5 is 414\nAssigning /dev/vfio/414 to $USER\nChanging permissions for /dev/vfio/414 to rw-rw----\n
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Read the GUID for the PF5 CSR stub.
                                                                           
                                                                          opae.io -d ad:00.5 -r 0 peek 0x8\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          0xaa31f54a3e403501\n
                                                                           
                                                                          opae.io -d ad:00.5 -r 0 peek 0x10\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          0x3e7b60a0df2d4850\n
                                                                           
                                                                          3.  
                                                                         
                                                                        15.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#444-pcie-configuration-using-ip-presets","title":"4.4.4 PCIe Configuration Using IP Presets","text":"
                                                                      The general flow for using IP Presets to modify he PCIe Sub-system is as follows:
                                                                       
                                                                         
                                                                      1. [OPTIONAL] Create a work directory using OFSS files if you wish to use OFSS configuration settings.
                                                                        2.  
                                                                      2. Open the PCIe-SS IP and make desired modifications.
                                                                        3.  
                                                                      3. Create an IP Presets file.
                                                                        4.  
                                                                      4. Create an PCIe OFSS file that uses the IP Presets file.
                                                                        5.  
                                                                      5. Build the FIM with the PCIe OFSS file from Step 4.
                                                                        6.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#4441-walkthrough-modify-pcie-configuration-using-ip-presets","title":"4.4.4.1 Walkthrough: Modify PCIe Configuration Using IP Presets","text":"
                                                                      Perform the following steps to use OFSS files to configure the PCIe Sub-system and PF/VF MUX. In this example, we will change the Revision ID of PF0. While this modification can be done with the OFSS flow, this walkthrough is intended to show the procedure for making any PCIe configuration change using IP presets.
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                         
                                                                      • To demonstrate updated PCIe PF/VF in hardware, use an OFS Agilex I-Series PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Run the setup stage of the build script using your desired OFSS configration to create a working directory for the iseries-dk design.
                                                                         
                                                                        ./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Open the PCIe-SS in the work directory using Quartus Parameter Editor.
                                                                         
                                                                        qsys-edit $OFS_ROOTDIR/work_iseries-dk/ipss/pcie/qip/pcie_ss.ip\n
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        In the IP Parameter Editor window, scroll down and select the PCIe Interfaces Port Settings -> Port 0 -> PCIe0 Device Identification Registers -> PCIe0 PF0 IDs tab. Modify the settings as desired. In this case, we are changing the Revision ID to 0x00000002. You may make any desired modifications.
                                                                         
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Once you are satisfied with your modifcations, create a new IP Preset file.
                                                                         
                                                                           
                                                                        1.  
                                                                          click New... in the Presets window.
                                                                           
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          In the New Preset window, set a unique Name for the preset; for example, iseries-dk-rev2.
                                                                           
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Click the ... button to set the save location for the IP Preset file to $OFS_ROOTDIR/ipss/pcie/presets. Set the File Name to match the name selected in Step 9. Click OK.
                                                                           
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          In the New Preset window, click Save. Click No when prompted to add the file to the IP search path.
                                                                           
                                                                           
                                                                          5.  
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Close IP Parameter Editor without saving or generating HDL.
                                                                         
                                                                        8.  
                                                                      8.  
                                                                        Create a new PCIe OFSS file in the $OFS_ROOTDIR/tools/ofss_config/pcie directory. For example:
                                                                         
                                                                        touch $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host_rtile_mod_preset.ofss\n
                                                                         
                                                                        Insert the following into the OFSS file to specify the IP Preset file created in Step 8. 
                                                                         
                                                                        [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\npreset = iseries-dk-rev2\n
                                                                         
                                                                        9.  
                                                                      9.  
                                                                        Edit the $OFS_ROOTDIR/tools/ofss_config/iseries-dk.ofss file to call new OFSS file created in Step 10.
                                                                         
                                                                        [include]  \"$OFS_ROOTDIR\"/tools/ofss_config/iseries-dk_base.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host_rtile_mod_preset.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory_rtile.ofss
                                                                         
                                                                        10.  
                                                                      10.  
                                                                        Compile the design with the modified iseries-dk.ofss file.
                                                                         
                                                                        cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk work_iseries-dk_pcie_mod\n
                                                                         
                                                                        11.  
                                                                      11.  
                                                                        Copy the resulting $OFS_ROOTDIR/work_iseries-dk_pcie_mod/syn/board/iseries-dk/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                                                                         
                                                                        12.  
                                                                      12.  
                                                                        Switch to your deployment environment.
                                                                         
                                                                        13.  
                                                                      13.  
                                                                        Program the ofs_top.sof image to the iseries-dk FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                                                                         
                                                                        14.  
                                                                      14.  
                                                                        Determing the PCIe B:D.F of your board. You may use the OPAE command fpgainfo fme to determine this.
                                                                         
                                                                        fpgainfo fme\n
                                                                         
                                                                        Example output:
                                                                         
                                                                        Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:84:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x501020226F0E360\nBitstream Version                : 5.0.1\nPr Interface Id                  : 7428037b-5408-52f2-8b1d-707870ec518a\nBoot Page                        : N/A\n
                                                                         
                                                                        Note: The errors related to the BMC are the result of the OFS BMC not being present on the iseries-dk design. These will be removed in a future release.
                                                                         
                                                                        15.  
                                                                      15.  
                                                                        Use lspci with the PCIe B:D.F of your board to verify that the PCIe changes have been implemented. In this example, the Rev for PF0 is 02.
                                                                         
                                                                        lspci -nvmms 84:00.0\n
                                                                         
                                                                        Example output:
                                                                         
                                                                        Slot:   84:00.0\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nRev:    02\nNUMANode:       1\nIOMMUGroup:     70\n
                                                                         
                                                                        16.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#45-minimal-fim","title":"4.5 Minimal FIM","text":"
                                                                      In a minimal FIM, the exercisers are removed and the PCIe SR-IOV PF/VF infrastructure is reduced to 1 PF and 1 VF. This minimal FIM is useful for oneAPI BSP applications.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#451-create-a-minimal-fim","title":"4.5.1 Create a Minimal FIM","text":"
                                                                      Perform the following steps to create a Minimal FIM.
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                         
                                                                      • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Create a new platform OFSS file that will use a 1PF/1VF PCIe configuration.
                                                                         
                                                                        cp $OFS_ROOTDIR/tools/ofss_config/pcie_host_rtile.ofss $OFS_ROOTDIR/tools/ofss_config/pcie_host_rtile_1pf_1vf.ofss\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Edit the $OFS_ROOTDIR/tools/ofss_config/pcie_host_rtile.ofss_1pf_1vf.ofss file to use the 1PF/1VF PCIe configuration as shown below:
                                                                         
                                                                        [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n# pcie_ss top_topology_hw.tcl parameter using the variables below\n# assuming x16 lane, following will map to  \"Gen<pcie_gen> <pcie_instances>x<16/pcie_instances>\"\n# i.e. for R-tile PCIe SS example \"Gen5 2x8\". For combinations supported.\n# Note:\n# - \"Gen5 2x8\" is supported at the moment\n# - \"Gen5 1x16\" is not supported in PCIe SS\n# - \"Gen4 1x16\" should work but not validated\npcie_gen = 5\npcie_instances = 2\n\n[pf0]\nnum_vfs = 1\nbar0_address_width = 20\nvf_bar0_address_width = 20\n
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Edit $OFS_ROOTDIR/tools/ofss_config/iseries-dk.ofss to use new PCIe file pcie_host_rtile_1pf_1vf.ofss as shown below:
                                                                         
                                                                        [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/iseries-dk_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host_rtile_1pf_1vf.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory_rtile.ofss\n
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Compile the FIM with Null HEs compile option, the No HSSI compile option, and 1PF/1VF configuration OFSS file.
                                                                         
                                                                        cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg,no_hssi work_iseries-dk_min\n
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Review the $OFS_ROOTDIR/work_iseries-dk_min/syn/board/iseries/syn_top/output_files/ofs_top.fit.rpt utilization report to see the utilization statistics for the Minimal fim. Refer to [Appendix A] Table A-4 for the expected utilization for this Minimal FIM.
                                                                         
                                                                        8.  
                                                                      8.  
                                                                        Copy the resulting $OFS_ROOTDIR/work_iseries-dk_min/syn/board/iseries/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                                                                         
                                                                        9.  
                                                                      9.  
                                                                        Switch to your deployment environment, if different than your development environment.
                                                                         
                                                                        10.  
                                                                      10.  
                                                                        Program the ofs_top.sof image to the iseries-dk FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step programming instructions.
                                                                         
                                                                        11.  
                                                                      11.  
                                                                        Use lspci to verify that the PCIe changes have been implemented.
                                                                         
                                                                        sudo lspci -vvv -s b1:00.0 | grep VF\n
                                                                         
                                                                        Example output: 
                                                                        Initial VFs: 1, Total VFs: 1, Number of VFs: 0, Function Dependency Link: 00\nVF offset: 1, stride: 1, Device ID: bcce\nVF Migration: offset: 00000000, BIR: 0\n
                                                                         
                                                                        12.  
                                                                      12.  
                                                                        You may wish to adjust the PR logic lock regions to maximize the resources available for the AFU. Refer to the Resize the Partial Reconfiguration Region section for instructions.
                                                                         
                                                                        13.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#46-migrating-to-a-different-agilex-device-number","title":"4.6 Migrating to a Different Agilex Device Number","text":"
                                                                      The following instructions enable a user to change the device part number of the Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile). Please be aware that this release works with Intel\u00ae Agilex\u00ae 7 FPGA devices that have F-tile for PCIe and Ethernet.  Other tiles will take further work.
                                                                       
                                                                      You may wish to change the device part number for the following reasons
                                                                       
                                                                         
                                                                      1. Migrate to same device package but with a different density
                                                                        2.  
                                                                      2. Migrate to a different package and with a different or same density
                                                                        3.  
                                                                       
                                                                      The default device for the Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) is AGIB027R29A1E2VR3
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#461-walkthrough-migrate-to-a-different-agilex-device-number","title":"4.6.1 Walkthrough: Migrate to a Different Agilex Device Number","text":"
                                                                      Perform the following steps to migrate to a different Agilex Device. In this example, we will migrate from the default AGIB027R29A1E2VR3 device to AGIB027R31A1E2VB. The package will change from R29A to R31A.
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Modify the part field in the $OFS_ROOTDIR/tools/ofss_config/iseries-dk_base.ofss file to use AGIB027R31A1E2VB.
                                                                         
                                                                        [ip]\ntype = ofs\n\n[settings]\nplatform = n6001\nfamily = agilex\nfim = base_x16\npart = AGIB027R31A1E2VB \ndevice_id = 6001\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Modify the DEVICE field in the $OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_top.qsf file to use AGIB027R31A1E2VB.
                                                                         
                                                                        ############################################################################################\n# FPGA Device\n############################################################################################\n\nset_global_assignment -name FAMILY \"Agilex 7\"\nset_global_assignment -name DEVICE AGIB027R31A1E2VB \n
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Modify the DEVICE field in the $OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_pr_afu.qsf file to use AGIB027R31A1E2VB.
                                                                         
                                                                        ############################################################################################\n# FPGA Device\n############################################################################################\n\nset_global_assignment -name FAMILY Agilex 7\nset_global_assignment -name DEVICE AGIB027R31A1E2VB \n
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        If the device you are migrating to uses the same package and pinout, you do not need to modify the pinout constraints. In this example, because we are migrating from package R29A to R31A, we need to modify the pinout to match the new device. If you would like Quartus to attempt to pin out the design automatically, you may remove all pin assignments. Typically, you will still need to set certain pins manually in order to guide Quartus for a successful compile (e.g. transceiver reference clocks).
                                                                         
                                                                           
                                                                        1.  
                                                                          Commment out all pin assignments in the following files:     * $OFS_ROOTDIR/syn/board/iseries-dk/setup/emif_loc.tcl     * $OFS_ROOTDIR/syn/board/iseries-dk/setup/top_loc.tcl
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Identify the pins in the design that will be constrained. In this example we will manually constrain the QSFP reference clock and the PCIe reference clock to help guide the fitter. The Device Migration Pinout Table shows th pin assignments that will be set, along with the pin number for both the old R24C package and the new R31C package.
                                                                           Net Name R29A Pin Name R31A Pin Name AGI 027 R29A Pin # AGI 027 R31A Pin # qsfp_ref_clk REFCLK_FGTL12A_Q2_RX_CH4p REFCLK_FGTL12C_Q2_RX_CH4p JD74 AM57 PCIE_REFCLK0 REFCLK_GXRL14C_CH0p REFCLK_GXRL14A_CH0p DR68 BU56 PCIE_REFCLK1 REFCLK_GXRL14C_CH1p REFCLK_GXRL14A_CH1p CU68 BP57 
                                                                          3.  
                                                                        3.  
                                                                          Constrain the pins identified in Step 6.B in the $OFS_ROOTDIR/syn/board/iseries-dk/setup/top_loc.tcl file for the new pinout for the AGF 027 R31C package. 
                                                                           
                                                                          set_location_assignment PIN_AM57 -to qsfp_ref_clk\n\nset_location_assignment PIN_BU56 -to PCIE_REFCLK0\nset_location_assignment PIN_BP57 -to PCIE_REFCLK1\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Uncomment the instance assignments related to he QSFP and PCIe reference clocks in the $OFS_ROOTDIR/syn/board/iseries-dk/setup/top_loc.tcl file.
                                                                           
                                                                          set_instance_assignment -name IO_STANDARD \"CURRENT MODE LOGIC (CML)\" -to qsfp_ref_clk\n\nset_instance_assignment -name IO_STANDARD HCSL -to PCIE_REFCLK0\nset_instance_assignment -name IO_STANDARD HCSL -to PCIE_REFCLK1\n
                                                                           
                                                                          5.  
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Compile a flat design. It is recommended to compile a flat design first before incorporating a PR region in the design.
                                                                         
                                                                        cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_8x25_ftile.ofss iseries-dk:flat work_iseries-dk_lrgdev\n
                                                                         
                                                                        8.  
                                                                      8.  
                                                                        Verify that the build completes successfuly. If there are timing violation, try building with a different seed. Refer to the Change the Compilation Seed section for instructions on changing the build seed.
                                                                         
                                                                        9.  
                                                                      9.  
                                                                        When you are satisfied with the pinout, preserve it by hard-coding the desired pinout to followig files:
                                                                         
                                                                           
                                                                        • $OFS_ROOTDIR/syn/board/iseries-dk/setup/emif_loc.tcl
                                                                          •  
                                                                        • $OFS_ROOTDIR/syn/board/iseries-dk/setup/top_loc.tcl
                                                                          •  
                                                                         
                                                                        10.  
                                                                      10.  
                                                                        When you are ready to re-incorporate PR into the design, modify the PR region to be compatible with the new device. Refer to the Resize the Partial Reconfiguration Region section for instructions.
                                                                         
                                                                        11.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#47-modify-the-ethernet-sub-system","title":"4.7 Modify the Ethernet Sub-System","text":"
                                                                      This section describes the flows for modifying the Ethernet Sub-System.
                                                                       
                                                                      Note: The default HSSI-SS configuration for the iseries-dk is 2x4x25GbE.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#471-walkthrough-modify-the-ethernet-sub-system-to-1x400gbe","title":"4.7.1 Walkthrough: Modify the Ethernet Sub-System to 1x400GbE","text":"
                                                                      OFS provides a preconfigured ofss file so the build script produces a FIM with a 1x400GbE Ethernet subsystem set for 400 GAUI-8.  You can build this system with the following:
                                                                       
                                                                         
                                                                      1.  
                                                                        Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Build the I-Series FIM with 1x400GbE FIM:
                                                                         
                                                                        ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_1x400_ftile.ofss iseries-dk work_iseries-dk_400\n
                                                                         
                                                                        4.  
                                                                       
                                                                      The resulting FIM with 1x400GbE has the following MAC settings:
                                                                       
                                                                      Table: 1x400GbE MAC Settings
                                                                       MAC Setting Value Client Interface Segmented FEC mode IEEE 802.3 RS(544.514) (CL 134) Auto Negotiation and link training Disabled Maximum Frame Size 1518 
                                                                      You can change the MAC settings by opening the Ethernet Subsystem IP in IP Parameter Editor, update the setting and then save the update as a new preset.  You will then edit the ofss_config/hssi/hssi_1x400_ftile.ofss to use the new preset.
                                                                       
                                                                      The following steps describe the steps to change the 400 GbE MAC frame size to 9600 bytes.  Note, the 2x200 and 8x25 GbE MAC implementations can be changed using this process.
                                                                       
                                                                         
                                                                      1.  
                                                                        Invoke IP Parameter editor to make changes to the Ethernet Subsystem IP.
                                                                         
                                                                        cd $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss\nqsys-edit hssi_ss.ip --quartus-project=$OFS_ROOTDIR/syn/board/iseries-dk/syn_top/ofs_top.qpf\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        In IP Parameter editor, load the 400g-fseries-dk preset by selecting and then click Apply
                                                                         
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        In the Device 0 Configuration tab, go to the F-Tile IP Configuration tab and scroll down to P8 MAC Options - P8 Basic and change the TX and RX maximum framesize to 9600.
                                                                         
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Click New in the Presets panel and in the New Preset pop up window, Name the new preset 400g-fseries-dk-9600 and in File enter $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss/presets/400g-fseries-dk-9600.qprs and click Save
                                                                         
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Close IP Parameter Editor and do not save changes to hssi_ss.ip.  The new preset captured the changes and this new preset will be used in the following updates to re-generate the Ethernet IP subsystem with the updated frame size.
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Create a new ofss file for the new preset.
                                                                         
                                                                        cp $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_1x400_ftile.ofss $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_1x400_ftile_9600.ofss\n
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Edit $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_1x400_ftile_9600.ofss to use the new 400g-fseries-dk-9600 preset as listed below:
                                                                         
                                                                        [ip]\ntype = hssi\n\n[settings]\noutput_name = hssi_ss\nnum_channels = 1\ndata_rate = 400GAUI-8\npreset = 400g-fseries-dk-9600\n
                                                                         
                                                                        8.  
                                                                      8.  
                                                                        Build the FIM with 9600 byte frame size by using the new hssi_1x400_ftile_9600.ofss file
                                                                         
                                                                        ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/iseries-dk.ofss,tools/ofss_config/hssi/hssi_1x400_ftile_9600.ofss iseries-dk work_iseries-dk_400_9600\n
                                                                         
                                                                        9.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#5-fpga-configuration","title":"5. FPGA Configuration","text":"
                                                                      Configuring the Agilex FPGA on the iseries-dk is done by programming a SOF image to the FPGA the embedded USvia JTAG using Quartus Programer.
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#51-walkthrough-set-up-jtag","title":"5.1 Walkthrough: Set up JTAG","text":"
                                                                      The Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) has an on-board FPGA Download Cable II module which is used to program the FPGA via JTAG.
                                                                       
                                                                      Perform the following steps to establish a JTAG connection with the iseries-dk.
                                                                       
                                                                      Pre-requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                                                        •  
                                                                       
                                                                         
                                                                      • This walkthrough requires a workstation with Quartus Prime Pro Version 23.3 tools installed, specifically the jtagconfig tool.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Refer to the following figure showing the location of the on-board Intel FPGA Download Cable II micro USB connector.
                                                                         
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Verify all switches are set to default as defined in Intel Agilex\u00ae 7 FPGA I-Series Development Kit User Guide.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Connect a Micro-USB to USB-A cable between the front panel J8 micro USB port and either the deployment server or an external computer that has Quartus Prime Pro Programming tools installed.
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Use the jtagconfig tool to check that the JTAG chain contains the AGIB027R29A1E2VR3 device.
                                                                         
                                                                        <QUARTUS_INSTALL_DIR>/23.3/quartus/bin/jtagconfig\n
                                                                         
                                                                        Example expected output:
                                                                         
                                                                        1) AGI FPGA Development Kit [1-13]\n  034BB0DD   AGIB027R29A(.|R2|R3)/..\n  020D10DD   VTAP10\n
                                                                         
                                                                        5.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#52-walkthrough-program-the-fpga-via-jtag","title":"5.2 Walkthrough: Program the FPGA via JTAG","text":"
                                                                      This walkthrough describes the steps to program the Agilex FPGA on the Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) with a SOF image via JTAG.
                                                                       
                                                                      Pre-Requisites:
                                                                       
                                                                         
                                                                      • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel Agilex 7 FPGA I-Series Development Kit (2xR-Tile, F-Tile)) for instructions on setting up a deployment environment.
                                                                        •  
                                                                       
                                                                         
                                                                      • This walkthrough requires a SOF image which will be programmed to the Agilex FPGA. Refer to the Compile OFS FIM Section for step-by-step instructions for generating a SOF image.
                                                                        •  
                                                                       
                                                                         
                                                                      • This walkthrough requires a JTAG connection to the iseries-dk. Refer to the Set up JTAG section for step-by-step instructions.
                                                                        •  
                                                                       
                                                                         
                                                                      • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) is connected via JTAG.
                                                                        •  
                                                                       
                                                                      Steps:
                                                                       
                                                                         
                                                                      1.  
                                                                        Start in your deployment environment.
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        If the card is already programmed with an OFS enabled design, determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                                                                         
                                                                        fpgainfo fme\n
                                                                         
                                                                        Example output:
                                                                         
                                                                        Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:84:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010202FAB46E6A\nBitstream Version                : 5.0.1\nPr Interface Id                  : b00b675b-a674-5849-9195-f662c92f5250\nBoot Page                        : N/A\n
                                                                         
                                                                        Note: The errors related to the BMC are the result of the OFS BMC not being present on the iseries-dk design. These will be removed in a future release.
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Remove the card from PCIe prior to programming. This will disable AER for the PCIe root port to prevent a surprise link-down event during programming.
                                                                         
                                                                        sudo pci_device 84:00.0 unplug\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Switch to the machine with JTAG connection to the iseries-dk, if different than your deployment machine.
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Open the Quartus programmer GUI
                                                                         
                                                                        quartus_pgmw\n
                                                                         
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Click Hardware Setup to open the Hardware Setup dialog window.
                                                                         
                                                                           
                                                                        1.  
                                                                          In the Currently selected hardware field select the iseries-dk.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          In the Hardware frequency field enter 16000000 Hz
                                                                           
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Click Close
                                                                           
                                                                          4.  
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        In the Quartus Prime Programmer window, click Auto Detect.
                                                                         
                                                                        8.  
                                                                      8.  
                                                                        If prompted, select the AGIB027R29A1E2VR3 device. The JTAG chain should show the device.
                                                                         
                                                                         
                                                                        9.  
                                                                      9.  
                                                                        Right click the AGIB027R29A1E2VR3 row and selct Change File.
                                                                         
                                                                         
                                                                        10.  
                                                                      10.  
                                                                        In the Select New Programming File window that opens, select the .sof image you wish to program and click Open.
                                                                         
                                                                        11.  
                                                                      11.  
                                                                        Check the Program/Configure box for the AGIB027R29A1E2VR3 row, then click Start. Wait for the Progress bar to show 100% (Success).
                                                                         
                                                                         
                                                                        12.  
                                                                      12.  
                                                                        Close the Quartus Programmer GUI.
                                                                         
                                                                        13.  
                                                                      13.  
                                                                        Switch to the deployment environment, if different than the JTAG connected machine.
                                                                         
                                                                        14.  
                                                                      14.  
                                                                        Replug the board PCIe
                                                                         
                                                                        sudo pci_device 84:00.0 plug\n
                                                                         
                                                                        15.  
                                                                      15.  
                                                                        Run fpgainfo fme to verify communication with the board, and to check the PR Interface ID.
                                                                         
                                                                        Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:84:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x5010202FAB46E6A\nBitstream Version                : 5.0.1\nPr Interface Id                  : ce31fe41-1d9b-572d-9ff1-9deae5c98b61\nBoot Page                        : N/A\n
                                                                         
                                                                        Note: The errors related to the BMC are the result of the OFS BMC not being present on the iseries-dk design. These will be removed in a future release.
                                                                         
                                                                        16.  
                                                                      "},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#appendix","title":"Appendix","text":""},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#appendix-a-resource-utilization-tables","title":"Appendix A: Resource Utilization Tables","text":"
                                                                      Table A-1 Default Flat FIM Resource Utilization
                                                                       Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 171138.4 18.75 767 5.78 afu_top 78519.0 8.6 249 1.88 auto_fab_0 1347.1 0.15 9 0.07 bpf 686.4 0.08 0 0.0 fme_top 633.0 0.07 6 0.05 hssi_wrapper 11887.5 1.3 81 0.61 mem_ss_top 5205.5 0.57 30 0.23 ofs_top_auto_tiles 4237.3 0.46 10 0.08 pcie_wrapper 66661.9 7.3 374 2.82 pmci_dummy_csr 670.5 0.07 0 0.0 qsfp_0 634.7 0.07 4 0.03 qsfp_1 633.6 0.07 4 0.03 rst_ctrl 17.3 0.0 0 0.0 sys_pll 0.5 0.0 0 0.0 
                                                                      Table A-2 Default Out-of-Tree FIM Resource Utilization
                                                                       Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 176794.4 19.37 767 5.78 afu_top 84201.6 9.22 249 1.88 auto_fab_0 1308.1 0.14 9 0.07 bpf 726.8 0.08 0 0.0 fme_top 630.2 0.07 6 0.05 hssi_wrapper 11957.6 1.31 81 0.61 mem_ss_top 5565.0 0.61 30 0.23 ofs_top_auto_tiles 4257.4 0.47 10 0.08 pcie_wrapper 66195.2 7.25 374 2.82 pmci_dummy_csr 671.0 0.07 0 0.0 qsfp_0 627.8 0.07 4 0.03 qsfp_1 631.0 0.07 4 0.03 rst_ctrl 18.6 0.0 0 0.0 sys_pll 0.5 0.0 0 0.0 
                                                                      Table A-3 Minimal FIM Resource Utilization
                                                                       Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 104858.0 11.49 524 3.95 afu_top 28211.9 3.09 107 0.81 auto_fab_0 1279.7 0.14 9 0.07 bpf 709.0 0.08 0 0.0 fme_top 617.9 0.07 6 0.05 hssi_dummy_csr 672.0 0.07 0 0.0 mem_ss_top 5513.7 0.6 30 0.23 pcie_wrapper 65808.3 7.21 372 2.8 pmci_dummy_csr 677.4 0.07 0 0.0 qsfp0_dummy_csr 677.3 0.07 0 0.0 qsfp1_dummy_csr 671.2 0.07 0 0.0 rst_ctrl 16.5 0.0 0 0.0 sys_pll 0.4 0.0 0 0.0"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#appendix-b-glossary","title":"Appendix B: Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/iseries_devkit/dev_guides/fim_dev/ug_ofs_iseries_dk_fim_dev/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                      Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                       
                                                                      OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/","title":"Getting Started Guide: Open FPGA Stack for Intel Agilex 7 FPGAs Targeting the Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile)","text":"
                                                                      Last updated: February 03, 2024 
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#10-about-this-document","title":"1.0 About This Document","text":"
                                                                      The purpose of this document is to help users get started in evaluating the 2023.3 version of the PCIe Attach release targeting the I-Series Development Kit. After reviewing this document, a user shall be able to:
                                                                       
                                                                         
                                                                      • Set up a server environment according to the Best Known Configuration (BKC)
                                                                        •  
                                                                      • Load and verify firmware targeting the FIM and AFU regions of the Agilex FPGA
                                                                        •  
                                                                      • Verify full stack functionality offered by the PCIe Attach OFS solution
                                                                        •  
                                                                      • Learn where to find additional information on other PCIe Attach ingredients
                                                                        •  
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#11-audience","title":"1.1 Audience","text":"
                                                                      The information in this document is intended for customers evaluating the PCIe Attach shell targeting Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile). This platform is a Development Kit intended to be used as a starting point for evaluation and development.
                                                                       
                                                                      Note: Code command blocks are used throughout the document. Commands that are intended for you to run are preceded with the symbol '$', and comments with '#'. Full command output may not be shown.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-1-terminology","title":"Table 1: Terminology","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-2-software-and-component-version-summary-for-ofs-pcie-attach-targeting-the-i-series-development-kit","title":"Table 2: Software and Component Version Summary for OFS PCIe Attach targeting the I-Series Development Kit","text":"
                                                                      The OFS 2023.3 PCIe Attach release is built upon tightly coupled software and Operating System version(s). The repositories listed below are used to manually build the Shell and the AFU portion of any potential workloads. Use this section as a general reference for the versions which compose this release. Specific instructions on building the FIM or AFU are discussed in their respective documents.
                                                                       Component Version Quartus https://www.intel.com/content/www/us/en/software-kit/782411/intel-quartus-prime-pro-edition-design-software-version-23-3-for-linux.html, patches: 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe) Host Operating System https://access.redhat.com/downloads/content/479/ver=/rhel---8/8.6/x86_64/product-software OneAPI-ASP https://github.com/OFS/oneapi-asp/releases/tag/ofs-2023.3-2, patches: 0.02 OFS Platform AFU BBB https://github.com/OFS/ofs-platform-afu-bbb/releases/tag/ofs-2023.3-2 OFS FIM Common Resources https://github.com/OFS/ofs-fim-common/releases/tag/ofs-2023.3-2 AFU Examples https://github.com/OFS/examples-afu/releases/tag/ofs-2023.3-2 OPAE-SIM https://github.com/OPAE/opae-sim"},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-3-programmable-firmware-version-summary-for-ofs-pcie-attach-targeting-the-i-series-development-kit","title":"Table 3: Programmable Firmware Version Summary for OFS PCIe Attach Targeting the I-Series Development Kit","text":"
                                                                      OFS releases include pre-built binaries for the FPGA, OPAE SDK and Linux DFL which can be programmed out-of-box (OOB) and include known identifiers shown below. Installation of artifacts provided with this release will be discussed in their relevant sections.
                                                                       Component Version Link FIM (shell) Pr Interface ID: TBD https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2 Host OPAE SDK https://github.com/OPAE/opae-sdk, tag: 2.10.0-1 https://github.com/OFS/opae-sdk/releases/tag/2.10.0-1 Host Linux DFL Drivers https://github.com/OPAE/linux-dfl, tag: ofs-2023.3-6.1-3 https://github.com/OFS/linux-dfl/releases/tag/ofs-2023.3-6.1-3"},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-4-hardware-bkc-for-ofs-pcie-attach","title":"Table 4: Hardware BKC for OFS PCIe Attach","text":"
                                                                      The following table highlights the hardware which composes the Best Known Configuation (BKC) for the OFS 2023.3 PCIe Attach release. The Intel FPGA Download Cable II is not required when using the I Series Dev Kit, as the device has an on-board blaster.
                                                                       Component Link Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) https://www.intel.com/content/www/us/en/products/details/fpga/development-kits/agilex/agi027.html (optional) Intel FPGA Download Cable II https://www.intel.com/content/www/us/en/products/sku/215664/intel-fpga-download-cable-ii/specifications.html"},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#12-server-requirements","title":"1.2 Server Requirements","text":""},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#121-host-server-specifications","title":"1.2.1 Host Server Specifications","text":"
                                                                      The host server must meet the following specifications:
                                                                       
                                                                         
                                                                      • The server platform should contain 128 GB of RAM to run certain demos, and to compile FIM Images
                                                                        •  
                                                                      • The server platform must be able to fit, power, and cool an Intel Agilex\u00ae 7 FPGA I-Series Development Kit (2x R-Tile and 1xF-Tile) as described by the product page
                                                                        •  
                                                                      • The server should be able to run PCIe at Gen 5 speeds to properly test designs and demos
                                                                        •  
                                                                      • The server must be able to properly power the I Series Dev Kit using an auxillary power cable as described in section 3.1 of the Intel Agilex\u00ae 7 FPGA I-Series Development Kit User Guide
                                                                        •  
                                                                       
                                                                      Note: Be sure to double check that purchased servers support the proper PCIe aux power cables.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#122-host-bios","title":"1.2.2 Host BIOS","text":"
                                                                      These are the host BIOS settings known to work with the I-Series Development Kit. Information about the server's currently loaded firmware and BIOS settings can be found through its remote access controller, or by manually entering the BIOS by hitting a specific key during power on. Your specific server platform will include instructions on proper BIOS configuration and should be followed when altering settings.
                                                                       
                                                                         
                                                                      • PCIe slot must be bifurcated x8 / x8
                                                                        •  
                                                                      • PCIe slot speed must be set to Gen 5
                                                                        •  
                                                                      • PCIe slot must have iommu enabled
                                                                        •  
                                                                      • Intel VT for Directed I/O (VT-d) must be enabled
                                                                        •  
                                                                       
                                                                      Specific BIOS paths are not listed here as they can differ between BIOS vendors and versions.
                                                                       
                                                                      In addition to BIOS settings required to support the operation of an OFS PCIe Attach solution targeting the I-Series Development Kit, server fan speed may need to be adjusted in the BIOS settings depending on local air temperature and air flow. The OFS PCIe Attach design does not automatically communicate cooling curve information with the on-board server management interface. Increasing air flow will mitigate possible thermal runaway and thermal throttling that may occur as a result. Refer to Board Thermal Requirements for more information.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#123-host-server-kernel-and-grub-configuration","title":"1.2.3 Host Server Kernel and GRUB Configuration","text":"
                                                                      While many host Linux kernel and OS distributions may work with this design, only the following configuration(s) have been tested:
                                                                       
                                                                         
                                                                      • OS: RedHat\u00ae Enterprise Linux\u00ae (RHEL) 8.6
                                                                        •  
                                                                      • Kernel: 6.1-lts
                                                                        •  
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#13-preparing-the-i-series-development-kit-for-installation-into-a-server","title":"1.3 Preparing the I-Series Development Kit for Installation into a Server","text":"
                                                                      Safety and Regulatory information can be found under the product page for this development kit.
                                                                       
                                                                      Ensure your device's switch configuration matches the default found in the Intel Agilex\u00ae 7 FPGA I-Series Development Kit User Guide. Physical installation of the board is covered in section 3.1 Applying Power to the Development Board.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#14-i-series-development-kit-jtag-setup","title":"1.4 I-Series Development Kit JTAG Setup","text":"
                                                                      A specific JTAG driver needs to be installed on the host OS. Follow the instructions under the driver setup for Red Hat 5+ on Intel\u00ae FPGA Download Cable (formerly USB-Blaster) Driver for Linux*.
                                                                       
                                                                      View the JTAG Chain after installing the proper driver and Quartus 23.3
                                                                       
                                                                      cd ~/intelFPGA_pro/quartus/bin\n./jtagconfig -D\n1) AGI FPGA Development Kit [1-13]\n   (JTAG Server Version 23.3.0 Build 104 09/20/2023 SC Pro Edition)\n  034BB0DD   AGIB027R29A(.|R2|R3)/.. (IR=10)\n  020D10DD   VTAP10 (IR=10)\n    Design hash    27AA3E0B7CE0A5B9F366\n    + Node 08586E00  (110:11) #0\n    + Node 0C006E00  JTAG UART #0\n    + Node 19104600  Nios II #0\n    + Node 30006E02  Signal Tap #2\n    + Node 30006E01  Signal Tap #1\n    + Node 30006E00  Signal Tap #0\n\n  Captured DR after reset = (0069761BB020D10DD) [65]\n  Captured IR after reset = (000D55) [21]\n  Captured Bypass after reset = (2) [3]\n  Captured Bypass chain = (0) [3]\n  JTAG clock speed auto-adjustment is enabled. To disable, set JtagClockAutoAdjust parameter to 0\n  JTAG clock speed 24 MHz\n
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#15-upgrading-the-i-series-development-kit-fim-via-jtag","title":"1.5 Upgrading the I-Series Development Kit FIM via JTAG","text":"
                                                                      Intel provides a pre-built FIM that can be used out-of-box for platform bring-up. This shell design is available on the OFS 2023.3 Release Page. After programming the shell and installing both the OPAE SDK and Linux DFL kernel drivers (as shown in sections 3.0 OFS DFL Kernel Drivers and 4.0 OPAE Software Development Kit), you can confirm the correct FIM has been configured by checking the output of fpgainfo fme against the following table:
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-5-fim-version","title":"Table 5: FIM Version","text":"Identifier Value Pr Interface ID 5bcd682f-5093-5fc7-8cd2-ae8073e19452 Bitstream ID TBD 
                                                                         
                                                                      1.  
                                                                        Download and unpack the artifacts from the 2023.3 release page. The file ofs_top_hps.sof is the base OFS FIM file. This file is loaded into the FPGA using the development kit built in USB Blaster. Please be aware this FPGA is not loaded into non-volatile storage, therefore if the server is power cycled, you will need to reload the FPGA .sof file.
                                                                         
                                                                        wget https://github.com/OFS/ofs-agx7-pcie-attach/releases/download/ofs-2023.3-2/iseries-dk-images.tar.gz\ntar xf iseries-dk-images.tar.gz\ncd iseries-dk-images/\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Remove the card from the PCIe bus to prevent a surprise link down. This step is not necessary if no image is currently loaded.
                                                                         
                                                                        sudo pci_device <PCIe BDF> unplug\n
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Start the Quartus Prime Programmer GUI interface, quartus_pgmw &, located in the bin directory of your Quartus installation. Select \"Hardware Setup\", double click the AGI FPGA Development Kit hardware item and change the hardware frequency to 16MHz.
                                                                         
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        On the home screen select \"Auto Detect\" and accept the default device.
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Left click on the line for device AGIB027R29A (the Agilex device) and hit \"Change File\". Load ofs_top.sof from the artifacts directory and check \"Program / Configure\". Hit start.
                                                                         
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Re-add the card to the PCIe bus. This step is not necessary if no image was loaded beforehand.
                                                                         
                                                                        sudo pci_device <BDF> plug\n
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        If this is the first time you've loaded an image into the board, you will need to restart (warm boot) the server (not power cycle / cold boot).
                                                                         
                                                                        8.  
                                                                      8.  
                                                                        Verify the PR Interface ID for your image matches expectation. When loading a FIM from the pre-compiled binary included in the artifacts archive, this ID will match the one listed in Table 5.
                                                                         
                                                                        9.  
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#20-ofs-stack-architecture-overview-for-reference-platform","title":"2.0 OFS Stack Architecture Overview for Reference Platform","text":""},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#21-hardware-components","title":"2.1 Hardware Components","text":"
                                                                      The OFS hardware architecture decomposes all designs into a standard set of modules, interfaces, and capabilities. Although the OFS infrastructure provides a standard set of functionality and capability, the user is responsible for making the customizations to their specific design in compliance with the specifications outlined in the FPGA Interface Manager Technical Reference Manual for Intel Agilex PCIe Attach: Open FPGA Stack.
                                                                       
                                                                      OFS is a hardware and software infrastructure that provides an efficient approach to developing a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#211-fpga-interface-manager","title":"2.1.1 FPGA Interface Manager","text":"
                                                                      The FPGA Interface Manager (FIM), or shell of the FPGA provides platform management functionality, clocks, resets, and interface access to the host and peripheral features on the acceleration platform. The OFS architecture for Intel Agilex 7 FPGA provides modularity, configurability, and scalability. The primary components of the FPGA Interface Manager or shell of the reference design are:
                                                                       
                                                                         
                                                                      • PCIe Subsystem - a hierarchical design that targets the P-tile PCIe hard IP and is configured to support bifurcated Gen 5 speeds
                                                                        •  
                                                                      • Ethernet Subsystem - provides portability to different Ethernet configurations across platforms and generations and reusability of the hardware framework and software stack.
                                                                        •  
                                                                      • Memory Subsystem - 2 x 8 GB DDR4 DIMMs, supporting 2666 MHz speeds, 64-bit width (no ECC)
                                                                        •  
                                                                      • Reset Controller
                                                                        •  
                                                                      • FPGA Management Engine - Provides a way to manage the platform and enable acceleration functions on the platform.
                                                                        •  
                                                                      • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                                                                        •  
                                                                      • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                                                                        •  
                                                                      • Platform Management Controller Interface (PMCI) to the board management controller
                                                                        •  
                                                                       
                                                                      The FPGA Management Engine (FME) provides management features for the platform and the loading/unloading of accelerators through partial reconfiguration. Each feature of the FME exposes itself to the kernel-level OFS drivers on the host through a Device Feature Header (DFH) register that is placed at the beginning of Control Status Register (CSR) space. Only one PCIe link can access the FME register space in a multi-host channel design architecture at a time.
                                                                       
                                                                      Note: For more information on the FIM and its external connections, refer to the FPGA Interface Manager Technical Reference Manual for Intel Agilex PCIe Attach: Open FPGA Stack.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#212-afu","title":"2.1.2 AFU","text":"
                                                                      An AFU is an acceleration workload that interfaces to the FIM. The AFU boundary in this reference design comprises both static and partial reconfiguration (PR) regions. You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region. A port gasket within the design provides all the PR specific modules and logic required to support partial reconfiguration. Only one partial reconfiguration region is supported in this design.
                                                                       
                                                                      Like the FME, the port gasket exposes its capability to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space.
                                                                       
                                                                      You can compile your design in one of the following ways:
                                                                       
                                                                         
                                                                      • Your entire AFU resides in a partial reconfiguration region of the FPGA.
                                                                        •  
                                                                      • The AFU is part of the static region and is compiled as a flat design.
                                                                        •  
                                                                      • Your AFU contains both static and PR regions.
                                                                        •  
                                                                       
                                                                      In this design, the AFU region is comprised of:
                                                                       
                                                                         
                                                                      • AFU Interface handler to verify transactions coming from AFU region.
                                                                        •  
                                                                      • PF/VF Mux to route transactions to and from corresponding AFU components: ST2MM module, Virtio LB stub, PCIe loopback host exerciser (HE-LB), HSSI host exerciser (HE-HSSI), Memory Host Exerciser (HE-MEM), Traffic Generator to memory (HE-MEM-TG), Port Gasket (PRG) and HPS Copy Engine.
                                                                        •  
                                                                      • AXI4 Streaming to Memory Map (ST2MM) Module that routes MMIO CSR accesses to FME and board peripherals.
                                                                        •  
                                                                      • Host exercisers to test PCIe, memory and HSSI interfaces (these can be removed from the AFU region after your FIM design is complete to provide more resource area for workloads)
                                                                        •  
                                                                      • Basic HPS Copy Engine to copy second-stage bootloader and Linux OS image from Host DDR to HPS DDR.
                                                                        •  
                                                                      • Port gasket and partial reconfiguration support.
                                                                        •  
                                                                      • Component for handling PLDM over MCTP over PCIe Vendor Defined Messages (VDM)
                                                                        •  
                                                                       
                                                                      The AFU has the option to consume native packets from the host or interface channels or to instantiate a shim provided by the Platform Interface Manager (PIM) to translate between protocols.
                                                                       
                                                                      Note: For more information on the Platform Interface Manager and AFU development and testing, refer to the AFU Development Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#22-ofs-software-overview","title":"2.2 OFS Software Overview","text":"
                                                                      The responsibility of the OFS kernel drivers is to act as the lowest software layer in the FPGA software stack, providing a minimalist driver implementation between the host software and functionality that has been implemented on the development platform. This leaves the implementation of IP-specific software in user-land, not the kernel. The OFS software stack also provides a mechanism for interface and feature discovery of FPGA platforms.
                                                                       
                                                                      The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space, and can be found on the OPAE SDK Github.
                                                                       
                                                                      The OFS drivers decompose implemented functionality, including external FIM features such as HSSI, EMIF and SPI, into sets of individual Device Features. Each Device Feature has its associated Device Feature Header (DFH), which enables a uniform discovery mechanism by software. A set of Device Features are exposed through the host interface in a Device Feature List (DFL). The OFS drivers discover and \"walk\" the Device Features in a Device Feature List and associate each Device Feature with its matching kernel driver.
                                                                       
                                                                      In this way the OFS software provides a clean and extensible framework for the creation and integration of additional functionalities and their features.
                                                                       
                                                                      Note: A deeper dive on available SW APIs and programming model is available in the Software Reference Manual: Intel\u00ae Open FPGA Stack, on kernel.org, and through the Linux DFL wiki pages.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#30-ofs-dfl-kernel-drivers","title":"3.0 OFS DFL Kernel Drivers","text":"
                                                                      OFS DFL driver software provides the bottom-most API to FPGA platforms. Libraries such as OPAE and frameworks like DPDK are consumers of the APIs provided by OFS. Applications may be built on top of these frameworks and libraries. The OFS software does not cover any out-of-band management interfaces. OFS driver software is designed to be extendable, flexible, and provide for bare-metal and virtualized functionality. An in depth look at the various aspects of the driver architecture such as the API, an explanation of the DFL framework, and instructions on how to port DFL driver patches to other kernel distributions can be found on the wiki.
                                                                       
                                                                      An in-depth review of the Linux device driver architecture can be found on opae.github.io.
                                                                       
                                                                      The DFL driver suite can be automatically installed using a supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3 Release Page.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#31-ofs-dfl-kernel-driver-installation-environment-setup","title":"3.1 OFS DFL Kernel Driver Installation Environment Setup","text":"
                                                                      All OFS DFL kernel driver primary release code resides in the Linux DFL GitHub repository. This repository is open source and should not require any permissions to access. It includes a snapshot of the Linux kernel with the OFS driver included in /drivers/fpga/*. Downloading, configuration, and compilation will be discussed in this section. Refer back to section 1.2.3 Host Server Kernel and GRUB Configuration for a list of supported Operating System(s).
                                                                       
                                                                      You can choose to install the DFL kernel drivers by either using pre-built binaries created for the BKC OS, or by building them on your local server. If you decide to use the pre-built packages available on the OFS 2023.3 Release Page, skip to section 3.3 Installing the OFS DFL Kernel Drivers from Pre-Built Packages. Regardless of your choice you will need to follow the steps in this section to prepare your server for installation.
                                                                       
                                                                      This installation process assumes the user has access to an internet connection to clone specific GitHub repositories, and to satisfy package dependencies.
                                                                       
                                                                      1. It is recommended you lock your Red Hat release version to 8.6 to prevent accidental upgrades. Update installed system packages to their latest versions.
                                                                       
                                                                      subscription-manager release --set=8.6\nsudo dnf update\nsubscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms\nsudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n
                                                                       
                                                                      2. Install the following package dependencies if building and installing drivers from source. If you do not require the use of a proxy to pull in downloads using dnf, you can safely remove those parameters from the following commands:
                                                                       
                                                                      If you require the use of a proxy, add it to DNF using by editing the following file\nsudo nano /etc/dnf/dnf.conf\n# Include your proxy by adding the following line, replacing the URL with your proxy's URL\n# proxy=http://proxy.server.com:port\n\nsudo dnf install  python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel python3-pybind11 numactl-devel\n\npython3 -m pip install --user jsonschema virtualenv pudb pyyaml setuptools pybind11\n\n# If setuptools and pybind11 were already installed\n\npython3 -m pip upgrade pybind11 setuptools\n
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#32-building-and-installing-the-ofs-dfl-kernel-drivers-from-source","title":"3.2 Building and Installing the OFS DFL Kernel Drivers from Source","text":"
                                                                      It is recommended you create an empty top level directory for your OFS related repositories to keep the working environment clean. All steps in this installation will use a generic top-level directory at /home/OFS/. If you have created a different top-level directory, replace this path with your custom path.
                                                                       
                                                                      1. Initialize an empty git repository and clone the DFL driver source code:
                                                                       
                                                                      mkdir /home/OFS/\ncd /home/OFS/\ngit init\ngit clone https://github.com/OFS/linux-dfl\ncd /home/OFS/linux-dfl\ngit checkout tags/ofs-2023.3-6.1-3\n
                                                                       
                                                                      Note: The linux-dfl repository is roughly 5 GB in size.
                                                                       
                                                                      2. Verify that the correct tag/branch have been checked out.
                                                                       
                                                                      git describe --tags\nofs-2023.3-6.1-3\n
                                                                       
                                                                      Note: If two different tagged releases are tied to the same commit, running git describe tags may report the other release's tag. This is why the match is made explicit.
                                                                       
                                                                      3. Copy an existing kernel configuration file from /boot and apply the minimal required settings changes.
                                                                       
                                                                      cd /home/OFS/linux-dfl\ncp /boot/config-`uname -r` .config\ncat configs/dfl-config >> .config\necho 'CONFIG_LOCALVERSION=\"-dfl\"' >> .config\necho 'CONFIG_LOCALVERSION_AUTO=y' >> .config\nsed -i -r 's/CONFIG_SYSTEM_TRUSTED_KEYS=.*/CONFIG_SYSTEM_TRUSTED_KEYS=\"\"/' .config\nsed -i '/^CONFIG_DEBUG_INFO_BTF/ s/./#&/' .config\necho 'CONFIG_DEBUG_ATOMIC_SLEEP=y' >> .config\nexport LOCALVERSION=\nmake olddefconfig\n
                                                                       
                                                                      Note: You can add a unique identifier to the kernel name by changing CONFIG_LOCALVERSION in .config. By default this is set to '-dfl'. This is useful if needing to differentiate between multiple installs. For example, you can add -dfl-2023.3
                                                                       
                                                                      4. The above command may report errors resembling symbol value 'm' invalid for CHELSIO_IPSEC_INLINE. These errors indicate that the nature of the config has changed between the currently executing kernel and the kernel being built. The option \"m\" for a particular kernel module is no longer a valid option, and the default behavior is to simply turn the option off. However, the option can likely be turned back on by setting it to 'y'. If the user wants to turn the option back on, change it to 'y' and re-run \"make olddefconfig\":
                                                                       
                                                                      cd /home/OFS/linux-dfl\necho 'CONFIG_CHELSIO_IPSEC_INLINE=y' >> .config\nmake olddefconfig\n
                                                                       
                                                                      Note: To use the built-in GUI menu for editing kernel configuration parameters, you can opt to run make menuconfig.
                                                                       
                                                                      5. Linux kernel builds take advantage of multiple processors to parallelize the build process. Display how many processors are available with the nproc command, and then specify how many make threads to utilize with the -j option. Note that number of threads can exceed the number of processors. In this case, the number of threads is set to the number of processors in the system.
                                                                       
                                                                      cd /home/OFS/linux-dfl\nmake -j $(nproc)\n
                                                                       
                                                                      6. The DFL Makefile contains a few options for package creation:
                                                                       
                                                                         
                                                                      • rpm-pkg: Build both source and binary RPM kernel packages
                                                                        •  
                                                                      • binrpm-pkg: Build only the binary kernel RPM package
                                                                        •  
                                                                      • deb-pkg: Build both source and binary deb kernel packages
                                                                        •  
                                                                      • bindeb-pkg: Build only the binary kernel deb package
                                                                        •  
                                                                       
                                                                      If you are concerned about the size of the resulting package and binaries, they can significantly reduce the size of the package and object files by using the make variable INSTALL_MOD_STRIP. If this is not a concern, feel free to skip this step. The below instructions will build a set of binary RPM packages:
                                                                       
                                                                      cd /home/OFS/linux-dfl\nmake INSTALL_MOD_STRIP=1 binrpm-pkg\n
                                                                       
                                                                      By default, a directory is created in your home directory called rpmbuild. This directory will house all the kernel packages which have been built. You need to navigate to the newly built kernel packages and install them. The following files were generated using the build command executed in the previous step:
                                                                       
                                                                      cd ~/rpmbuild/RPMS/x86_64\nls\nkernel-6.1.41_dfl.x86_64.rpm  kernel-headers-6.1.41_dfl.x86_64.rpm\nsudo dnf localinstall kernel*.rpm\n
                                                                       
                                                                      7. The system will need to be rebooted in order for changes to take effect. After a reboot, select the newly built kernel as the boot target. This can be done pre-boot using the command grub2-reboot, which removes the requirement for user intervention. After boot, verify that the currently running kernel matches expectation.
                                                                       
                                                                      uname -r\n6.1.41-dfl\n
                                                                       
                                                                      8. Verify the DFL drivers have been successfully installed by reading version information directly from /lib/modules. Recall that the name of the kernel built as a part of this section is 6.1.41-dfl. If the user set a different name for their kernel, change this path as needed:
                                                                       
                                                                      cd /usr/lib/modules/6.1.41-dfl/kernel/drivers/fpga\nls\ndfl-afu.ko     dfl-fme.ko      dfl-fme-region.ko  dfl.ko             dfl-pci.ko      fpga-mgr.ko     intel-m10-bmc-sec-update.ko\ndfl-fme-br.ko  dfl-fme-mgr.ko  dfl-hssi.ko        dfl-n3000-nios.ko  fpga-bridge.ko  fpga-region.ko\n
                                                                       
                                                                      If an OFS device that is compatible with these drivers is installed on the server, you can double check the driver versions by listing the currently loaded kernel modules with lsmod:
                                                                       
                                                                      lsmod | grep dfl\nuio_dfl                20480  0\ndfl_emif               16384  0\nuio                    20480  1 uio_dfl\nptp_dfl_tod            16384  0\ndfl_intel_s10_iopll    20480  0\n8250_dfl               20480  0\ndfl_fme_region         20480  0\ndfl_fme_br             16384  0\ndfl_fme_mgr            20480  2\ndfl_fme                49152  0\ndfl_afu                36864  0\ndfl_pci                20480  0\ndfl                    40960  11 dfl_pci,uio_dfl,dfl_fme,intel_m10_bmc_pmci,dfl_fme_br,8250_dfl,qsfp_mem,ptp_dfl_tod,dfl_afu,dfl_intel_s10_iopll,dfl_emif\nfpga_region            20480  3 dfl_fme_region,dfl_fme,dfl\nfpga_bridge            20480  4 dfl_fme_region,fpga_region,dfl_fme,dfl_fme_br\nfpga_mgr               20480  4 dfl_fme_region,fpga_region,dfl_fme_mgr,dfl_fme\n
                                                                       
                                                                      9. Two kernel parameters must be added to the boot command line for the newly installed kernel. First, open the file grub:
                                                                       
                                                                      sudo vim /etc/default/grub\n
                                                                       
                                                                      10.  In the variable GRUB_CMDLINE_LINUX add the following parameters in bold: GRUB_CMDLINE_LINUX=\"crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\".
                                                                       
                                                                      Note: If you wish to instead set hugepages on a per session basis, you can perform the following steps. These settings will be lost on reboot.
                                                                       
                                                                      mkdir -p /mnt/huge \nmount -t hugetlbfs nodev /mnt/huge \necho 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages \necho 2048 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages \n
                                                                       
                                                                      11.  Save your edits, then apply them to the GRUB2 configuration file.
                                                                       
                                                                      sudo grub2-mkconfig\n
                                                                       
                                                                      12.  Warm reboot. Your kernel parameter changes should have taken affect.
                                                                       
                                                                      cat /proc/cmdline\nBOOT_IMAGE=(hd1,gpt2)/vmlinuz-6.1.41-dfl root=/dev/mapper/cl-root ro crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200 rhgb quiet\n
                                                                       
                                                                      A list of all DFL drivers and their purpose is maintained on the DFL Wiki.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#33-installing-the-ofs-dfl-kernel-drivers-from-pre-built-packages","title":"3.3 Installing the OFS DFL Kernel Drivers from Pre-Built Packages","text":"
                                                                      To use the pre-built Linux DFL packages, you first need to download the files from the OFS 2023.3 Release Page. You can choose to either install using the SRC or RPM packages.
                                                                       
                                                                      tar xf kernel-6.1.41_dfl-1.x86_64-version.tar.gz\n\nsudo dnf localinstall kernel-6.1.41_dfl_*.x86_64.rpm \\\nkernel-devel-6.1.41_dfl_*.x86_64.rpm \\\nkernel-headers-6.1.41_dfl_*.x86_64.rpm\n\n### OR\n\nsudo dnf localinstall kernel-6.1.41_dfl_*.src.rpm\n
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#40-opae-software-development-kit","title":"4.0 OPAE Software Development Kit","text":"
                                                                      The OPAE SDK software stack sits in user space on top of the OFS kernel drivers. It is a common software infrastructure layer that simplifies and streamlines integration of programmable accelerators such as FPGAs into software applications and environments. OPAE consists of a set of drivers, user-space libraries, and tools to discover, enumerate, share, query, access, manipulate, and reconfigure programmable accelerators. OPAE is designed to support a layered, common programming model across different platforms and devices. To learn more about OPAE, its documentation, code samples, an explanation of the available tools, and an overview of the software architecture, visit opae.github.io.
                                                                       
                                                                      The OPAE SDK source code is contained within a single GitHub repository hosted at the OPAE Github. This repository is open source and does not require any permissions to access.
                                                                       
                                                                      You can choose to install the OPAE SDK by either using pre-built binaries created for the BKC OS, or by building them on your local server. If you decide to use the pre-built packages available on the OFS 2023.3 Release Page, skip to section 4.3 Installing the OPAE SDK with Pre-built Packages. Regardless of your choice you will need to follow the steps in this section to prepare your server for installation.
                                                                       
                                                                      You may also choose to use the supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3 Release Page.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#41-opae-sdk-installation-environment-setup","title":"4.1 OPAE SDK Installation Environment Setup","text":"
                                                                      This installation process assumes you have access to an internet connection in order to pull specific GitHub repositories, and to satisfy package dependencies.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-6-opae-package-description","title":"Table 6: OPAE Package Description","text":"Package Name Description opae OPAE SDK is a collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. It provides a library implementing the OPAE C API for presenting a streamlined and easy-to-use interface for software applications to discover, access, and manage FPGA devices and accelerators using the OPAE software stack. opae-debuginfo This package provides debug information for package opae. Debug information is useful when developing applications that use this package or when debugging this package. opae-debugsource This package provides debug sources for package opae. Debug sources are useful when developing applications that use this package or when debugging this package. opae-devel OPAE headers, tools, sample source, and documentation opae-devel-debuginfo This package provides debug information for package opae-devel. Debug information is useful when developing applications that use this package or when debugging this package. opae-tools This package contains OPAE base tools binaries opae-extra-tools Additional OPAE tools opae-extra-tools-debuginfo This package provides debug information for package opae-extra-tools. Debug information is useful when developing applications that use this package or when debugging this package. 
                                                                         
                                                                      1.  
                                                                        Remove any currently installed OPAE packages.
                                                                         
                                                                        sudo dnf remove opae*\n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Initialize an empty git repository and clone the tagged OPAE SDK source code.
                                                                         
                                                                        cd /home/OFS/\ngit init\ngit clone https://github.com/OFS/opae-sdk opae-sdk\ncd /home/OFS/opae-sdk\ngit checkout tags/2.10.0-1\n
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Verify that the correct tag/branch have been checkout out.
                                                                         
                                                                        git describe --tags\n2.10.0-1\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Update your system, then set up a temporary podman container to build OPAE, which will allow you to customize the python installation without affecting system packages.
                                                                         
                                                                        sudo subscription-manager release --set=8.6\nsudo dnf update\n\ncd /home/OFS\npodman pull registry.access.redhat.com/ubi8:8.6\npodman run -ti -v \"$PWD\":/src:Z -w /src registry.access.redhat.com/ubi8:8.6\n\n# Everything after runs within container:\n\n# Enable EPEL\ndnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n\ndnf install --enablerepo=codeready-builder-for-rhel-8-x86_64-rpms -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel make numactl-devel\n\npip3 install --upgrade --prefix=/usr pip setuptools pybind11\n\n./opae-sdk/packaging/opae/rpm/create unrestricted\n\nexit\n
                                                                         
                                                                        The following packages will be built in the same directory as create:
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Install the packages you just created.
                                                                         
                                                                        cd /home/OFS/opae-sdk/packaging/opae/rpm\nrm -rf opae-2.10.0-1.el8.src.rpm \nsudo dnf localinstall -y opae*.rpm\n
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Check that all packages have been installed and match expectation:
                                                                         
                                                                        rpm -qa | grep opae\nopae-2.10.0-1.el8.x86_64.rpm\nopae-debuginfo-2.10.0-1.el8.x86_64.rpm\nopae-debugsource-2.10.0-1.el8.x86_64.rpm\nopae-devel-2.10.0-1.el8.x86_64.rpm\nopae-devel-debuginfo-2.10.0-1.el8.x86_64.rpm\nopae-extra-tools-2.10.0-1.el8.x86_64.rpm\nopae-extra-tools-debuginfo-2.10.0-1.el8.x86_64.rpm\n
                                                                         
                                                                        7.  
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#43-installing-the-opae-sdk-with-pre-built-packages","title":"4.3 Installing the OPAE SDK with Pre-Built Packages","text":"
                                                                      You can skip the entire build process and use a set of pre-built binaries supplied by Intel. Visit the OFS 2023.3 Release Page and navigate to the bottom of the page, under the Assets tab you will see a file named opae-2.10.0-1.x86_64-*_*.tar.gz. Download this package and extract its contents:
                                                                       
                                                                      tar xf opae-2.10.0-1.x86_64-<<date>>_<<build>>.tar.gz\n
                                                                       
                                                                      For a fast installation you can delete the source RPM as it isn't necessary, and install all remaining OPAE RPMs:
                                                                       
                                                                      rm opae-*.src.rpm\nsudo dnf localinstall opae*.rpm\n
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#44-opae-tools-overview","title":"4.4 OPAE Tools Overview","text":"
                                                                      The following section offers a brief introduction including expected output values for the utilities included with OPAE. A full explanation of each command with a description of its syntax is available in the opae-sdk GitHub repo.
                                                                       
                                                                      A list of all tools included in the OPAE SDK release can be found on the OPAE FPGA Tools tab of ofs.github.io.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#441-board-management-with-fpgainfo","title":"4.4.1 Board Management with fpgainfo","text":"
                                                                      The fpgainfo utility displays FPGA information derived from sysfs files. As this release targets a development kit platform, it does not have access to an on-board BMC. Subcommands that target specific BMC functionality (such as reading temeletry) are not supported for this release.
                                                                       
                                                                      Displays FPGA information derived from sysfs files. The command argument is one of the following: errors, power, temp, port, fme, bmc, phy or mac, security. Some commands may also have other arguments or options that control their behavior.
                                                                       
                                                                      For systems with multiple FPGA devices, you can specify the BDF to limit the output to the FPGA resource with the corresponding PCIe configuration. If not specified, information displays for all resources for the given command.
                                                                       
                                                                      Note: Your Bitstream ID and PR Interface Id may not match the below examples.
                                                                       
                                                                      The following examples walk through sample outputs generated by fpgainfo. As the I-Series Development Kit does not contain a traditional BMC as used by other OFS products, those lines in fpgainfo's output will not return valid objects. The subcommand fpgainfo bmc will likewise fail to report telemetry data.
                                                                       
                                                                      Intel Acceleration Development Platform N6001\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : TBD\nBitstream Version                : 5.0.1\nPr Interface Id                  : TBD\nBoot Page                        : N/A\n
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#443-updating-with-fpgasupdate","title":"4.4.3 Updating with fpgasupdate","text":"
                                                                      The fpgasupdate tool is used to program AFU workloads into an open slot in a FIM. The fpgasupdate tool only accepts images that have been formatted using PACsign.
                                                                       
                                                                      As the I-Series Development Kit does not contain a traditional BMC, you do not have access to a factory, user1, and user2 programmed image for both the FIM and BMC FW and RTL. Only the programming of a GBS workload is supported for this release.
                                                                       
                                                                      The process of programming a SOF with a new FIM version is shown in section 1.5 Upgrading the I-Series Development Kit FIM via JTAG
                                                                       
                                                                      sudo fpgasupdate ofs_pr_afu.gbs   <PCI ADDRESS>\n[2022-04-14 16:42:31.58] [WARNING ] Update starting. Please do not interrupt.                                           \n[2022-04-14 16:42:31.58] [INFO    ] updating from file ofs_pr_afu.gbs with size 19928064               \n[2022-04-14 16:42:31.60] [INFO    ] waiting for idle                                                                 \n[2022-04-14 16:42:31.60] [INFO    ] preparing image file                                                                \n[2022-04-14 16:42:38.61] [INFO    ] writing image file                                                                 \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [19928064/19928064 bytes][Elapsed Time: 0:00:16.01]                                       \n[2022-04-14 16:42:54.63] [INFO    ] programming image file                                                              \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588][Elapsed Time: 0:06:16.40]                                                                 \n[2022-04-14 16:49:11.03] [INFO    ] update of 0000:b1:00.0 complete                                                     \n[2022-04-14 16:49:11.03] [INFO    ] Secure update OK                                                                   \n[2022-04-14 16:49:11.03] [INFO    ] Total time: 0:06:39.45\n
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#444-verify-fme-interrupts-with-hello_events","title":"4.4.4 Verify FME Interrupts with hello_events","text":"
                                                                      The hello_events utility is used to verify FME interrupts. This tool injects FME errors and waits for error interrupts, then clears the errors.
                                                                       
                                                                      Sample output from sudo hello_events.
                                                                       
                                                                      sudo hello_events\nWaiting for interrupts now...\ninjecting error\nFME Interrupt occurred\nSuccessfully tested Register/Unregister for FME events!\nclearing error\n
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#445-host-exercisor-modules","title":"4.4.5 Host Exercisor Modules","text":"
                                                                      The reference FIM and unchanged FIM compilations contain Host Exerciser Modules (HEMs). These are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. There are three HEMs present in the Intel OFS Reference FIM - HE-LPBK, HE-MEM, and HE-HSSI. These exercisers are tied to three different VFs that must be enabled before they can be used. Execution of these exercisers requires you bind specific VF endpoint to vfio-pci. The host-side software looks for these endpoints to grab the correct FPGA resource.
                                                                       
                                                                      Refer to the Intel FPGA Interface Manager Technical Reference Manual for Intel Agilex PCIe Attach: Open FPGA Stack for a full description of these modules.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-7-module-pfvf-mappings","title":"Table 7: Module PF/VF Mappings","text":"Module PF/VF ST2MM PF0 HE-MEM PF0-VF0 HE-HSSI PF0-VF1 HE-MEM_TG PF0-VF2 HE-LB Stub PF1-VF0 HE-LB PF2 VirtIO LB Stub PF3 HPS Copy Engine PF4"},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#4451-he-mem-he-lb","title":"4.4.5.1 HE-MEM / HE-LB","text":"
                                                                      The host exerciser used to exercise and characterize the various host-FPGA interactions eg. MMIO, Data transfer from host to FPGA , PR, host to FPGA memory etc. Host Exerciser Loopback (HE-LBK) AFU can move data between host memory and FPGA.
                                                                       
                                                                      HE-LBK supports: - Latency (AFU to Host memory read) - MMIO latency (Write+Read) - MMIO BW (64B MMIO writes) - BW (Read/Write, Read only, Wr only)
                                                                       
                                                                      Host Exerciser Loopback Memory (HE-MEM) AFU is used to exercise use of FPGA connected DDR, data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host.
                                                                       
                                                                      HE-LB is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth. HE-MEM is used to exercise use of FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host. HE-MEM uses external DDR memory (i.e. EMIF) to store data. It has a customized version of the AVMM interface to communicate with the EMIF memory controller. Both exercisers rely on the user-space tool host_exerciser. When using the I-Series Development Kit SmartNIC Platform, optimal performance requires the exercisers be run at 400 MHz.
                                                                       
                                                                      Execution of these exercisers requires you to bind specific VF endpoint to vfio-pci. The following commands will bind the correct endpoint for a device with B/D/F 0000:b1:00.0 and run through a basic loopback test.
                                                                       
                                                                      Note: While running the opae.io init command listed below, the command has failed if no output is present after completion. Double check that Intel VT-D and IOMMU have been enabled in the kernel as discussed in section 3.0 OFS DFL Kernel Drivers.
                                                                       
                                                                      sudo pci_device  0000:b1:00.0 vf 3\n\nsudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci                                                             \nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci \niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188                                                                  \nAssigning /dev/vfio/188 to user                                                                 \nChanging permissions for /dev/vfio/188 to rw-rw----\n\n\nsudo host_exerciser --clock-mhz 400 lpbk\n    starting test run, count of 1\nAPI version: 1\nAFU clock from command line: 400 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Number of clocks: 5224\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1022\n    Bandwidth: 5.018 GB/s\n    Test lpbk(1): PASS\n
                                                                       
                                                                      The following example will run a loopback throughput test using one cache line per request.
                                                                       
                                                                      sudo pci_device  0000:b1:00.0 vf 3\n\nsudo opae.io init -d 0000:b1:00.2 user:user\n\nsudo host_exerciser --clock-mhz 400 --mode trput --cls cl_1 lpbk\n    starting test run, count of 1\nAPI version: 1\nAFU clock from command line: 400 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 512\n    Host Exerciser numWrites: 513\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Number of clocks: 3517\n    Total number of Reads sent: 512\n    Total number of Writes sent: 512\n    Bandwidth: 7.454 GB/s\n    Test lpbk(1): PASS\n
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#4452-traffic-generator-afu-test-application","title":"4.4.5.2 Traffic Generator AFU Test Application","text":"
                                                                      Beginning in OPAE version 2.0.11-1+ the TG AFU has an OPAE application to access & exercise traffic, targeting a specific bank. The supported arguments for test configuration are:
                                                                       
                                                                         
                                                                      • Number of test loops: --loops
                                                                        •  
                                                                      • Number of read transfers per test loop: -r,--read
                                                                        •  
                                                                      • Number of write transfers per test loop: -w,--write
                                                                        •  
                                                                      • Burst size of each transfer: -b,--bls
                                                                        •  
                                                                      • Address stride between each transfer: --stride
                                                                        •  
                                                                      • Target memory TG: -m,--mem-channel
                                                                        •  
                                                                       
                                                                      Below are some example commands for how to execute the test application. To run the preconfigured write/read traffic test on channel 0:
                                                                       
                                                                      mem_tg tg_test\n
                                                                       
                                                                      Target channel 1 with a 1MB single-word write only test for 1000 iterations
                                                                       
                                                                      mem_tg --loops 1000 -r 0 -w 2000 -m 1 tg_test\n
                                                                       
                                                                      Target channel 2 with 4MB write/read test of max burst length for 10 iterations
                                                                       
                                                                      mem_tg --loops 10 -r 8 -w 8 --bls 255 -m 2 tg_test\n
                                                                       
                                                                      sudo mem_tg --loops 1000 -r 2000 -w 2000 --stride 2 --bls 2  -m 1 tg_test\n[2022-07-15 00:13:16.349] [tg_test] [info] starting test run, count of 1\nMemory channel clock frequency unknown. Assuming 300 MHz.\nTG PASS\nMem Clock Cycles: 17565035\nWrite BW: 4.37232 GB/s\nRead BW: 4.37232 GB/s\n
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#4453-he-hssi","title":"4.4.5.3 HE-HSSI","text":"
                                                                      HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic.
                                                                       
                                                                      The hssi application provides a means of interacting with the 10G and with the 100G HSSI AFUs. In both 10G and 100G operating modes, the application initializes the AFU, completes the desired transfer as described by the mode- specific options, and displays the ethernet statistics by invoking ethtool --statistics INTERFACE.
                                                                       
                                                                      sudo fpgainfo phy b1:00.0\nboard_n6000.c:306:read_bmcfw_version() **ERROR** : Failed to get read object\nboard_n6000.c:482:print_board_info() **ERROR** : Failed to read bmc version\nboard_n6000.c:332:read_max10fw_version() **ERROR** : Failed to get read object\nboard_n6000.c:488:print_board_info() **ERROR** : Failed to read max10 version\nBoard Management Controller NIOS FW version:\nBoard Management Controller Build version:\n//****** PHY ******//\nInterface                        : DFL\nObject Id                        : 0xEF00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x501020241BF165B\nBitstream Version                : TBD\nPr Interface Id                  : TBD\nQSFP0                            : Connected\nQSFP1                            : Connected\n//****** HSSI information ******//\nHSSI version                     : 2.0\nNumber of ports                  : 8\nPort0                            :25GbE        UP\nPort1                            :25GbE        UP\nPort2                            :25GbE        UP\nPort3                            :25GbE        DOWN\nPort4                            :25GbE        UP\nPort5                            :25GbE        UP\nPort6                            :25GbE        DOWN\nPort7                            :25GbE        DOWN\n
                                                                       
                                                                      The following example walks through the process of binding the VF corresponding with the HE-HSSI exerciser to vfio-pci, sending traffic, and verifying that traffic was received.
                                                                      "},{"location":"hw/iseries_devkit/user_guides/ug_qs_ofs_iseries/ug_qs_ofs_iseries/#table-8-accelerator-pfvf-and-guid-mappings","title":"Table 8: Accelerator PF/VF and GUID Mappings","text":"Component VF Accelerator GUID I Series Dev Kit base PF XXXX:XX:XX.0 N/A VirtIO Stub XXXX:XX:XX.1 3e7b60a0-df2d-4850-aa31-f54a3e403501 HE-MEM Stub XXXX:XX:XX.2 56e203e9-864f-49a7-b94b-12284c31e02b Copy Engine XXXX:XX:XX.4 44bfc10d-b42a-44e5-bd42-57dc93ea7f91 HE-MEM XXXX:XX:XX.5 8568ab4e-6ba5-4616-bb65-2a578330a8eb HE-HSSI XXXX:XX:XX.6 823c334c-98bf-11ea-bb37-0242ac130002 MEM-TG XXXX:XX:XX.7 4dadea34-2c78-48cb-a3dc-5b831f5cecbb 
                                                                         
                                                                      1.  
                                                                        Create 3 VFs in the PR region.
                                                                         
                                                                        sudo pci_device b1:00.0 vf 3 \n
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Verify all 3 VFs were created.
                                                                         
                                                                        lspci -s b1:00 \nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01) \nb1:00.1 Processing accelerators: Intel Corporation Device bcce \nb1:00.2 Processing accelerators: Intel Corporation Device bcce \nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device \nb1:00.4 Processing accelerators: Intel Corporation Device bcce \nb1:00.5 Processing accelerators: Intel Corporation Device bccf \nb1:00.6 Processing accelerators: Intel Corporation Device bccf \nb1:00.7 Processing accelerators: Intel Corporation Device bccf \n
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Bind all the PF/VF endpoints to the vfio-pci driver.
                                                                         
                                                                        sudo opae.io init -d 0000:b1:00.1 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 187\nAssigning /dev/vfio/187 to user\nChanging permissions for /dev/vfio/187 to rw-rw----\n\nsudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188\nAssigning /dev/vfio/188 to user\nChanging permissions for /dev/vfio/188 to rw-rw----\n\n...\n\nsudo opae.io init -d 0000:b1:00.7 user:user\nBinding (0x8086,0xbccf) at 0000:b1:00.7 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.7 is 319\nAssigning /dev/vfio/319 to user\nChanging permissions for /dev/vfio/319 to rw-rw----\n
                                                                         
                                                                        4.  
                                                                      4.  
                                                                        Check that the accelerators are present using fpgainfo. Note your port configuration may differ from the below.
                                                                         
                                                                        sudo fpgainfo port \n//****** PORT ******//\nObject Id                        : 0xEC00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0xE0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.7\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 4dadea34-2c78-48cb-a3dc-5b831f5cecbb\n//****** PORT ******//\nObject Id                        : 0xC0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.6\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 823c334c-98bf-11ea-bb37-0242ac130002\n//****** PORT ******//\nObject Id                        : 0xA0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.5\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n//****** PORT ******//\nObject Id                        : 0x80B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.4\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 44bfc10d-b42a-44e5-bd42-57dc93ea7f91\n//****** PORT ******//\nObject Id                        : 0x40B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n//****** PORT ******//\nObject Id                        : 0x20B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 3e7b60a0-df2d-4850-aa31-f54a3e403501\n
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Check Ethernet PHY settings with fpgainfo.
                                                                         
                                                                        sudo fpgainfo phy -B 0xb1 \n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : TBD\nBitstream Version                : 5.0.1\nPr Interface Id                  : TBD\n//****** HSSI information ******//\nHSSI version                     : 1.0\nNumber of ports                  : 8\nPort0                            :25GbE        DOWN\nPort1                            :25GbE        DOWN\nPort2                            :25GbE        DOWN\nPort3                            :25GbE        DOWN\nPort4                            :25GbE        DOWN\nPort5                            :25GbE        DOWN\nPort6                            :25GbE        DOWN\nPort7                            :25GbE        DOWN\n
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        Set loopback mode.
                                                                         
                                                                        sudo hssiloopback --loopback enable  --pcie-address 0000:b1:00.0 \nargs Namespace(loopback='enable', pcie_address='0000:b1:00.0', port=0)\nsbdf: 0000:b1:00.0\nFPGA dev: {'segment': 0, 'bus': 177, 'dev': 0, 'func': 0, 'path': '/sys/class/fpga_region/region0', 'pcie_address': '0000:b1:00.0'}\nargs.hssi_grps{0: ['dfl_dev.6', ['/sys/bus/pci/devices/0000:b1:00.0/fpga_region/region0/dfl-fme.0/dfl_dev.6/uio/uio0']]}\nfpga uio dev:dfl_dev.6\n\n--------HSSI INFO START-------\nDFH                     :0x3000000010002015\nHSSI ID                 :0x15\nDFHv                    :0.5\nguidl                   :0x99a078ad18418b9d\nguidh                   :0x4118a7cbd9db4a9b\nHSSI version            :1.0\nFirmware Version        :1\nHSSI num ports          :8\nPort0                   :25GbE\nPort1                   :25GbE\nPort2                   :25GbE\nPort3                   :25GbE\nPort4                   :25GbE\nPort5                   :25GbE\nPort6                   :25GbE\nPort7                   :25GbE\n--------HSSI INFO END-------\n\nhssi loopback enabled to port0\n
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Send traffic through the 10G AFU.
                                                                         
                                                                        sudo hssi --pci-address b1:00.6 hssi_10g --num-packets 100       \n10G loopback test\n  port: 0\n  eth_loopback: on\n  he_loopback: none\n  num_packets: 100\n  packet_length: 64\n  src_address: 11:22:33:44:55:66\n    (bits): 0x665544332211\n  dest_address: 77:88:99:aa:bb:cc\n    (bits): 0xccbbaa998877\n  random_length: fixed\n  random_payload: incremental\n  rnd_seed0: 5eed0000\n  rnd_seed1: 5eed0001\n  rnd_seed2: 25eed\n  eth:\n\nNo eth interface, so not honoring --eth-loopback.\n0x40000           ETH_AFU_DFH: 0x1000010000001000\n0x40008          ETH_AFU_ID_L: 0xbb370242ac130002\n0x40010          ETH_AFU_ID_H: 0x823c334c98bf11ea\n0x40030      TRAFFIC_CTRL_CMD: 0x0000000000000000\n0x40038     TRAFFIC_CTRL_DATA: 0x0000000100000000\n0x40040 TRAFFIC_CTRL_PORT_SEL: 0x0000000000000000\n0x40048        AFU_SCRATCHPAD: 0x0000000045324511\n\n0x3c00         number_packets: 0x00000064\n0x3c01          random_length: 0x00000000\n0x3c02         random_payload: 0x00000000\n0x3c03                  start: 0x00000000\n0x3c04                   stop: 0x00000000\n0x3c05           source_addr0: 0x44332211\n0x3c06           source_addr1: 0x00006655\n0x3c07             dest_addr0: 0xaa998877\n0x3c08             dest_addr1: 0x0000ccbb\n0x3c09        packet_tx_count: 0x00000064\n0x3c0a              rnd_seed0: 0x5eed0000\n0x3c0b              rnd_seed1: 0x5eed0001\n0x3c0c              rnd_seed2: 0x00025eed\n0x3c0d             pkt_length: 0x00000040\n0x3cf4          tx_end_tstamp: 0x000003d2\n0x3d00                num_pkt: 0xffffffff\n0x3d01               pkt_good: 0x00000064\n0x3d02                pkt_bad: 0x00000000\n0x3d07            avst_rx_err: 0x00000000\n0x3d0b          rx_sta_tstamp: 0x00000103\n0x3d0c          rx_end_tstamp: 0x0000053b\n0x3e00               mac_loop: 0x00000000\n\nHSSI performance:\n        Selected clock frequency : 402.832 MHz\n        Latency minimum : 642.948 ns\n        Latency maximum : 896.155 ns\n        Achieved Tx throughput : 18.4528 GB/s\n        Achieved Rx throughput : 16.7101 GB/s\n\nNo eth interface, so not showing stats.\n
                                                                         
                                                                        8.  
                                                                       
                                                                      The hssi_loopback utility works in conjunction with a packet generator accelerator function unit (AFU) to test high-speed serial interface (HSSI) cards. hssi_loopback tests both external and internal loopbacks.
                                                                       
                                                                      The hssistats tool provides the MAC statistics.
                                                                      "},{"location":"hw/n6001/","title":"Index","text":"
                                                                      Repository folder for Agilex OFS PCIe Attach collateral targeting Intel FPGA SmartNIC N6001-PL.
                                                                      "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/","title":"FPGA Interface Manager Developer Guide for Open FPGA Stack: Intel\u00ae FPGA SmartNIC N6001-PL PCIe Attach","text":"
                                                                      Last updated: February 03, 2024 
                                                                      "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1-introduction","title":"1. Introduction","text":""},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#11-about-this-document","title":"1.1. About This Document","text":"
                                                                      This document serves as a guide for OFS Agilex PCIe Attach developers targeting the Intel\u00ae FPGA SmartNIC N6001-PL. The following topics are covered in this guide:
                                                                       
                                                                         
                                                                      • Compiling the OFS Agilex PCIe Attach FIM design
                                                                        •  
                                                                      • Simulating the OFS Agilex PCIe Attach design
                                                                        •  
                                                                      • Customizing the OFS Agilex PCIe Attach FIM design
                                                                        •  
                                                                      • Configuring the FPGA with an OFS Agilex PCIe Attach FIM design
                                                                        •  
                                                                       
                                                                      The FIM Development Walkthroughs Table lists all of the walkthroughs provided in this guide. These walkthroughs provide step-by-step instructions for performing different FIM Development tasks.
                                                                       
                                                                      Table: FIM Development Walkthroughs
                                                                       Walkthrough Name Category Install Quartus Prime Pro Software Setup Install Git Large File Storage Extension Setup Clone FIM Repository Setup Set Development Environment Variables Setup Set Up Development Environment Setup Compile OFS FIM Compilation Manually Generate OFS Out-Of-Tree PR FIM Compilation Change the Compilation Seed Compilation Run Individual Unit Level Simulation Simulation Run Regression Unit Level Simulation Simulation Add a new module to the OFS FIM Customization Modify and run unit tests for a FIM that has a new module Customization Modify and run UVM tests for a FIM that has a new module Customization Hardware test a FIM that has a new module Customization Debug the FIM with Signal Tap Customization Compile the FIM in preparation for designing your AFU Customization Resize the Partial Reconfiguration Region Customization Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS Customization Modify PCIe Sub-System and PF/VF MUX Configuration Using IP Presets Customization Create a Minimal FIM Customization Migrate to a Different Agilex Device Number Customization Modify the Memory Sub-System Using IP Presets With OFSS Customization Modify the Ethernet Sub-System Channels With Pre-Made HSSI OFSS Customization Add Channels to the Ethernet Sub-System Channels With Custom HSSI OFSS Customization Modify the Ethernet Sub-System With Pre-Made HSSI OFSS Plus Additional Modifications Customization Modify the Ethernet Sub-System Without HSSI OFSS Customization Remove the HPS Customization Set up JTAG Configuration Program the FPGA via JTAG Configuration Program the FPGA via RSU Configuration"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#111-knowledge-pre-requisites","title":"1.1.1 Knowledge Pre-Requisites","text":"
                                                                      It is recommended that you have the following knowledge and skills before using this developer guide.
                                                                       
                                                                         
                                                                      • Basic understanding of OFS and the difference between OFS designs. Refer to the OFS Welcome Page.
                                                                        •  
                                                                      • Review the release notes for the Intel Agilex 7 PCIe Attach Reference Shells, with careful consideration of the Known Issues.
                                                                        •  
                                                                      • Review of Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL).
                                                                        •  
                                                                      • FPGA compilation flows using Intel\u00ae Quartus\u00ae Prime Pro Edition.
                                                                        •  
                                                                      • Static Timing closure, including familiarity with the Timing Analyzer tool in Intel\u00ae Quartus\u00ae Prime Pro Edition, applying timing constraints, Synopsys* Design Constraints (.sdc) language and Tcl scripting, and design methods to close on timing critical paths.
                                                                        •  
                                                                      • RTL (System Verilog) and coding practices to create synthesized logic.
                                                                        •  
                                                                      • RTL simulation tools.
                                                                        •  
                                                                      • Intel\u00ae Quartus\u00ae Prime Pro Edition Signal Tap Logic Analyzer tool software.
                                                                        •  
                                                                      "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#12-fim-development-theory","title":"1.2. FIM Development Theory","text":"
                                                                      This section will help you understand how the OFS Agilex PCIe Attach FIM can be developed to fit your design goals.
                                                                       
                                                                      The Default FIM Features section provides general information about the default features of the OFS Agilex PCIe Attach FIM so you can become familiar with the default design. For more detailed information about the FIM architecture, refer to the [OFS Agilex PCIe Attach FIM Technical Reference Manual].
                                                                       
                                                                      The Customization Options section then gives suggestions of how this default design can be customized. Step-by-step walkthroughs for many of the suggested customizations are later described in the FIM Customization section.
                                                                       
                                                                      FIM development for a new acceleration card generally consists of the following steps:
                                                                       
                                                                         
                                                                      1. Install OFS and familiarize yourself with provided scripts and source code
                                                                        2.  
                                                                      2. Develop high level design with your specific functionality
                                                                           
                                                                        1. Determine requirements and key performance metrics
                                                                          2.  
                                                                        2. Select IP cores
                                                                          3.  
                                                                        3. Select FPGA device
                                                                          4.  
                                                                        4. Develop software memory map
                                                                          5.  
                                                                         
                                                                        3.  
                                                                      3. Select and implement FIM Physical interfaces including:
                                                                           
                                                                        1. External clock sources and creation of internal PLL clocks
                                                                          2.  
                                                                        2. General I/O
                                                                          3.  
                                                                        3. Ethernet modules
                                                                          4.  
                                                                        4. External memories
                                                                          5.  
                                                                        5. FPGA programming methodology
                                                                          6.  
                                                                         
                                                                        4.  
                                                                      4. Develop device physical implementation
                                                                           
                                                                        1. FPGA device pin assignment
                                                                          2.  
                                                                        2. Create logic lock regions
                                                                          3.  
                                                                        3. Create of timing constraints
                                                                          4.  
                                                                        4. Create Intel Quartus Prime Pro FIM test project and validate:
                                                                             
                                                                          1. Placement
                                                                            2.  
                                                                          2. Timing constraints
                                                                            3.  
                                                                          3. Build script process
                                                                            4.  
                                                                          4. Review test FIM FPGA resource usage
                                                                            5.  
                                                                           
                                                                          5.  
                                                                         
                                                                        5.  
                                                                      5. Select FIM to AFU interfaces and development of PIM
                                                                        6.  
                                                                      6. Implement FIM design
                                                                           
                                                                        1. Develop RTL
                                                                          2.  
                                                                        2. Instantiate IPs
                                                                          3.  
                                                                        3. Develop test AFU to validate FIM
                                                                          4.  
                                                                        4. Develop unit and device level simulation
                                                                          5.  
                                                                        5. Develop timing constraints and build scripts
                                                                          6.  
                                                                        6. Perform timing closure and build validation
                                                                          7.  
                                                                         
                                                                        7.  
                                                                      7. Create FIM documentation to support AFU development and synthesis
                                                                        8.  
                                                                      8. Software Device Feature discovery
                                                                        9.  
                                                                      9. Integrate, validate, and debug hardware/software
                                                                        10.  
                                                                      10. Prepare for high volume production
                                                                        11.  
                                                                      "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#121-default-fim-features","title":"1.2.1 Default FIM Features","text":""},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1211-top-level","title":"1.2.1.1 Top Level","text":"
                                                                      Figure: OFS Agilex PCIe Attach n6001 FIM Top-Level Diagram
                                                                       
                                                                      "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1212-interfaces","title":"1.2.1.2 Interfaces","text":"
                                                                      The key interfaces in the OFS Agilex PCIe Attach design are listed in the Release Capabilities Table. It describes the capabilities of the n6001 hardware as well as the capabilities of the default OFS Agilex PCIe Attach design targeting the n6001.
                                                                       
                                                                      Table: Release Capabilities
                                                                       Interface n6001 Hardware Capabilities OFS Agilex PCIe Attach Default Design Implementation Host Interface PCIe Gen4x16 PCIe Gen4x16 Network Interface 2 x QSFP-28/56 cages 2x4x25GbE | 2x2x100GbE | 2x4x10GbE External Memory 5xDDR4 DIMMs sockets - 40-bits (1 available for HPS) 4xDDR4 - 2400MHz - 4GB (1Gb x 32) - 32-bits - No ECC1xDDR4 - 2400MHz - 1GB (256Mb x 32 with 256 Mb x8 ECC) - 40-bits - With ECC - For HPS"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1213-subsystems","title":"1.2.1.3 Subsystems","text":"
                                                                      The FIM Subsystems Table  describes the Platform Designer IP subsystems used in the OFS Agilex PCIe Attach n6001 FIM.
                                                                       
                                                                      Table: FIM Subsystems
                                                                       Subsystem User Guide Document ID PCIe Subsystem PCIe Subsystem Intel FPGA IP User Guide for Intel Agilex OFS N/A Memory Subsystem Memory Subsystem Intel FPGA IP User Guide for Intel Agilex OFS 686148[1] Ethernet Subsystem Ethernet Subsystem Intel FPGA IP User Guide 773413[1] 
                                                                      [1] You must log in to myIntel and request entitled access.
                                                                      "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1214-host-exercisers","title":"1.2.1.4 Host Exercisers","text":"
                                                                      The default AFU workload in the OFS Agilex PCIe Attach n6001 FIM contains several modules called Host Exercisers which are used to exercise the interfaces on the board. The Host Exerciser Descriptions Table describes these modules.
                                                                       
                                                                      Table: Host Exerciser Descriptions
                                                                       Name Acronym Description OPAE Command Host Exerciser Loopback HE-LB Used to exercise and characterize host to FPGA data transfer. host_exerciser Host Exerciser Memory HE_MEM Used to exercise and characterize host to Memory data transfer. host_exerciser Host Exerciser Memory Traffic Generator HE_MEM_TG Used to exercise and test available memory channels with a configurable traffic pattern. mem_tg Host Exerciser High Speed Serial Interface HE-HSSI Used to exercise and characterize HSSI interfaces. hssi 
                                                                      The host exercisers can be removed from the design at compile-time using command line arguments for the build script.
                                                                      "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1215-module-access-via-apfbpf","title":"1.2.1.5 Module Access via APF/BPF","text":"
                                                                      The OFS Agilex PCIe Attach n6001 FIM uses AXI4-Lite interconnect logic named the AFU Peripheral Fabric (APF) and Board Peripheral Fabric (BPF) to access the registers of the various modules in the design. The APF/BPF modules define master/slave interactions, namely between the host software and AFU and board peripherals. The APF Address Map Table describes the address mapping of the APF, followed by the BPF Address Map Table which describes the address mapping of the BPF.
                                                                       
                                                                      Table: APF Address Map
                                                                       Address Size (Bytes) Feature 0x00000\u20130x3FFFF 256K Board Peripherals (See BPF Address Map table) 0x40000 \u2013 0x4FFFF 64K ST2MM 0x50000 \u2013 0x5FFFF 64K Reserved 0x60000 \u2013 0x60FFF 4K UART 0x61000 \u2013 0x6FFFF 4K Reserved 0x70000 \u2013 0x7FFFF 56K PR Gasket:4K= PR Gasket DFH, control and status4K= Port DFH4K=User Clock52K=Remote STP 0x80000 \u2013 0x80FFF 4K AFU Error Reporting 
                                                                      Table: BPF Address Mapping
                                                                       Address Size (Bytes) Feature 0x00000 - 0x0FFFF 64K FME 0x10000 - 0x10FFF 4K PCIe 0x11000 - 0x11FFF 4K Reserved 0x12000 - 0x12FFF 4K QSFP0 0x13000 - 0x13FFF 4K QSFP1 0x14000 - 0x14FFF 4K HSSI 0x15000 - 0x15FFF 4K EMIF 0x20000 - 0x3FFFF 128K PMCI Controller"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#122-customization-options","title":"1.2.2 Customization Options","text":"
                                                                      OFS is designed to be easily customizable to meet your design needs. The OFS FIM Customization Examples Table lists the general user flows for OFS Agilex PCIe Attach n6001 FIM development, along with example customizations for each user flow, plus links to step-by-step walkthroughs where available.
                                                                       
                                                                      Table: OFS FIM Customization Examples
                                                                       Walkthrough Name Add a new module to the OFS FIM Modify and run unit tests for a FIM that has a new module Modify and run UVM tests for a FIM that has a new module Hardware test a FIM that has a new module Debug the FIM with Signal Tap Compile the FIM in preparation for designing your AFU Resize the Partial Reconfiguration Region Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS Modify PCIe Sub-System and PF/VF MUX Configuration Using IP Presets Create a Minimal FIM Migrate to a Different Agilex Device Number Modify the Memory Sub-System Using IP Presets With OFSS Modify the Ethernet Sub-System Channels With Pre-Made HSSI OFSS Add Channels to the Ethernet Sub-System Channels With Custom HSSI OFSS Modify the Ethernet Sub-System With Pre-Made HSSI OFSS Plus Additional Modifications Modify the Ethernet Sub-System Without HSSI OFSS Remove the HPS"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#13-development-environment","title":"1.3 Development Environment","text":"
                                                                      This section describes the components required for OFS FIM development, and provides a walkthrough for setting up the environment on your development machine.
                                                                       
                                                                      Note that your development machine may be different than your deployment machine where the FPGA acceleration card is installed. FPGA development work and deployment work can be performed either on the same machine, or on different machines as desired. Please see the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up the environment for deployment machines.
                                                                      "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#131-development-tools","title":"1.3.1 Development Tools","text":"
                                                                      The Development Environment Table describes the Best Known Configuration (BKC) for the tools that are required for OFS FIM development.
                                                                       
                                                                      Table: Development Environment BKC
                                                                       Component Version Installation Walkthrough Operating System RedHatEnterprise Linux\u00ae (RHEL) 8.6 N/A Intel Quartus Prime Software Quartus Prime Pro Version 23.3 for Linux + Patches 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe) Section 1.3.1.1 Python 3.6.8 or later N/A GCC 7.4.0 or later N/A cmake 3.15 or later N/A git with git-lfs 1.8.3.1 or later Section 1.3.1.2 FIM Source Files ofs-2023.3-2 Section 1.3.2.1"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1311-walkthrough-install-quartus-prime-pro-software","title":"1.3.1.1 Walkthrough: Install Quartus Prime Pro Software","text":"
                                                                      Intel Quartus Prime Pro Version 23.3 is verified to work with the latest OFS release ofs-2023.3-2.  However, you have the option to port and verify the release on newer versions of Intel Quartus Prime Pro software.
                                                                       
                                                                      Use RedHatEnterprise Linux\u00ae (RHEL) 8.6 for compatibility with your development flow and also testing your FIM design in your platform. 
                                                                       
                                                                      Prior to installing Quartus:
                                                                       
                                                                         
                                                                      1.  
                                                                        Ensure you have at least 64 GB of free space for Quartus Prime Pro installation and your development work.
                                                                         
                                                                           
                                                                        • Intel\u00ae recommends that your system be configured to provide virtual memory equal in size or larger than the recommended physical RAM size that is required to process your design.
                                                                          •  
                                                                        • The disk space may be significantly more based on the device families included in the install. Prior to installation, the disk space should be enough to hold both zipped tar files and uncompressed installation files. After successful installation, delete the downloaded zipped files and uncompressed zip files to release the disk space.
                                                                          •  
                                                                         
                                                                        2.  
                                                                      2.  
                                                                        Perform the following steps to satisfy the required dependencies.
                                                                         
                                                                        $ sudo dnf install -y gcc gcc-c++ make cmake libuuid-devel rpm-build autoconf automake bison boost boost-devel libxml2 libxml2-devel make ncurses grub2 bc csh flex glibc-locale-source libnsl ncurses-compat-libs \n
                                                                         
                                                                        Apply the following configurations.
                                                                         
                                                                        $ sudo localedef -f UTF-8 -i en_US en_US.UTF-8 \n$ sudo ln -s /usr/lib64/libncurses.so.6 /usr/lib64/libncurses.so.5 \n$ sudo ln -s /usr/bin/python3 /usr/bin/python\n
                                                                         
                                                                        3.  
                                                                      3.  
                                                                        Create the default installation path: /intelFPGA_pro/, where  is the default path of the Linux workstation, or as set by the system administrator and  is your Quartus version number. 
                                                                        The installation path must satisfy the following requirements:
                                                                         
                                                                           
                                                                        • Contain only alphanumeric characters
                                                                          •  
                                                                        • No special characters or symbols, such as !$%@^&*<>,
                                                                          •  
                                                                        • Only English characters
                                                                          •  
                                                                        • No spaces
                                                                          •  
                                                                         
                                                                      4.  
                                                                        Download your required Quartus Prime Pro Linux version here.
                                                                         
                                                                        5.  
                                                                      5.  
                                                                        Install required Quartus patches. The Quartus patch .run files can be found in the Assets tab on the OFS Release GitHub page. The patches for this release are 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe).
                                                                         
                                                                        6.  
                                                                      6.  
                                                                        After running the Quartus Prime Pro installer, set the PATH environment variable to make utilities quartus, jtagconfig, and quartus_pgm discoverable. Edit your bashrc file ~/.bashrc to add the following line:
                                                                         
                                                                        export PATH=<Quartus install directory>/quartus/bin:$PATH\nexport PATH=<Quartus install directory>/qsys/bin:$PATH\n
                                                                         
                                                                        For example, if the Quartus install directory is /home/intelFPGA_pro/23.3 then the new line is:
                                                                         
                                                                        export PATH=/home/intelFPGA_pro/23.3/quartus/bin:$PATH\nexport PATH=/home/intelFPGA_pro/23.3/qsys/bin:$PATH\n
                                                                         
                                                                        7.  
                                                                      7.  
                                                                        Verify, Quartus is discoverable by opening a new shell:
                                                                         
                                                                        $ which quartus\n/home/intelFPGA_pro/23.3/quartus/bin/quartus\n
                                                                         
                                                                        8. "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1312-walkthrough-install-git-large-file-storage-extension","title":"1.3.1.2 Walkthrough: Install Git Large File Storage Extension","text":"
                                                                        To install the Git Large File Storage (LFS) extension, execute the following commands:
                                                                         
                                                                           
                                                                        1. Obtain Git LFS package     
                                                                          curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | sudo bash\n
                                                                          2.  
                                                                        2. Install Git LFS package     
                                                                          sudo dnf install git-lfs\n
                                                                          3.  
                                                                        3. Install Git LFS     
                                                                          git lfs install\n
                                                                          4.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#132-fim-source-files","title":"1.3.2 FIM Source Files","text":"
                                                                        The source files for the OFS Agilex PCIe Attach FIM are provided in the following repository: https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2.
                                                                         
                                                                        Some essential directories in the repository are described as follows:
                                                                         
                                                                        ofs-agx7-pcie-attach\n|  syn                      // Contains files related to synthesis\n|  |  board                     // Contains synthesis files for several cards, including the n6001 \n|  |  |  n6001                  // Contains synthesis files for n6001\n|  |  |  |  setup                       // Contains setup files, including pin constraints and location constraints\n|  |  |  |  syn_top                     // Contains Quartus project files\n|  verification             // Contains files for UVM testing\n|  ipss                     // Contains files for IP Sub-Systems\n|  |  qsfp                      // Contains source files for QSFP Sub-System\n|  |  hssi                      // Contains source files for HSSI Sub-System\n|  |  pmci                      // Contains source files for PMCI Sub-System (not used in F-Tile FIM)\n|  |  pcie                      // Contains source files for PCIe Sub-System\n|  |  mem                       // Contains source files for Memory Sub-System\n|  sim                      // Contains simulation files\n|  |  unit_test                 // Contains files for all unit tests\n|  |  |  scripts                    // Contains script to run regression unit tests\n|  license                  // Contains Quartus patch\n|  ofs-common               // Contains files which are common across OFS platforms\n|  |  verification              // Contains common UVM files\n|  |  scripts                   // Contains common scripts\n|  |  |  common\n|  |  |  |  syn                         // Contains common scripts for synthesis, including build script\n|  |  |  |  sim                         // Contains common scripts for simulation\n|  |  tools                     // Contains common tools files\n|  |  |  mk_csr_module              // Contains common files for CSR modules\n|  |  |  fabric_generation          // Contains common files for APF/BPF fabric generation\n|  |  |  ofss_config                // Contains common files for OFSS configuration tool\n|  |  |  |  ip_params                   // Contains default IP parameters for certain Sub-Systems when using OFSS\n|  |  src                       // Contains common source files, including host exercisers\n|  tools                    //\n|  |  ofss_config               // Contains top level OFSS files for each pre-made board configuration\n|  |  |  hssi                       // Contains OFSS files for Ethernet-SS configuraiton\n|  |  |  memory                     // Contains OFSS files for Memory-SS configuration\n|  |  |  pcie                       // Contains OFSS files for PCIe-SS configuration\n|  |  |  iopll                      // Contains OFSS files for IOPLL configuration\n|  src                      // Contains source files for Agilex PCIe Attach FIM\n|  |  pd_qsys                   // Contains source files related to APF/BPF fabric\n|  |  includes                  // Contains source file header files\n|  |  top                       // Contains top-level source files, including design top module\n|  |  afu_top                   // Contains top-level source files for AFU\n
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1321-walkthrough-clone-fim-repository","title":"1.3.2.1 Walkthrough: Clone FIM Repository","text":"
                                                                        Perform the following steps to clone the OFS Agilex PCIe Attach FIM Repository:
                                                                         
                                                                           
                                                                        1. Create a new directory to use as a clean starting point to store the retrieved files.     
                                                                          mkdir OFS_BUILD_ROOT\ncd OFS_BUILD_ROOT\nexport OFS_BUILD_ROOT=$PWD\n
                                                                          2.  
                                                                        2. Clone GitHub repository using the HTTPS git method     
                                                                          git clone --recurse-submodules https://github.com/OFS/ofs-agx7-pcie-attach.git\n
                                                                          3.  
                                                                        3. Check out the correct tag of the repository     
                                                                          cd ofs-agx7-pcie-attach\ngit checkout --recurse-submodules tags/ofs-2023.3-2\n
                                                                          4.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#133-environment-variables","title":"1.3.3 Environment Variables","text":"
                                                                        The OFS FIM compilation and simulation scripts require certain environment variables be set prior to execution.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#1331-walkthrough-set-development-environment-variables","title":"1.3.3.1 Walkthrough: Set Development Environment Variables","text":"
                                                                        Perform the following steps to set the required environment variables. These environment variables must be set prior to simulation or compilation tasks so it is recommended that you create a script to set these variables.
                                                                         
                                                                           
                                                                        1.  
                                                                          Navigate to the top level directory of the cloned OFS FIM repository.
                                                                           
                                                                          cd ofs-agx7-pcie-attach\n
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set project variables   
                                                                          # Set OFS Root Directory - e.g. this is the top level directory of the cloned OFS FIM repository\nexport OFS_ROOTDIR=$PWD\n
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Set variables based on your development environment   
                                                                          # Set proxies if required for your server\nexport http_proxy=<YOUR_HTTP_PROXY>\nexport https_proxy=<YOUR_HTTPS_PROXY>\nexport ftp_proxy=<YOUR_FTP_PROXY>\nexport socks_proxy=<YOUR_SOCKS_PROXY>\nexport no_proxy=<YOUR_NO_PROXY>\n\n# Set Quartus license path\nexport LM_LICENSE_FILE=<YOUR_LM_LICENSE_FILE>\n\n# Set Synopsys License path (if using Synopsys for simulation)\nexport DW_LICENSE_FILE=<YOUR_DW_LICENSE_FILE>\nexport SNPSLMD_LICENSE_FILE=<YOUR_SNPSLMD_LICENSE_FILE>\n\n# Set Quartus Installation Directory - e.g. $QUARTUS_ROOTDIR/bin contains Quartus executables\nexport QUARTUS_ROOTDIR=<YOUR_QUARTUS_INSTALLATION_DIRECTORY>\n\n# Set the Tools Directory - e.g. $TOOLS_LOCATION contains the 'synopsys' directory if you are using Synopsys. Refer to the $VCS_HOME variable for an example.\nexport TOOLS_LOCATION=<YOUR_TOOLS_LOCATION>\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Set generic environment variables
                                                                           
                                                                          # Set Work directory \nexport WORKDIR=$OFS_ROOTDIR\n\n# Set Quartus Tools variables\nexport QUARTUS_HOME=$QUARTUS_ROOTDIR\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport QUARTUS_VER_AC=$QUARTUS_ROOTDIR\nexport IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport IMPORT_IP_ROOTDIR=$IP_ROOTDIR\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\n\n# Set Verification Tools variables (if running simulations)\nexport DESIGNWARE_HOME=$TOOLS_LOCATION/synopsys/vip_common/vip_Q-2020.03A\nexport UVM_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=$TOOLS_LOCATION/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport MTI_HOME=$QUARTUS_ROOTDIR/../questa_fse\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n\n# Set PATH to include compilation and simulation tools\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/../qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$IOFS_BUILD_ROOT/opae-sdk/install-opae-sdk/bin:$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$DESIGNWARE_HOME/bin:$VCS_HOME/bin:$PATH\n
                                                                           
                                                                          5.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#134-walkthrough-set-up-development-environment","title":"1.3.4 Walkthrough: Set Up Development Environment","text":"
                                                                        This walkthrough guides you through the process of setting up your development environment in preparation for FIM development. This flow only needs to be done once on your development machine.
                                                                         
                                                                           
                                                                        1.  
                                                                          Ensure that Quartus Prime Pro Version 23.3 for Linux with Intel Agilex FPGA device support is installed on your development machine. Refer to the Install Quartus Prime Pro Software section for step-by-step installation instructions.
                                                                           
                                                                             
                                                                          1. Verify version number
                                                                            quartus_sh --version\n
                                                                             
                                                                            Example Output:
                                                                             
                                                                            Quartus Prime Shell\nVersion 23.3 Build 94 06/14/2023 SC Pro Edition\nCopyright (C) 2023  Intel Corporation. All rights reserved.\n
                                                                             
                                                                            2.  
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Ensure that all support tools are installed on your development machine, and that they meet the version requirements.
                                                                           
                                                                             
                                                                          1.  
                                                                            Python 3.6.8 or later
                                                                             
                                                                               
                                                                            1.  
                                                                              Verify version number
                                                                               
                                                                              python --version\n
                                                                               
                                                                              Example Output:
                                                                               
                                                                              Python 3.6.8\n
                                                                               
                                                                              2.  
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            GCC 7.4.0 or later
                                                                             
                                                                               
                                                                            1.  
                                                                              Verify version number
                                                                               
                                                                              gcc --version\n
                                                                               
                                                                              Example output:
                                                                               
                                                                              gcc (GCC) 7.4.0\n
                                                                               
                                                                              2.  
                                                                             
                                                                            3.  
                                                                          3.  
                                                                            cmake 3.15 or later
                                                                             
                                                                               
                                                                            1.  
                                                                              Verify version number
                                                                               
                                                                              cmake --version\n
                                                                               
                                                                              Example output:
                                                                               
                                                                              cmake version 3.15\n
                                                                               
                                                                              2.  
                                                                             
                                                                            4.  
                                                                          4.  
                                                                            git with git-lfs 1.8.3.1 or later. Refer to the Install Git Large File Storage Extension section for step-by-step instructions on installing the Git Large File Storage (LFS) extension.
                                                                             
                                                                               
                                                                            1.  
                                                                              Verify version number
                                                                               
                                                                              git --version\n
                                                                               
                                                                              Example output:
                                                                               
                                                                              git version 1.8.3.1\n
                                                                               
                                                                              2.  
                                                                             
                                                                            5.  
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Clone the ofs-agx7-pcie-attach repository. Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Install UART IP license patch .02.
                                                                           
                                                                             
                                                                          1.  
                                                                            Navigate to the license directory
                                                                             
                                                                            cd $IOFS_BUILD_ROOT/license\n
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            Install Patch 0.02
                                                                             
                                                                            sudo ./quartus-0.0-0.02iofs-linux.run\n
                                                                             
                                                                            3.  
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Install Quartus Patches 0.13 patch (Generic Serial Flash Interface IP), 0.21 patch (PCIe). All required patches are provided in the Assets of the OFS FIM Release: https://github.com/OFS/ofs-agx7-pcie-attach/releases/tag/ofs-2023.3-2
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Verify that patches have been installed correctly. They should be listed in the output of the following command.
                                                                           
                                                                          quartus_sh --version\n
                                                                           
                                                                          7.  
                                                                        7.  
                                                                          Set required environment variables. Refer to the [Set Environment Variables] section for step-by-step instructions.
                                                                           
                                                                          8.  
                                                                         
                                                                        This concludes the walkthrough for setting up your development environment. At this point you are ready to begin FIM development.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2-fim-compilation","title":"2. FIM Compilation","text":"
                                                                        This section describes the process of compiling OFS FIM designs using the provided build scripts. It contains two main sections:
                                                                         
                                                                           
                                                                        • Compilation Theory - Describes the theory behind FIM compilation
                                                                          •  
                                                                        • Compilation Flows - Describes the process of compiling a FIM
                                                                          •  
                                                                         
                                                                        The walkthroughs provided in this section are:
                                                                         
                                                                           
                                                                        • Compile OFS FIM
                                                                          •  
                                                                        • Manually Generate OFS Out-Of-Tree PR FIM
                                                                          •  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#21-compilation-theory","title":"2.1 Compilation Theory","text":"
                                                                        This section describes the theory behind FIM compilation.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#211-fim-build-script","title":"2.1.1 FIM Build Script","text":"
                                                                        The OFS Common Repository contains a script named build_top.sh which is used to build OFS FIM designs and generate output files that can be programmed to the board. After cloning the OFS FIM repository (with the ofs-common repository included), the build script can be found in the following location:
                                                                         
                                                                        $OFS_ROOTDIR/ofs-common/scripts/common/syn/build_top.sh\n
                                                                         
                                                                        The usage of the build_top.sh script is as follows:
                                                                         
                                                                        build_top.sh [-k] [-p] [-e] [--stage=<action>] [--ofss=<ip_config>] <build_target>[:<fim_options>] [<work_dir_name>]\n
                                                                         Field Options Description Requirement -k None Keep. Preserves and rebuilds within an existing work tree instead of overwriting it. Optional -p None When set, and if the FIM supports partial reconfiguration, a PR template tree is generated at the end of the FIM build. The PR template tree is located in the top of the work directory but is relocatable and uses only relative paths. See $OFS_ROOTDIR/syn/common/scripts generate_pr_release.sh for details. Optional -e None Run only Quartus analysis and elaboration. It completes the setup stage, passes -end synthesis to the Quartus compilation flow and exits without running the finish stage. Optional --stage all | setup | compile | finish Controls which portion of the OFS build is run.\u00a0\u00a0- all: Run all build stages (default)\u00a0\u00a0- setup: Initialize a project in the work directory\u00a0\u00a0- compile: Run the Quartus compilation flow on a project that was already initialized with setup\u00a0\u00a0- finish: Complete OFS post-compilation tasks, such as generating flash images and, if -p is set, generating a release. Optional --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. This parameter is consumed during the setup stage and IP is updated only inside the work tree. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. Optional <build_target> n6001 Specifies which board is being targeted. Required <fim_options> flat | null_he_lb | null_he_hssi | null_he_mem | null_he_mem_tg Used to change how the FIM is built.\u00a0\u00a0- flat - Compiles a flat design (no PR assignments). This is useful for bringing up the design on a new board without dealing with PR complexity.\u00a0\u00a0- null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null.\u00a0\u00a0- null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null.\u00a0\u00a0- null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null.\u00a0\u00a0- null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null. More than one FIM option may be passed included in the <fim_options> list by concatenating them separated by commas. For example: <build_target>:flat,null_he_lb,null_he_hssi Optional <work_dir_name> String Specifies the name of the work directory in which the FIM will be built. If not specified, the default target is $OFS_ROOTDIR/work Optional 
                                                                        Note: The he_null is a minimal block with CSRs that responds to PCIe MMIO requests in order to keep PCIe alive.
                                                                         
                                                                        The build script copies source files from the existing cloned repository into the specified work directory, which are then used for compilation. As such, any changes made in the base source files will be included in all subsequent builds, unless the -k option is used, in which case an existing work directories files are used as-is. Likewise, any changes made in a work directory is only applied to that work directory, and will not be updated in the base repository by default.
                                                                         
                                                                        Refer to Compile OFS FIM which provides step-by-step instructions for running the build_top.sh script with some of the different available options.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#212-ofss-file-usage","title":"2.1.2 OFSS File Usage","text":"
                                                                        The OFS FIM build script can use OFSS files to easily customize design IP prior to compilation using preset configurations. The OFSS files specify certain parameters for different IPs. Using OFSS is provided as a convenience feature for building FIMs. The Provided OFSS Files table below describes the pre-made OFSS files for the n6001 that can be found in the $OFS_ROOTDIR/tools/ofss_config directory. Only the OFSS files listed in this table are compatible with the n6001 In order to compile an n6001 FIM, you must supply OFSS files corresponding to each IP that is present in your design.
                                                                         
                                                                        Table: Provided OFSS Files
                                                                         OFSS File Name Location Type Description n6001.ofss $OFS_ROOTDIR/tools/ofss_config Top Includes the following OFSS files: \u00a0\u00a0\u2022 n6001_base.ofss \u00a0\u00a0\u2022 pcie_host.ofss \u00a0\u00a0\u2022 iopll.ofss \u00a0\u00a0\u2022 memory.ofss n6001_2pf.ofss $OFS_ROOTDIR/tools/ofss_config Top Includes the following OFSS files: \u00a0\u00a0\u2022 n6001_base.ofss \u00a0\u00a0\u2022 pcie_2pf.ofss \u00a0\u00a0\u2022 iopll.ofss \u00a0\u00a0\u2022 memory.ofss n6001_1pf_1vf.ofss $OFS_ROOTDIR/tools/ofss_config Top Includes the following OFSS files: \u00a0\u00a0\u2022 n6001_base.ofss \u00a0\u00a0\u2022 pcie_1pf_1vf.ofss \u00a0\u00a0\u2022 iopll.ofss \u00a0\u00a0\u2022 memory.ofss n6001_base.ofss $OFS_ROOTDIR/tools/ofss_config ofs Defines certain attributes of the design, including the platform name, device family, fim type, part number, and device ID. pcie_host.ofss $OFS_ROOTDIR/tools/ofss_config/pcie pcie Defines the PCIe Subsystem with the following configuration:\u00a0\u00a0\u2022 PF0 (3 VFs)\u00a0\u00a0\u2022 PF1 (0 VFs)\u00a0\u00a0\u2022 PF2 (0 VFs)\u00a0\u00a0\u2022 PF3 (0 VFs)\u00a0\u00a0\u2022 PF4 (0 VFs) pcie_2pf.ofss $OFS_ROOTDIR/tools/ofss_config/pcie pcie Defines the PCIe Subsystem with the following configuration:\u00a0\u00a0\u2022 PF0 (0 VFs)\u00a0\u00a0\u2022 PF1 (0 VFs) pcie_1pf_1vf.ofss $OFS_ROOTDIR/tools/ofss_config/pcie pcie Defines the PCIe Subsystem with the following configuration:\u00a0\u00a0\u2022 PF0 (1 VF) iopll.ofss $OFS_ROOTDIR/tools/ofss_config/iopll iopll Sets the IOPLL frequency to 470 MHz memory.ofss $OFS_ROOTDIR/tools/ofss_config/memory memory Defines the memory IP preset file to be used during the build as n6001 hssi_8x25.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP configuration to be 8x25 GbE hssi_8x10.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP configuration to be 8x10 GbE hssi_2x100.ofss $OFSS_ROOTDIR/tools/ofss_config/hssi hssi Defines the Ethernet-SS IP configuration to be 2x100 GbE 
                                                                        There can typically be three sections contained within an OFSS file.
                                                                         
                                                                           
                                                                        •  
                                                                          [include]
                                                                           
                                                                             
                                                                          • This section of an OFSS file contains elements separated by a newline, where each element is the path to an OFSS file that is to be included for configuration by the OFSS Configuration Tool. Ensure that any environment variables (e.g. $OFS_ROOTDIR) is set correctly. The OFSS Config tool uses breadth first search to include all of the specified OFSS files; the ordering of OFSS files does not matter
                                                                            •  
                                                                           
                                                                          •  
                                                                         
                                                                           
                                                                        •  
                                                                          [ip]
                                                                           
                                                                             
                                                                          • This section of an OFSS file contains a key value pair that allows the OFSS Config tool to determine which IP configuration is being passed in. The currently supported values of IP are ofs, iopll, pcie, memory, and hssi.
                                                                            •  
                                                                           
                                                                          •  
                                                                         
                                                                           
                                                                        •  
                                                                          [settings]
                                                                           
                                                                             
                                                                          • This section of an OFSS file contains IP specific settings. Refer to an existing IP OFSS file to see what IP settings are set. For the IP type `ofss``, the settings will be information of the OFS device (platform, family, fim, part #, device_id)
                                                                            •  
                                                                           
                                                                          •  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2121-platform-ofss-file","title":"2.1.2.1 Platform OFSS File","text":"
                                                                        The <platform>.ofss file (e.g. $OFS_ROOTDIR/tools/ofss_config/n6001.ofss) is the platform level OFSS wrapper file. This is typically the OFSS file that is provided to the build script. It only contains an include section which lists all other OFSS files that are to be used when the <platform>.ofss file is passed to the build script.
                                                                         
                                                                        The generic structure of a <platform>.ofss file is as follows:
                                                                         
                                                                        [include]\n<PATH_TO_PLATFORM_BASE_OFSS_FILE>\n<PATH_TO_PCIE_OFSS_FILE>\n<PATH_TO_IOPLL_OFSS_FILE>\n<PATH_TO_MEMORY_OFSS_FILE>\n<PATH_TO_HSSI_OFSS_FILE>\n
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2122-ofs-ip-ofss-file","title":"2.1.2.2 OFS IP OFSS File","text":"
                                                                        An OFSS file with IP type ofs (e.g. $OFS_ROOTDIR/tools/ofss_config/n6001_base.ofss) contains board specific information for the target board.
                                                                         
                                                                        Currently supported configuration options for an OFSS file with IP type ofs are described in the OFS IP OFSS File Options table.
                                                                         
                                                                        Table: OFS IP OFSS File Options
                                                                         Section Parameter n6001 Default Value [ip] type ofs [settings] platform N6001 family agilex fim base_x16 part AGFB014R24A2E2V device_id 6001"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2123-pcie-ip-ofss-file","title":"2.1.2.3 PCIe IP OFSS File","text":"
                                                                        An OFSS file with IP type pcie (e.g. $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss) is used to configure the PCIe-SS in the FIM.
                                                                         
                                                                        The PCIe OFSS file has a special section type ([pf*]) which is used to define physical functions (PFs) in the FIM. Each PF has a dedicated section, where the * character is replaced with the PF number. For example, [pf0], [pf1], etc. For reference FIM configurations, you must have at least 1 PF with 1VF, or 2PFs. This is because the PR region cannot be left unconnected. PFs must be consecutive. The PFVF Limitations table describes the supported number of PFs and VFs.
                                                                         
                                                                        Table: PF/VF Limitations
                                                                         Parameter Value Min # of PFs 1 (on PF0) Max # of PFs 8 Min # of VFs 0 (if 2PFs are used) Max # of VFs 2000 distributed across all PFs 
                                                                        Currently supported configuration options for an OFSS file with IP type pcie are described in the PCIe IP OFSS File Options table.
                                                                         
                                                                        Table: PCIe IP OFSS File Options
                                                                         Section Parameter Options Default Description [ip] type pcie N/A Specifies that this OFSS file configures the PCIe-SS [settings] output_name pcie_ss N/A Specifies the output name of the PCIe-SS IP preset String N/A OPTIONAL - Specifies the name of a PCIe-SS IP presets file to use when building the FIM. When used, a presets file will take priority over any other parameters set in this OFSS file. [pf*] num_vfs Integer 0 Specifies the number of Virtual Functions in the current PF bar0_address_width Integer 12 bar4_address_width Integer 14 vf_bar0_address_width Integer 12 ats_cap_enable 0 | 1 0 vf_ats_cap_enable 0 | 1 0 prs_ext_cap_enable 0 | 1 0 pasid_cap_enable 0 | 1 0 pci_type0_vendor_id 32'h Value 0x00008086 pci_type0_device_id 32'h Value 0x0000bcce revision_id 32'h Value 0x00000001 class_code 32'h Value 0x00120000 subsys_vendor_id 32'h Value 0x00008086 subsys_dev_id 32'h Value 0x00001771 sriov_vf_device_id 32'h Value 0x0000bccf exvf_subsysid 32'h Value 0x00001771 
                                                                        The default values for all PCIe-SS parameters (that are not defined in the PCIe IP OFSS file) are defined in $OFS_ROOTDIR/ofs-common/tools/ofss_config/ip_params/pcie_ss_component_parameters.py. When using a PCIe IP OFSS file during compilation, the PCIe-SS IP that is used will be defined based on the values in the PCIe IP OFSS file plus the parameters defined in pcie_ss_component_parameters.py.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2124-iopll-ip-ofss-file","title":"2.1.2.4 IOPLL IP OFSS File","text":"
                                                                        An OFSS file with IP type iopll (e.g. $OFS_ROOTDIR/tools/ofss_config/iopll/iopll.ofss) is used to configure the IOPLL in the FIM.
                                                                         
                                                                        The IOPLL OFSS file has a special section type ([p_clk]) which is used to define the IOPLL clock frequency.
                                                                         
                                                                        Currently supported configuration options for an OFSS file with IP type iopll are described in the IOPLL OFSS File Options table.
                                                                         
                                                                        Table: IOPLL OFSS File Options
                                                                         Section Parameter Options Description [ip] type iopll Specifies that this OFSS file configures the IOPLL [settings] output_name sys_pll Specifies the output name of the IOPLL. instance_name iopll_0 Specifies the instance name of the IOPLL. [p_clk] freq Integer: 250 - 470 Specifies the IOPLL clock frequency in MHz. 
                                                                        Note: The following frequencies have been tested on reference boards: 350MHz, 400MHz, 470MHz.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2125-memory-ip-ofss-file","title":"2.1.2.5 Memory IP OFSS File","text":"
                                                                        An OFSS file with IP type memory (e.g. $OFS_ROOTDIR/tools/ofss_config/memory/memory.ofss) is used to configure the Memory-SS in the FIM.
                                                                         
                                                                        The Memory OFSS file specifies a preset value, which selects a presets file (.qprs) to configure the Memory-SS.
                                                                         
                                                                        Currently supported configuration options for an OFSS file with IP type memory are described in the Memory OFSS File Options table.
                                                                         
                                                                        Table: Memory OFSS File Options
                                                                         Section Parameter Options Description [ip] type memory Specifies that this OFSS file configures the Memory-SS [settings] output_name mem_ss_fm Specifies the output name of the Memory-SS. preset n6001 | String[1] Specifies the name of the .qprs presets file that will be used to build the Memory-SS. 
                                                                        [1] You may generate your own .qprs presets file with a unique name using Quartus. 
                                                                         
                                                                        Memory-SS presets files are stored in the $OFS_ROOTDIR/ipss/mem/qip/presets directory.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2126-hssi-ip-ofss-file","title":"2.1.2.6 HSSI IP OFSS File","text":"
                                                                        An OFSS file with IP type hssi (e.g. $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_8x25.ofss) is used to configure the Ethernet-SS in the FIM.
                                                                         
                                                                        Currently supported configuration options for an OFSS file with IP type hssi are described in the HSSI OFSS File Options table.
                                                                         
                                                                        Table: HSSI OFSS File Options
                                                                         Section Parameter Options Description [ip] type hssi Specifies that this OFSS file configures the Ethernet-SS [settings] output_name hssi_ss Specifies the output name of the Ethernet-SS num_channels Integer Specifies the number of channels. data_rate 10GbE | 25GbE | 100GCAUI-4 Specifies the data rate preset None | String[1] OPTIONAL - Selects the platform whose preset .qprs file will be used to build the Ethernet-SS. When used, this will overwrite the other settings in this OFSS file. 
                                                                        [1] You may generate your own .qprs presets file with a unique name using Quartus. 
                                                                         
                                                                        Ethernet-SS presets are stored in  $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss/presets directory.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#213-ofs-build-script-outputs","title":"2.1.3 OFS Build Script Outputs","text":"
                                                                        The output files resulting from running the the OFS FIM build_top.sh build script are copied to a single directory during the finish stage of the build script. The path for this directory is: $OFS_ROOTDIR/<WORK_DIRECTORY>/syn/board/n6001/syn_top/output_files.
                                                                         
                                                                        The output files include programmable images and compilation reports. The OFS Build Script Output Descriptions table describes the images that are generated by the build script.
                                                                         
                                                                        Table: OFS Build Script Output Descriptions
                                                                         File Description ofs_top[_hps].bin This is an intermediate, raw binary file. This intermediate raw binary file is produced by taking the Quartus generated .sof file, and converting it to *.pof using quartus_pfg, then converting the *.pof to *.hexout using quartus_cpf, and finally converting the *.hexout to *.bin using objcopy. Depending on whether the FPGA design contains an HPS block, a different file will be generated. **ofs_top.bin* - Raw binary image of the FPGA generated if there is no HPS present in the design. ofs_top_hps.bin - Raw binary image of the FPGA generated if there is an HPS present in the design. ofs_top_page1.bin This is the binary of the Factory Image and is the input to PACSign utility to generate ofs_top_page1_unsigned.bin binary image file. This image will carry binary content for the HPS if it is included in the SOF image. ofs_top_page0_factory.bin This is an input file to PACSign to generate ofs_top_page0_unsigned_factory.bin. ofs_top_page0_unsigned_factory.bin This is the unsigned PACSign output generated for the Factory Image. ofs_top_page1_user1.bin This is an input file to PACSign to generate ofs_top_page1_unsigned_user1.bin. This file is created by taking the ofs_top_[hps].bin file and assigning the User1 or appending factory block information. ofs_top_page1_unsigned_user1.bin This is the unsigned FPGA binary image generated by the PACSign utility for the User1 Image. This file is used to load the FPGA flash User1 Image using the fpgasupdate tool. ofs_top_page2_user2.bin This is an input file to PACSign to generate ofs_top_page2_unsigned_user2.bin. This file is created by taking the ofs_top_[hps].bin file and assigning the User2 or appending factory block information. ofs_top_page2_unsigned_user2.bin This is the unsigned FPGA binary image generated by the PACSign utility for the User2 Image. This file is used to load the FPGA flash User2 Image using the fpgasupdate tool. ofs_top_hps.sof If your design contains an Intel\u00ae Agilex\u00ae 7 FPGA Hard Processor System, then the build assembly process combines the FPGA ofs_top.sof programming file with u-boot-spl-dtb.hex to produce this file."},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#22-compilation-flows","title":"2.2 Compilation Flows","text":"
                                                                        This section provides information for using the build script to generate different FIM types. Walkthroughs are provided for each compilation flow. These walkthroughs require that the development environment has been set up as described in the Set Up Development Environment section.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#221-flat-fim","title":"2.2.1 Flat FIM","text":"
                                                                        A flat FIM is compiled such that there is no partial reconfiguration region, and the entire design is built as a flat design. This is useful for compiling new designs without worrying about the complexity introduced by partial reconfiguration. The flat compile removes the PR region and PR IP; thus, you cannot use the -p build flag when using the flat compile setting. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#222-in-tree-pr-fim","title":"2.2.2 In-Tree PR FIM","text":"
                                                                        An In-Tree PR FIM is the default compilation if no compile flags or compile settings are used. This flow will compile the design with the partial reconfiguration region, but it will not create a relocatable PR directory tree to aid in AFU development. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#223-out-of-tree-pr-fim","title":"2.2.3 Out-of-Tree PR FIM","text":"
                                                                        An Out-of-Tree PR FIM will compile the design with the partial reconfiguration region, and will create a relocatable PR directory tree to aid in AFU workload development. This is especially useful if you are developing a FIM to be used by another team developing AFU workloads. This is the recommended build flow in most cases. There are two ways to create the relocatable PR directory tree:
                                                                         
                                                                           
                                                                        • Run the FIM build script with the -p option. Refer to the Compile OFS FIM Section for step-by-step instructions for this flow.
                                                                          •  
                                                                        • Run the generate_pr_release.sh script after running the FIM build script. Refer to the Walkthrough: Manually Generate OFS Out-Of-Tree PR FIM Section step-by-step instructions for this flow.
                                                                          •  
                                                                         
                                                                        In both cases, the generate_pr_release.sh is run to create the relocatable build tree. This script is located at $OFS_ROOTDIR/ofs-common/scripts/common/syn/generate_pr_release.sh. Usage for this script is as follows:
                                                                         
                                                                        generate_pr_release.sh -t <PATH_OF_RELOCATABLE_PR_TREE> <BOARD_TARGET> <WORK_DIRECTORY>\n
                                                                         
                                                                        The Generate PR Release Script Options table describes the options for the generate_pr_release.sh script.
                                                                         
                                                                        Table: Generate PR Release Script Options
                                                                         Parameter Options Description <PATH_OF_RELOCATABLE_PR_TREE> String Specifies the location of the relocatable PR directory tree to be created. <BOARD_TARGET> n6001 Specifies the name of the board target. <WORK_DIRECTORY> String Specifies the existing work directory from which the relocatable PR directory tree will be created from. 
                                                                        After generating the relocatable build tree, it is located in the $OFS_ROOTDIR/<WORK_DIRECTORY>/pr_build_template directory (or the directory you specified if generated separately). The contents of this directory have the following structure:
                                                                         
                                                                        \u251c\u2500\u2500 bin\n\u251c\u2500\u2500 \u251c\u2500\u2500 afu_synth\n\u251c\u2500\u2500 \u251c\u2500\u2500 qar_gen\n\u251c\u2500\u2500 \u251c\u2500\u2500 update_pim\n\u251c\u2500\u2500 \u251c\u2500\u2500 run.sh\n\u251c\u2500\u2500 \u251c\u2500\u2500 build_env_config\n\u251c\u2500\u2500 README\n\u251c\u2500\u2500 hw\n\u251c\u2500\u2500 \u251c\u2500\u2500 lib\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 build\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 fme-ifc-id.txt\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 platform\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 fme-platform-class.txt\n\u251c\u2500\u2500 \u251c\u2500\u2500 blue_bits\n\u251c\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 ofs_top_hps.sof\n\u2514\u2500\u2500 \u251c\u2500\u2500 \u251c\u2500\u2500 ofs_top.sof\n
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#223-he_null-fim","title":"2.2.3 HE_NULL FIM","text":"
                                                                        An HE_NULL FIM refers to a design with one, some, or all of the Host Exercisers replaced by he_null blocks. The he_null is a minimal block with CSRs that responds to PCIe MMIO requests in order to keep PCIe alive. You may use any of the build flows (flat, in-tree, out-of-tree) with the HE_NULL compile options. The HE_NULL compile options are as follows:
                                                                         
                                                                           
                                                                        • null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null
                                                                          •  
                                                                        • null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null
                                                                          •  
                                                                        • null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null
                                                                          •  
                                                                        • null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null
                                                                          •  
                                                                         
                                                                        The Compile OFS FIM section gives step-by-step instructions for this flow.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#225-walkthrough-compile-ofs-fim","title":"2.2.5 Walkthrough: Compile OFS FIM","text":"
                                                                        Perform the following steps to compile the OFS Agilex PCIe Attach FIM for n6001:
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Navigate to the root directory.
                                                                           
                                                                          cd $OFS_ROOTDIR\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Run the build_top.sh script with the desired compile options. Some examples are provided:
                                                                           
                                                                             
                                                                          •  
                                                                            Default FIM
                                                                             
                                                                            ./ofs-common/scripts/common/syn/build_top.sh n6001 work_n6001\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            Flat FIM using OFSS
                                                                             
                                                                            ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001:flat work_n6001_flat\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            In-Tree PR FIM using OFSS
                                                                             
                                                                            ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001 work_n6001_in_tree_pr\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            Out-of-Tree PR FIM using OFSS
                                                                             
                                                                            ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001 work_n6001_oot_pr\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            HE_NULL Flat FIM using OFSS
                                                                             
                                                                            ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001:flat,null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_n6001_flat\n
                                                                             
                                                                            •  
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Once the build script is complete, the build summary should report that the build is complete and passes timing. For example:
                                                                           
                                                                          ***********************************\n***\n***        OFS_PROJECT: n6001\n***        OFS_BOARD: n6001\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 6\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                                                           
                                                                          6.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#226-walkthrough-manually-generate-ofs-out-of-tree-pr-fim","title":"2.2.6 Walkthrough: Manually Generate OFS Out-Of-Tree PR FIM","text":"
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Navigate to the root directory.
                                                                           
                                                                          cd $OFS_ROOTDIR\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Run the build_top.sh script with the desired compile options using the n6001 OFSS presets. In order to create the relocatable PR tree, you may not compile with the flat option. For example:
                                                                           
                                                                          ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001 work_n6001\n
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Run the generate_pr_release.sh script to create the relocatable PR tree.
                                                                           
                                                                          ./ofs-common/scripts/common/syn/generate_pr_release.sh -t work_n6001/pr_build_template n6001 work_n6001\n
                                                                           
                                                                          6.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#227-compilation-seed","title":"2.2.7 Compilation Seed","text":"
                                                                        You may change the seed which is used by the build script during Quartus compilation to change the starting point of the fitter. Trying different seeds is useful when your design is failing timing by a small amount.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#2271-walkthrough-change-the-compilation-seed","title":"2.2.7.1 Walkthrough: Change the Compilation Seed","text":"
                                                                        Perform the following steps to change the compilation seed for the FIM build.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Edit the SEED assignment in the $OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_top.qsf file to your desired seed value. The value can be any non-negative integer value.
                                                                           
                                                                          set_global_assignment -name SEED 1\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Build the FIM. Refer to the Compile OFS FIM section for instructions.
                                                                           
                                                                          5.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#3-fim-simulation","title":"3. FIM Simulation","text":"
                                                                        Unit level simulation of key components in the FIM is provided for verification of the following areas:
                                                                         
                                                                           
                                                                        • Ethernet
                                                                          •  
                                                                        • PCIe
                                                                          •  
                                                                        • External Memory
                                                                          •  
                                                                        • Core FIM
                                                                          •  
                                                                         
                                                                        The Unit Level simulations work with Synopsys VCS/VCSMX or Mentor Graphics Questasim simulators. The scripts to run each unit level simulation are located in $OFS_ROOTDIR/sim/unit_test. Each unit test directory contains a README which describes the test in detail.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#31-simulation-file-generation","title":"3.1 Simulation File Generation","text":"
                                                                        The simulation files must be generated prior to running unit level simulations. The script to generate simulation files is in the following location:
                                                                         
                                                                        $OFS_ROOTDIR/ofs-common/scripts/common/sim/gen_sim_files.sh\n
                                                                         
                                                                        The usage of the gen_sim_files.sh script is as follows:
                                                                         
                                                                        gen_sim_files.sh [--ofss=<ip_config>] <build_target>[:<fim_options>] [<device>] [<family>]\n
                                                                         
                                                                        The Gen Sim Files Script Options table describes the options for the gen_sim_files.sh script.
                                                                         
                                                                        Table: Gen Sim Files Script Options
                                                                         Field Options Description Requirement --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. Platform Dependent[1] <build_target> n6001 Specifies which board is being targeted. Required <fim_options> null_he_lb | null_he_hssi | null_he_mem | null_he_mem_tg Used to change how the FIM is built.\u00a0\u00a0- null_he_lb - Replaces the Host Exerciser Loopback (HE_LBK) with he_null.\u00a0\u00a0- null_he_hssi - Replaces the Host Exerciser HSSI (HE_HSSI) with he_null.\u00a0\u00a0- null_he_mem - Replaces the Host Exerciser Memory (HE_MEM) with he_null.\u00a0\u00a0- null_he_mem_tg - Replaces the Host Exerciser Memory Traffic Generator with he_null. More than one FIM option may be passed included in the <fim_options> list by concatenating them separated by commas. For example: <build_target>:null_he_lb,null_he_hssi Optional <device> string Specifies the device ID for the target FPGA. If not specified, the default device is parsed from the QSF file for the project. Optional <family> string Specifies the family for the target FPGA. If not specified, the default family is parsed from the QSF file for the project. Optional 
                                                                        [1] Using OFSS is required for the F-Tile Development Kit.
                                                                         
                                                                        Refer to the Run Individual Unit Level Simulation section for an example of the simulation files generation flow.
                                                                         
                                                                        When running regression tests, you may use the -g command line argument to generate simulation files; refer to the Run Regression Unit Level Simulation section for step-by-step instructions.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#32-individual-unit-tests","title":"3.2 Individual Unit Tests","text":"
                                                                        Each unit test may be run individually using the run_sim.sh script located in the following directory:
                                                                         
                                                                        $OFS_ROOTDIR/ofs-common/scripts/common/sim/run_sim.sh\n
                                                                         
                                                                        The usage for the run_sim.sh script is as follows:
                                                                         
                                                                        sh run_sim.sh TEST=<test> [VCSMX=<0|1> | MSIM=<0|1>]\n
                                                                         
                                                                        The Run Sim Script Options table describes the options for the run_sim.sh script.
                                                                         
                                                                        Table: Run Sim Script Options
                                                                         Field Options Description TEST String Specify the name of the test to run, e.g. dfh_walker VCSMX 0 | 1 When set, the VCSMX simulator will be used MSIM 0 | 1 When set, the QuestaSim simulator will be used 
                                                                        Note: The default simulator is VCS if neither VCSMX nor MSIM are set.
                                                                         
                                                                        The log for a unit test is stored in a transcript file in the simulation directory of the test that was run.
                                                                         
                                                                        $OFS_ROOTDIR/sim/unit_test/<TEST_NAME>/<SIMULATOR>/transcript\n
                                                                         
                                                                        For example, the log for the DFH walker test using VCSMX would be found at:
                                                                         
                                                                        $OFS_ROOTDIR/sim/unit_test/dfh_walker/sim_vcsmx/transcript\n
                                                                         
                                                                        The simulation waveform database is saved as vcdplus.vpd for post simulation review.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#321-walkthrough-run-individual-unit-level-simulation","title":"3.2.1 Walkthrough: Run Individual Unit Level Simulation","text":"
                                                                        Perform the following steps to run an individual unit test.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Generate the simulation files for the n6001
                                                                           
                                                                          cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/n6001.ofss n6001\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Navigate to the common simulation directory   
                                                                          cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Run the desired unit test using your desired simulator
                                                                           
                                                                             
                                                                          •  
                                                                            Using VCS
                                                                             
                                                                            sh run_sim.sh TEST=<test_name>\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            Using VCSMX
                                                                             
                                                                            sh run_sim.sh TEST=<test_name> VCSMX=1\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            Using QuestaSim
                                                                             
                                                                            sh run_sim.sh TEST=<test_name> MSIM=1\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            For example, to run the DFH walker test using VCSMX:
                                                                             
                                                                            sh run_sim.sh TEST=dfh_walker VCSMX=1\n
                                                                             
                                                                            •  
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Once the test is complete, check the output for the simulation results. Review the log for detailed test results.
                                                                           
                                                                          Test status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/ofs-agx7-pcie-attach/sim/unit_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time         356233750000\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 356233750000 fs\nCPU Time:     57.730 seconds;       Data structure size:  47.2Mb\nTue Sep  5 09:44:19 2023\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                                                                           
                                                                          7.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#33-regression-unit-tests","title":"3.3 Regression Unit Tests","text":"
                                                                        You may use the regression script regress_run.py to run some or all of the unit tests available with a single command. The regression script is in the following location:
                                                                         
                                                                        $OFS_ROOTDIR/sim/unit_test/scripts/regress_run.py\n
                                                                         
                                                                        The usage of the regression script is as follows:
                                                                         
                                                                        regress_run.py [-h] [-l] [-n <num_procs>] [-k <test_package>] [-s <simulator>] [-g] [--ofss <ip_config>] [-b <board_name>] [-e]\n
                                                                         
                                                                        The Regression Unit Test Script Options table describes the options for the regress_run.py script.
                                                                         
                                                                        Table: Regression Unit Test Script Options
                                                                         Field Options Description -h | --help N/A Show the help message and exit -l | --local N/A Run regression locally -n | --n_procs Integer Maximum number of processes/tests to run in parallel when run locally. This has no effect on farm runs. -k | --pack all | fme | he | hssi | list | mem | pmci Test package to run during regression. The \"list\" option will look for a text file named \"list.txt\" in the \"unit_test\" directory for a text list of tests to run (top directory names). The default test package is all. -s | --sim vcs | vcsmx | msim Specifies the simulator used for the regression tests. The default simulator is vcs -g | --gen_sim_files N/A Generate simulation files. This only needs to be done once per repo update. This is the equivalent of running the gen_sim_files.sh script. -o | --ofss <ip_config> Used to modify IP, such as the PCIe SS, using .ofss configuration files. More than one .ofss file may be passed to the --ofss switch by concatenating them separated by commas. For example: --ofss config_a.ofss,config_b.ofss. -b | --board_name n6001 Specifies the board target -e | --email_list String Specifies email list to send results to multiple recipients 
                                                                        The log for each unit test that is run by the regression script is stored in a transcript file in the simulation directory of the test that was run.
                                                                         
                                                                        $OFS_ROOTDIR/sim/unit_test/<TEST_NAME>/<SIMULATOR>/transcript\n
                                                                         
                                                                        For example, the log for the DFH walker test using VCSMX would be found at:
                                                                         
                                                                        $OFS_ROOTDIR/sim/unit_test/dfh_walker/sim_vcsmx/transcript\n
                                                                         
                                                                        The simulation waveform database is saved as vcdplus.vpd for post simulation review.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#331-walkthrough-run-regression-unit-level-simulation","title":"3.3.1 Walkthrough: Run Regression Unit Level Simulation","text":"
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Run regression test with the your desired options. For example, to simulate with the options to generate simulation files, run locally, use 8 processes, run all tests, use VCS simulator, and target the n6001:
                                                                           
                                                                          cd $OFS_ROOTDIR/sim/unit_test/scripts\n\npython regress_run.py --ofss $OFS_ROOTDIR/tools/ofss_config/n6001.ofss -g -l -n 8 -k all -s vcs -b n6001\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Once all tests are complete, check that the tests have passed.
                                                                           
                                                                          2024-02-01 11:58:56,220: Passing Unit Tests:37/37 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n  2024-02-01 11:58:56,220:    bfm_test:............................. PASS -- Time Elapsed:0:06:02.414524\n  2024-02-01 11:58:56,220:    csr_test:............................. PASS -- Time Elapsed:0:06:15.337707\n  2024-02-01 11:58:56,220:    dfh_walker:........................... PASS -- Time Elapsed:0:05:58.861082\n  2024-02-01 11:58:56,220:    flr:.................................. PASS -- Time Elapsed:0:05:56.581355\n  2024-02-01 11:58:56,220:    fme_csr_access:....................... PASS -- Time Elapsed:0:00:53.277935\n  2024-02-01 11:58:56,220:    fme_csr_directed:..................... PASS -- Time Elapsed:0:01:01.278407\n  2024-02-01 11:58:56,220:    he_lb_test:........................... PASS -- Time Elapsed:0:06:49.305322\n  2024-02-01 11:58:56,220:    he_mem_lb_test:....................... PASS -- Time Elapsed:0:42:19.990309\n  2024-02-01 11:58:56,220:    he_null_test:......................... PASS -- Time Elapsed:0:49:40.190675\n  2024-02-01 11:58:56,220:    hssi_csr_test:........................ PASS -- Time Elapsed:0:51:48.399285\n  2024-02-01 11:58:56,220:    hssi_kpi_test:........................ PASS -- Time Elapsed:2:14:31.673783\n  2024-02-01 11:58:56,220:    hssi_test:............................ PASS -- Time Elapsed:2:09:01.947835\n  2024-02-01 11:58:56,220:    indirect_csr:......................... PASS -- Time Elapsed:0:05:20.803518\n  2024-02-01 11:58:56,220:    mem_ss_csr_test:...................... PASS -- Time Elapsed:0:29:34.454916\n  2024-02-01 11:58:56,220:    mem_ss_rst_test:...................... PASS -- Time Elapsed:1:00:13.989947\n  2024-02-01 11:58:56,220:    mem_tg_test:.......................... PASS -- Time Elapsed:0:38:11.820365\n  2024-02-01 11:58:56,220:    pcie_csr_test:........................ PASS -- Time Elapsed:0:05:52.164841\n  2024-02-01 11:58:56,220:    pf_vf_access_test:.................... PASS -- Time Elapsed:0:05:50.003577\n  2024-02-01 11:58:56,220:    pmci_csr_test:........................ PASS -- Time Elapsed:0:06:00.823947\n  2024-02-01 11:58:56,220:    pmci_mailbox_test:.................... PASS -- Time Elapsed:0:07:33.270359\n  2024-02-01 11:58:56,220:    pmci_multi_master_test:............... PASS -- Time Elapsed:0:31:19.277990\n  2024-02-01 11:58:56,220:    pmci_qsfp_telemetry_test:............. PASS -- Time Elapsed:0:31:43.263409\n  2024-02-01 11:58:56,220:    pmci_rd_default_value_test:........... PASS -- Time Elapsed:0:06:12.833577\n  2024-02-01 11:58:56,220:    pmci_ro_mailbox_test:................. PASS -- Time Elapsed:0:09:07.148307\n  2024-02-01 11:58:56,220:    pmci_vdm_b2b_drop_err_scenario_test:.. PASS -- Time Elapsed:0:15:30.884401\n  2024-02-01 11:58:56,220:    pmci_vdm_len_err_scenario_test:....... PASS -- Time Elapsed:0:21:48.295012\n  2024-02-01 11:58:56,220:    pmci_vdm_mctp_mmio_b2b_test:.......... PASS -- Time Elapsed:0:07:40.951160\n  2024-02-01 11:58:56,220:    pmci_vdm_multipkt_error_scenario_test: PASS -- Time Elapsed:1:54:24.778252\n  2024-02-01 11:58:56,220:    pmci_vdm_multipkt_tlp_err_test:....... PASS -- Time Elapsed:0:28:16.872490\n  2024-02-01 11:58:56,220:    pmci_vdm_tlp_error_scenario_test:..... PASS -- Time Elapsed:0:28:45.735583\n  2024-02-01 11:58:56,221:    pmci_vdm_tx_rx_all_random_lpbk_test:.. PASS -- Time Elapsed:0:18:10.509131\n  2024-02-01 11:58:56,221:    pmci_vdm_tx_rx_all_toggle_test:....... PASS -- Time Elapsed:0:14:59.887002\n  2024-02-01 11:58:56,221:    pmci_vdm_tx_rx_lpbk_test:............. PASS -- Time Elapsed:0:15:07.519350\n  2024-02-01 11:58:56,221:    port_gasket_test:..................... PASS -- Time Elapsed:0:05:54.059223\n  2024-02-01 11:58:56,221:    qsfp_test:............................ PASS -- Time Elapsed:0:06:08.632942\n  2024-02-01 11:58:56,221:    remote_stp_test:...................... PASS -- Time Elapsed:0:05:58.309660\n  2024-02-01 11:58:56,221:    uart_csr:............................. PASS -- Time Elapsed:0:06:19.757272\n  2024-02-01 11:58:56,221: Failing Unit Tests: 0/37 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n  2024-02-01 11:58:56,221: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n  2024-02-01 11:58:56,221:       Number of Unit test results captured: 37\n  2024-02-01 11:58:56,221:       Number of Unit test results passing.: 37\n  2024-02-01 11:58:56,221:       Number of Unit test results failing.:  0\n  2024-02-01 11:58:56,221:     End Unit regression running at date/time................: 2024-02-01 11:58:56.221252\n  2024-02-01 11:58:56,221:     Elapsed time for Unit regression run....................: 3:19:03.056457\n
                                                                           
                                                                          5.  
                                                                         
                                                                        --->
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#4-fim-customization","title":"4. FIM Customization","text":"
                                                                        This section describes how to perform specific customizations of the FIM, and provides step-by-step walkthroughs for these customizations. Each walkthrough can be done independently. These walkthroughs require a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment. The FIM Customization Walkthroughs table lists the walkthroughs that are provided in this section. Some walkthroughs include steps for testing on hardware. Testing on hardware requires that you have a deployment environment set up. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                                                         
                                                                        Table: OFS FIM Customization Examples
                                                                         Walkthrough Name Add a new module to the OFS FIM Modify and run unit tests for a FIM that has a new module Modify and run UVM tests for a FIM that has a new module Hardware test a FIM that has a new module Debug the FIM with Signal Tap Compile the FIM in preparation for designing your AFU Resize the Partial Reconfiguration Region Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS Modify PCIe Sub-System and PF/VF MUX Configuration Using IP Presets Create a Minimal FIM Migrate to a Different Agilex Device Number Modify the Memory Sub-System Using IP Presets With OFSS Modify the Ethernet Sub-System Channels With Pre-Made HSSI OFSS Add Channels to the Ethernet Sub-System Channels With Custom HSSI OFSS Modify the Ethernet Sub-System With Pre-Made HSSI OFSS Plus Additional Modifications Modify the Ethernet Sub-System Without HSSI OFSS Remove the HPS"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#41-adding-a-new-module-to-the-fim","title":"4.1 Adding a new module to the FIM","text":"
                                                                        This section provides a information for adding a custom module to the FIM, simulating the new design, compiling the new design, implementing and testing the new design on hardware, and debugging the new design on hardware.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#411-hello-fim-theory-of-operation","title":"4.1.1 Hello FIM Theory of Operation","text":"
                                                                        If you intend to add a new module to the FIM area, then you will need to inform the host software of the new module. The FIM exposes its functionalities to host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). This set of CSR registers and their operation is described in FIM MMIO Regions.
                                                                         
                                                                        See FPGA Device Feature List (DFL) Framework Overview for a description of the software process to read and process the linked list of Device Feature Header (DFH) CSRs within a FPGA.
                                                                         
                                                                        This example will add a hello_fim module to the design. The Hello FIM example adds a simple DFH register and 64bit scratchpad register connected to the Board Peripheral Fabric (BPF) that can be accessed by the Host. You can use this example as the basis for adding a new feature to your FIM.
                                                                         
                                                                        The Hello FIM design can be verified by Unit Level simulation, Universal Verification Methodology (UVM) simulation, and running in hardware on the n6001 card. The process for these are described in this section.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#4111-hello-fim-board-peripheral-fabric-bpf","title":"4.1.1.1 Hello FIM Board Peripheral Fabric (BPF)","text":"
                                                                        The Hello FIM module will be connected to the Board Peripheral Fabric (BPF), and will be connected such that it can be mastered by the Host. The BPF is an interconnect generated by Platform Designer. The Hello FIM BPF Interface Diagram figure shows the APF/BPF Master/Slave interactions, as well as the added Hello FIM module.
                                                                         
                                                                        Figure: Hello FIM BPF Interface Diagram
                                                                         
                                                                         
                                                                        The BPF fabric is defined in $OFS_ROOTDIR/src/pd_qsys/fabric/bpf.txt.
                                                                         
                                                                        We will add the Hello FIM module to an un-used address space in the MMIO region. The Hello FIM MMIO Address Layout table below shows the MMIO region for the Host with the Hello FIM module added at base address 0x16000.
                                                                         
                                                                        Table: Hello FIM MMIO Address Layout
                                                                         Offset Feature CSR set 0x00000 FME AFU 0x10000 PCIe Interface 0x12000 QSFP Controller 0 0x13000 QSFP Controller 1 0x14000 E-Tile Ethernet  Interface 0x15000 EMIF 0x16000 Hello FIM 0x20000 PMCI Controller 0x40000 ST2MM (Streaming to Memory-Mapped) 0x60000 VUART 0x70000 PR Control & Status (Port Gasket) 0x71000 Port CSRs (Port Gasket) 0x72000 User Clock (Port Gasket) 0x74000 Remote SignalTap (Port Gasket) 0x80000 AFU Errors (AFU Interface Handler)"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#4112-hello-fim-csr","title":"4.1.1.2 Hello FIM CSR","text":"
                                                                        The Hello FIM CSR will consist of the three registers shown in the Hello FIM CSR table below. The DFH and Hello FIM ID registers are read-only. The Scratchpad register supports read and write accesses.
                                                                         
                                                                        Table: Hello FIM CSR
                                                                         Offset Attribute Description Default Value 0x016000 RO DFH(Device Feature Headers) register 0x30000006a0000100 0x016030 RW Scrachpad register 0x0 0x016038 RO Hello FIM ID register 0x6626070150000034"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#412-walkthrough-add-a-new-module-to-the-ofs-fim","title":"4.1.2 Walkthrough: Add a new module to the OFS FIM","text":"
                                                                        Perform the following steps to add a new module to the OFS FIM that can be accessed by the Host.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Make hello_fim source directory
                                                                           
                                                                          mkdir $OFS_ROOTDIR/src/hello_fim\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Create hello_fim_top.sv file.
                                                                           
                                                                          touch $OFS_ROOTDIR/src/hello_fim/hello_fim_top.sv\n
                                                                           
                                                                          Copy the following code into hello_fim_top.sv:
                                                                           
                                                                          // ***************************************************************************\n//                               INTEL CONFIDENTIAL\n//\n//        Copyright (C) 2023 Intel Corporation All Rights Reserved.\n//\n// The source code contained or described herein and all  documents related to\n// the  source  code  (\"Material\")  are  owned  by  Intel  Corporation  or its\n// suppliers  or  licensors.    Title  to  the  Material  remains  with  Intel\n// Corporation or  its suppliers  and licensors.  The Material  contains trade\n// secrets  and  proprietary  and  confidential  information  of  Intel or its\n// suppliers and licensors.  The Material is protected  by worldwide copyright\n// and trade secret laws and treaty provisions. No part of the Material may be\n// used,   copied,   reproduced,   modified,   published,   uploaded,  posted,\n// transmitted,  distributed,  or  disclosed  in any way without Intel's prior\n// express written permission.\n//\n// No license under any patent,  copyright, trade secret or other intellectual\n// property  right  is  granted  to  or  conferred  upon  you by disclosure or\n// delivery  of  the  Materials, either expressly, by implication, inducement,\n// estoppel or otherwise.  Any license under such intellectual property rights\n// must be express and approved by Intel in writing.\n//\n// You will not, and will not allow any third party to modify, adapt, enhance, \n// disassemble, decompile, reverse engineer, change or create derivative works \n// from the Software except and only to the extent as specifically required by \n// mandatory applicable laws or any applicable third party license terms \n// accompanying the Software.\n//\n// -----------------------------------------------------------------------------\n// Engineer     : \n// Create Date  : September 2023\n// Module Name  : hello_fim_top.sv\n// Project      : OFS\n// -----------------------------------------------------------------------------\n//\n// Description: \n// This is a simple module that implements DFH registers and \n// AVMM address decoding logic.\n\nmodule hello_fim_top  #(\n   parameter ADDR_WIDTH  = 12, \n   parameter DATA_WIDTH = 64, \n   parameter bit [11:0] FEAT_ID = 12'h001,\n   parameter bit [3:0]  FEAT_VER = 4'h1,\n   parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n   parameter bit END_OF_LIST = 1'b0\n)(\n   input  logic    clk,\n   input  logic    reset,\n// -----------------------------------------------------------\n//  AXI4LITE Interface\n// -----------------------------------------------------------\n   ofs_fim_axi_lite_if.slave   csr_lite_if\n);\n\nimport ofs_fim_cfg_pkg::*;\nimport ofs_csr_pkg::*;\n\n//-------------------------------------\n// Signals\n//-------------------------------------\n   logic [ADDR_WIDTH-1:0]              csr_waddr;\n   logic [DATA_WIDTH-1:0]              csr_wdata;\n   logic [DATA_WIDTH/8-1:0]            csr_wstrb;\n   logic                               csr_write;\n   logic                               csr_slv_wready;\n   csr_access_type_t                   csr_write_type;\n\n   logic [ADDR_WIDTH-1:0]              csr_raddr;\n   logic                               csr_read;\n   logic                               csr_read_32b;\n   logic [DATA_WIDTH-1:0]              csr_readdata;\n   logic                               csr_readdata_valid;\n   logic [ADDR_WIDTH-1:0]              csr_addr;\n\n   logic [63:0]                        com_csr_writedata;\n   logic                               com_csr_read;\n   logic                               com_csr_write;\n   logic [63:0]                        com_csr_readdata;\n   logic                               com_csr_readdatavalid;\n   logic [5:0]                         com_csr_address;\n\n// AXI-M CSR interfaces\nofs_fim_axi_mmio_if #(\n   .AWID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .AWADDR_WIDTH (ADDR_WIDTH),\n   .WDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH),\n   .ARID_WIDTH   (ofs_fim_cfg_pkg::MMIO_TID_WIDTH),\n   .ARADDR_WIDTH (ADDR_WIDTH),\n   .RDATA_WIDTH  (ofs_fim_cfg_pkg::MMIO_DATA_WIDTH)\n) csr_if();\n\n// AXI4-lite to AXI-M adapter\naxi_lite2mmio axi_lite2mmio (\n   .clk       (clk),\n   .rst_n     (~reset),\n   .lite_if   (csr_lite_if),\n   .mmio_if   (csr_if)\n);\n\n//---------------------------------\n// Map AXI write/read request to CSR write/read,\n// and send the write/read response back\n//---------------------------------\nofs_fim_axi_csr_slave #(\n   .ADDR_WIDTH (ADDR_WIDTH),\n   .USE_SLV_READY (1'b1)\n\n   ) csr_slave (\n   .csr_if             (csr_if),\n\n   .csr_write          (csr_write),\n   .csr_waddr          (csr_waddr),\n   .csr_write_type     (csr_write_type),\n   .csr_wdata          (csr_wdata),\n   .csr_wstrb          (csr_wstrb),\n   .csr_slv_wready     (csr_slv_wready),\n   .csr_read           (csr_read),\n   .csr_raddr          (csr_raddr),\n   .csr_read_32b       (csr_read_32b),\n   .csr_readdata       (csr_readdata),\n   .csr_readdata_valid (csr_readdata_valid)\n);\n\n// Address mapping\nassign csr_addr             = csr_write ? csr_waddr : csr_raddr;\nassign com_csr_address      = csr_addr[5:0];  // byte address\nassign csr_slv_wready       = 1'b1 ;\n// Write data mapping\nassign com_csr_writedata    = csr_wdata;\n\n// Read-Write mapping\nalways_comb\nbegin\n   com_csr_read             = 1'b0;\n   com_csr_write            = 1'b0;\n   casez (csr_addr[11:6])\n      6'h00 : begin // Common CSR\n         com_csr_read       = csr_read;\n         com_csr_write      = csr_write;\n      end   \n      default: begin\n         com_csr_read       = 1'b0;\n         com_csr_write      = 1'b0;\n      end\n   endcase\nend\n\n// Read data mapping\nalways_comb begin\n   if (com_csr_readdatavalid) begin\n      csr_readdata       = com_csr_readdata;\n      csr_readdata_valid = 1'b1;\n   end\n   else begin\n      csr_readdata       = '0;\n      csr_readdata_valid = 1'b0;\n   end\nend\n\nhello_fim_com  #(\n   .FEAT_ID          (FEAT_ID),\n   .FEAT_VER         (FEAT_VER),\n   .NEXT_DFH_OFFSET  (NEXT_DFH_OFFSET),\n   .END_OF_LIST      (END_OF_LIST)\n) hello_fim_com_inst (\n   .clk                   (clk                     ),\n   .reset                 (reset                   ),\n   .writedata             (com_csr_writedata       ),\n   .read                  (com_csr_read            ),\n   .write                 (com_csr_write           ),\n   .byteenable            (4'hF                    ),\n   .readdata              (com_csr_readdata        ),\n   .readdatavalid         (com_csr_readdatavalid   ),\n   .address               (com_csr_address         )\n   );\nendmodule\n
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Create hello_fim_com.sv file.
                                                                           
                                                                          touch $OFS_ROOTDIR/src/hello_fim/hello_fim_com.sv\n
                                                                           
                                                                          Copy the following code to hello_fim_com.sv:
                                                                           
                                                                          module hello_fim_com #(\n  parameter bit [11:0] FEAT_ID = 12'h001,\n  parameter bit [3:0]  FEAT_VER = 4'h1,\n  parameter bit [23:0] NEXT_DFH_OFFSET = 24'h1000,\n  parameter bit END_OF_LIST = 1'b0\n)(\ninput clk,\ninput reset,\ninput [63:0] writedata,\ninput read,\ninput write,\ninput [3:0] byteenable,\noutput reg [63:0] readdata,\noutput reg readdatavalid,\ninput [5:0] address\n);\n\nwire reset_n = !reset;  \nreg [63:0] rdata_comb;\nreg [63:0] scratch_reg;\n\nalways @(negedge reset_n ,posedge clk)  \n   if (!reset_n) readdata[63:0] <= 64'h0; else readdata[63:0] <= rdata_comb[63:0];\n\nalways @(negedge reset_n , posedge clk)\n   if (!reset_n) readdatavalid <= 1'b0; else readdatavalid <= read;\n\nwire wr = write;\nwire re = read;\nwire [5:0] addr = address[5:0];\nwire [63:0] din  = writedata [63:0];\nwire wr_scratch_reg = wr & (addr[5:0]  == 6'h30)? byteenable[0]:1'b0;\n\n// 64 bit scratch register\nalways @( negedge  reset_n,  posedge clk)\n   if (!reset_n)  begin\n      scratch_reg <= 64'h0;\n   end\n   else begin\n   if (wr_scratch_reg) begin \n      scratch_reg <=  din;  \n   end\nend\n\nalways @ (*)\nbegin\nrdata_comb = 64'h0000000000000000;\n   if(re) begin\n      case (addr)  \n        6'h00 : begin\n                rdata_comb [11:0]   = FEAT_ID ;  // dfh_feature_id  is reserved or a constant value, a read access gives the reset value\n                rdata_comb [15:12]  = FEAT_VER ;  // dfh_feature_rev    is reserved or a constant value, a read access gives the reset value\n                rdata_comb [39:16]  = NEXT_DFH_OFFSET ;  // dfh_dfh_ofst is reserved or a constant value, a read access gives the reset value\n                rdata_comb [40]     = END_OF_LIST ;        //dfh_end_of_list\n                rdata_comb [59:40]  = 20'h00000 ;  // dfh_rsvd1     is reserved or a constant value, a read access gives the reset value\n                rdata_comb [63:60]  = 4'h3 ;  // dfh_feat_type  is reserved or a constant value, a read access gives the reset value\n        end\n        6'h30 : begin\n                rdata_comb [63:0]   = scratch_reg; \n        end\n        6'h38 : begin\n                rdata_comb [63:0]       = 64'h6626_0701_5000_0034;\n        end\n        default : begin\n                rdata_comb = 64'h0000000000000000;\n        end\n      endcase\n   end\nend\n\nendmodule\n
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Create hello_fim_design_files.tcl file.
                                                                           
                                                                          touch $OFS_ROOTDIR/src/hello_fim/hello_fim_design_files.tcl\n
                                                                           
                                                                          Copy the following code into hello_fim_design_files.tcl
                                                                           
                                                                          # Copyright 2023 Intel Corporation.\n#\n# THIS SOFTWARE MAY CONTAIN PREPRODUCTION CODE AND IS PROVIDED BY THE\n# COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED\n# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE\n# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR\n# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\n# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR\n# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,\n# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE\n# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,\n# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# Hello FIM Files\n#--------------------\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_com.sv\nset_global_assignment -name SYSTEMVERILOG_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_top.sv\n
                                                                           
                                                                          7.  
                                                                        7.  
                                                                          Modify $OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_top.qsf to include Hello FIM module
                                                                           
                                                                          ######################################################\n# Verilog Macros\n######################################################\n.....\nset_global_assignment -name VERILOG_MACRO \"INCLUDE_HELLO_FIM\"     # Includes Hello FIM\n
                                                                           
                                                                          8.  
                                                                        8.  
                                                                          Modify $OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_top_sources.tcl to include Hello FIM design files
                                                                           
                                                                          ############################################\n# Design Files\n############################################\n...\n# Subsystems\n...\nset_global_assignment -name SOURCE_TCL_SCRIPT_FILE $::env(BUILD_ROOT_REL)/src/hello_fim/hello_fim_design_files.tclset_global_assignment -name SOURCE_TCL_SCRIPT_FILE ../setup/hello_fim_design_files.tcl\n
                                                                           
                                                                          9.  
                                                                        9.  
                                                                          Modify $OFS_ROOTDIR/src/pd_qsys/fabric/fabric_design_files.tcl to include BPF Hello FIM Slave IP.
                                                                           
                                                                          #--------------------\n# BPF\n#--------------------\n...\nset_global_assignment -name IP_FILE   $::env(BUILD_ROOT_REL)/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip\n
                                                                           
                                                                          10.  
                                                                        10.  
                                                                          Modify $OFS_ROOTDIR/src/includes/fabric_width_pkg.sv to add Hello FIM slave information and update EMIF slave next offset.
                                                                           
                                                                          localparam bpf_hello_fim_slv_baseaddress = 'h16000;    // New\nlocalparam bpf_hello_fim_slv_address_width = 12;       // New\nlocalparam bpf_emif_slv_next_dfh_offset = 'h1000;      // Old value: 'hB000\nlocalparam bpf_hello_fim_slv_next_dfh_offset = 'hA000; // New\nlocalparam bpf_hello_fim_slv_eol = 'b0;                // New\n
                                                                           
                                                                          11.  
                                                                        11.  
                                                                          Modify $OFS_ROOTDIR/src/top/top.sv
                                                                           
                                                                             
                                                                          1.  
                                                                            Add bpf_hello_fim_slv_if to AXI interfaces
                                                                             
                                                                            // AXI4-lite interfaces\n...\nofs_fim_axi_lite_if #(.AWADDR_WIDTH(fabric_width_pkg::bpf_hello_fim_slv_address_width), .ARADDR_WIDTH(fabric_width_pkg::bpf_hello_fim_slv_address_width)) bpf_hello_fim_slv_if();\n
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            Add Hello FIM instantiation
                                                                             
                                                                            //*******************************\n// Hello FIM Subsystem\n//*******************************\n\n`ifdef INCLUDE_HELLO_FIM\nhello_fim_top #(\n   .ADDR_WIDTH       (fabric_width_pkg::bpf_hello_fim_slv_address_width),\n   .DATA_WIDTH       (64),\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset),\n   .END_OF_LIST      (fabric_width_pkg::bpf_hello_fim_slv_eol)\n) hello_fim_top_inst (\n    .clk (clk_csr),\n    .reset(~rst_n_csr),\n    .csr_lite_if    (bpf_hello_fim_slv_if)\n);\n`else\ndummy_csr #(\n   .FEAT_ID          (12'h100),\n   .FEAT_VER         (4'h0),\n   .NEXT_DFH_OFFSET  (fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset),\n   .END_OF_LIST      (fabric_width_pkg::bpf_hello_fim_slv_eol)\n) hello_fim_dummy (\n   .clk         (clk_csr),\n   .rst_n       (rst_n_csr),\n   .csr_lite_if (bpf_hello_fim_slv_if)\n);\n\n`endif\n
                                                                             
                                                                            3.  
                                                                          3.  
                                                                            Add interfaces for Hello FIM slv to bpf instantiation
                                                                             
                                                                            bpf bpf (\n...\n  .bpf_hello_fim_slv_awaddr   (bpf_hello_fim_slv_if.awaddr  ),\n  .bpf_hello_fim_slv_awprot   (bpf_hello_fim_slv_if.awprot  ),\n  .bpf_hello_fim_slv_awvalid  (bpf_hello_fim_slv_if.awvalid ),\n  .bpf_hello_fim_slv_awready  (bpf_hello_fim_slv_if.awready ),\n  .bpf_hello_fim_slv_wdata    (bpf_hello_fim_slv_if.wdata   ),\n  .bpf_hello_fim_slv_wstrb    (bpf_hello_fim_slv_if.wstrb   ),\n  .bpf_hello_fim_slv_wvalid   (bpf_hello_fim_slv_if.wvalid  ),\n  .bpf_hello_fim_slv_wready   (bpf_hello_fim_slv_if.wready  ),\n  .bpf_hello_fim_slv_bresp    (bpf_hello_fim_slv_if.bresp   ),\n  .bpf_hello_fim_slv_bvalid   (bpf_hello_fim_slv_if.bvalid  ),\n  .bpf_hello_fim_slv_bready   (bpf_hello_fim_slv_if.bready  ),\n  .bpf_hello_fim_slv_araddr   (bpf_hello_fim_slv_if.araddr  ),\n  .bpf_hello_fim_slv_arprot   (bpf_hello_fim_slv_if.arprot  ),\n  .bpf_hello_fim_slv_arvalid  (bpf_hello_fim_slv_if.arvalid ),\n  .bpf_hello_fim_slv_arready  (bpf_hello_fim_slv_if.arready ),\n  .bpf_hello_fim_slv_rdata    (bpf_hello_fim_slv_if.rdata   ),\n  .bpf_hello_fim_slv_rresp    (bpf_hello_fim_slv_if.rresp   ),\n  .bpf_hello_fim_slv_rvalid   (bpf_hello_fim_slv_if.rvalid  ),\n  .bpf_hello_fim_slv_rready   (bpf_hello_fim_slv_if.rready  ),\n...\n);\n
                                                                             
                                                                            4.  
                                                                           
                                                                          12.  
                                                                        12.  
                                                                          Modify $OFS_ROOTDIR/src/pd_qsys/fabric/bpf.txt to add the hello_fim module as a slave to the apf.
                                                                           
                                                                          # NAME   FABRIC      BASEADDRESS    ADDRESS_WIDTH SLAVES\napf         mst     n/a             18            fme,pcie,pmci,qsfp0,qsfp1,emif,hssi,hello_fim\n...\nhello_fim   slv     0x16000         12            n/a\n
                                                                           
                                                                          13.  
                                                                        13.  
                                                                          Execute helper script to generate BPF design files
                                                                           
                                                                          cd $OFS_ROOTDIR/src/pd_qsys/fabric/\n\nsh gen_fabrics.sh\n
                                                                           
                                                                          14.  
                                                                        14.  
                                                                          Once the script completes, the following new IP is created: $OFS_ROOTDIR/src/pd_qsys/fabric/ip/bpf/bpf_hello_fim_slv.ip.
                                                                           
                                                                          15.  
                                                                        15.  
                                                                          [OPTIONAL] You may verify the BPF changes have been made correctly by opening bpf.qsys to analyze the BPF.
                                                                           
                                                                          cd $OFS_ROOTDIR/src/pd_qsys/fabric\n\nqsys-edit bpf.qsys --quartus-project=$OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_top.qpf\n
                                                                           
                                                                          Find the bpf_hello_fim_slv instance:
                                                                           
                                                                           
                                                                          16.  
                                                                        16.  
                                                                          Compile the Hello FIM design
                                                                           
                                                                          cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001 work_n6001_hello_fim\n
                                                                           
                                                                          17.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#413-walkthrough-modify-and-run-unit-tests-for-a-fim-that-has-a-new-module","title":"4.1.3 Walkthrough: Modify and run unit tests for a FIM that has a new module","text":"
                                                                        Perform the following steps to modify the unit test files to support a FIM that has had a new module added to it.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design in order to simulate.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Modify $OFS_ROOTDIR/sim/unit_test/dfh_walker/test_csr_defs.sv
                                                                           
                                                                             
                                                                          1.  
                                                                            Add HELLO_FIM_IDX entry to t_dfh_idx enumeration.
                                                                             
                                                                            ...\ntypedef enum {\n    FME_DFH_IDX,\n    THERM_MNGM_DFH_IDX,\n    GLBL_PERF_DFH_IDX,\n    GLBL_ERROR_DFH_IDX,\n    QSFP0_DFH_IDX,\n    QSFP1_DFH_IDX,\n    HSSI_DFH_IDX,\n    EMIF_DFH_IDX,\n    HELLO_FIM_DFH_IDX,  // New\n    PMCI_DFH_IDX,\n    ST2MM_DFH_IDX,\n    VUART_DFH_IDX,\n    PG_PR_DFH_IDX,\n    PG_PORT_DFH_IDX,\n    PG_USER_CLK_DFH_IDX,\n    PG_REMOTE_STP_DFH_IDX,\n    AFU_ERR_DFH_IDX,\n    MAX_DFH_IDX\n} t_dfh_idx;\n...\n
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            Add HELLO_FIM_DFH to get_dfh_names function.
                                                                             
                                                                            ...\nfunction automatic dfh_name[MAX_DFH_IDX-1:0] get_dfh_names();\n...\n  dfh_names[PMCI_DFH_IDX]        = \"PMCI_DFH\";\n  dfh_names[HELLO_FIM_DFH_IDX]   = \"HELLO_FIM_DFH\";  // New\n  dfh_names[ST2MM_DFH_IDX]       = \"ST2MM_DFH\";\n...\nreturn dfh_names;\n...\n
                                                                             
                                                                            3.  
                                                                          3.  
                                                                            Add expected DFH value for Hello FIM to the get_dfh_values function.
                                                                             
                                                                            ...\nfunction automatic [MAX_DFH_IDX-1:0][63:0] get_dfh_values();\n...\n  dfh_values[PMCI_DFH_IDX]       = 64'h3_00000_xxxxxx_1012;\n  dfh_values[PMCI_DFH_IDX][39:16] = fabric_width_pkg::bpf_pmci_slv_next_dfh_offset;\n\n  dfh_values[HELLO_FIM_DFH_IDX]  = 64'h3_00000_xxxxxx_0100;  // New\n  dfh_values[HELLO_FIM_DFH_IDX][39:16] = fabric_width_pkg::bpf_hello_fim_slv_next_dfh_offset; // New\n\n  dfh_values[ST2MM_DFH_IDX]      = 64'h3_00000_xxxxxx_0014;\n  dfh_values[ST2MM_DFH_IDX][39:16] = fabric_width_pkg::apf_st2mm_slv_next_dfh_offset;\n...\nreturn dfh_values;\n...\n
                                                                             
                                                                            4.  
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Generate simulation files
                                                                           
                                                                          cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\n./gen_sim_files.sh --ofss=$OFS_ROOTDIR/tools/ofss_config/n6001.ofss n6001\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Run DFH Walker Simulation   
                                                                          cd $OFS_ROOTDIR/ofs-common/scripts/common/sim\n\nsh run_sim.sh TEST=dfh_walker\n
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Verify that the test passes, and that the output shows the Hello FIM in the DFH sequence
                                                                           
                                                                          ********************************************\n Running TEST(0) : test_dfh_walking\n********************************************\n...\n\nREAD64: address=0x00015000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x3000000010001009\n\nEMIF_DFH\n   Address   (0x15000)\n   DFH value (0x3000000010001009)\n\nREAD64: address=0x00016000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x30000000a0000100\n\nHELLO_FIM_DFH\n   Address   (0x16000)\n   DFH value (0x30000000a0000100)\n\nREAD64: address=0x00020000 bar=0 vf_active=0 pfn=0 vfn=0\n\n   ** Sending TLP packets **\n   ** Waiting for ack **\n   READDATA: 0x3000000200001012\n\nPMCI_DFH\n   Address   (0x20000)\n   DFH value (0x3000000200001012)\n\n...\n\nTest status: OK\n\n********************\n  Test summary\n********************\n   test_dfh_walking (id=0) - pass\nTest passed!\nAssertion count: 0\n$finish called from file \"/home/ofs-agx7-pcie-attach/sim/unit_test/scripts/../../bfm/rp_bfm_simple/tester.sv\", line 210.\n$finish at simulation time         356791250000\n           V C S   S i m u l a t i o n   R e p o r t\nTime: 356791250000 fs\nCPU Time:     61.560 seconds;       Data structure size:  47.4Mb\nTue Aug 15 16:29:45 2023\nrun_sim.sh: USER_DEFINED_SIM_OPTIONS +vcs -l ./transcript\nrun_sim.sh: run_sim.sh DONE!\n
                                                                           
                                                                          6.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#414-walkthrough-modify-and-run-uvm-tests-for-a-fim-that-has-a-new-module","title":"4.1.4 Walkthrough: Modify and run UVM tests for a FIM that has a new module","text":"
                                                                        Perform the following steps to modify the UVM simulation files to support the Hello FIM design.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design in order to simulate.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Modify $OFS_ROOTDIR/verification/tests/sequences/dfh_walking_seq.svh
                                                                           
                                                                             
                                                                          1.  
                                                                            Modify the dfh_offset_array to insert the Hello FIM.
                                                                             
                                                                            dfh_offset_array = new[16];\ndfh_offset_array[ 0] = tb_cfg0.PF0_BAR0;                    // FME_DFH                0x8000_0000\ndfh_offset_array[ 1] = dfh_offset_array[ 0] + 64'h0_1000;   // THERM_MNGM_DFH         0x8000_1000\ndfh_offset_array[ 2] = dfh_offset_array[ 1] + 64'h0_2000;   // GLBL_PERF_DFH          0x8000_3000\ndfh_offset_array[ 3] = dfh_offset_array[ 2] + 64'h0_1000;   // GLBL_ERROR_DFH         0x8000_4000\ndfh_offset_array[ 4] = dfh_offset_array[ 3] + 64'h0_E000;   // QSFP0_DFH              0x8001_2000\ndfh_offset_array[ 5] = dfh_offset_array[ 4] + 64'h0_1000;   // QSFP1_DFH              0x8001_3000\ndfh_offset_array[ 6] = dfh_offset_array[ 5] + 64'h0_1000;   // HSSI_DFH               0x8001_4000\ndfh_offset_array[ 7] = dfh_offset_array[ 6] + 64'h0_1000;   // EMIF_DFH               0x8001_5000\ndfh_offset_array[ 8] = dfh_offset_array[ 7] + 64'h0_1000;   // HELLO_FIM_DFH          0x8001_6000\ndfh_offset_array[ 9] = dfh_offset_array[ 8] + 64'h6_a000;   // PMCI_DFH               0x8008_0000\ndfh_offset_array[ 10] = dfh_offset_array[ 9] + 64'h8_0000;  // ST2MM_DFH              0x8010_0000\ndfh_offset_array[ 11] = dfh_offset_array[10] + 64'h3_0000;  // PG_PR_DFH_IDX          0x8013_0000\ndfh_offset_array[ 12] = dfh_offset_array[11] + 64'h0_1000;  // PG_PORT_DFH_IDX        0x8013_1000\ndfh_offset_array[ 13] = dfh_offset_array[12] + 64'h0_1000;  // PG_USER_CLK_DFH_IDX    0x8013_2000\ndfh_offset_array[ 14] = dfh_offset_array[13] + 64'h0_1000;  // PG_REMOTE_STP_DFH_IDX  0x8013_3000\ndfh_offset_array[ 15] = dfh_offset_array[14] + 64'h0_D000;  // PG_AFU_ERR_DFH_IDX     0x8014_0000\n
                                                                             
                                                                            2.  
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Modify `$OFS_ROOTDIR/verification/tests/sequences/mmio_seq.svh``
                                                                           
                                                                             
                                                                          1.  
                                                                            Add test code related to the Hello FIM. This code will verify the scratchpad register at 0x16030 and read only the register at 0x16038.
                                                                             
                                                                            // HELLO_FIM_Scratchpad 64 bit access\n`uvm_info(get_name(), $psprintf(\"////Accessing PF0 HELLO_FIM_Scratchpad Register %0h+'h16030////\", tb_cfg0.PF0_BAR0), UVM_LOW)\n\nassert(std::randomize(wdata));\naddr = tb_cfg0.PF0_BAR0+'h1_6000+'h30;\n\nmmio_write64(.addr_(addr), .data_(wdata));\nmmio_read64 (.addr_(addr), .data_(rdata));\n\nif(wdata !== rdata)\n    `uvm_error(get_name(), $psprintf(\"Data mismatch 64! Addr = %0h, Exp = %0h, Act = %0h\", addr, wdata, rdata))\nelse\n    `uvm_info(get_name(), $psprintf(\"Data match 64! addr = %0h, data = %0h\", addr, rdata), UVM_LOW)\n\naddr = tb_cfg0.PF0_BAR0+'h1_6000+'h38;\nwdata = 64'h6626_0701_5000_0034;\nmmio_read64 (.addr_(addr), .data_(rdata));\nif(wdata !== rdata)\n    `uvm_error(get_name(), $psprintf(\"Data mismatch 64! Addr = %0h, Exp = %0h, Act = %0h\", addr, wdata, rdata))\nelse\n    `uvm_info(get_name(), $psprintf(\"Data match 64! addr = %0h, data = %0h\", addr, rdata), UVM_LOW)\n
                                                                             
                                                                            Note: uvm_info and uvm_error statements will put a message into log file.
                                                                             
                                                                            2.  
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Modify $OFS_ROOTDIR/verification/scripts/Makefile_VCS.mk
                                                                           
                                                                             
                                                                          1.  
                                                                            Add INCLUDE_HELLO_FIM define option to enable Hello FIM on UVM
                                                                             
                                                                            VLOG_OPT += +define+INCLUDE_HELLO_FIM\n
                                                                             
                                                                            2.  
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Re-generate the UVM files
                                                                           
                                                                             
                                                                          1.  
                                                                            Navigate to the verification scripts directory
                                                                             
                                                                            cd $VERDIR/scripts\n
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            Clean the output of previous builds
                                                                             
                                                                            gmake -f Makefile_VCS.mk clean\n
                                                                             
                                                                            3.  
                                                                          3.  
                                                                            Compile the IP files
                                                                             
                                                                            gmake -f Makefile_VCS.mk cmplib_adp\n
                                                                             
                                                                            4.  
                                                                          4.  
                                                                            Build the RTL and Test Benches
                                                                             
                                                                            gmake -f Makefile_VCS.mk build_adp DUMP=1 \n
                                                                             
                                                                            5.  
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Run the UVM DFH Walker Simulation 
                                                                           
                                                                             
                                                                          1.  
                                                                            Run the DFH Walker simulation
                                                                             
                                                                            cd $VERDIR/scripts\ngmake -f Makefile_VCS.mk run TESTNAME=dfh_walking_test DUMP=1\n
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            The output logs are stored in the $VERDIR/sim/dfh_walking_test directory. The main files to note are described in Table 5-3:
                                                                             
                                                                            Table 5-3 UVM Output Logs
                                                                             File Name Description runsim.log A log file of UVM trans.log A log file of transactions on PCIe bus inter.vpd A waveform for VCS 
                                                                            3.  
                                                                          3.  
                                                                            Run the following command to quickly verify- that the Hello FIM module was successfully accessed. In the example below, the message DFH offset Match! Exp = 80016000 Act = 80016000 shows that the Hello FIM module was successfully accessed.
                                                                             
                                                                            cd $VERDIR/sim/dfh_walking_test\ncat runsim.log | grep \"DFH offset\"\n
                                                                             
                                                                            Expected output:
                                                                             
                                                                            UVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 111950000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp = 80000000 Act = 80000000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 112586000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80001000 Act = 80001000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 113222000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80003000 Act = 80003000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 113858000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80004000 Act = 80004000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 114494000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80012000 Act = 80012000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 115147000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80013000 Act = 80013000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 115801000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80014000 Act = 80014000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 116628000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80015000 Act = 80015000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 117283000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80016000 Act = 80016000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 117928000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80080000 Act = 80080000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 118594000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80100000 Act = 80100000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 119248000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80130000 Act = 80130000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 119854000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80131000 Act = 80131000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 120460000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80132000 Act = 80132000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 121065000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80133000 Act = 80133000\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/dfh_walking_seq.svh(73) @ 121672000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] DFH offset Match! Exp= 80140000 Act = 80140000\n
                                                                             
                                                                            4.  
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Run the UVM MMIO Simulation 
                                                                           
                                                                             
                                                                          1.  
                                                                            Run the MMIO test
                                                                             
                                                                            cd $VERDIR/scripts\ngmake -f Makefile_VCS.mk run TESTNAME=mmio_test DUMP=1\n
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            Run the following commands to show the result of the scratchpad register and Hello FIM ID register. You can see the \"Data match\" message indicating that the registers are successfuly verified.
                                                                             
                                                                            cd $VERDIR/sim/mmio_test\ncat runsim.log | grep \"Data\" | grep 1603\n
                                                                             
                                                                            Expected output:
                                                                             
                                                                            UVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/mmio_seq.svh(68) @ 115466000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] Data match 64! addr = 80016030, data = 880312f9558c00e1\nUVM_INFO /home/ofs-agx7-pcie-attach/verification/tests/sequences/mmio_seq.svh(76) @ 116112000000: uvm_test_top.tb_env0.v_sequencer@@m_seq [m_seq] Data match 64! addr = 80016038, data = 6626070150000034\n
                                                                             
                                                                            3.  
                                                                           
                                                                          7.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#415-walkthrough-hardware-test-a-fim-that-has-a-new-module","title":"4.1.5 Walkthrough: Hardware test a FIM that has a new module","text":"
                                                                        Perform the following steps to program and hardware test a FIM that has had a new module added to it.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough uses a FIM design that has been generated with a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for generating a Hello FIM design.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          [OPTIONAL] In the work directory where the FIM was compiled, determine the PR Interface ID of your design. You can use this value at the end of the walkthrough to verify that the design has been configured to the FPGA.
                                                                           
                                                                          cd $OFS_ROOTDIR/<work_directory>/syn/board/n6001/syn_top/\n\ncat fme-ifc-id.txt\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          1d6beb4e-86d7-5442-a763-043701fb75b7\n
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Switch to your deployment environment.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Program the FPGA with the Hello FIM image. Refer to the Program the FPGA via RSU Section for step-by-step programming instructions.
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Run fpgainfo to determine the PCIe B:D.F of your board, and to verify the PR Interface ID matches the ID you found in Step 1.
                                                                           
                                                                          fpgainfo fme\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Initialize opae.io
                                                                           
                                                                          sudo opae.io init -d <B:D.F>\n
                                                                           
                                                                          For example:
                                                                           
                                                                          sudo opae.io init -d 98:00.0\n
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Run DFH walker. Note the value read back from offset 0x16000 indicates the DFH ID is 0x100 which matches the Hello FIM module.
                                                                           
                                                                          sudo opae.io walk -d <B:D.F>\n
                                                                           
                                                                          For example:
                                                                           
                                                                          sudo opae.io walk -d 98:00.0\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          ...\noffset: 0x15000, value: 0x3000000010001009\n    dfh: id = 0x9, rev = 0x1, next = 0x1000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x16000, value: 0x30000000a0000100\n    dfh: id = 0x100, rev = 0x0, next = 0xa000, eol = 0x0, reserved = 0x0, feature_type = 0x3\noffset: 0x20000, value: 0x3000000200001012\n    dfh: id = 0x12, rev = 0x1, next = 0x20000, eol = 0x0, reserved = 0x0, feature_type = 0x3\n...\n
                                                                           
                                                                          7.  
                                                                        7.  
                                                                          Read all of the registers in the Hello FIM module
                                                                           
                                                                             
                                                                          1.  
                                                                            Read the DFH Register
                                                                             
                                                                            opae.io -d 98:00.0 -r 0 peek 0x16000\n
                                                                             
                                                                            Example Output:
                                                                             
                                                                            0x30000006a0000100\n
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            Read the Scratchpad Register
                                                                             
                                                                            opae.io -d 98:00.0 -r 0 peek 0x16030\n
                                                                             
                                                                            Example Output:
                                                                             
                                                                            0x0\n
                                                                             
                                                                            3.  
                                                                          3.  
                                                                            Read the ID Register
                                                                             
                                                                            opae.io -d 98:00.0 -r 0 peek 0x16038\n
                                                                             
                                                                            Example Output:
                                                                             
                                                                            0x6626070150000034\n
                                                                             
                                                                            4.  
                                                                           
                                                                          8.  
                                                                        8.  
                                                                          Verify the scratchpad register at 0x16030 by writing and reading back from it.
                                                                           
                                                                             
                                                                          1.  
                                                                            Write to Scratchpad register
                                                                             
                                                                            opae.io -d 0000:15:00.0 -r 0 poke 0x16030 0x123456789abcdef\n
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            Read from Scratchpad register
                                                                             
                                                                            opae.io -d 15:00.0 -r 0 peek 0x16030\n
                                                                             
                                                                            Expected output:
                                                                             
                                                                            0x123456789abcdef\n
                                                                             
                                                                            3.  
                                                                          3.  
                                                                            Write to Scratchpad register
                                                                             
                                                                            opae.io -d 15:00.0 -r 0 poke 0x16030 0xfedcba9876543210\n
                                                                             
                                                                            4.  
                                                                          4.  
                                                                            Read from Scratchpad register
                                                                             
                                                                            opae.io -d 15:00.0 -r 0 peek 0x16030\n
                                                                             
                                                                            Expected output:
                                                                             
                                                                            0xfedcba9876543210\n
                                                                             
                                                                            5.  
                                                                           
                                                                          9.  
                                                                        9.  
                                                                          Release the opae.io tool
                                                                           
                                                                          opae.io release -d 15:00.0\n
                                                                           
                                                                          10.  
                                                                        10.  
                                                                          Confirm the driver has been set back to dfl-pci
                                                                           
                                                                          opae.io ls\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          [0000:98:00.0] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                                                                           
                                                                          11.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#416-walkthrough-debug-the-fim-with-signal-tap","title":"4.1.6 Walkthrough: Debug the FIM with Signal Tap","text":"
                                                                        The following steps guide you through the process of adding a Signal Tap instance to your design. The added Signal Tap instance provides hardware to capture the desired internal signals and connect the stored trace information via JTAG. Please be aware that the added Signal Tap hardware will consume FPGA resources and may require additional floorplanning steps to accommodate these resources. Some areas of the FIM use logic lock regions and these regions may need to be re-sized.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough uses a FIM design that has had a Hello FIM module added to it. Refer to the Add a new module to the OFS FIM section for step-by-step instructions for creating a Hello FIM design. You do not need to compile the design.
                                                                          •  
                                                                         
                                                                        Perform the following steps in your development environment:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Synthesize the design using the -e build script option. You may skip this step if you are using a pre-synthesized design.
                                                                           
                                                                          cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -e --ofss tools/ofss_config/n6001.ofss n6001 work_hello_fim_with_stp\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Open the design in Quartus. The Compilation Dashboard should show that the Analysis & Synthesis step has completed.
                                                                           
                                                                          quartus $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/n6001/syn_top/ofs_top.qpf\n
                                                                           
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Open Tools -> Signal Tap Logic Analyzer
                                                                           
                                                                           
                                                                             
                                                                          1.  
                                                                            Select the Default template and click Create
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            Assign the clock for sampling the Signal Tap instrumented signals of interest. Note that the clock selected should correspond to the signals you want to view for best trace fidelity. Different clocks can be used, however, there maybe issues with trace inaccuracy due to sampling time differences. This example instruments the hello_fim_top module previously intetegrated into the FIM. If unfamiliar with code, it is helpful to use the Quartus Project Navigator to find the block of interest and open the design instance for review.
                                                                             
                                                                               
                                                                            1.  
                                                                              In the Signal Configuration -> Clock box of the Signal Tap Logic Analyzer window, click the \"...\" button
                                                                               
                                                                               
                                                                              2.  
                                                                            2.  
                                                                              In the Node Finder tool that opens, type hello_fim_top_inst|clock into the Named field, then click Search. Select the clk signal from the Matching Nodes box and click the \">\" button to move it to the Nodes Found box. Click OK to close the Node Finder dialog.
                                                                               
                                                                               
                                                                              3.  
                                                                             
                                                                            3.  
                                                                          3.  
                                                                            Update the sample depth and other Signal Tap settings as needed for your debugging criteria.
                                                                             
                                                                             
                                                                            4.  
                                                                          4.  
                                                                            In the Signal Tap GUI add the nodes to be instrumented by double-clicking on the \"Double-click to add nodes\" legend.
                                                                             
                                                                             
                                                                            5.  
                                                                          5.  
                                                                            This brings up the Node Finder to add the signals to be traced. In this example we will monitor the memory mapped interface to the Hello FIM. Select the signals that appear from the  search patterns hello_fim_top_inst|reset and hello_fim_top_inst|csr_lite_if\\*. Click Insert and close the Node Finder dialog.
                                                                             
                                                                             
                                                                            6.  
                                                                          6.  
                                                                            To provide a unique name for your Signal Tap instance, select \"auto_signaltap_0\", right-click, and select Rename Instance (F2). Provide a descriptive name for your instance, for example, stp_for_hello_fim.
                                                                             
                                                                             
                                                                            7.  
                                                                          7.  
                                                                            Save the newly created Signal Tap file, in the $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/n6001/syn_top/ directory, and give it the same name as the instance. Ensure that the Add file to current project checkbox is ticked.
                                                                             
                                                                             
                                                                            8.  
                                                                          8.  
                                                                            In the dialog that pops up, click \"Yes\" to add the newly created Signal Tap file to the project settings files.
                                                                             
                                                                             
                                                                            This will automatically add the following lines to $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/n6001/syn_top/ofs_top.qsf:
                                                                             
                                                                            set_global_assignment -name ENABLE_SIGNALTAP ON\nset_global_assignment -name USE_SIGNALTAP_FILE stp_for_hello_fim.stp\nset_global_assignment -name SIGNALTAP_FILE stp_for_hello_fim.stp\n
                                                                             
                                                                            9.  
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Close all Quartus GUIs.
                                                                           
                                                                          7.  
                                                                        7.  
                                                                          Compile the project with the Signal Tap file added to the project. Use the -k switch to perform the compilation using the files in a specified working directory and not the original ones from the cloned repository. 
                                                                           
                                                                          ./ofs-common/scripts/common/syn/build_top.sh -p -k --ofss tools/ofss_config/n6001.ofss n6001 work_hello_fim_with_stp\n
                                                                           
                                                                          Alternatively, you can copy the ofs_top.qsf and stp_for_hello_fim.stp files from the Hello FIM with STP work directory to replace the original files in the cloned OFS repository. In this scenario, all further FIM compilation projects will include the Signal Tap instance integrated into the design. Execute the following commands for this alternative flow:
                                                                           
                                                                          Copy the modified file work_hello_fim_with_stp/syn/board/n6001/syn_top/ofs_top.qsf to the source OFS repository, into syn/board/n6001/syn_top/.
                                                                           
                                                                          cd $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/n6001/syn_top\n\ncp ofs_top.qsf $OFS_ROOTDIR/syn/board/n6001/syn_top\n\ncp stp_for_hello_fim.stp $OFS_ROOTDIR/syn/board/n6001/syn_top\n
                                                                           
                                                                          Compile the FIM to create a new work directory.
                                                                           
                                                                          cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001 work_hello_fim_with_stp_src_repo\n
                                                                           
                                                                          8.  
                                                                        8.  
                                                                          Ensure that the compile completes successfully and meets timing:
                                                                           
                                                                          ***********************************\n***\n***        OFS_PROJECT: n6001\n***        OFS_BOARD: n6001\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 6\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                                                           
                                                                          9.  
                                                                        9.  
                                                                          Set up a JTAG connection to the n6001. Refer to Set up JTAG section for step-by-step instructions.
                                                                           
                                                                          10.  
                                                                        10.  
                                                                          Copy the ofs_top.sof and stp_for_hello_fim.stp files to the machine which is connected to the n6001 via JTAG.
                                                                           
                                                                          11.  
                                                                        11.  
                                                                          From the JTAG connected machine, program the $OFS_ROOTDIR/work_hello_fim_with_stp/syn/board/n6001/syn_top/output_files/ofs_top.sof image to the n6001 FPGA. Refer to the Program the FPGA via JTAG section for step-by-step programming instructions.
                                                                           
                                                                          12.  
                                                                        12.  
                                                                          Open the Quartus Signal Tap GUI
                                                                           
                                                                          $QUARTUS_ROOTDIR/bin/quartus_stpw\n
                                                                           
                                                                          13.  
                                                                        13.  
                                                                          In the Signal Tap Logic Analyzer window, select File -> Open, and choose the stp_for_hello_fim.stp file.
                                                                           
                                                                           
                                                                          14.  
                                                                        14.  
                                                                          In the right pane of the Signal Tap GUI, in the Hardware: selection box select the cable for the n6001. In the Device: selection box select the Agilex device.
                                                                           
                                                                           
                                                                          15.  
                                                                        15.  
                                                                          If the Agilex Device is not displayed in the Device: list, click the 'Scan Chain' button to re-scan the JTAG device chain.
                                                                           
                                                                          16.  
                                                                        16.  
                                                                          Create the trigger conditions. In this example, we will capture data on a rising edge of the Read Address Valid signal.
                                                                           
                                                                           
                                                                          17.  
                                                                        17.  
                                                                          Start analysis by selecting the 'stp_for_hello_fim' instance and pressing 'F5' or clicking the Run Analysis icon in the toolbar. You should see a green message indicating the Acquisition is in progress. Then, move to the Data Tab to observe the signals captured.
                                                                           
                                                                           
                                                                          18.  
                                                                        18.  
                                                                          To generate traffic in the csr_lite_if signals of the Hello FIM module, walk the DFH list or peek/poke the Hello FIM registers.
                                                                           
                                                                          opae.io init -d 0000:98:00.0\nopae.io walk -d 0000:98:00.0\nopae.io release -d 0000:98:00.0\n
                                                                           
                                                                          The signals should be captured on the rising edge of arvalid in this example. Zoom in to get a better view of the signals.
                                                                           
                                                                           
                                                                          19.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#42-preparing-fim-for-afu-development","title":"4.2 Preparing FIM for AFU Development","text":"
                                                                        To save area, the default Host Excercisers in the FIM can be replaced by a \"he_null\" blocks. There are a few things to note:
                                                                         
                                                                           
                                                                        • \"he_null\" is a minimal block with registers that responds to PCIe MMIO request. MMIO responses are required to keep PCIe alive (end points enabled in PCIe-SS needs service downstream requests).
                                                                          •  
                                                                        • If an exerciser with other I/O connections such as \"he_mem\" or \"he_hssi\" is replaced, then then those I/O ports are simply tied off.
                                                                          •  
                                                                        • The options supported are null_he_lb, null_he_hssi, null_he_mem and null_he_mem_tg. Any combination, order or all can be enabled at the same time. 
                                                                          •  
                                                                        • Finer grain control is provided for you to, for example, turn off only the exercisers in the Static Region in order to save area.
                                                                          •  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#421-walkthrough-compile-the-fim-in-preparation-for-designing-your-afu","title":"4.2.1 Walkthrough: Compile the FIM in preparation for designing your AFU","text":"
                                                                        Perform the following steps to compile a FIM for where the exercisers are removed and replaced with an he_null module while keeping the PF/VF multiplexor connections.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the [Walkthrough: Set Up Development Environment] Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the FIM repository (or use an existing cloned repository). Refer to the [Walkthrough: Clone FIM Repository] section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the [Walkthrough: Set Development Environment Variables] section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Compile the FIM with the HE_NULL compile options
                                                                           
                                                                          cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_n6001\n
                                                                           
                                                                          4.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#43-partial-reconfiguration-region","title":"4.3 Partial Reconfiguration Region","text":"
                                                                        To take advantage of the available resources in the Intel\u00ae Agilex\u00ae 7 FPGA for an AFU design, you can adjust the size of the AFU PR partition. An example reason for the changing the size of PR region is if you add more logic to the FIM region, then you may need to adjust the PR region to fit the additional logic into the static region.  Similarly, if you reduce logic in the FIM region, then you can adjust the PR region to provide more logic resources for the AFU.
                                                                         
                                                                        After the compilation of the FIM, the resources usage broken down by partitions as reported in the following two files 
                                                                        $OFS_ROOTDIR/<WORDK_DIRECTORY>/syn/board/n6001/syn_top/output_files/ofs_top.fit.place.rpt\n$OFS_ROOTDIR/<WORDK_DIRECTORY>/syn/board/n6001/syn_top/output_files/ofs_top.fit.rpt\n
                                                                         
                                                                        The next is a report of the resources usage by partitions defined for the FIM. 
                                                                         
                                                                         
                                                                        In this case, the default size for the afu_top|port_gasket|pr_slot|afu_main PR partition is large enough to hold the logic of the default AFU, which is mainly occupied by the Host Exercisers. However, larger designs might require additional resources.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#431-walkthrough-resize-the-partial-reconfiguration-region","title":"4.3.1 Walkthrough: Resize the Partial Reconfiguration Region","text":"
                                                                        Perform the following steps to customize the resources allocated to the AFU in the PR regions:
                                                                         
                                                                           
                                                                        1.  
                                                                          The OFS flow provides the TCL file $OFS_ROOTDIR/syn/board/n6001/setup/pr_assignments.tcl which defines the PR partition where the AFU is allocated.
                                                                           
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Use Quartus Chip Planner to identify the locations of the resources available within the Intel\u00ae Agilex\u00ae 7 FPGA chip for placement and routing your AFU. You need to identify a pair of coordinates, the origin (X0, Y0) and top right corner (X1, Y1) of the new or existing rectangles to modify as shown in the following image. 
                                                                           
                                                                           
                                                                          3.  
                                                                         
                                                                        The coordinates of the top right corner of the lock regions are computed indirectly based on the Width and Height, as follows.
                                                                         
                                                                        X1 = X0 + Width \nY1 = Y0 + Height\n
                                                                         
                                                                           
                                                                        1.  
                                                                          Make changes to the pr_assignments.tcl based on your findings in Quartus Chip Planner. You can modify the size and location of existing lock regions or add new ones and assign them to the AFU PR partition.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Recompile your FIM and create the PR relocatable build tree using the following commands.
                                                                           
                                                                          cd $OFS_ROOTDIR    \nofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001 work_n6001_resize_pr\n
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Analyze the resource utilization report per partition produced after recompiling the FIM.
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Perform further modification to the PR regions until the results are satisfactory. Make sure timing constraints are met.
                                                                           
                                                                          5.  
                                                                         
                                                                        For more information on how to optimize the floor plan of your Partial Reconfiguration design refer to the following online documentation.
                                                                         
                                                                           
                                                                        • Analyzing and Optimizing the Design Floorplan
                                                                          •  
                                                                         
                                                                           
                                                                        • Partial Reconfiguration Design Flow - Step 3 - Floorplan the Design
                                                                          •  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#44-pcie-configuration","title":"4.4 PCIe Configuration","text":"
                                                                        The PCIe sub-system IP and PF/VF MUX can be modified either using the OFSS flow or the IP Presets flow. The OFSS flow supports a subset of all available PCIe Sub-system settings, while the IP Preset flow can make any available PCIe Sub-system settings change. With PCIe-SS modifcations related to the PFs and VFs, the PF/VF MUX logic is automatically configured based on the PCIe-SS configuration. The sections below describe each flow.
                                                                         
                                                                           
                                                                        • PCIe Configuration Using OFSS
                                                                          •  
                                                                        • [PCIe Configuration Using IP Presets]
                                                                          •  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#441-pfvf-mux-configuration","title":"4.4.1 PF/VF MUX Configuration","text":"
                                                                        The default PF/VF MUX configuration for OFS PCIe Attach FIM for the n6001 can support up to 8 PFs and 2000 VFs distributed accross all PFs.
                                                                         
                                                                        For reference FIM configurations, you must have at least 1 PF with 1VF, or 2PFs. This is because the PR region cannot be left unconnected. PFs must be consecutive. The PFVF Limitations table describes the supported number of PFs and VFs.
                                                                         
                                                                        Table: PF/VF Limitations
                                                                         Parameter Value Min # of PFs 1 (on PF0) Max # of PFs 8 Min # of VFs 1 on PF0 Max # of VFs 2000 distributed across all PFs 
                                                                        New PF or VF instances will automatically have a null_afu module instantiated and connected to the new PF or VF.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#442-pcie-ss-configuration-registers","title":"4.4.2 PCIe-SS Configuration Registers","text":"
                                                                        The PCIe-SS configuration registers contain the Vendor, Device and Subsystem Vendor ID registers which are used in PCIe add-in cards to uniquely identify the card for assignment to software drivers. OFS has these registers set with Intel values for out of the box usage. If you are using OFS for a PCIe add in card that your company manufactures, then update the PCIe Subsytem Subsystem ID and Vendor ID registers as described below and change OPAE provided software code to properly operate with your company's register values.
                                                                         
                                                                        The Vendor ID is assigned to your company by the PCI-SIG (Special Interest Group). The PCI-SIG is the only body officially licensed to give out IDs. You must be a member of PCI-SIG to request your own ID. Information about joining PCI-SIG is available here: PCI-SIG. You select the Subsystem Device ID for your PCIe add in card.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#443-pcie-configuration-using-ofss","title":"4.4.3 PCIe Configuration Using OFSS","text":"
                                                                        The general flow for using OFSS to modify the PCIe Sub-system and PF/VF MUX is as follows:
                                                                         
                                                                           
                                                                        1. Create or modify a PCIe OFSS file with the desired PCIe configuration. 
                                                                          2.  
                                                                        2. Call this PCIe OFSS file when running the FIM build script.
                                                                          3.  
                                                                         
                                                                        The PCIe IP OFSS File Options table lists all of the configuration options supported by the OFSS flow. Any other customizations to the PCIe sub-system must be done with the IP Presets Flow.
                                                                         
                                                                        Table: PCIe IP OFSS File Options
                                                                         Section Parameter Options Default Description [ip] type pcie N/A Specifies that this OFSS file configures the PCIe-SS [settings] output_name pcie_ss N/A Specifies the output name of the PCIe-SS IP preset String N/A OPTIONAL - Specifies the name of a PCIe-SS IP presets file to use when building the FIM. When used, a presets file will take priority over any other parameters set in this OFSS file. [pf*] num_vfs Integer 0 Specifies the number of Virtual Functions in the current PF bar0_address_width Integer 12 bar4_address_width Integer 14 vf_bar0_address_width Integer 12 ats_cap_enable 0 | 1 0 vf_ats_cap_enable 0 | 1 0 prs_ext_cap_enable 0 | 1 0 pasid_cap_enable 0 | 1 0 pci_type0_vendor_id 32'h Value 0x00008086 pci_type0_device_id 32'h Value 0x0000bcce revision_id 32'h Value 0x00000001 class_code 32'h Value 0x00120000 subsys_vendor_id 32'h Value 0x00008086 subsys_dev_id 32'h Value 0x00001771 sriov_vf_device_id 32'h Value 0x0000bccf exvf_subsysid 32'h Value 0x00001771"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#4431-walkthrough-modify-the-pcie-sub-system-and-pfvf-mux-configuration-using-ofss","title":"4.4.3.1 Walkthrough: Modify the PCIe Sub-System and PF/VF MUX Configuration Using OFSS","text":"
                                                                        Perform the following steps to modify the PF/VF MUX configuration.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          View the default OFS PCIe Attach FIM for the n6001 PF/VF configuration in the the $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss file.
                                                                           
                                                                          [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n\n[pf0]\nnum_vfs = 3\nbar0_address_width = 20\nvf_bar0_address_width = 20\n\n[pf1]\n\n[pf2]\nbar0_address_width = 18\n\n[pf3]\n\n[pf4]\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Create a new PCIe OFSS file from the existing pcie_host.ofss file
                                                                           
                                                                          cp $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host.ofss $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_pfvf_mod.ofss\n
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Configure the new pcie_pfvf_mod.ofss with your desired settings. In this example we will add PF5 with 2 VFs.
                                                                           
                                                                          [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\n\n[pf0]\nnum_vfs = 3\nbar0_address_width = 20\nvf_bar0_address_width = 20\n\n[pf1]\n\n[pf2]\nbar0_address_width = 18\n\n[pf3]\n\n[pf4]\n\n[pf5]\nnum_vfs = 2\n
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Edit the $OFS_ROOTDIR/tools/ofss_config/n6001.ofss file to use the new PCIe configuration file pcie_pfvf_mod.ofss
                                                                           
                                                                          [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/n6001_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_pfvf_mod.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory.ofss\n
                                                                           
                                                                          7.  
                                                                        7.  
                                                                          Compile the FIM. 
                                                                           
                                                                          cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001 work_n6001_pfvf_mod\n
                                                                           
                                                                          8.  
                                                                        8.  
                                                                          Copy the resulting $OFS_ROOTDIR/work_n6001_pfvf_mod/syn/board/n6001/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                                                                           
                                                                          9.  
                                                                        9.  
                                                                          Switch to your deployment environment.
                                                                           
                                                                          10.  
                                                                        10.  
                                                                          Program the .bin image to the n6001 FPGA. Refer to the Program the FPGA via RSU Section for step-by-step programming instructions.
                                                                           
                                                                          11.  
                                                                        11.  
                                                                          Verify the number of VFs on the newly added PF8. In this example, we defined 2 VFs on PF5 in Step 5.
                                                                           
                                                                          sudo lspci -vvv -s 98:00.5 | grep VF\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          Initial VFs: 2, Total VFs: 2, Number of VFs: 0, Function Dependency Link: 05\nVF offset: 4, stride: 1, Device ID: bccf\nVF Migration: offset: 00000000, BIR: 0\n
                                                                           
                                                                          12.  
                                                                        12.  
                                                                          Verify communication with the newly added PF5. New PF/VF are seamlessly connected to their own CSR stub, which can be read at DFH Offset 0x0. You can bind to the function and perform opae.io peek commands to read from the stub CSR. Similarly, perform opae.io poke commands to write into the stub CSRs. Use this mechanism to verify that the new PF/VF Mux configuration allows to write and read back values from the stub CSRs. 
                                                                           
                                                                          13.  
                                                                         
                                                                        The GUID for every new PF/VF CSR stub is the same.
                                                                         
                                                                        * NULL_GUID_L           = 64'haa31f54a3e403501\n* NULL_GUID_H           = 64'h3e7b60a0df2d4850\n
                                                                         
                                                                        1. Initialize the driver on PF5
                                                                         
                                                                        ```bash\nsudo opae.io init -d 98:00.5\n```\n
                                                                         
                                                                        2. Read the GUID for the PF5 CSR stub.
                                                                         
                                                                        ```bash\nsudo opae.io -d 98:00.5 -r 0 peek 0x8\nsudo opae.io -d 98:00.5 -r 0 peek 0x10\n```\n\nExample output:\n\n```bash\n0xaa31f54a3e403501\n0x3e7b60a0df2d4850\n```\n
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#444-pcie-sub-system-configuration-using-ip-presets","title":"4.4.4 PCIe Sub-System configuration Using IP Presets","text":"
                                                                        The general flow for using IP Presets to modify he PCIe Sub-system is as follows:
                                                                         
                                                                           
                                                                        1. [OPTIONAL] Create a work directory using OFSS files if you wish to use OFSS configuration settings as a starting point.
                                                                          2.  
                                                                        2. Open the PCIe-SS IP and make desired modifications.
                                                                          3.  
                                                                        3. Create an IP Presets file.
                                                                          4.  
                                                                        4. Create an PCIe OFSS file that uses the IP Presets file.
                                                                          5.  
                                                                        5. Build the FIM with the PCIe OFSS file from Step 4.
                                                                          6.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#4441-walkthrough-modify-pcie-sub-system-and-pfvf-mux-configuration-using-ip-presets","title":"4.4.4.1 Walkthrough: Modify PCIe Sub-System and PF/VF MUX Configuration Using IP Presets","text":"
                                                                        Perform the following steps to use an IP preset file to configure the PCIe Sub-system and PF/VF MUX. In this example, we will change the Revision ID of PF0. While this modification can be done with the OFSS flow, this walkthrough is intended to show the procedure for making any PCIe configuration change using IP presets.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment to build the FIM. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough requires an OFS Agilex PCIe Attach deployment environment to test the design. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          [OPTIONAL] Run the setup stage of the build script using your desired OFSS configration to create a working directory for the n6001 design.
                                                                           
                                                                          ./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/n6001.ofss n6001 work_n6001\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Open the PCIe-SS in the work directory using Quartus Parameter Editor. If you performed Step 3, open the PCIe-SS IP from the work directory; otherwise, open the PCIe-SS IP from the source files.
                                                                           
                                                                          qsys-edit $OFS_ROOTDIR/work_n6001/ipss/pcie/qip/pcie_ss.ip\n
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Modify the settings as desired. In this example we are changing the Device ID to 0xbccf and the Revision ID to 0x2. In the IP Parameter Editor, scroll down and expand the PCIe Interfaces Ports Settings -> Port 0 -> PCIe0 Device Identification Registers -> PCIe0 PF0 IDs tab and make these changes.
                                                                           
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Once you are satisfied with your modifcations, create a new IP Preset file.
                                                                           
                                                                             
                                                                          1.  
                                                                            click New... in the Presets window.
                                                                             
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            In the New Preset window, set a unique Name for the preset; for example, n6001-rev2.
                                                                             
                                                                             
                                                                            3.  
                                                                          3.  
                                                                            Click the ... button to set the save location for the IP Preset file to $OFS_ROOTDIR/ipss/pcie/presets. Set the File Name to match the name selected in Step 9. Click OK.
                                                                             
                                                                             
                                                                            4.  
                                                                          4.  
                                                                            In the New Preset window, click Save. Click No when prompted to add the file to the IP search path.
                                                                             
                                                                             
                                                                            5.  
                                                                           
                                                                          7.  
                                                                        7.  
                                                                          Close IP Parameter Editor without saving or generating HDL.
                                                                           
                                                                          8.  
                                                                        8.  
                                                                          Create a new PCIe OFSS file in the $OFS_ROOTDIR/tools/ofss_config/pcie directory. For example:
                                                                           
                                                                          touch $OFS_ROOTDIR/tools/ofss_config/pcie/pcie_host_mod_preset.ofss\n
                                                                           
                                                                          Insert the following into the OFSS file to specify the IP Preset file created in Step 6. 
                                                                           
                                                                          [ip]\ntype = pcie\n\n[settings]\noutput_name = pcie_ss\npreset = n6001-rev2\n
                                                                           
                                                                          9.  
                                                                        9.  
                                                                          Edit the $OFS_ROOTDIR/tools/ofss_config/n6001.ofss file to call new OFSS file created in Step 10.
                                                                           
                                                                          [include]  \"$OFS_ROOTDIR\"/tools/ofss_config/n6001_base.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host_mod_preset.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss  \"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory.ofss
                                                                           
                                                                          10.  
                                                                        10.  
                                                                          Compile the design with the modified n6001.ofss file.
                                                                           
                                                                          cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss,tools/ofss_config/hssi/hssi_8x25.ofss n6001 work_n6001_pcie_mod\n
                                                                           
                                                                          11.  
                                                                        11.  
                                                                          Copy the resulting $OFS_ROOTDIR/work_n6001_pcie_mod/syn/board/n6001/syn_top/output_files/ofs_top_hps.sof image to your deployment environmenment for JTAG programming, or copy a bin file (e.g. ofs_top_page1_unsigned_user1.bin) for RSU programming.
                                                                           
                                                                          Note: OPAE FPGA management commands require recognition of the FPGA PCIe Device ID for control.  If there is a problem between OPAE management recognition of FPGA PCIe values, then control of the card will be lost.  For this reason, you are strongly encouraged to program the FPGA via JTAG to load the test FPGA image.  If there is a problem with the SOF image working with your host software that is updated for the new PCIe settings, then you can load a known good SOF file to recover.  Once you sure that both the software and FPGA work properly, you can load the FPGA into FPGA flash using the OPAE command fpgasupdate.
                                                                           
                                                                          12.  
                                                                        12.  
                                                                          Switch to your deployment environment.
                                                                           
                                                                          13.  
                                                                        13.  
                                                                          Program the image to the n6001 FPGA. Refer to the Program the FPGA via JTAG Section for step-by-step JTAG programming instructions, or the Program the FPGA via RSU Section for step-by-step RSU programming instructions.
                                                                           
                                                                          14.  
                                                                        14.  
                                                                          Use lspci to verify that the PCIe changes have been implemented.
                                                                           
                                                                          lspci -nvmms 98:00.0\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          Slot:   98:00.0\nClass:  1200\nVendor: 8086\nDevice: bcce\nSVendor:        8086\nSDevice:        1771\nPhySlot:        1-1\nRev:    02\nNUMANode:       1\nIOMMUGroup:     8\n
                                                                           
                                                                          15.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#45-minimal-fim","title":"4.5 Minimal FIM","text":"
                                                                        In a minimal FIM, the exercisers and Ethernet subsystem are removed from the design, and the AFU PR area is expanded to make use of the available area previously used by the removed components. This minimal FIM is useful for HDL applications.
                                                                         
                                                                        There are two types of provided minimal FIMs:
                                                                         
                                                                           
                                                                        • 2PF: this minimal FIM has two physical functions, with the APF/BPF on PF0, and the AFU PR region on PF1.
                                                                           
                                                                          •  
                                                                         
                                                                           
                                                                        • 1PF/1VF: This minmal FIM has a single physical function, with the APF/BPF on PF0, and the AFU PR region on PF0-VF0.
                                                                           
                                                                          •  
                                                                         
                                                                        The FIM source repository contains OFSS file that can be used to build the two different types of minimal FIM.
                                                                         
                                                                           
                                                                        • $OFS_ROOTDIR/tools/ofss_config/n6001_2pf.ofss
                                                                          •  
                                                                        • $OFS_ROOTDIR/tools/ofss_config/n6001_1pf_1vf.ofss
                                                                          •  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#451-walkthrough-create-a-minimal-fim","title":"4.5.1 Walkthrough: Create a Minimal FIM","text":"
                                                                        Perform the following steps to create a Minimal FIM.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          The OFS FIM repo provides a PR assignments TCL file which optimizes the PR region for the minimal FIM. Copy the minimal PR assignments TCL file into the pr_assignments.tcl file location for use in the FIM build process.
                                                                           
                                                                             
                                                                          1.  
                                                                            Rename the current pr_assignments.tcl file to pr_assignments_base.tcl for future use
                                                                             
                                                                            mv $OFS_ROOTDIR/syn/board/n6001/setup/pr_assignments.tcl $OFS_ROOTDIR/syn/board/n6001/setup/pr_assignments_base.tcl\n
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            Copy the pr_assignments_slim.tcl file to pr_assignments.tcl to be used in the current build
                                                                             
                                                                            cp $OFS_ROOTDIR/syn/board/n6001/setup/pr_assignments_slim.tcl $OFS_ROOTDIR/syn/board/n6001/setup/pr_assignments.tcl\n
                                                                             
                                                                            3.  
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Compile the FIM with Null HEs compile option, the No HSSI compile option, and 1PF/1VF configuration OFSS file.
                                                                           
                                                                          cd $OFS_ROOTDIR\n
                                                                           
                                                                             
                                                                          •  
                                                                            For 2PF Minimal FIM
                                                                             
                                                                            ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001_2pf.ofss n6001:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_n6001_minimal_fim\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            For 1PF/1VF Minimal FIM
                                                                             
                                                                            ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001_1pf_1vf.ofss n6001:null_he_lb,null_he_hssi,null_he_mem,null_he_mem_tg work_n6001_minimal_fim\n
                                                                             
                                                                            •  
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Review the $OFS_ROOTDIR/work_n6001_minimal_fim/syn/board/n6001/syn_top/output_files/ofs_top.fit.rpt utilization report to see the utilization statistics for the Minimal fim. Refer to [Appendix A] Table A-4 for the expected utilization for this Minimal FIM.
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Copy the resulting $OFS_ROOTDIR/work_n6001_minimal_fim/syn/board/n6001/syn_top/output_files/ofs_top.sof image to your deployment environmenment.
                                                                           
                                                                          7.  
                                                                        7.  
                                                                          Switch to your deployment environment, if different than your development environment.
                                                                           
                                                                          8.  
                                                                        8.  
                                                                          Program the .bin image to the n6001 FPGA. Refer to the Program the FPGA via RSU Section for step-by-step programming instructions.
                                                                           
                                                                          9.  
                                                                        9.  
                                                                          Verify the minimal FIM design on the board.
                                                                           
                                                                             
                                                                          •  
                                                                            For 2PF Minimal FIM:
                                                                             
                                                                               
                                                                            1. Use opae.io ls to verify that there are two PFs
                                                                              2.  
                                                                             
                                                                            sudo opae.io ls\n
                                                                             
                                                                            Example output:
                                                                             
                                                                            [0000:98:00.1] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n[0000:98:00.0] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                                                                             
                                                                               
                                                                            1.  
                                                                              Use fpgainfo port to verify the ports.
                                                                               
                                                                              sudo fpgainfo port\n
                                                                               
                                                                              Example output:
                                                                               
                                                                              //****** PORT ******//\nInterface                        : DFL\nObject Id                        : 0xEE00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nInterface                        : UIO\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 00000000-0000-0000-0000-000000000000\n
                                                                               
                                                                              2.  
                                                                            2.  
                                                                              Bind the VFIO driver on PF1.
                                                                               
                                                                              sudo opae.io init -d 98:00.1\n
                                                                               
                                                                              Example output:
                                                                               
                                                                              Unbinding (0x8086,0xbcce) at 0000:98:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:98:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:98:00.1 is 15\n
                                                                               
                                                                              3.  
                                                                            3.  
                                                                              Use fpgainfo port to verify the ports. There should now be a port entry for PF1. The Accelerator GUID for PF1 should be as shown below, which is the GUID for the HE-MEM.
                                                                               
                                                                                //****** PORT ******//\n  Interface                        : DFL\n  Object Id                        : 0xEE00000\n  PCIe s:b:d.f                     : 0000:98:00.0\n  Vendor Id                        : 0x8086\n  Device Id                        : 0xBCCE\n  SubVendor Id                     : 0x8086\n  SubDevice Id                     : 0x1771\n  Socket Id                        : 0x00\n  //****** PORT ******//\n  Interface                        : VFIO\n  Object Id                        : 0x2098000000000000\n  PCIe s:b:d.f                     : 0000:98:00.1\n  Vendor Id                        : 0x8086\n  Device Id                        : 0xBCCE\n  SubVendor Id                     : 0x8086\n  SubDevice Id                     : 0x1771\n  Socket Id                        : 0x01\n  Accelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n  //****** PORT ******//\n  Interface                        : UIO\n  Object Id                        : 0xED00000\n  PCIe s:b:d.f                     : 0000:98:00.0\n  Vendor Id                        : 0x8086\n  Device Id                        : 0xBCCE\n  SubVendor Id                     : 0x8086\n  SubDevice Id                     : 0x1771\n  Socket Id                        : 0x01\n  Accelerator GUID                 : 00000000-0000-0000-0000-000000000000\n
                                                                               
                                                                              4.  
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            For 1PF/1VF Minimal FIM:
                                                                             
                                                                               
                                                                            1. Use opae.io ls to verify that there is one PF.
                                                                              2.  
                                                                             
                                                                            sudo opae.io ls\n
                                                                             
                                                                            Example output:
                                                                             
                                                                            [0000:98:00.0] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                                                                             
                                                                               
                                                                            1.  
                                                                              Use lspci to verify that there is 1 VF on PF0.
                                                                               
                                                                              sudo lspci -vvv -s 98:00.0 | grep VF\n
                                                                               
                                                                              Example output:
                                                                               
                                                                              Initial VFs: 1, Total VFs: 1, Number of VFs: 0, Function Dependency Link: 00\nVF offset: 1, stride: 1, Device ID: bcce\nVF Migration: offset: 00000000, BIR: 0\n
                                                                               
                                                                              2.  
                                                                            2.  
                                                                              Use fpgainfo port to verify the ports.
                                                                               
                                                                              sudo fpgainfo port\n
                                                                               
                                                                              Example output:
                                                                               
                                                                              //****** PORT ******//\nInterface                        : DFL\nObject Id                        : 0xEE00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nInterface                        : UIO\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 00000000-0000-0000-0000-000000000000\n
                                                                               
                                                                              3.  
                                                                            3.  
                                                                              Initialize one virtual function
                                                                               
                                                                              sudo pci_device 0000:98:00.0 vf 1\n
                                                                               
                                                                              4.  
                                                                            4.  
                                                                              Use opae.io ls to verify the VF has been initialized.
                                                                               
                                                                              sudo opae.io ls\n
                                                                               
                                                                              Example output:
                                                                               
                                                                              [0000:98:00.0] (0x8086:0xbcce 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n[0000:98:00.1] (0x8086:0xbccf 0x8086:0x1771) Intel Acceleration Development Platform N6001 (Driver: dfl-pci)\n
                                                                               
                                                                              5.  
                                                                            5.  
                                                                              Bind the VFIO driver on VF0
                                                                               
                                                                              sudo opae.io init -d 0000:98:00.1\n
                                                                               
                                                                              6.  
                                                                            6.  
                                                                              Use fpgainfo port to verify the ports. There should now be a port entry for VF0. The Accelerator GUID for VF0 should be as shown below, which is the GUID for the HE-MEM.
                                                                               
                                                                              sudo fpgainfo port\n
                                                                               
                                                                              Example output:
                                                                               
                                                                              //****** PORT ******//\nInterface                        : DFL\nObject Id                        : 0xEE00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nInterface                        : VFIO\nObject Id                        : 0x2098000000000000\nPCIe s:b:d.f                     : 0000:98:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n//****** PORT ******//\nInterface                        : UIO\nObject Id                        : 0xED00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 00000000-0000-0000-0000-000000000000\n
                                                                               
                                                                              7.  
                                                                             
                                                                            •  
                                                                           
                                                                          10.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#46-migrate-to-a-different-agilex-device-number","title":"4.6 Migrate to a Different Agilex Device Number","text":"
                                                                        The following instructions enable a user to change the device part number of the Intel\u00ae FPGA SmartNIC N6001-PL. Please be aware that this release works with Intel\u00ae Agilex\u00ae 7 FPGA devices that have P tile for PCIe and E tile for Ethernet.  Other tiles will take further work.
                                                                         
                                                                        You may wish to change the device part number for the following reasons
                                                                         
                                                                           
                                                                        1. Migrate to same device package but with a different density
                                                                          2.  
                                                                        2. Migrate to a different package and with a different or same density
                                                                          3.  
                                                                         
                                                                        The default device for the Intel\u00ae FPGA SmartNIC N6001-PL is AGFB014R24A2E2V
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#461-walkthrough-migrate-to-a-different-agilex-device-number","title":"4.6.1 Walkthrough: Migrate to a Different Agilex Device Number","text":"
                                                                        Perform the following steps to migrate your design to target a different Agilex device using the OFSS build flow. In this example we will change the device from the default AGFB014R24A2E2V to a new device AGFB022R25A2E2V.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Modify the part field in the $OFS_ROOTDIR/tools/ofss_config/n6001_base.ofss file to use AGFB022R25A2E2V. This is only necessary if you are using the OFSS flow.
                                                                           
                                                                          [ip]\ntype = ofs\n\n[settings]\nplatform = n6001\nfamily = agilex\nfim = base_x16\npart = AGFB022R25A2E2V\ndevice_id = 6001\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Modify the DEVICE field in the $OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_top.qsf file.
                                                                           
                                                                          ############################################################################################\n# FPGA Device\n############################################################################################\n\nset_global_assignment -name FAMILY Agilex\nset_global_assignment -name DEVICE AGFB022R25A2E2V\n
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Modify the DEVICE field in the $OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_pr_afu.qsf file.
                                                                           
                                                                          ############################################################################################\n# FPGA Device\n############################################################################################\n\nset_global_assignment -name FAMILY Agilex\nset_global_assignment -name DEVICE AGFB022R25A2E2V\n
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Modify the DEVICE field in te $OFS_ROOTDIR/ipss/pmci/pmci_ss.qsf file.
                                                                           
                                                                          set_global_assignment -name DEVICE AGFB022R25A2E2V\n
                                                                           
                                                                          7.  
                                                                        7.  
                                                                          If you are changing to a device with a different package, you must change the pin assignments in the location files. If you would like Quartus to attempt to pin out the design automatically, you may remove all pin assignments instead. Typically you will be required to set certain pins manually in order to guide Quartus for a successful compile (e.g. transceiver reference clocks). In this example we will start by commenting out all of the pin constraints in the following files and attempt to let Quartus pin out the design as much as possibe:
                                                                           
                                                                          $OFS_ROOTDIR/syn/board/n6001/setup/emif_loc.tcl\n$OFS_ROOTDIR/syn/board/n6001/setup/hps_loc.tcl\n$OFS_ROOTDIR/syn/board/n6001/setup/pmci_loc.tcl\n$OFS_ROOTDIR/syn/board/n6001/setup/top_loc.tcl\n
                                                                           
                                                                          For example:
                                                                           
                                                                          #set_location_assignment PIN_CU26 -to hssi_rec_clk[0]\n
                                                                           
                                                                          8.  
                                                                        8.  
                                                                          Identify the pins you wish to assign prior to compiling. In this example, we will re-pin some of the reference clocks to help guide the fitter. Refer to the Pin-Out Files for Intel FPGAs for the pin list of your device. In this example, the Migration Re-Pin Mapping table below shows the pins we will re-pin in the constraints files.
                                                                           
                                                                          Table: Migration Re-Pin Mapping
                                                                           Pin Name FIM Signal Name AGF 014 R24A Pin # AGF 022 R25A Pin # REFCLK_GXER9A_CH0p cr3_cpri_reflclk_clk[0] PIN_AT13 PIN_CE18 REFCLK_GXER9A_CH0n \"cr3_cpri_reflclk_clk[0](n)\" PIN_AP13 PIN_CA18 REFCLK_GXER9A_CH1p cr3_cpri_refclk_clk[1] PIN_AR14 PIN_CC19 REFCLK_GXER9A_CH1n \"cr3_cpri_refclk_clk[1](n)\" PIN_AN14 PIN_BW19 REFCLK_GXER9A_CH2p cr3_cpri_refclk_clk[2] PIN_AJ12 PIN_BL17 REFCLK_GXER9A_CH2n \"cr3_cpri_refclk_clk[2](n)\" PIN_AH11 PIN_BJ15 REFCLK_GXER9A_CH3p qsfp_ref_clk PIN_AK13 PIN_BN18 REFCLK_GXER9A_CH3n \"qsfp_ref_clk(n)\" PIN_AH13 PIN_BJ18 REFCLK_GXER9A_CH4p cr3_cpri_reflclk_clk_184_32m PIN_AJ14 PIN_BL19 REFCLK_GXER9A_CH4n \"cr3_cpri_reflclk_clk_184_32m(n)\" PIN_AL14 PIN_BR19 REFCLK_GXER9A_CH5p cr3_cpri_reflclk_clk_153_6m PIN_AR16 PIN_CC21 REFCLK_GXER9A_CH5n \"cr3_cpri_reflclk_clk_153_6m(n)\" PIN_AN16 PIN_BW21 REFCLK_GXPL10A_CH0n \"PCIE_REFCLK0(n)\" PIN_AH49 PIN_DD56 REFCLK_GXPL10A_CH0p PCIE_REFCLK0 PIN_AJ48 PIN_DF57 REFCLK_GXPL10A_CH2n \"PCIE_REFCLK1(n)\" PIN_AD49 PIN_CT56 REFCLK_GXPL10A_CH2p PCIE_REFCLK1 PIN_AE48 PIN_CV57 
                                                                          9.  
                                                                        9.  
                                                                          Re-pin the reference clocks defined in $OFS_ROOTDIR/syn/board/n6001/setup/top_loc.tcl
                                                                           
                                                                          set_location_assignment PIN_BN18 -to qsfp_ref_clk\nset_location_assignment PIN_BJ18 -to \"qsfp_ref_clk(n)\"\nset_location_assignment PIN_CC19 -to cr3_cpri_refclk_clk[1]\nset_location_assignment PIN_BW19 -to \"cr3_cpri_refclk_clk[1](n)\"\nset_location_assignment PIN_BL17 -to cr3_cpri_refclk_clk[2]\nset_location_assignment PIN_BJ15 -to \"cr3_cpri_refclk_clk[2](n)\"\nset_location_assignment PIN_CE18 -to cr3_cpri_reflclk_clk[0]\nset_location_assignment PIN_CA18 -to \"cr3_cpri_reflclk_clk[0](n)\"\nset_location_assignment PIN_BL19 -to cr3_cpri_reflclk_clk_184_32m\nset_location_assignment PIN_BR19 -to \"cr3_cpri_reflclk_clk_184_32m(n)\"\nset_location_assignment PIN_CC21 -to cr3_cpri_reflclk_clk_153_6m\nset_location_assignment PIN_BW21 -to \"cr3_cpri_reflclk_clk_153_6m(n)\"\n\nset_location_assignment PIN_DD56 -to \"PCIE_REFCLK0(n)\"\nset_location_assignment PIN_DF57 -to PCIE_REFCLK0\nset_location_assignment PIN_CT56 -to \"PCIE_REFCLK1(n)\"\nset_location_assignment PIN_CV57 -to PCIE_REFCLK1\n
                                                                           
                                                                          10.  
                                                                        10.  
                                                                          Un-comment the instance assignemnts for the transceiver reference clocks defined in $OFS_ROOTDIR/syn/board/n6001/setup/top_loc.tcl.
                                                                           
                                                                          set_instance_assignment -name IO_STANDARD \"DIFFERENTIAL LVPECL\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=disable_3p3v_tol\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=156250000\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to qsfp_ref_clk\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=TRUE\" -to qsfp_ref_clk\nset_instance_assignment -name IO_STANDARD \"DIFFERENTIAL LVPECL\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=disable_3p3v_tol\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=184320000\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=FALSE\" -to cr3_cpri_refclk_clk[1]\nset_instance_assignment -name IO_STANDARD \"DIFFERENTIAL LVPECL\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=disable_3p3v_tol\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=153600000\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=FALSE\" -to cr3_cpri_refclk_clk[2]\nset_instance_assignment -name IO_STANDARD \"DIFFERENTIAL LVPECL\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=disable_3p3v_tol\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=245760000\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=FALSE\" -to cr3_cpri_reflclk_clk[0]\nset_instance_assignment -name IO_STANDARD \"DIFFERENTIAL LVPECL\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=enable_3p3v_tol\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=184320000\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=FALSE\" -to cr3_cpri_reflclk_clk_184_32m\nset_instance_assignment -name IO_STANDARD \"DIFFERENTIAL LVPECL\" -to cr3_cpri_reflclk_clk_153_6m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_termination=enable_term\" -to cr3_cpri_reflclk_clk_153_6m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_enable_3p3v=enable_3p3v_tol\" -to cr3_cpri_reflclk_clk_153_6m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_disable_hysteresis=enable_hyst\" -to cr3_cpri_reflclk_clk_153_6m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_input_freq=153600000\" -to cr3_cpri_reflclk_clk_153_6m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_powerdown_mode=false\" -to cr3_cpri_reflclk_clk_153_6m\nset_instance_assignment -name HSSI_PARAMETER \"refclk_divider_use_as_bti_clock=FALSE\" -to cr3_cpri_reflclk_clk_153_6m\n
                                                                           
                                                                          11.  
                                                                        11.  
                                                                          Compile a flat design. It is recommended to compile a flat design first before incorporating a PR region in the design. This reduces design complexity while you determine the correct pinout for your design.
                                                                           
                                                                          cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001:flat work_n6001_migrate_device_ofss\n
                                                                           
                                                                          12.  
                                                                        12.  
                                                                          The compile should succeed. If the compile fails with errors relating to the pinout, review the error messages and modify the pinout.
                                                                           
                                                                          ***********************************\n***\n***        OFS_PROJECT: n6001\n***        OFS_BOARD: n6001\n***        Q_PROJECT:  ofs_top\n***        Q_REVISION: ofs_top\n***        SEED: 3\n***        Build Complete\n***        Timing Passed!\n***\n***********************************\n
                                                                           
                                                                          13.  
                                                                        13.  
                                                                          After a successful compile, to preserve pin assignemnts youmust hard code the new pin asigments back to the constraints files.
                                                                           
                                                                          $OFS_ROOTDIR/syn/board/n6001/setup/emif_loc.tcl\n$OFS_ROOTDIR/syn/board/n6001/setup/hps_loc.tcl\n$OFS_ROOTDIR/syn/board/n6001/setup/pmci_loc.tcl\n$OFS_ROOTDIR/syn/board/n6001/setup/top_loc.tcl\n
                                                                           
                                                                          14.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#47-modify-the-memory-sub-system","title":"4.7 Modify the Memory Sub-System","text":"
                                                                        OFS allows modifications on the Memory Sub-System in the FIM. This section provides examples walkthroughs for modifiying the Memory-SS.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#471-walkthrough-modify-the-memory-sub-system-using-ip-presets-with-ofss","title":"4.7.1 Walkthrough: Modify the Memory Sub-System Using IP Presets With OFSS","text":"
                                                                        This walkthrough will go through the flow of modifying the Memory-SS in the OFS FIM. In this example, we will enable ECC on memory channels 0-3. Note that routes for the ECC pins on Channels 0 and 1 are not physiclly present on standard n6001 board hardware; the purpose of this walkthrough is only to show an example of how to make modifications to the IP.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Open the Memory Subsystem mem_ss.ip in IP Parameter Editor
                                                                           
                                                                          qsys-edit $OFS_ROOTDIR/ipss/mem/qip/mem_ss/mem_ss.ip\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          The Memory Subsystem IP will open in IP Parameter Editor. Click Dive Into Packaged Subsystem.
                                                                           
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          The Platform Designer mem_ss view opens. All of the EMIFs are shown in the Filter window.
                                                                           
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Click each EMIF 0 through 3 and perform the following actions.
                                                                           
                                                                             
                                                                          1.  
                                                                            In the Parameters window, click the Memory tab and change the DQ width to 40.
                                                                             
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            In the Parameters window, click the Controller tab.
                                                                             
                                                                             
                                                                            3.  
                                                                          3.  
                                                                            Scroll down and check the box for Enable Error Detection and Correction Logic with ECC. 
                                                                             
                                                                             
                                                                            4.  
                                                                           
                                                                          7.  
                                                                        7.  
                                                                          Once Step 6 has been done for each EMIF 0-3, click File -> Save. Close the Platform Designer window.
                                                                           
                                                                          8.  
                                                                        8.  
                                                                          In the IP Parameter Editor Presets window, click New to create an IP Presets file.
                                                                           
                                                                           
                                                                          9.  
                                                                        9.  
                                                                          In the New Preset window, set the Name for the preset. In this case we will name it n6001-ecc.
                                                                           
                                                                           
                                                                          10.  
                                                                        10.  
                                                                          Click the ... button to select the location for the Preset file.
                                                                           
                                                                          11.  
                                                                        11.  
                                                                          In the Save As window, change the save location to $OFS_ROOTDIR/ipss/mem/qip/presets and change the File Name to n6001-ecc.qprs. Click OK.
                                                                           
                                                                           
                                                                          12.  
                                                                        12.  
                                                                          Click Save in the New Preset window. Click No when prompted to add the file to the IP search path.
                                                                           
                                                                           
                                                                          13.  
                                                                        13.  
                                                                          Close the IP Parameter Editor. You do not need to generate or save the IP.
                                                                           
                                                                          14.  
                                                                        14.  
                                                                          Edit the $OFS_ROOTDIR/syn/board/n6001/setup/emif_loc.tcl file to add pin assignments for the new signals supporting ECC on Channels 0-3. Note that routes for the ECC pins on Channels 0 and 1 are not physiclly present on a standard n6001 board.
                                                                           
                                                                          # CH0 DQS4 (ECC)\nset_location_assignment PIN_CG48  -to ddr4_mem[0].dbi_n[4]\nset_location_assignment PIN_CF47  -to ddr4_mem[0].dqs_n[4]\nset_location_assignment PIN_CH47  -to ddr4_mem[0].dqs[4]\nset_location_assignment PIN_CE50  -to ddr4_mem[0].dq[32]\nset_location_assignment PIN_CG50  -to ddr4_mem[0].dq[33]\nset_location_assignment PIN_CF49  -to ddr4_mem[0].dq[34]\nset_location_assignment PIN_CH49  -to ddr4_mem[0].dq[35]\nset_location_assignment PIN_CE46  -to ddr4_mem[0].dq[36]\nset_location_assignment PIN_CG46  -to ddr4_mem[0].dq[37]\nset_location_assignment PIN_CF45  -to ddr4_mem[0].dq[38]\nset_location_assignment PIN_CH45  -to ddr4_mem[0].dq[39]\n\n# CH1 DQS4 (ECC)\nset_location_assignment PIN_DC34  -to ddr4_mem[1].dbi_n[4]\nset_location_assignment PIN_CY33  -to ddr4_mem[1].dqs_n[4]\nset_location_assignment PIN_DB33  -to ddr4_mem[1].dqs[4]\nset_location_assignment PIN_DA36  -to ddr4_mem[1].dq[32]\nset_location_assignment PIN_DC36  -to ddr4_mem[1].dq[33]\nset_location_assignment PIN_CY35  -to ddr4_mem[1].dq[34]\nset_location_assignment PIN_DB35  -to ddr4_mem[1].dq[35]\nset_location_assignment PIN_DA32  -to ddr4_mem[1].dq[36]\nset_location_assignment PIN_DC32  -to ddr4_mem[1].dq[37]\nset_location_assignment PIN_CY31  -to ddr4_mem[1].dq[38]\nset_location_assignment PIN_DB31  -to ddr4_mem[1].dq[39]\n\n\n# CH2 DQS4 (ECC)\nset_location_assignment PIN_G36  -to ddr4_mem[2].dbi_n[4]\nset_location_assignment PIN_H35  -to ddr4_mem[2].dqs_n[4]\nset_location_assignment PIN_F35  -to ddr4_mem[2].dqs[4]\nset_location_assignment PIN_G38  -to ddr4_mem[2].dq[32]\nset_location_assignment PIN_J38  -to ddr4_mem[2].dq[33]\nset_location_assignment PIN_H33  -to ddr4_mem[2].dq[34]\nset_location_assignment PIN_J34  -to ddr4_mem[2].dq[35]\nset_location_assignment PIN_F33  -to ddr4_mem[2].dq[36]\nset_location_assignment PIN_H37  -to ddr4_mem[2].dq[37]\nset_location_assignment PIN_F37  -to ddr4_mem[2].dq[38]\nset_location_assignment PIN_G34  -to ddr4_mem[2].dq[39]\n\n# CH3 DQS4 (ECC)\nset_location_assignment PIN_L50 -to ddr4_mem[3].dbi_n[4]\nset_location_assignment PIN_P49 -to ddr4_mem[3].dqs_n[4]\nset_location_assignment PIN_M49 -to ddr4_mem[3].dqs[4]\nset_location_assignment PIN_M51 -to ddr4_mem[3].dq[32]\nset_location_assignment PIN_N48 -to ddr4_mem[3].dq[33]\nset_location_assignment PIN_M47 -to ddr4_mem[3].dq[34]\nset_location_assignment PIN_L48 -to ddr4_mem[3].dq[35]\nset_location_assignment PIN_P47 -to ddr4_mem[3].dq[36]\nset_location_assignment PIN_P51 -to ddr4_mem[3].dq[37]\nset_location_assignment PIN_N52 -to ddr4_mem[3].dq[38]\nset_location_assignment PIN_L52 -to ddr4_mem[3].dq[39]\n
                                                                           
                                                                          15.  
                                                                        15.  
                                                                          Edit the $OFS_ROOTDIR/tools/ofss_config/memory/memory.ofss file to use the n6001-ecc preset that was generated previously.
                                                                           
                                                                          [ip]\ntype = memory\n\n[settings]\noutput_name = mem_ss_fm\npreset = n6001-ecc\n
                                                                           
                                                                          16.  
                                                                        16.  
                                                                          Compile the design with the n6001.ofss file, which will use the modified memory.ofss file.
                                                                           
                                                                          ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss,tools/ofss_config/hssi/hssi_8x25.ofss n6001 work_n6001_mem_ecc_preset\n
                                                                           
                                                                          17.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#48-modify-the-ethernet-sub-system","title":"4.8 Modify the Ethernet Sub-System","text":"
                                                                        This section describes the flows for modifying the Ethernet Sub-System. There are three flows you may use to make modifications.
                                                                         
                                                                           
                                                                        • Modify the Ethernet Sub-System with OFSS supported changes only. These modifications are supported natively by the build script, and are made at run-time of the build script. This flow is useful for users who only need to leverage natively supported HSSI OFSS settings.
                                                                          •  
                                                                        • Modify the Ethernet Sub-System with OFSS supported changes, then make additional custom modifications not covered by OFSS. These modifications will be captured in a presets file which can be used in future compiles. This flow is useful for users who whish to leverage pre-made HSSI OFSS settings, but make additional modifications not natively supported by HSSI OFSS.
                                                                          •  
                                                                        • Modify the Ethernet Sub-System without HSSI OFSS. These modification will be made directly in the source files.
                                                                          •  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#481-walkthrough-modify-the-ethernet-sub-system-channels-with-pre-made-hssi-ofss","title":"4.8.1 Walkthrough: Modify the Ethernet Sub-System Channels With Pre-Made HSSI OFSS","text":"
                                                                        This walkthrough describes how to use OFSS to configure the Ethernet-SS. Refer to section HSSI IP OFSS File for detailed information about modifications supported by Ethernet-SS OFSS files. This walkthrough is useful for users who only need to leverage the pre-made, natively supported HSSI OFSS settings.
                                                                         
                                                                        Pre-Requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Edit the $OFS_ROTDIR/tools/ofss_config/n6001.ofss file to use the desired Ethernet-SS OFSS configuration. The pre-provided OFSS configurations are as follows:
                                                                           
                                                                             
                                                                          •  
                                                                            To select 2x4x25GbE configuration, add the following line
                                                                             
                                                                            \"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_8x25.ofss\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            To select 2x4x10GbE configuration, add the following line
                                                                             
                                                                            \"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_8x10.ofss\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            To select 2x1x100GbE configuration, add the following line
                                                                             
                                                                            \"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_2x100.ofss\n
                                                                             
                                                                            •  
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Compile the FIM using the n6001.ofss file.
                                                                           
                                                                          cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss n6001 work_n6001\n
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          The resulting FIM will contain the Ethernet-SS configuration specified in Step 3. The Ethernet-SS IP in the resulting work directory shows the parameter settings that are used.
                                                                           
                                                                          6.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#482-walkthrough-add-channels-to-the-ethernet-sub-system-channels-with-custom-hssi-ofss","title":"4.8.2 Walkthrough: Add Channels to the Ethernet Sub-System Channels With Custom HSSI OFSS","text":"
                                                                        This walkthrough describes how to create an use a custom OFSS file to add channels to the Ethernet-SS and compile a design with a 3x4x10GbE Ethernet-SS configuration. This walkthrough is useful for users who wish to leverage the natively supported HSSI OFSS settings.
                                                                         
                                                                        Pre-Requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Create a new HSSI OFSS file $OFS_ROOTDIR/tools/ofss_config/hssi/hssi_12x10.ofss with the following contents.
                                                                           
                                                                          [ip]\ntype = hssi\n\n[settings]\noutput_name = hssi_ss\nnum_channels = 12\ndata_rate = 10GbE\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Edit the $OFS_ROOTDIR/tools/ofss_config/n6001.ofss file to use the new HSSI OFSS file generated in Step 3.
                                                                           
                                                                          [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/n6001_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_12x10.ofss\n
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Identify the which channels will be added. You may use the E-Tile Channel Placement Tool to aid in your design. In this example we will add the 4 new 10GbE channels to Channels 8-11.
                                                                           
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Based on your channel selection, identify which pins will be used. Refer to the Pin-Out Files for Intel FPGAs determine the required pins for your device. In this example we are targeting the AGFB014R24A2E2V device. Set the pin assignments in the $OFS_ROOTDIR/syn/board/n6001/setup/top_loc.tcl file.
                                                                           
                                                                          set_location_assignment PIN_AV7  -to qsfp_serial[2].rx_p[0]\nset_location_assignment PIN_AW10 -to qsfp_serial[2].rx_p[1]\nset_location_assignment PIN_BB7  -to qsfp_serial[2].rx_p[2]\nset_location_assignment PIN_BC10 -to qsfp_serial[2].rx_p[3]\n\nset_location_assignment PIN_AV1 -to qsfp_serial[2].tx_p[0]\nset_location_assignment PIN_AW4 -to qsfp_serial[2].tx_p[1]\nset_location_assignment PIN_BB1 -to qsfp_serial[2].tx_p[2]\nset_location_assignment PIN_BC4 -to qsfp_serial[2].tx_p[3]\n
                                                                           
                                                                          7.  
                                                                        7.  
                                                                          Edit the NUM_QSFP_PORTS value in the $OFS_ROOTDIR/ipss/hssi/rtl/inc/ofs_fim_eth_plat_if_pkg.sv file to 3.
                                                                           
                                                                          localparam NUM_QSFP_PORTS       = 3; // QSFP cage on board\n
                                                                           
                                                                          8.  
                                                                        8.  
                                                                          Compile the design. It is recommended to compile a flat design first before incorporating a PR region in the design. This reduces design complexity while you determine the correct pinout for your design.
                                                                           
                                                                          cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001:flat work_n6001_12x10\n
                                                                           
                                                                          9.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#483-walkthrough-modify-the-ethernet-sub-system-with-pre-made-hssi-ofss-plus-additional-modifications","title":"4.8.3 Walkthrough: Modify the Ethernet Sub-System With Pre-Made HSSI OFSS Plus Additional Modifications","text":"
                                                                        This walkthrough describes how to use OFSS to first modify the Ethernet-SS, then make additional modifications on top. Refer to section HSSI IP OFSS File for detailed information about modifications supported by Ethernet-SS OFSS files. This flow is useful for users who whish to leverage pre-made OFSS settings, but make additional modifications not natively supported by OFSS.
                                                                         
                                                                        Pre-Requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Edit the $OFS_ROTDIR/tools/ofss_config/n6001.ofss file to use the desired Ethernet-SS OFSS configuration starting point.
                                                                           
                                                                             
                                                                          •  
                                                                            To select 2x4x25GbE configuration, add the following line
                                                                             
                                                                            \"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_8x25.ofss\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            To select 2x4x10GbE configuration, add the following line
                                                                             
                                                                            \"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_8x10.ofss\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            To select 2x1x100GbE configuration, add the following line
                                                                             
                                                                            \"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_2x100.ofss\n
                                                                             
                                                                            •  
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Run the setup stage of the build script with the OFSS file to create a work directory which contains the Ethernet-SS IP configuration specified in Step 3.
                                                                           
                                                                          cd $OFS_ROOTDIR\n\n./ofs-common/scripts/common/syn/build_top.sh --stage setup --ofss tools/ofss_config/n6001.ofss n6001 work_n6001\n
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Open the Ethernet-SS IP in Quartus Parameter Editor. The IP settings will match te configuration of the OFSS file defined in Step 3. Make any additional modifications in the Parameter Editor.
                                                                           
                                                                          qsys-edit $OFS_ROOTDIR/work_n6001/ipss/hssi/qip/hssi_ss/hssi_ss.ip\n
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Once you are satisfied with your changes, click the New... button in the Presets pane of IP Parameter Editor.
                                                                           
                                                                           
                                                                          7.  
                                                                        7.  
                                                                          In the New Preset window, create a unique Name. In this example the name is n6001-hssi-presets.
                                                                           
                                                                           
                                                                          8.  
                                                                        8.  
                                                                          Click the ... button to select where to save the preset file. Give it a name, and save it to $OFS_ROOTDIR/ipss/hssi/qip/hssi_ss/presets
                                                                           
                                                                           
                                                                          9.  
                                                                        9.  
                                                                          Click Save in the New Preset window. Click No when prompted to add the file to the IP search path.
                                                                           
                                                                          10.  
                                                                        10.  
                                                                          Close out of all Quartus GUIs. You do not need to save or compile the IP.
                                                                           
                                                                          11.  
                                                                        11.  
                                                                          Create a new HSSI OFSS file in the $OFS_ROOTDIR/tools/ofss_config/hssi directory named hssi_preset_n6001.ofss with the following contents. Note that the num_channels and data_rate settings will be overwritten by the contents of the preset file. The preset setting must match the name you selected in Step 7.
                                                                           
                                                                          [ip]\ntype = hssi\n\n[settings]\noutput_name = hssi_ss\nnum_channels = 8\ndata_rate = 25GbE\npreset = n6001-hssi-presets\n
                                                                           
                                                                          12.  
                                                                        12.  
                                                                          Edit the $OFS_ROOTDIR/tools/ofss_config/n6001.ofss file to use the new HSSI OFSS file created in Step 10.
                                                                           
                                                                          [include]\n\"$OFS_ROOTDIR\"/tools/ofss_config/n6001_base.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/pcie/pcie_host.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/iopll/iopll.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/memory/memory.ofss\n\"$OFS_ROOTDIR\"/tools/ofss_config/hssi/hssi_preset_n6001.ofss\n
                                                                           
                                                                          13.  
                                                                        13.  
                                                                          Compile the design using the n6001 OFSS file. It is recommended to compile a flat design first before incorporating a PR region in the design. This reduces design complexity while you modify the FIM.
                                                                           
                                                                          ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001:flat work_n6001_hssi_preset\n
                                                                           
                                                                          14.  
                                                                        14.  
                                                                          The resulting FIM will contain the Ethernet-SS configuration specified by the presets file. The Ethernet-SS IP in the resulting work directory shows the parameter settings that are used.
                                                                           
                                                                          15.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#484-walkthrough-modify-the-ethernet-sub-system-without-hssi-ofss","title":"4.8.4 Walkthrough: Modify the Ethernet Sub-System Without HSSI OFSS","text":"
                                                                        This walkthrough describes how to modify the Ethernet-SS wihout using OFSS. This flow will edit the Ethernet-SS IP source directly. This walkthrough is useful for users who wish to make all Ethernet-SS modifications manually, without leveraging HSSI OFSS.
                                                                         
                                                                        Pre-Requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Open the Ethernet-SS IP in Quartus Parameter Editor. Make your modifications in the Parameter Editor.
                                                                           
                                                                          qsys-edit $OFS_ROOTDIR/work_n6001/ipss/hssi/qip/hssi_ss/hssi_ss.ip\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Once you are satisfied with your changes, click the Generate HDL. Save the design if prompted.
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Compile the design.
                                                                           
                                                                             
                                                                          •  
                                                                            If you are not using any other OFSS files in your compilation flow, use the following command to compile. It is recommended to compile a flat design before incorporating a PR region in the design. This reduces design complexity while you modify the FIM.
                                                                             
                                                                            ./ofs-common/scripts/common/syn/build_top.sh n6001:flat work_n6001\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          • If you are using OFSS files for other IP in the design, ensure that the top level OFSS file (e.g. $OFS_ROOTDIR/tools/ofss_config/n6001.ofss) does not specify an HSSI OFSS file. Then use the following command to compile. It is recommended to compile a flat design first before incorporating a PR region in the design. This reduces design complexity while you modify the FIM.
                                                                            •  
                                                                           
                                                                          ./ofs-common/scripts/common/syn/build_top.sh --ofss tools/ofss_config/n6001.ofss n6001:flat work_n6001\n
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          The resulting FIM will contain the Ethernet-SS configuration contained in the hssi_ss.ip source file.
                                                                           
                                                                          7.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#49-modifying-the-hps","title":"4.9 Modifying the HPS","text":"
                                                                        This section describes ways to modify the HPS.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#491-walkthrough-remove-the-hps","title":"4.9.1 Walkthrough: Remove the HPS","text":"
                                                                        Perform the following steps to remove the HPS from the FIM design.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires a development environment. Refer to the Set Up Development Environment Section for instructions on setting up a development environment.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Clone the OFS PCIe Attach FIM repository (or use an existing cloned repository). Refer to the Clone FIM Repository section for step-by-step instructions.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Set development environment variables. Refer to the Set Development Environment Variables section for step-by-step instructions.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Create a Memory Sub-system IP presets file with the connection to the HPS removed.
                                                                           
                                                                             
                                                                          1.  
                                                                            Open the the Memory Sub-System IP
                                                                             
                                                                            qsys-edit $OFS_ROOTDIR/ipss/mem/qip/mem_ss/mem_ss.ip\n
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            In the IP Parameter Editor window that opens, remove the entries corresponding to the HPS (row #4) in the Memory Interfaces and Application Interfaces sections. To do this, click the row to be removed, then click the minus (-) button.
                                                                             
                                                                             
                                                                            3.  
                                                                          3.  
                                                                            In the Presets pane, click New... to create a new IP presets file.
                                                                             
                                                                             
                                                                            4.  
                                                                          4.  
                                                                            In the New Preset window, create a unique preset name. For example, n6001-mem-no-hps.
                                                                             
                                                                             
                                                                            5.  
                                                                          5.  
                                                                            Click the ... button to select the save location of the IP presets file. In the Save As window, set the Look In field to the memory IP presets directory $OFS_ROOTDIR/ipss/mem/qip/presets. Set the File Name field to match the name selected in Step 4. Click OK.
                                                                             
                                                                             
                                                                            6.  
                                                                          6.  
                                                                            Click Save in the New Preset window. Click No when prompted to add the file to the IP search path.
                                                                             
                                                                             
                                                                            7.  
                                                                          7.  
                                                                            Close IP Parameter Editor without saving or generating HDL.
                                                                             
                                                                            8.  
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Edit the Memory OFSS file $OFS_ROOTDIR/tools/ofss_config/memory/memory.ofss to use the IP presets file generated in Step 4.
                                                                           
                                                                          [ip]\ntype = memory\n\n[settings]\noutput_name = mem_ss\npreset = n6001-mem-no-hps\n
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Edit $OFS_ROOTDIR/syn/board/n6001/syn_top/ofs_top.qsf to comment out the INCLUDE_HPS and INCLUDE_UART macro definitions.
                                                                           
                                                                          #set_global_assignment -name VERILOG_MACRO \"INCLUDE_HPS\"\n#set_global_assignment -name VERILOG_MACRO \"INCLUDE_UART\"\n
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Build the FIM.
                                                                           
                                                                          ./ofs-common/scripts/common/syn/build_top.sh -p --ofss tools/ofss_config/n6001.ofss,tools/ofss_config/hssi/hssi_8x25.ofss n6001 work_n6001_no_hps\n
                                                                           
                                                                          7.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#5-fpga-configuration","title":"5. FPGA Configuration","text":"
                                                                        Configuring the Agilex FPGA on the n6001 can be done by Remote System Update (RSU) using OPAE commands, or by programming a SOF image to the FPGA via JTAG using Quartus Programer.
                                                                         
                                                                        Programming via RSU will program the flash device on the board for non-volatile image updates. Programming via JTAG will configure the FPGA for volatile image updates.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#51-walkthrough-set-up-jtag","title":"5.1 Walkthrough: Set up JTAG","text":"
                                                                        Perform the following steps to set up a JTAG connection to the Intel\u00ae FPGA SmartNIC N6001-PL.
                                                                         
                                                                        Pre-requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough requires a workstation with Quartus Prime Pro Version 23.3 tools installed, specifically the jtagconfig tool.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough requires an Intel FPGA Download Cable II.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Set the board switches to dynamically select either the Intel\u00ae Agilex\u00ae 7 FPGA or MAX\u00ae 10 device on the JTAG chain.
                                                                           
                                                                             
                                                                          1. Set SW1.1=ON as shown in the next image. The switches are located at the back of the Intel\u00ae FPGA SmartNIC N6001-PL.
                                                                            2.  
                                                                           
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          The Intel\u00ae FPGA SmartNIC N6001-PL has a 10 pin JTAG header on the top side of the board. Connect an Intel\u00ae FPGA Download II Cable to the JTAG header of the Intel\u00ae FPGA SmartNIC N6001-PL as shown in picture below. This picture shows the Intel\u00ae FPGA SmartNIC N6001-PL card installed in the middle bay, top slot of a SuperMicro\u00ae SYS-220HE-FTNR server where the lower slot does not have card installed allowing the Intel\u00ae Download II cables to pass through removed the slot access. 
                                                                           
                                                                           
                                                                          Note: If using the Intel FGPA download Cable on Linux, add the udev rule as described in Intel FPGA Download Cable Driver for Linux.
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Set the JTAG chain to select the Intel\u00ae Agilex\u00ae 7 FPGA as the target by writing to the JTAG enable register in the BMC (Register 0x378). This is done via PMCI registers 0x2040C and 0x20400.
                                                                           
                                                                          Note: The commands below are targeted to a board with PCIe B:D.F of 98:00.0. Use the correct PCIe B:D.F of your card.
                                                                           
                                                                          sudo opae.io init -d 0000:98:00.0 $USER\nsudo opae.io -d 0000:98:00.0 -r 0 poke 0x2040c 0x100000000\nsudo opae.io -d 0000:98:00.0 -r 0 poke 0x20400 0x37800000002\nsudo opae.io release -d 0000:98:00.0\n
                                                                           
                                                                          Note: To later re-direct the JTAG back to the MAX 10 device, execute the following commands.
                                                                           
                                                                          sudo opae.io init -d 0000:b1:00.0 $USER\nsudo opae.io -d 0000:b1:00.0 -r 0 poke 0x2040c 0x000000000\nsudo opae.io -d 0000:b1:00.0 -r 0 poke 0x20400 0x37800000002\nsudo opae.io release -d 0000:b1:00.0\n
                                                                           
                                                                          Optionally, rather than dynamically commanding Intel\u00ae Agilex\u00ae 7 FPGA/MAX10 selection with the PMCI register settings, you can fix the selection with the following switch settings shown in the table below:
                                                                           SW1.1 SW2 JTAG Target OFF OFF Intel\u00ae Agilex\u00ae 7 FPGA OFF ON MAX\u00ae 10 FPGA ON X Intel\u00ae Agilex\u00ae 7 FPGA if BMC register 0x378=0x1 ON X MAX\u00ae 10 FPGA if BMC register 0x378=0x0 
                                                                          4.  
                                                                        4.  
                                                                          Use the jtagconfig tool to check that the JTAG chain contains the AGFB014R24A2E2V device.
                                                                           
                                                                          <QUARTUS_INSTALL_DIR>/23.3/quartus/bin/jtagconfig\n
                                                                           
                                                                          Example expected output:
                                                                           
                                                                          TBD\n
                                                                           
                                                                          5.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#52-walkthrough-program-the-fpga-via-jtag","title":"5.2 Walkthrough: Program the FPGA via JTAG","text":"
                                                                        This walkthrough describes the steps to program the Agilex FPGA on the Intel\u00ae FPGA SmartNIC N6001-PL with a SOF image via JTAG.
                                                                         
                                                                        Pre-Requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough requires a SOF image which will be programmed to the Agilex FPGA. Refer to the Compile OFS FIM Section for step-by-step instructions for generating a SOF image.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough requires a JTAG connection to the n6001. Refer to the Set up JTAG section for step-by-step instructions.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel\u00ae FPGA SmartNIC N6001-PL is connected via JTAG.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Start in your deployment environment.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          If the card is already programmed with an OFS enabled design, determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                                                                           
                                                                          fpgainfo fme\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Remove the card from PCIe prior to programming. This will disable AER for the PCIe root port to prevent a surprise link-down event during programming.
                                                                           
                                                                          sudo pci_device b1:00.0 unplug\n
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Switch to the machine with JTAG connection to the n6001, if different than your deployment machine.
                                                                           
                                                                          5.  
                                                                        5.  
                                                                          Open the Quartus programmer GUI
                                                                           
                                                                          quartus_pgmw\n
                                                                           
                                                                           
                                                                          6.  
                                                                        6.  
                                                                          Click Hardware Setup to open the Hardware Setup dialog window.
                                                                           
                                                                             
                                                                          1.  
                                                                            In the Currently selected hardware field select the n6001.
                                                                             
                                                                            2.  
                                                                          2.  
                                                                            In the Hardware frequency field enter 16000000 Hz
                                                                             
                                                                             
                                                                            3.  
                                                                          3.  
                                                                            Click Close
                                                                             
                                                                            4.  
                                                                           
                                                                          7.  
                                                                        7.  
                                                                          In the Quartus Prime Programmer window, click Auto Detect.
                                                                           
                                                                          8.  
                                                                        8.  
                                                                          If prompted, select the AGFB014R24A2E2V device. The JTAG chain should show the divice.
                                                                           
                                                                           
                                                                          9.  
                                                                        9.  
                                                                          Right click the AGFB014R24A2E2V row and selct Change File.
                                                                           
                                                                           
                                                                          10.  
                                                                        10.  
                                                                          In the Select New Programming File window that opens, select the .sof image you wish to program and click Open.
                                                                           
                                                                          11.  
                                                                        11.  
                                                                          Check the Program/Configure box for the AGFB014R24A2E2V row, then click Start. Wait for the Progress bar to show 100% (Success).
                                                                           
                                                                          12.  
                                                                        12.  
                                                                          Close the Quartus Programmer GUI. You can answer 'No' if a dialog pops up asking to save the 'Chain.cdf' file
                                                                           
                                                                          13.  
                                                                        13.  
                                                                          Switch to the deployment environment, if different than the JTAG connected machine.
                                                                           
                                                                          14.  
                                                                        14.  
                                                                          Replug the board PCIe
                                                                           
                                                                          sudo pci_device b1:00.0 plug\n
                                                                           
                                                                          15.  
                                                                        15.  
                                                                          Run fpgainfo fme to verify communication with the board, and to check the PR Interface ID.
                                                                           
                                                                          Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                                                           
                                                                          16.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#53-remote-system-update","title":"5.3 Remote System Update","text":"
                                                                        The OPAE fpgasupdate tool can be used to update the Intel Max10 Board Management Controller (BMC) image and firmware (FW), root entry hash, and FPGA Static Region (SR) and user image (PR). The fpgasupdate tool only accepts images that have been formatted using PACsign. If a root entry hash has been programmed onto the board, then you must also sign the image using the correct keys. Refer to the Security User Guide: Intel Open FPGA Stack for information on created signed images and on programming and managing the root entry hash.  
                                                                         
                                                                        The Intel\u00ae FPGA SmartNIC N6001-PL ships with a factory, user1, and user2 programmed image for both the FIM and BMC FW and RTL on all cards. The platform ships with a single FIM image that can be programmed into either user1 or user2, depending in the image selected.
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#531-walkthrough-program-the-fpga-via-rsu","title":"5.3.1 Walkthrough: Program the FPGA via RSU","text":"
                                                                        This walkthrough describes the steps to program the Agilex FPGA on the Intel\u00ae FPGA SmartNIC N6001-PL with a BIN image via JTAG.
                                                                         
                                                                        Pre-Requisites:
                                                                         
                                                                           
                                                                        • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex\u00ae 7 PCIe Attach FPGAs (Intel FPGA SmartNIC N6001-PL) for instructions on setting up a deployment environment.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough requires a BIN image which will be programmed to the Agilex FPGA. Refer to the Compile OFS FIM Section for step-by-step instructions for generating a BIN image. The image used for programming must be formatted with PACsign before programming. This is done automatically by the build script.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough requires a JTAG connection to the n6001. Refer to the Set up JTAG section for step-by-step instructions.
                                                                          •  
                                                                         
                                                                           
                                                                        • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel\u00ae FPGA SmartNIC N6001-PL is connected via JTAG.
                                                                          •  
                                                                         
                                                                        Steps:
                                                                         
                                                                           
                                                                        1.  
                                                                          Start in your deployment environment.
                                                                           
                                                                          2.  
                                                                        2.  
                                                                          Determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                                                                           
                                                                          fpgainfo fme\n
                                                                           
                                                                          Example output:
                                                                           
                                                                          Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                                                           
                                                                          3.  
                                                                        3.  
                                                                          Use the OPAE fpgasupdate command to program a PACsign signed image to flash. The flash slot which will be programmed is determined by the PACsign header.
                                                                           
                                                                          sudo fpgasupdate <IMAGE> <PCIe B:D.F>\n
                                                                           
                                                                             
                                                                          •  
                                                                            Example: update User Image 1 in flash
                                                                             
                                                                            sudo fpgasupdate ofs_top_page1_unsigned_user1.bin 98:00.0\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            Example: update User Image 2 in flash
                                                                             
                                                                            sudo fpgasupdate ofs_top_page2_unsigned_user2.bin 98:00.0\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            Example: update Factory Image in flash
                                                                             
                                                                            sudo fpgasupdate ofs_top_page0_unsigned_factory.bin 98:00.0\n
                                                                             
                                                                            •  
                                                                           
                                                                          4.  
                                                                        4.  
                                                                          Use the OPAE rsu command to reconfigure the FPGA with the new image. You may select which image to configure from (User 1, User 2, Factory).
                                                                           
                                                                          sudo rsu fpga --page <PAGE> <PCIe B:D.F>\n
                                                                           
                                                                             
                                                                          •  
                                                                            Example: configure FPGA with User 1 Image
                                                                             
                                                                            sudo rsu fpga --page user1 98:00.0\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            Example: configure FPGA with User 2 Image
                                                                             
                                                                            sudo rsu fpga --page user2 98:00.0\n
                                                                             
                                                                            •  
                                                                           
                                                                             
                                                                          •  
                                                                            Example: configure FPGA with Factory Image
                                                                             
                                                                            sudo rsu fpga --page factory 98:00.0\n
                                                                             
                                                                            •  
                                                                           
                                                                          5.  
                                                                        "},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#appendix","title":"Appendix","text":""},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#appendix-a-resource-utilization-tables","title":"Appendix A: Resource Utilization Tables","text":"
                                                                        Table A-1 Default Out-of-Tree FIM Resource Utilization
                                                                         Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % top 181,018.30 37.15 784 11.03 afu_top 104,994.20 21.55 287 4.04 pcie_wrapper 36,565.00 7.51 195 2.74 hssi_wrapper 20,132.10 4.13 173 2.43 mem_ss_top 9,092.80 1.87 76 1.07 pmci_wrapper 4,269.30 0.88 26 0.37 alt_sld_fab_0 2,726.90 0.56 13 0.18 bpf 1,364.60 0.28 0 0.00 qsfp_top 620.10 0.13 4 0.06 fme_top 615.30 0.13 6 0.08 qsfp_top 614.00 0.13 4 0.06 rst_ctrl 17.90 0.00 0 0.00 sys_pll 0.50 0.00 0 0.00 hps_ss 0.00 0.00 0 0.00 
                                                                        Table A-2 Minimal FIM Resource Utilization
                                                                         Compilation Hierarchy Node ALMs needed ALM Utilization % M20Ks M20K Utilization % 103192.5 21.18 410 5.77 afu_top 45848.7 9.41 150 2.11 auto_fab_0 1770.1 0.36 9 0.13 bpf 1265.8 0.26 0 0.0 fme_top 662.4 0.14 6 0.08 hps_ss 0.0 0.0 0 0.0 hssi_dummy_csr 675.5 0.14 0 0.0 mem_ss_top 8723.9 1.79 60 0.84 pcie_wrapper 38361.7 7.87 159 2.24 pmci_wrapper 4509.9 0.93 26 0.37 qsfp0_dummy_csr 672.2 0.14 0 0.0 qsfp1_dummy_csr 681.7 0.14 0 0.0 rst_ctrl 18.0 0.0 0 0.0 sys_pll 0.5 0.0 0 0.0"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#appendix-b-glossary","title":"Appendix B: Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                        Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                         
                                                                        OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/","title":"Hard Processor System Software Developer Guide: OFS for Intel Agilex FPGAs Targeting Intel\u00ae N6000/1-PL FPGA SmartNIC Platform","text":"
                                                                        Quartus Prime Pro Version: 23.1
                                                                         
                                                                        Last updated: Last updated: February 03, 2024 
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#glossary","title":"Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel Max10 or Intel Cyclone10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implemented on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Management Component Transport Protocol MCTP A standardized model for communication with management controllers. Defines the transport protocol carrying PLDM messages through the BMC. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE-SDK The OPAE-SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Level Data Model PLDM A specification for reporting telemetry data to the host, such as board temperature, voltage, and current. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#file-types","title":"File Types","text":"Extension Description ITS File (*.its) The image source file which describes the contents of the image and defines various properties used during boot. Actual contents to be included in the image (kernel, ramdisk, etc.) are specified in the image source file as paths to the appropriate data files. ITB File (*.itb) Produced as output from mkimage, using an image source file. Contains all the referenced data (kernel, ramdisk, SSBL, etc.) and other information needed by UBoot to handle the image properly. This image is transferred to the target and booted. DTB File (*.dtb) The Device Tree Blob is loaded into memory by U-Boot during the boot process, and a pointer to it is shared with the kernel. This file describes the system's hardware layout to the kernel. FIT Image (*.fit) Format used for uImage payloads developed by U-Boot. On aarch64 the kernel must be in image format and needs a device tree to boot. SPL (*.spl) The Secondary Program Loader is a small binary which is embedded in a FIM SOF and loaded into board DDR4 RAM when the FPGA is initially configured. This file is responsible for loading U-Boot into system RAM."},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#10-introduction","title":"1.0 Introduction","text":"
                                                                        The Open FPGA Stack (OFS) is a modular collection of hardware platform components, open source upstreamed software, and broad ecosystem support that enables an efficient path to develop a custom FPGA platform. OFS Provides a framework of FPGA synthesizable code, a simulation environment and synthesis/simulation scripts.  The updated OFS architecture for Intel Agilex FPGA devices improves upon the modularity, configurability and scalability of the first release of the OFS architecture while maintaining compatibility with the original design.  The primary components of the FPGA Interface Manager or shell of the reference design are:
                                                                         
                                                                           
                                                                        • PCIe Subsystem
                                                                          •  
                                                                        • HSSI Subsystem
                                                                          •  
                                                                        • Memory Subsystem
                                                                          •  
                                                                        • Hard Processor System (HPS)
                                                                          •  
                                                                        • Reset Controller
                                                                          •  
                                                                        • FPGA Management Engine (FME)
                                                                          •  
                                                                        • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                                                                          •  
                                                                        • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                                                                          •  
                                                                        • SPI Interface to BMC controller
                                                                          •  
                                                                         
                                                                        The Intel\u00ae N6000-PL and N6001-PL FPGA SmartNIC Platforms are acceleration cards that use the OFS infrastructure. The key difference between these two platforms is:
                                                                         
                                                                           
                                                                        • Intel\u00ae N6000-PL SmartNIC Platform has a bifurcated PCIe bus with Gen4x8 interfacing to the the Intel Agilex FPGA and Gen4x8 interfacing to an Intel E810 SmartNIC.  This platform is targeted specifically for VRAN, UPF and vCSR applications.  The FPGA designs targeting these vertical market applications were generated using the OFS infrastructure.
                                                                          •  
                                                                        • Intel\u00ae N6001-PL SmartNIC Platform has a Gen4x16 interface directly to the Intel Agilex FPGa and is not populated with an Intel E810 SmartNIC.  This platform is the reference platform for the OFS reference designs for Intel Agilex FPGA.  
                                                                          •  
                                                                         
                                                                        Note: throughout this document \"Intel N6000/1-PL FPGA SmartNIC Platform\" denotes both cards. This document describes the software package that runs on the Hard Processor System (HPS) which is a key component within both platforms.
                                                                         
                                                                        The Intel N6000/1-PL FPGA SmartNIC Platform has a customized build script that can be used to both set up a development environment and build the essential pieces of the HPS software image. This script, meta-bake.py, has its own dedicated Section 3.1 Building U-Boot which can be used to quickly get started with the HPS build flow. It is recommended you use this build script to construct the first and second stage bootloader files, as it will handle all of the setup and patching required to build out your complete Yocto image. You can familiarize yourself with the contents of this package in its public GitHub repository located at https://github.com/OPAE/meta-opae-fpga/tree/main/tools/meta-bake. All other information included for individual components is included for learning purposes only and is not meant as a recipe for image creation.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#11-reference-documents","title":"1.1 Reference Documents","text":"
                                                                        This document pulls much of its information from related Agilex FPGA documentation hosted on intel.com. Reading these references is not required for initial platform bring up, but will serve to further your knowledge of the FPGA SoC boot and configuration process.
                                                                         
                                                                        Table 1. Reference Documents
                                                                         Document Title Intel\u00ae Agilex\u2122 Hard Processor System Technical Reference Manual Intel\u00ae Agilex\u2122 SoC FPGA Boot User Guide Intel\u00ae Agilex\u2122 Configuration User Guide"},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#12-reference-images","title":"1.2 Reference Images","text":"
                                                                        Intel has provided a set of two pre-compiled ITB images that can be used for exploration and evaluation of the HPS bring-up flow. These images contain the complete SSBL package specific to the board and can be copied to the N6000/1-PL SmartNIC Platform with an HPS enabled FIM loaded. Refer to Section 4.1 Example Boot for an example on how to use the built-in copy engine IP in tandem with the host-side cpeng software to transfer an SSBL.
                                                                         
                                                                        The package is found on the official OFS 2023.3-2 Release on GitHub. Two ITB artifacts are included at the bottom of the page under Assets - one with the Vendor Authorized Boot (VAB) certificate included, and one without. Which you choose to load depends on whether the currently loaded FIM requires VAB authentication. Section 4.3 Example Boot contains instructions on the boot flow using these files for platform bring up.
                                                                         
                                                                        The default username for these two images is root and the password is empty. A good place to start after loading the ITB is to set up SSH for file transfers and the remote console, as seen in 8.0 Connecting remotely to the HPS using ssh.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#20-architecture-overview","title":"2.0 Architecture Overview","text":"
                                                                        The OFS architecture is classified into:
                                                                         
                                                                           
                                                                        • 1. Host Interface Adapters (PCIe)
                                                                          •  
                                                                        • 2. Low Performance Peripherals
                                                                          •  
                                                                             
                                                                          • 2.1. Slow speed peripherals (example: JTAG, I2C, SMBus, and so on)
                                                                            •  
                                                                          • 2.2. Management peripherals (example: FPGA FME)
                                                                            •  
                                                                           
                                                                        • 3. High Performance Peripherals
                                                                          •  
                                                                             
                                                                          • 3.1. Memory peripherals
                                                                            •  
                                                                          • 3.2. Acceleration Function Units (AFUs)
                                                                            •  
                                                                          • 3.3. HPS Peripheral
                                                                            •  
                                                                           
                                                                        • 4. Fabrics
                                                                          •  
                                                                             
                                                                          • 4.1. Peripheral Fabric (multi drop)
                                                                            •  
                                                                          • 4.2. AFU Streaming fabric (point to point)
                                                                            •  
                                                                           
                                                                         
                                                                        The HPS is connected to the AFU and implements all the board specific flows that customers require to begin the application development using the HPS such as host communication, firmware load and update, integration with OFS, and memory. The HPS implements a basic Hello World software application and is intended as a starting point for customers to begin development with HPS.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#21-hps-peripherals","title":"2.1 HPS Peripherals","text":"
                                                                        Figure 1  Intel Agilex FPGA HPS Peripherals
                                                                         
                                                                         
                                                                        The Intel Agilex\u2122 SoC integrates a full-featured Arm\u00ae Cortex-A53\u00ae MPCore Processor.
                                                                         
                                                                        The Cortex-A53 MPCore supports high-performance applications and provides the capability for secure processing and virtualization.
                                                                         
                                                                        Each CPU in the processor has the following features:
                                                                         
                                                                           
                                                                        • Support for 32-bit and 64-bit instruction sets.
                                                                          •  
                                                                        • To pipeline with symmetric dual issue of most instructions.
                                                                          •  
                                                                        • Arm NEON\u00ae Single Instruction Multiple Data (SIMD) co-processor with a Floating-Point Unit (FPU)
                                                                          •  
                                                                        • Single and double-precision IEEE-754 floating point math support
                                                                          •  
                                                                        • Integer and polynomial math support.
                                                                          •  
                                                                        • Symmetric Multiprocessing (SMP) and Asymmetric Multiprocessing (AMP) modes.
                                                                          •  
                                                                        • Armv8 Cryptography Extension.
                                                                          •  
                                                                        • Level 1 (L1) cache:
                                                                          •  
                                                                        • 32 KB two-way set associative instruction cache.
                                                                          •  
                                                                        • Single Error Detect (SED) and parity checking support for L1 instruction cache.
                                                                          •  
                                                                        • 32 KB four-way set associative data cache.
                                                                          •  
                                                                        • Error checking and correction (ECC), Single Error Correct, Double Error Detect (SECDED) protection for L1 data cache.
                                                                          •  
                                                                        • Memory Management Unit (MMU) that communicates with the System MMU (SMMU).
                                                                          •  
                                                                        • Generic timer.
                                                                          •  
                                                                        • Governor module that controls clock and reset.
                                                                          •  
                                                                        • Debug modules:
                                                                          •  
                                                                        • Performance Monitor Unit.
                                                                          •  
                                                                        • Embedded Trace Macrocell (ETMv4).
                                                                          •  
                                                                        • Arm CoreSight\u00ae cross trigger interface, the four CPUs share a 1 MB L2 cache with ECC, SECDED protection.
                                                                          •  
                                                                         
                                                                        A Snoop Control Unit (SCU) maintains coherency between the CPUs and communicates with the system Cache Coherency Unit (CCU). At a system level, the Cortex-A53 MPCore interfaces to a Generic Interrupt Controller (GIC), CCU, and System Memory Management Unit (SMMU).
                                                                         
                                                                        Beyond the Arm Cortex-A53 MPCore Processor, the HPS integrates a variety of useful peripherals for use in your design, such as Ethernet, USB, Memory Controller, on-chip RAM, SPI, UART and more. Refer to the Intel\u00ae Agilex\u2122 Hard Processor System Technical Reference Manual for more information.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#22-zarlink-device","title":"2.2 Zarlink Device","text":"
                                                                        The Microchip\u00ae Zarlink device ZL30793 is used for time synchronization. It acts as the protocol engine that drives IEEE 1588-2008 PTP protocol. The Zarlink device is connected to the HPS side and the programming interface is SPI. The FPGA bitstream containing the HPS has the First Stage Bootloader (FSBL) only. This enable commands to be given from a terminal program connected through UART.
                                                                         
                                                                        The software in HPS can access the Clock generator through SPI to enable write and read operations controlled by the terminal program. It can also read the status of the hold over and Loss of Lock signals and control the LED.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#23-copy-engine","title":"2.3 Copy Engine","text":"
                                                                        A host with OPAE SDK and Linux DFL installed will provide the hps OPAE command with related options to transfer images from host to the HPS image. The module in the OFS FIM and HPS software that performs this transfer is called the Copy Engine (CPE), which is included by default within the HPS image.
                                                                         
                                                                        Refer to the Getting Started Guide: \u00ae Open FPGA Stack for Intel Agilex FPGAs for platform and software installation instructions.
                                                                         
                                                                        The CPE software is patched into Linux on the HPS in Yocto through the meta-intel-fpga-refdes layer. This service is daemonized and requires systemd in order to operate. This service will communicate with the HPS IP integrated in the FIM in order to coordinate and monitor file transfers from the host CPE software to DDR connected the HPS. The CPE HPS-side software takes advantage of the built-in I/O lightweight kernel module to communicate with the FIM's HPS IP. It can restart the transfer if the initial transfer of the image is not successful. The CPE can also serve as reference on how to integrate your own systemd service in the Linux build running on the HPS.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#24-boot-flow","title":"2.4 Boot Flow","text":"
                                                                        The boot flow for the Agilex OFS design for the Intel N6000/1-PL FPGA SmartNIC Platform is an FPGA-first boot flow, meaning the Intel Agilex Secure Device Manager (SDM) configures the FPGA first and then boots the HPS. Alternatively, you can boot the HPS first and then configure the FPGA core as part of the Second-Stage Boot Loader (SSBL) or after the Operating System (OS) boots. HPS-first boot is not covered in this document, but for more information please refer to the Intel\u00ae Agilex\u2122 SoC FPGA Boot User Guide.
                                                                         
                                                                        For the FPGA-first boot flow supported by the Intel Agilex OFS FIM, the First Stage Bootloader is part of FPGA bitstream. The available Secure Device Manager (SDM) in the FPGA initially configures the FPGA core and periphery in this mode. The first stage bootloader is produced as a part of a U-Boot build and cna be manually inserted into a Quartus generated SOF file as shown in step 7 of Section 9.2 Configuring the HPS.
                                                                         
                                                                        After completion, the HPS boots. All the I/O, including the HPS-allocated I/O, are configured, and brought out of tri-state. If the HPS is not booted:
                                                                         
                                                                           
                                                                        • The HPS is held in reset
                                                                          •  
                                                                        • HPS-dedicated I/O are held in reset
                                                                          •  
                                                                        • HPS-allocated I/O are driven with reset values from the HPS.
                                                                          •  
                                                                        • If the FPGA is configured before the HPS boots, then the boot flow looks as shown in the example figure below.
                                                                          •  
                                                                         
                                                                        Figure 2. Typical FPGA First Configuration Steps
                                                                         
                                                                         
                                                                        The flow includes the Time from Power-on-Reset (TPOR) to boot completion (TBoot_Complete).
                                                                         
                                                                        Table 2. FPGA Configuration First Stages
                                                                         Time Boot Stage Device State TPOR to T1 POR Power-on reset T1 to T2 SDM: Boot ROM 1. SDM samples the MSEL pins to determine the configuration scheme and boot source. 2. SDM establishes the device security level based on eFuse values. 3. SDM initializes the device by reading the configuration firmware (initial part of the bitstream) from the boot source. 4. SDM authenticates and decrypts the configuration firmware (this process occurs as necessary throughout the configuration). 5. SDM starts executing the configuration firmware. T2 to T3 SDM: Configuration Firmware 1. SDM I/O are enabled. 2. SDM configures the FPGA I/O and core (full configuration) and enables the rest of your configured SDM I/O. 3. SDM loads the FSBL from the bitstream into HPS on-chip RAM. 4. SDM enables HPS SDRAM I/O and optionally enables HPS debug. 5. FPGA is in user mode. 6. HPS is released from reset. CPU1-CPU3 are in a wait-for-interrupt (WFI) state. T3 to T4 First-Stage Boot Loader (FSBL) 1. HPS verifies the FPGA is in user mode. 2. The FSBL initializes the HPS, including the SDRAM. 3. The user application through the host must request the copy engine using the OPAE command hps to transfer the itb image (SSBL +Linux) to the HPS DRAM. 4. HPS peripheral I/O pin mux and buffers are configured. Clocks, resets, and bridges are also configured. 5. HPS I/O peripherals are available. T4 to T5 Second-Stage Boot Loader (SSBL) 1. HPS bootstrap completes. 2. OS is loaded into SDRAM. T5 to TBoot\\Complete Operating System (OS) The OS boots and applications are scheduled for runtime launch. 
                                                                        When using the Pre-boot Execution Environment (PXE) boot mechanism, you must use an option ROM. OFS FIM does not have PXE boot implemented in the HPS.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#25-authorization","title":"2.5 Authorization","text":"
                                                                        The HPS FSBL is part of the static region (SR) FPGA bitstream. Intel provides the capability to sign the FPGA bitstream binaries so that they can be authenticated when remotely updated and when configuring the FPGA. Signing of the SR bitstream is a two-stage process where you must sign with:
                                                                         
                                                                        1. `quartus_sign` tool\n2. OPAE `PACSign` tool\n
                                                                         
                                                                        Signing with PACSign ensures the security of the BMC RSU update process to the flash, and requires a compatible binary file. Quartus signing provides ensures security when the bitstream is configured through the SDM into the Intel Agilex FPGA device using Vendor Authorized Boot.
                                                                         
                                                                        Vendor Authorized Bootloader (VAB) considers the SDM as a trusted root entity such that when firmware is authenticated and booted and is running on the SDM with dedicated crypto HW IP blocks, it is considered a trusted entity. As such it is trusted to perform the authentication and authorization steps for subsequent bitstreams.
                                                                         
                                                                        Each subsequent loaded object after the SDM boot firmware does not need to re-implement the authentication and authorization functions. The authentication and authorization functions are centralized. Arm Trusted Firmware (ATF) is used to make a trusted execution environment (TEE) in the HPS. The source code for both Arm Trusted firmware and the First Stage Boot Loader (FSBL) is provided in the GitHub.
                                                                         
                                                                        The SSBL + Linux is a part of an itb file and may also be signed with Quartus_sign and PACSign for authentication. This process is demonstrated in Section 9.2 Configuring the HPS.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#30-environment-setup","title":"3.0 Environment Setup","text":""},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#31-building-u-boot","title":"3.1 Building U-Boot","text":"
                                                                        When creating a development environment using meta-bake.py both U-Boot and the patches required to work with the Intel N6000/1-PL FPGA SmartNIC Platform are located at /meta-opae-fpga/tools/meta-bake/build/agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/build/socfpga_agilex_n6000_defconfig. To review the required patches applied to U-Boot, navigate to /meta-opae-fpga/tools/meta-bake/build/agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/git/patches. From there, using git commands such as git status and git branch will show changes to the build environment. 
                                                                         
                                                                        Currently the meta-bake build flow requires a specific environment and software dependencies. Refer to section 6.1 Running meta-bake.py for more information. 
                                                                         
                                                                        Invoke the meta-bake.py build script to build your entire image, including U-Boot.
                                                                         
                                                                        $ cd /meta-opae-fpga/tools/meta-bake\n$ ./meta-bake.py --conf n6000/layers.yaml builddir\n
                                                                         
                                                                        This build process is highly system dependent and can take upwards of 1 hour to complete. Make sure you have at least 200 GB of free space on the system before running a build.
                                                                         
                                                                        To build U-Boot manually after execution of meta-bake.py navigate to /meta-bake/build/agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-srcfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/build/socfpga_agilex_n6000_defconfig and run make. After running meta-bake.py, you can rebuild U-Boot to incorporate any changes you have made. Navigate to the U-Boot directory at /meta-opae-fpga/tools/meta-bake/build/agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/build/socfpga_agilex_n6000_defconfig and run the following commands to rebuild.
                                                                         
                                                                        $ wget https://developer.arm.com/-/media/Files/downloads/gnu-a/10.2-2020.11/binrel/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz\n$ tar xf gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz\n$ rm gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz\n$ export CROSS_COMPILE=`pwd`/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-\n$ export ARCH=arm64\n$ make -j `nproc`\n
                                                                         
                                                                        This recompile will result in a new ITB SSBL which may be loaded on an Intel FPGA SmartNIC N6000/1 platform. Several components of the ITB image are present under the U-Boot directory but are not rebuilt as a part of this flow. These files will need to be replaced before rebuilding U-Boot for changes to take affect.
                                                                         
                                                                        U-Boot comes with its own dumpimage tool, which can be used to identify an image and extract and identify its contents. This tool is built by default under /u-boot-socfpga/tools, and in the meta-bake.py environment setup under /meta-opae-fpga/tools/meta-bake/build/agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/build/socfpga_agilex_n6000_defconfig/tools. This tool can also be used to extract specific components of the ITB file.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#32-yocto","title":"3.2 Yocto","text":"
                                                                        Yocto is an open source toolkit used to create Linux distributions and commonly used for creating Linux images and bootloaders for embedded environments. A Yocto build environment is made up of one or more layers, with each layer consisting of recipes that get processed to build/install components in a layer. The workhorse of a Yocto build is the program called bitbake. This program processes the recipes to compile and build packages and images for the target platform. For SoC platforms, like the HPS, the ARM cross compiler is required.
                                                                         
                                                                        The build script used for the Agilex SoC GSRD, create-linux-distro-release, is a bash script that automates the build of Linux images of different types (gsrd, nand, etc.) that are compatible with a target FPGA platform (agilex, stratix10, etc.). This script has been ported to Python 3 and modified to build an environment for the Intel FPGA SmartNIC N6000/1 platform, named meta-bake.py. This script pulls in the necessary patches and additional changes needed to support the platform.
                                                                         
                                                                        In general, meta-bake.py pulls Yocto layers/recipes from public repositories, configures a Yocto build environment, and builds an image for a supported FPGA platform. The Yocto layer is always the first to be built, and includes the bitbake utility. The following table lists remote repositories hosting Yocto meta data source used by meta-bake.py and create-linux-distro as well as source code used for building binaries that make up the Linux image (kernel and rootfs).
                                                                         
                                                                        Note: Not all repositories can be viewed in a web browser. All can be cloned using git.
                                                                         Repository Description https://git.yoctoproject.org/git/poky Base build tools and meta data layers https://git.openembedded.org/meta-openembedded Layers for OE-core packages https://git.yoctoproject.org/git/meta-intel-fpga Meta data layers for Intel FPGA SoC platforms https://github.com/altera-opensource/meta-intel-fpga-refdes BSP layer for Intel SoC FPGA GSRD https://github.com/altera-opensource/linux-socfpga Linux kernel source repository for socfpga https://github.com/altera-opensource/u-boot-socfpga U-Boot bootloader source repository for socfpga https://github.com/altera-opensource/arm-trusted-firmware Source for ATF 
                                                                        Recipes in the meta-intel-fpga-refdes layer mostly inherit from and extend recipes in other layers. The following table lists the new or modified recipes (in meta-intel-fpga-refdes) necessary to support an Intel FPGA SmartNIC N6000/1 HPS boot image.
                                                                         Component Recipe Description Linux Kernel recipes-kernel/linux/linux-socfpga-lts_5.10.bbappend Recipe to append the GSRC SoC FPGA device tree to the Yocto build U-Boot recipes-bsp/u-boot/u-boot-socfpga_v2021.07.bbappend Recipe to fetch and build socfpga U-Boot. Modified to support N6000/1 in U-Boot. This also creates a shell script, *mkuboot-fit.sh. copy-engine recipes-bsp/copy-engine/copy-engine-0.1.bb New recipe to build copy-engine daemon in rootfs. N6000/1 Image recipes-images/poky/n6000-image-minimal.bb New recipe to create the N6000/1 image with copy-engine and linuxptp packages installed. 
                                                                        mkuboot-fit.sh is meant to be called after a Yocto build to create the U-Boot FIT image for N6000/1, and is called automatically by meta-bake.py. This is a workaround for the Yocto build order which builds the bootloader (U-Boot) before building the Linux image rootfs. Because the rootfs is part of the U-Boot FIT image, the rootfs must be built before building the bootloader. The result of calling this script is copying the rootfs (as a .cpio file) to the U-Boot source file tree and calling make in the U-Boot build tree. When called again with the rootfs present, the resulting image will contain the rootfs. This is performed automatically as a part of the meta-bake.py build flow.
                                                                         
                                                                        See here for more information regarding Yocto. Several reference designs found in rocketboards.org use Yocto for building the Linux image and/or bootloader. For the N6000/1 image and boot flow, the Yocto build script for the Agilex SoC Golden System Reference Design has been adapted to automate building the boot loader, Linux Image, and filesystem needed to support N6000/1 devices.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#321-customizing-the-yocto-image","title":"3.2.1 Customizing the Yocto Image","text":"
                                                                        The following is a list of customizations made for building Yocto to run on the Intel FPGA SmartNIC N6000/1-PL platform.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#3211-extending-the-u-boot-recipe","title":"3.2.1.1 Extending the U-Boot recipe","text":"
                                                                        A recipe extension file (recipes-bsb/u-boot/u-boot-socfpga_v2021.07.bbappend) has been added to the meta-intel-fpga-refdes layer which accomplishes the following:
                                                                         
                                                                           
                                                                        • Adds patches using Yocto's patching mechanism
                                                                          •  
                                                                        • Introduces a new U-Boot config, socfpga_agilex_n6000_defconfig, and associates it with a keyword, agilex-n6000, that can be referenced in Yocto configuration files. These patches are necessary until those changes are merged into the public u-boot-socfpga repository. This config works for both Smartnic Platforms.
                                                                          •  
                                                                        • Creates mkuboot-fit.sh script file with variables for U-Boot source and build directories that will get expanded to the actual paths that Yocto uses for fetching/building U-Boot. Along with this recipe file, relevant patch files have been added. Once the changes are in the U-Boot repository, the patches and any references to them must be removed.
                                                                          •  
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#3212-patching-the-linux-kernel","title":"3.2.1.2 Patching The Linux Kernel","text":"
                                                                        The kernel extension recipe, meta-intel-fpga-refdes/recipes-kernel/linux/linux-socfpga-lts_5.10.bbappend, in the meta-intel-fpga-refdes layer, has been modified to add a patch file using Yocto's patching mechanism. This patch file adds the device tree for N6000/1 and is only necessary until this change is merged into the public linux-socfpga repository.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#3213-adding-custom-user-space-software","title":"3.2.1.3 Adding Custom User Space Software","text":"
                                                                        A new recipe, meta-intel-fpga-refdes/recipes-bsp/copy-engine-0.1.bb and relevant source files, have been added to the meta-intel-fpga-refdes layer. This recipe includes instructions for building the copy-engine program as well as installing it as a systemd service. Yocto will build this into an RPM package that gets installed into any image that includes it in the IMAGE_INSTALL variable. This recipe may be used as a guide for installing additional user space software.
                                                                         
                                                                        You may also create a new Hello World application and add it to the Yocto build as shown below.
                                                                         
                                                                           
                                                                        1. Generate a BSD 3-Clause License and create an MD5 hash of it.
                                                                          2.  
                                                                         
                                                                        Copyright (c) 2023, User's Name \nAll rights reserved. \n\nRedistribution and use in source and binary forms, with or without \nmodification, are permitted provided that the following conditions are met: \n\n * Redistributions of source code must retain the above copyright notice, \n   this list of conditions and the following disclaimer. \n * Redistributions in binary form must reproduce the above copyright \n   notice, this list of conditions and the following disclaimer in the \n   documentation and/or other materials provided with the distribution. \n * Neither the name of Company Name nor the names of its contributors may \n   be used to endorse or promote products derived from this software \n   without specific prior written permission. \n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" \nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE \nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE \nARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE \nLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR \nCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF \nSUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS \nINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN \nCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) \nARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE \nPOSSIBILITY OF SUCH DAMAGE. \n
                                                                         
                                                                        After license creation you will need to create an MD5 Hash of the clause. On Linux you can either pipe the raw text into echo \"text\" | md5sum, or create a new file and point to it md5sum licensefile.txt.
                                                                         
                                                                           
                                                                        1. Create a BB recipe file, following the same directory structure as other Yocto recipes.
                                                                          2.  
                                                                         
                                                                        $ cd /meta-opae-fpga/tools/meta-bake/build/meta-intel-fpga-refdes\n$ mkdir -p recipe-example/helloworld && cd recipe-example/helloworld\n
                                                                         
                                                                        Create recipe file helloworld.bb in directory helloworld.
                                                                         
                                                                        SUMMARY = \"Example hello world\" DESCRIPTION = \"helloworld in HPS\" \n\nAUTHOR = \"Your Name <your.email@address.com>\" \n\nLICENSE = \"BSD-3-Clause\" \n\nLIC_FILES_CHKSUM = \"file://${COMMON_LICENSE_DIR}/BSD-3-Clause;md5=<Your MD5 Hash>\" \n\ninherit systemd pkgconfig \n\nSRC_URI = \"file://helloworld.c\" \n\nS = \"${WORKDIR}\" \n\ndo_compile() { \n        ${CC} ${CFLAGS} ${LDFLAGS} helloworld.c -o helloworld \n}\n\ndo_install() { \n        install -d ${D}${bindir} \n        install -m 0755 helloworld ${D}${bindir} \n} \n
                                                                         
                                                                           
                                                                        1. Create source file helloworld.c in the same helloworld directory.
                                                                          2.  
                                                                         
                                                                        #include <stdio.h> \n\nvoid main void() \n{ \n    Printf(\u201c\\nHello World\\n\u201d) \n} \n
                                                                         
                                                                           
                                                                        1. Re-run ./meta-bake.py --conf n6000/layers.yaml <Build Directory>. This will a new programmable SSBL that contains your Hello World program. Program the resulting ITB file as shown in Section 4.3 Example Boot and verify the application has been included in your build.
                                                                          2.  
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#3214-adding-kernel-driver-software","title":"3.2.1.4 Adding Kernel Driver Software","text":"
                                                                        New recipes for custom kenel modules can be created at /build/meta-intel-fpga-refdes/recipes-kernel/linux/, and instructed to include custom module code. These can be patched in, included as a part of a new branch, or included by default if upstreamed. For more information visit the YoctoProject's Linux Kernel Development Manual. An example file from N6000/1 that can be used as an example is /build/meta-intel-fpga-refdes/recipes-kernel/linux/linux-socfpga-lts_5.10.bbappend.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#3215-creating-an-image-type","title":"3.2.1.5 Creating an Image Type","text":"
                                                                        A new recipe, meta-intel-fpga-refdes/recipes-images/poky/n6000-image-minimal.bb, has been added that includes directives to install the copy-engine package (built in this layer) as well as the linuxptp package (available in other layers). In addition to including these packages, this recipe includes a rootfs post processing command that removes the Linux kernel image files from the rootfs. This is done because the Linux kernel is part of the U-Boot FIT image and therefore not used from the rootfs. Removing this redundant file reduces the final U-Boot FIT image by about 30Kb. This recipe may be modified or used as a guide to add additional user space software.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#3216-testing-and-debugging","title":"3.2.1.6 Testing and Debugging","text":"
                                                                        As mentioned previously, the script will erase source files every time it is executed. This means that any changes made locally will be lost when the script is run again after making these changes. The example below shows how to test local changes without executing the script again.
                                                                         
                                                                        $ cd build\n$ source poky/oe-init-build-env agilex-gsrd-rootfs/\n$ bitbake n6000-image-minimal\n$ ./agilex-n6000-rootfs/tmp/deploy/images/agilex/mkuboot-fit.sh\n
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#40-booting-the-hps","title":"4.0 Booting the HPS","text":""},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#41-ofs-fim-boot-overview","title":"4.1 OFS FIM Boot Overview","text":"
                                                                        This implementation of an FPGA First boot flow requires that the FSBL poll on a given register before continuing to boot the HPS. Once this register indicates it is (copy engine) ready, the FSBL loads a monolithic U-Boot FIT image at a given offset 0x02000000.
                                                                         
                                                                        This image is made up of the following components:
                                                                         
                                                                           
                                                                        • U-Boot bootloader also referred to as second stage bootloader
                                                                          •  
                                                                        • Linux Kernel image
                                                                          •  
                                                                        • Root filesystem (rootfs) consisting of kernel modules as well as user space software.
                                                                          •  
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#42-booting-ofs-fim-for-intel-agilex-fpga","title":"4.2 Booting OFS FIM for Intel Agilex FPGA","text":"
                                                                        As mentioned before, the Intel N6000/1-PL FPGA SmartNIC boot flow is an FPGA-first boot flow which requires that the Intel Agilex FPGA to be configured with the necessary components (SPL/FSBL, copyengine) in order for the HPS to boot.
                                                                         
                                                                        SD/eMMC is not supported for FSBL for HPS first
                                                                         
                                                                           
                                                                        • First Stage Bootloader (FSBL): u-boot-spl-dtb.hex is embedded into FPGA image
                                                                          •  
                                                                        • Monolithic FIT Image Downloaded from Host, u-boot.itb contains the following components
                                                                          1) Second Stage Bootloader (SSBL): U-Boot + U-Boot Device Tree
                                                                           
                                                                          2) Linux kernel, Image, + Linux Device Tree
                                                                           
                                                                          3) Linux RAM based Root File System
                                                                           
                                                                          •  
                                                                         
                                                                           
                                                                        • First Stage Bootloader (FSBL) is U-boot-spl
                                                                          •  
                                                                        • U-boot-spl is built when U-Boot is built
                                                                          •  
                                                                        • Artifact is u-boot-spl-dtb.hex
                                                                             
                                                                          • The user has to check into build location : ofs-n6001/syn/setup/vab_sw/u-boot-spl-dtb.hex
                                                                            •  
                                                                          • Then run the command
                                                                            •  
                                                                          • quartus/pfg -c -o hps/path=u-boot-spl-dtb.hex orig.sof orig/fsbl.sof
                                                                            •  
                                                                           
                                                                          •  
                                                                         
                                                                        Things to consider while developing your ITB image:
                                                                         
                                                                        - The size of the u-boot.itb matters.\n- FIT is downloaded to [0x2000000](https://github.com/altera-opensource/u-boot-socfpga/blob/541b6afcb183ddb350ad367c9b63cc6db94c1f6e/configs/socfpga_agilex_n6010_defconfig#L4)\n- Linux Device Tree and RootFS are unpacked to high memory\n- Linux is unpacked to an address specified in the FIT, [0xb600000](https://github.com/altera-opensource/u-boot-socfpga/blob/541b6afcb183ddb350ad367c9b63cc6db94c1f6e/arch/arm/dts/socfpga_agilex_n6010-u-boot.dtsi#L4)\n- If size of u-boot.itb is greater than 0xb600000 \u2013 0x2000000, then FIT will be corrupted mid extraction, resulting in unpredictable kernel crashes.\n
                                                                         
                                                                        This example assumes the following preconditions have been met prior to booting HPS:
                                                                         
                                                                        1) A SOF file synthesized with the SPL (u-boot-spl-dtb.hex).
                                                                         
                                                                        2) Copy engine IP with relevant registers accessible to host and HPS.
                                                                         
                                                                        Once the host FPGA boots with the required bitstream, the SPL in the HPS begins polling a register in the copy engine. One way to get an indication that the HPS is ready to continue past the SPL is to use a terminal emulator on a host with a serial cable connected to the FPGA's UART port. To transfer the U-Boot FIT image, use the hps cpeng subcommand from the host. Note, the hps program can be installed as part of installing the OPAE SDK and Linux DFL suite of packages.
                                                                         
                                                                        hps command details are located in Section 5.0 HPS Command Usage.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#43-example-boot","title":"4.3 Example Boot","text":"
                                                                        This example assumes the following preconditions have been met prior to booting HPS:
                                                                         
                                                                           
                                                                        • A SOF file synthesized with the SPL (u-boot-spl-dtb.hex).
                                                                          •  
                                                                        • Copy engine IP with relevant registers accessible to host and HPS.
                                                                          •  
                                                                         
                                                                        Once the host FPGA boots with the required bitstream, the SPL in the HPS will begin polling a register in the copy engine. One way to get an indication that the HPS is ready to continue past the SPL is to use a terminal emulator on a host with a serial cable connected to the FPGA's UART port.
                                                                         
                                                                        To transfer the U-Boot FIT image, use the hps program with cpeng subcommand from the host. Note, the hps program is installed as part of installing the OPAE SDK suite of packages. See here for information on running the hps program. The following example assumes your N6000/1 board is at PCIe BDF 0000:b1:00.0.
                                                                         
                                                                        # Bind vfio-pci driver to Copy Engine PCIe Physical Function\n$ sudo opae.io init -d b1:00.4 root\n# Load the HPS SSBL\n$ hps cpeng -f u-boot.itb\n[2021-09-25 01:59:25.538] [cpeng] [info] starting copy of file:u-boot.itb, size: 116725656, chunk size: 4096\n[2021-09-25 01:59:29.935] [cpeng] [info] last chunk 1944, aligned 2048\n[2021-09-25 01:59:29.935] [cpeng] [info] transferred file in 28498 chunk(s)\n[2021-09-25 01:59:29.935] [cpeng] [info] waiting for ssbl verify...\n[2021-09-25 01:59:33.848] [cpeng] [info] ssbl verified\n[2021-09-25 01:59:33.848] [cpeng] [info] waiting for kernel verify...\n[2021-09-25 01:59:39.626] [cpeng] [info] kernel verified\n
                                                                         
                                                                        This will transfer the U-Boot FIT image via the copy engine IP to the HPS DDR and then signal completion of the transfer to the copy engine. Once the copy engine completes the actual transfer, it will write to the register the HPS SPL is polling on allowing the SPL to load the U-Boot bootloader which will in turn boot into the Linux image embedded in the U-Boot FIT image. If a terminal emulator is connected to the UART as described above, a user can observe U-Boot and Linux running on the HPS.
                                                                         
                                                                           
                                                                        1. Validate the HPS SSBL has been loaded by checking for its heartbeat.
                                                                          2.  
                                                                         
                                                                        $ hps heartbeat\n[2021-09-25 01:59:42.722] [heartbeat] [info] heartbeat value: 0x30015\n[2021-09-25 01:59:43.722] [heartbeat] [info] heartbeat value: 0x40015\n[2021-09-25 01:59:44.722] [heartbeat] [info] heartbeat value: 0x50015\n[2021-09-25 01:59:45.723] [heartbeat] [info] heartbeat value: 0x60015\n[2021-09-25 01:59:46.723] [heartbeat] [info] heartbeat value: 0x70015\n
                                                                         
                                                                           
                                                                        1. Login to HPS as user, root, with no password over serial connection. This process is covered in 8.0 Connecting remotely to the HPS using ssh.
                                                                          2.  
                                                                         
                                                                        agilex login: root\nroot@agilex:~# ls\nroot@agilex:~# ls /\nbin dev home lib mnt root sbin sys usr\nboot etc init media proc run srv tmp var\nroot@agilex:~#\n
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#50-hps-command-usage","title":"5.0 HPS Command Usage","text":""},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#51-synopsis","title":"5.1 Synopsis","text":"
                                                                        hps OPTIONS SUBCOMMAND SUBCOMMAND\\_OPTIONS
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#52-description","title":"5.2 Description","text":"
                                                                        hps is an application to aid in the development, deployment, and debugging of an HPS (hard processor system) on an Intel Agilex device using OFS. The current version of the hps program assumes an AFU (accelerator functional unit) is configured into the FPGA that can be discovered/accessed through an OPAE library and used for communicating with the HPS. When invoked with one of its subcommands, hps will enumerate the AFU for that subcommand before executing it.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#53-options","title":"5.3 Options","text":"
                                                                        -h,--help\n\nPrint this help message and exit\n\n-p,--pci-address address\n\nUse address in the filter for locating the AFU where address must be in\n\nthe following format: [domain]\\bus\\:\\device\\.\\function\\\n\n-l,--log-level \\level\\\n\nstdout logging level. Must be one of:\n\n{trace,debug,info,warning,error,critical,off}\n\nDefault is info.\n\n-s,--shared\n\nopen in shared mode, default is off\n\n-t,--timeout timeout\n\nProgram timeout in milliseconds. Default is 60000 ms.
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#54-subcommands","title":"5.4 Subcommands","text":""},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#541-cpeng","title":"5.4.1 cpeng","text":"
                                                                        The copy engine command is used to program copy engine AFU registers to copy an image file from the host into the FPGA DDR. When the HPS boots, the first stage boot loader loads an image from a specific offset in DDR that will be used to transition into the second stage boot loader and subsequently boot into the embedded Linux that is also part of this image.
                                                                         cpeng  options description -h,--help Print this help message and exit -f,--filename filename Path to image file to copy. Default is u-boot.itb -d,--destination offset DDR Offset. Default is 0x2000000. -t,--timeout cpeng timeout Timeout of cpeng command in microseconds. Default is 1 sec (1000000 usec). -r,--data-request-limit size Can be 64, 128, 512, or 1024 and represents the PCIe request size in bytes that the copy engine IP will use. This is encoded to 0, 1, 2, or 3 and written to the copy engine DATA\\REQUEST\\LIMIT register. Default is 512. -c,--chunk size Split the copy into chunks of size size. 0 indicates no chunks. Chunk sizes must be aligned with data request limit. Default is 4096. --soft-reset Issue a soft reset only. --skip-ssbl-verify Do not wait for ssbl verify. --skip-kernel-verify Do now wait for kernel verify."},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#542-heartbeat","title":"5.4.2 heartbeat","text":"
                                                                        This subcommand reads the value in the HPS2HOST register to check for a hearbeat. This compares the value to previous value read and determines the HPS is alive if the value is incrementing. This relies on the hps running the hello-cpeng program in heartbeat mode which will increment the upper 16 bits in the HPS2HOST register. Please see a typical sequence of using the rsu and hps commands as below for a device with BDF 15:00:0
                                                                         
                                                                        rsu fpga -p user1 15:00.0\nsudo opae.io release -d 15:00.0\nsudo opae.io init -d 15:00.4 root:root\nhps cpeng -f u-boot-userkey-vab.itb\ntimeout 5 hps heartbeat\nsudo opae.io release -d 15:00.0\nhps cpeng -f u-boot-userkey-vab.itb\n
                                                                         
                                                                        The above command will transfer the U-Boot FIT image via the copy engine IP to the HPS DDR and then signal completion of the transfer to the copy engine. After the copy engine completes the actual transfer, it writes to the register the HPS SPL is polling on allowing the SPL to load the U-Boot bootloader which in turn boots into the Linux image embedded in the U-Boot FIT image. If a terminal emulator is connected to the UART as described above, a user can observe U-Boot and Linux running on the HPS.
                                                                         
                                                                        First FSBL is loaded and executed by FPGA configuration. Then Board/Server gets powered on. FPGA Configuration is done via JTAG followed by a reboot
                                                                         
                                                                        The FSBL will send the following output the serial port: 
                                                                        U-Boot SPL 2021.07-00312-g32c0556437 (Sep 17 2021 - 08:42:45 -0700)\nReset state: Cold\nMPU 1200000 kHz\nL4 Main 400000 kHz\nL4 sys free 100000 kHz\nL4 MP 200000 kHz\nL4 SP 100000 kHz\nSDMMC 50000 kHz\nDDR: Warning: DRAM size from device tree (1024 MiB)\nmismatch with hardware (2048 MiB).\nDDR: 1024 MiB\nSDRAM-ECC: Initialized success with 239 ms\nwaiting for host to copy image\n
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#60-meta-bakepy","title":"6.0 meta-bake.py","text":"
                                                                        A script called meta-bake.py has been added to allow for more control of configuration/customization of recipes and their dependencies. This script separates the data from the logic by requiring that data be expressed in a yaml configuration file. This file contains the following confiration data:
                                                                         
                                                                           
                                                                        • machine - The FPGA/SoC platform to build for, choices are agilex, stratix10, arria10, cyclone5
                                                                          •  
                                                                        • image - The image type to build, choices are gsrd, nand, pcie, pr, qsqpi, sgmii, tse, n6000
                                                                          •  
                                                                        • target - The build target to build. This is typically a Yocto image to build.
                                                                          •  
                                                                        • fit - Make a monolothic FIT image after the Yocto build. This will use U-Boot source and binaries as well as the rootfs made for the image.
                                                                          •  
                                                                        • repos - A list of repositories to pull for Yocto recipes. This information is made up of:
                                                                             
                                                                          • name - The project name (this is also the directory where source is clone to)
                                                                            •  
                                                                          • url - The URL to pull the source from
                                                                            •  
                                                                          • branch - The branch to checkout
                                                                            •  
                                                                          • add_layers - Can be either True or a list of sub-directories to add as layers in bblayers.conf
                                                                            •  
                                                                          • patch - Path to a file to use to patch the source code
                                                                            •  
                                                                          • keep - When set to true, this will leave the source tree untouched on subsequent runs
                                                                            •  
                                                                           
                                                                          •  
                                                                        • upstream_versions - Dependencies/versions to use for either Linux kernel, U-Boot, and/or ATF. This information is made up of:
                                                                             
                                                                          • name - Project name
                                                                            •  
                                                                          • version - version to configure recipes that use it
                                                                            •  
                                                                          • branch - branch to use, will use git hash in recipe
                                                                            •  
                                                                          • url - URL to pull the source from
                                                                            •  
                                                                          • disabled - when set to True, this project will be ignored
                                                                            •  
                                                                           
                                                                          •  
                                                                        • local - Used to configure local.conf used by Yocto/bitbake build. This information is made up of:
                                                                             
                                                                          • remove - List of keys to remove from local.conf
                                                                            •  
                                                                          • values - Dictionary of key/value pairs to use to insert into local.conf. Any existing key/value pairs will be overwritten.
                                                                            •  
                                                                           
                                                                          •  
                                                                        "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#61-running-meta-bakepy","title":"6.1 Running meta-bake.py","text":"
                                                                        To create an U-Boot fit and spl image for N6000/1 platforms, run the following command after meeting these setup conditions:
                                                                         
                                                                           
                                                                        • Host PC with Ubuntu 20.04 LTS
                                                                             
                                                                          • ARM cross compiler
                                                                               
                                                                            • set CROSS_COMPILE environment variable to: /bin/aarch64-linux-gnu-     - e.x. export CROSS_COMPILE=aarch64-linux-gnu- 
                                                                            • set ARCH to: arm64     - e.x. export ARCH=arm64
                                                                              •  
                                                                            • At least 100 Gb of disk space
                                                                              •  
                                                                            • Tested on OFS 3.0.0
                                                                              •  
                                                                              This script does not require any bare-metal accesses to perform its build and can be run from a VM with no alterations. Ensure that the Ubuntu 20.04 Guest VM you create has enough to space to perform the entire build (recommend at least 200 GiB total space), as does the drive it is stored on. You will need to configure proxy access in the VM if required for your environment. You can use any VM technology; having ssh enabled in the VM will allow you to more easily transfer the completed build files back to the host system but is not required.
                                                                               
                                                                              Package dependencies to build Yocto on each supported OS can be found on the Yocto Quick Start page.
                                                                               
                                                                              wget https://developer.arm.com/-/media/Files/downloads/gnu-a/10.2-2020.11/binrel/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz\ntar xf gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz\nrm gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz\nexport CROSS_COMPILE=`pwd`/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-\nexport ARCH=arm64\n./meta-bake.py build\n
                                                                               
                                                                              After running this build, the images you need to boot the HPS are located under build/agilex-n6000-images. Follow the steps in Section 4.3 Example Boot to finish bringing up your board.
                                                                               
                                                                              This script will do the following:
                                                                               
                                                                                 
                                                                              • Parse layers.yaml for configuration to use for build
                                                                                •  
                                                                              • Download recipe repositories (including poky) listed in repos secion of layers.yaml
                                                                                •  
                                                                              • Apply refdes-n6000 patch to meta-intel-fpga-refdes source tree
                                                                                •  
                                                                              • Configure Yocto build in build directory
                                                                                •  
                                                                              • Source build/poky/oe-init-build-env passing in agilex-n6000-rootfs. This will initialize conf files.
                                                                                •  
                                                                              • Configure build/agilex-n6000-rootfs/conf/local.conf using values in local section of layers.yaml          * Note: IMAGE_FSTYPES is configured to include cpio
                                                                                •  
                                                                              • Configure build/agilex-n6000-rootfs/conf/bblayers.conf using layer specification in repos section of layers.yaml
                                                                                •  
                                                                              • Run Yocto build for target listed in layers.conf
                                                                                •  
                                                                              • Call bitbake n6000-image-minimal
                                                                                •  
                                                                              • Get environment variables to locate rootfs cpio file as well as U-Boot source and build directories
                                                                                •  
                                                                              • Copy rootfs created by Yocto build for U-Boot
                                                                                •  
                                                                              • Copy rootfs cpio file (n6000-image-minimal-agilex*.rootfs.cpio) to U-Boot build directory for selected configuration (socfpga_agilex_n6000_defconfig)
                                                                                •  
                                                                              • Call U-Boot build in directory for selected configuration
                                                                                •  
                                                                              • Copy FIT image (u-boot.itb) to images directory, build/agilex-n6000-images
                                                                                •  
                                                                              • Many important images are copied to build/agilex-n6000-images, which may be useful if using VAB
                                                                                •  
                                                                              "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#62-required-changes","title":"6.2 Required Changes","text":"
                                                                              The patch file applied on top of the meta-intel-fpga-refdes repository introduces patches to:
                                                                               
                                                                                 
                                                                              • Add patch files so that Yocto can modify Linux kernel to add configuration for creating a device tree binary (DTB) compatible with N6000/1
                                                                                •  
                                                                              • Add patch files so that Yocto can modify the bootloader in U-Boot to support booting with the assistance of the copy engine IP
                                                                                •  
                                                                              • Modify rootfs to include copy-engine daemon as well as other packages that can be useful for debug
                                                                                •  
                                                                               
                                                                              These changes may eventually be merged into upstream repositories for linux-socfpga, u-boot-socfpga, and meta-intel-fpga-refdes. Once all changes make their way into the repositories for the aforementioned projects, it will no longer be necessary to apply patches.
                                                                              "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#63-manual-build","title":"6.3 Manual Build","text":"
                                                                              One may use meta-bake.py to only pull down required repositories and configure a Yocto build environment by using the --skip-build command line argument. To initiate a build after this, source poky/oe-init-build-env passing in a directory as the only argument. This will set up the user's environment to be able to run bitbake. To build the Yocto image, run bitbake n6000-image-minimal. This will build all the components necessary to build a FIT image. Once the build is complete, U-Boot make system may be used to make the FIT. The U-Boot build directory for the selected configuration can be found in the Yocto build environment directory at: 
                                                                              $ cd tmp/work/agilex-poky-linux/u-boot-socfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/build/socfpga_agilex_n6000_defconfig\n
                                                                               Once in this directory, ensure that the necessary files are present in here in order to assemble the FIT image (u-boot.itb) 
                                                                              $ cp ../../../../../../deploy/images/agilex/n6000-image-minimal-agilex.cpio rootfs.cpio\n$ ls Image linux.dtb rootfs.cpio\nImage  linux.dtb  rootfs.cpio\n$ make\n
                                                                              "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#64-manual-vab-signing","title":"6.4 Manual VAB Signing","text":"
                                                                                 
                                                                              • By default, meta-bake.py will sign and certify the proper files for use with VAB. Below is an example on how to perform the manual VAB Signing Process.
                                                                                •  
                                                                               
                                                                              Make sure Quartus already installed and its tools added to environment. Example PATH=$PATH:/home/intelFPGA\\pro/21.3/quartus/bin/
                                                                               
                                                                              $ cd HPS_VAB\n$ quartus_sign --family=agilex --operation=make_private_pem --curve=secp384r1 --no_passphrase userkey_root_private.pem\n$ quartus_sign --family=agilex --operation=make_public_pem userkey_root_private.pem userkey_root_public.pem\n$ quartus_sign --family=agilex --operation=make_rootuserkey_root_public.pem userkey_root_public.qky\n$ chmod +x fcs_prepare\n$ ./fcs_prepare --hps_cert bl31.bin -v\n$ quartus_sign --family=agilex --operation=SIGN --qky=userkey_root_public.qky --pem=userkey_root_private.pem unsigned_cert.ccert signed_cert_bl31.bin.ccert\n\n# ATF Sign\n\n$ ./fcs_prepare --finish signed_cert_bl31.bin.ccert --imagefile bl31.bin\n$ mv hps_image_signed.vab signed-bl31.bin\n$ rm unsigned_cert.ccert\n\n# u-boot-nodtb\n\n$ ./fcs_prepare --hps_cert u-boot-nodtb.bin -v\n\n#signed_u-boot-nodtb.bin.ccert\n\n$ quartus_sign --family=agilex --operation=SIGN --qky=userkey_root_public.qky --pem=userkey_root_private.pem unsigned_cert.ccert signed_u-boot-nodtb.bin.ccert\n\n# u-boot-nodtb.bin Sign\n\n$ ./fcs_prepare --finish signed_u-boot-nodtb.bin.ccert --imagefile u-boot-nodtb.bin\n$ mv hps_image_signed.vab signed-u-boot-nodtb.bin\n$ rm unsigned\\_cert.ccert\n\n# u-boot.dtb\n\n$ ./fcs_prepare --hps_cert u-boot.dtb -v\n\n#signed_u-boot.dtb.ccert\n\n$ quartus_sign --family=agilex --operation=SIGN --qky=userkey_root_public.qky --pem=userkey_root_private.pem unsigned_cert.ccert signed_u-boot.dtb.ccert\n\n# u-boot.dtb Sign\n\n$ ./fcs_prepare --finish signed_u-boot.dtb.ccert --imagefile u-boot.dtb\n$ mv hps_image_signed.vab signed-u-boot.dtb\n$ rm unsigned_cert.ccert\n\n# Image\n\n$ ./fcs_prepare --hps/cert Image -v\n\n#signed_Image.ccert\n\n$ quartus_sign --family=agilex --operation=SIGN --qky=userkey_root_public.qky --pem=userkey_root_private.pem unsigned_cert.ccert signed_Image.ccert\n\n# Image Sign\n\n$ ./fcs_prepare --finish signed_Image.ccert --imagefile Image\n$ mv hps_image_signed.vab signed-Image\n$ rm unsigned_cert.ccert\n\n# linux.dtb\n\n$ ./fcs_prepare --hps_cert linux.dtb -v\n\n#signed_linux.dtb.ccert\n\n$ quartus_sign --family=agilex --operation=SIGN --qky=userkey_root_public.qky --pem=userkey_root_private.pem unsigned_cert.ccert signed_linux.dtb.ccert\n\n# linux.dtb Sign\n\n$ ./fcs_prepare --finish signed_linux.dtb.ccert --imagefile linux.dtb\n$ mv hps_image_signed.vab signed-linux.dtb\n$ rm unsigned_cert.ccert\n\n# rootfs.cpio\n\n$ ./fcs_prepare --hps_cert rootfs.cpio -v\n\n#signed_rootfs.cpio.ccert\n\n$ quartus_sign --family=agilex --operation=SIGN --qky=userkey_root_public.qky --pem=userkey_root_private.pem unsigned_cert.ccert signed_rootfs.cpio.ccert\n\n# rootfs.cpio\n\n$ ./fcs_prepare --finish signed_rootfs.cpio.ccert --imagefile rootfs.cpio\n$ mv hps_image_signed.vab signed-rootfs.cpio\n$ rm unsigned_cert.ccert\n
                                                                               
                                                                              Copy the following files to u-boot-socfpga folder:
                                                                               
                                                                              #Copy the image back to uboot folder\n$ cp signed-bl31.bin ../u-boot-socfpga/\n$ cp signed-u-boot-nodtb.bin ../u-boot-socfpga/\n$ cp signed-u-boot.dtb ../u-boot-socfpga/\n$ cp signed-Image ../u-boot-socfpga/\n$ cp signed-linux.dtb ../u-boot-socfpga/\n$ cp signed-root\n$ fs.cpio ../u-boot-socfpga/\n
                                                                               Recompile the U-Boot
                                                                               
                                                                              $ git clone https://github.com/altera-opensource/u-boot-socfpga\n$ cd u-boot-socfpga\n$ export CROSS\\COMPILE=aarch64-none-linux-gnu-; export ARCH=arm\n$ make socfpga/agilex/n6000/vab/defconfig\n$ make -j 24\n$ cd ..\n
                                                                               
                                                                              Figure 3.1 N6000/1 Configs
                                                                               
                                                                               
                                                                              If you not see the defconfig desired, please checkout the correct branch version. Example config shown above is socfpga_v2021.10.
                                                                               
                                                                              If the memory device tree it mismatches with your hardware (figure below), change the memory device tree at u-boot-socfpga/arch/arm/dts/socfpga_agilex_n6000-u-boot.dtsi
                                                                               
                                                                              To make it 2GB, change as
                                                                               
                                                                              memory {\n\n\\* 2GB \\*\n\nreg = <0 0x00000000 0 0x40000000>,<0 0x00000000 0 0x40000000>;\n\n};\n
                                                                               Figure 3.2 Device tree mismatches example
                                                                               
                                                                               
                                                                              Refer to 6. Host Side Startup
                                                                               
                                                                              $ sudo opae.io init -d 4b:00.4 root:root\n$ hps cpeng -f u-boot.itb\n$ timeout 5 hps heartbeat\n
                                                                               The error happen (Figure below) when the Images do not sign with VAB.
                                                                               
                                                                              Figure 3.3 VAB certificate error example
                                                                               
                                                                              "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#70-debugging","title":"7.0 Debugging","text":"
                                                                              Debugging the HPS from the host side is standard HPS debugging. The primary debug tool is UART on the HPS and Arm DS-5 debugger.
                                                                               
                                                                              A UART connection can be enabled on the board using the following procedure:
                                                                               
                                                                                 
                                                                              1.  
                                                                                Connect the HPS Debug Card and HPS UART to the Intel N6000/1-PL FPGA SmartNIC Platform board
                                                                                 
                                                                                2.  
                                                                              2.  
                                                                                Open Putty with the following setting 
                                                                                    Port:COM4\n\n    Baudrate:115200\n\n    Data bits : 8\n\n    Stop bits : 1\n\n    Parity : None\n\n    Flow Control : None\n
                                                                                 
                                                                                3.  
                                                                              3.  
                                                                                Reboot the Intel N6000/1-PL FPGA SmartNIC Platform board by typing reboot in the shell. You will be able to see the HPS UART traffic in the putty. If any issues are encountered in this step, check the HPS UART connection and the UART driver.
                                                                                 
                                                                                4.  
                                                                              4.  
                                                                                Check the PCI bdf ( lspci | grep acc ) or fpgainfo fme at the shell prompt.
                                                                                 
                                                                                5.  
                                                                              5.  
                                                                                Run the rsu and fpga\\reconfig scripts with respective arguments to print the logs.
                                                                                 
                                                                                6.  
                                                                              "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#80-connecting-remotely-to-the-hps-using-ssh","title":"8.0 Connecting remotely to the HPS using ssh","text":"
                                                                              The HPS running on the Intel FPGA N6000/1-PL SmartNIC Platform can be remotely accessed to via the utility ssh, allowing the user to run commands and copy files. SSH must be run over a Point-To-Point Protocol daemon that has been included in the HPS software (as a part of the meta-openembedded layer, in the recipes-daemons/ippool recipe). In this example, the HPS is set up as a PPP Server, and the host OS is set up as a PPP Client. Serial communication between the host and HPS is accomplished via HPS UART1, which communicates through the FIM to the Soft UART on the FPGA, who in turn communicates with the host over PCIe.
                                                                               
                                                                              The following steps assume the SSBL has not yet been loaded onto the HPS. If it has, a cold boot will reset the system.
                                                                               
                                                                                 
                                                                              1. The HPS Copy Engine Module is available for access on PF 4 via the PF/VF Mux on the FPGA. This port needs to be bound to driver vfio-pci (the following example assumes PCIE BDF 0000:b1:00.0). Substitute your device's BDF address and desired user/group access permissions into the following command.
                                                                                2.  
                                                                               
                                                                              $ sudo opae.io init -d 0000:b1:00.4 <USER>[:<GROUP>]\nUnbinding (0x8086,0xbcce) at 0000:b1:00.4 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.4 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.4 is 190\nAssigning /dev/vfio/190 to DCPsupport\nChanging permissions for /dev/vfio/190 to rw-rw----\n
                                                                               
                                                                                 
                                                                              1. When an HPS enabled SOF or BIN with the FSBL is loaded onto the FPGA, a message will be displayed on the host OS (seen via dmesg) after boot once the serial port has been registered with the dfl-uart driver. The UART driver is included as a part of the linux-dfl driver package. An example output from dmesg is shown below (search dmesg using dmesg | grep dfl-uart):
                                                                                2.  
                                                                               
                                                                              [    7.343014] dfl-uart dfl_dev.7: serial8250_register_8250_port 2\n
                                                                               
                                                                              The device file that corresponds with serial UART port 2 is /dev/ttyS2 (format is /dev/ttyS<port number>). A serial communication program can be used to view the HPS boot in realtime, then log in and run commands when boot has completed. Minicom is the program that will be used in this example, although others will work. Install Minicom using DNF sudo dnf install minicom. 
                                                                               
                                                                                 
                                                                              1. Minicom requires configuration changes before it can listen to the serial device. Using the built-in menu accessed by sudo minicom -s, ensure the information under \"Serial port setup\" matches the following, where the serial device corresponds with the serial port discussed previously:
                                                                                2.  
                                                                               
                                                                               +-----------------------------------------------------------------------+\n    | A -    Serial Device      : /dev/ttyS2                                |\n    |                                                                       |\n    | C -   Callin Program      :                                           |\n    | D -  Callout Program      :                                           |\n    | E -    Bps/Par/Bits       : 115200 8N1                                |\n    | F - Hardware Flow Control : Yes                                       |\n    | G - Software Flow Control : No                                        |\n    |                                                                       |\n    |    Change which setting?                                              |\n    +-----------------------------------------------------------------------+\n
                                                                               
                                                                                 
                                                                              1.  
                                                                                Save and exit the configuration menu. Run Minicom using the command sudo minicom and keep the terminal open and connected.
                                                                                 
                                                                                2.  
                                                                              2.  
                                                                                Load the SSBL onto the HPS using a second terminal. This requires a built ITB image.
                                                                                 
                                                                                3.  
                                                                               
                                                                              $ hps cpeng -f u-boot.itb\n
                                                                               
                                                                                 
                                                                              1. You should see the HPS boot sequence continue through your Minicom terminal. Once boot has completed, log in using the user root with an empty password.
                                                                                2.  
                                                                               
                                                                              ...\n...\n...\n[  OK  ] Finished Load/Save Random Seed.\n[  OK  ] Finished OpenSSH Key Generation.\n\nPoky (Yocto Project Reference Distro) 3.3.6 agilex ttyS0\n\nagilex login: root\nroot@agilex:~#\n
                                                                               
                                                                                 
                                                                              1. Configure the running Yocto image on the HPS as a PPP server. Run the following command through Minicom on the HPS (connects address 192.168.250.2 on the HPS to 192.168.250.1 on the host):
                                                                                2.  
                                                                               
                                                                              root@agilex:~# pppd noauth passive 192.168.250.1:192.168.250.2\n[  410.465450] PPP generic driver version 2.4.2\n...\n
                                                                               
                                                                                 
                                                                              1. Exit the Minicom program running on the host using ^A X. Execute the following command on the host to establish a PPP connection as the client (if not installed on the host, run sudo dnf install ppp):
                                                                                2.  
                                                                               
                                                                              $ sudo pppd ttyS2 115200 crtscts lock local noauth passive debug\n
                                                                               
                                                                                 
                                                                              1. A new network interface device registered to ppp should be visible.
                                                                                2.  
                                                                               
                                                                              $ ip -4 addr\n...\n8: ppp0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UNKNOWN group default qlen 3\n    inet 192.168.250.2 peer 192.168.250.1/32 scope global ppp0\n       valid_lft forever preferred_lft forever\n
                                                                               
                                                                              With both the client and server communicating, ssh and scp can be used to run commands and transfer files using IPv4 address 192.168.250.1 on the host. An example operation run on the host OS is shown below:
                                                                               
                                                                              [user@localhost ]: scp file_package.tar.gz root@192.168.250.1\n
                                                                               
                                                                              Note: If you are developing software for the HPS and altering system settings, it is possible for ssh to prohibit a connection due to a false man-in-the-middle attack warning. The flag <ssh/scp> -o StrictHostKeyChecking=no can be used to ignore the warning.
                                                                              "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#90-example-design-enabling-emmc-i2c-and-uart-in-platform-design","title":"9.0 Example Design - Enabling eMMC, I2C and UART in Platform Design","text":"
                                                                              The following section will walk through the process by which eMMC, I2C, and UART can be added to the FIM and the HPS image. The goal of this section is to allow the HPS to configure eMMC memory on boot and uses WNC's FPGA SmartNIC Series as a reference.
                                                                              "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#91-configuring-the-fim","title":"9.1 Configuring the FIM","text":"
                                                                                 
                                                                              1. Configure eMMC, I2C, and UAET in Platform Designer. ACtual pin assignments are determined by the WNC board schematic. In Quartus, navigate to the HPS Processor Subsystem Intel Agilex FPGA IP -> Pin Mux and Peripherals -> Advanced -> Advanced IP Placement.
                                                                                2.  
                                                                               
                                                                              Check your pin assigments for the eMMC, UART and I2C in the Pin Planner. If these assignments are not present, then they can be found at the following link. Based on the changes shown above, the UART pins are removed on HPS IO3 and IO4 what are mapped on AG6 and AB1.
                                                                               
                                                                               
                                                                                 
                                                                              1. Click Apply Selections->Generate HDL
                                                                                2.  
                                                                               
                                                                               
                                                                                 
                                                                              1. Check for instantiation in top.sv. Click Generate -> Show Instatiation Template.
                                                                                2.  
                                                                               
                                                                               
                                                                              The following image demonstrates eMMC and I2C properly instatiated in top.sv.
                                                                               
                                                                               
                                                                                 
                                                                              1. Add the following to the hps_ss modules in top.sv.
                                                                                2.  
                                                                               
                                                                               
                                                                               
                                                                                 
                                                                              1. Compile the design.
                                                                                2.  
                                                                               
                                                                              "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#92-configuring-the-hps","title":"9.2 Configuring the HPS","text":"
                                                                                 
                                                                              1. Enable mmc and DesignWare Memory Card interface flags in U-Boot (.config). After building U-Boot with meta-bake.py, this file is located at /agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga//build/socfpga_agilex_n6000_defconfig/.config. 
                                                                                CONFIG_CMD_MMC=y //Enable mmc command tool in uboot \nCONFIG_MMC_DW=y // support DesignWare Memory Card Interface\n
                                                                                 
                                                                                   
                                                                                1. Enable and configure I2C and eMMC in U-Boot (socfpga_agilex_n6000.dts). After building U-Boot with meta-bake.py, this file is located at /agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga/git/arch/arm/dts/socfpga_agilex_n6000.dts. 
                                                                                     
                                                                                  1. Enable and configure I2C and eMMC in the Linux device tree (socfpga_agilex_n6000.dts). After building U-Boot with meta-bake.py, this file is located at /agilex-n6000-rootfs/tmp/work-shared/agilex/kernel-source/arch/arm64/boot/dts/intel/socfpga_agilex_n6000.dts. 
                                                                                       
                                                                                    1. Compile Linux by navigating to the directory agilex-n6000-rootfs/tmp/work/agilex-poky-linux/linux-socfpga-lts/5.10.60-lts+gitAUTOINC+c35d63f9c7-r0/linux-agilex-standard-build and running the following:
                                                                                      2.  
                                                                                     
                                                                                    $ make -j `nproc` Image dtbs  \n
                                                                                     
                                                                                       
                                                                                    1. Add the software utilitiesutil-linux-mkfs e2fsprogs.mke2fs e2fsprogs in to the Linux RootFS. Thes utilities will be used to create a filsystem (ext4, FAT32, etc.) and partition the eMMC. Make the following changes in meta-intel-fpga-refdes/recipes-images/poky/n6000-image-minimal.bb.
                                                                                      2.  
                                                                                     
                                                                                       
                                                                                    1.  
                                                                                      As an output from the Linux compilation from step 4 you will produce the files Image and socfpga_agilex_n6000.dtb. Transfer both over to the socfpga_agilex_n6000_defconfig directory. Rename socfpga_agilex_n6000.dtb to linux.dtb. Compile U-Boot by running make in directory agilex-n6000-rootfs/tmp/work/agilex-poky-linux/u-boot-socfpga/1_v2021.07+gitAUTOINC+24e26ba4a0-r0/build/socfpga_agilex_n6000_defconfig/. This compilation will produce both spl/u-boot-spl-dtb.hex and u-boot.itb. 
                                                                                       
                                                                                      2.  
                                                                                    2.  
                                                                                      Combines these files with your SOF FSBL bootloader (created in Section 9.1 Configuring the FIM).
                                                                                       
                                                                                      3.  
                                                                                     
                                                                                    $ quartus_pfg -c ofs_top.sof ofs_top_hps.sof -o hps_path=u-boot-spl-dtb.hex\n$ quartus_pfg -c  ofs_top_hps_pof_flash.pfg //to pof file and flash to qspi\n
                                                                                     
                                                                                    Program the FSBL enabled SOF onto the FPGA and warm reboot the server for changes to take affect. After reboot has completed use the following commands to program the SSBL.
                                                                                     
                                                                                    $ sudo opae.io release -d d8:00.0\n$ sudo opae.io init -d d8:00.4 root:root\n$ hps cpeng -f u-boot.itb\n
                                                                                     
                                                                                    During HPS boot you should see the following message if the eMMC has been properly configured.
                                                                                     
                                                                                    "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#93-emmc-testing-from-the-hps","title":"9.3 eMMC Testing from the HPS","text":"
                                                                                    The following memory test was run from the U-Boot shell.
                                                                                     
                                                                                       
                                                                                    1. Erase eMMC using a start block offset 0x400.
                                                                                      2.  
                                                                                     
                                                                                    $ mmc erase 0x400 20\n
                                                                                     
                                                                                     
                                                                                    Write test data (at 0x00A00000) into the eMMC offset 0x400.
                                                                                     
                                                                                    $ mmc write 0x00A00000 0x400 10\n
                                                                                     
                                                                                     
                                                                                    Read test data (at 0x00A00040) back from eMMC offset 0x400.
                                                                                     
                                                                                    $ mmc read 0x00A00040 0x400 10\n
                                                                                     
                                                                                     
                                                                                    Data comparison at memory offset 0x00A00000 and 0x00A00040. Data should match.
                                                                                     
                                                                                    $ cmp.l 0x00A00000 0x00A00040 10.\n
                                                                                     
                                                                                     
                                                                                    The following memory test was run from Linux running on the HPS.
                                                                                     
                                                                                       
                                                                                    1. Display the eMMC and its partitions.
                                                                                      2.  
                                                                                     
                                                                                    $ fdisk -l\n
                                                                                     
                                                                                     
                                                                                       
                                                                                    1. Create a primary partition on the eMMC.
                                                                                      2.  
                                                                                     
                                                                                     
                                                                                       
                                                                                    1. Verify the partition has been created.
                                                                                      2.  
                                                                                     
                                                                                     
                                                                                       
                                                                                    1. Format the ext3 filesystem in the partition you just created (p1).
                                                                                      2.  
                                                                                     
                                                                                     
                                                                                       
                                                                                    1. Create the directory mydata in /mnt. Mount the eMMC p1 partition to /mnt/mydata and verify the filsystem mount was successful.
                                                                                      2.  
                                                                                     
                                                                                    $ mkdir -p /mnt/mydata\n$ mount /dev/mmcblk0p1 /mnt/mydata\n$ df\n
                                                                                     
                                                                                     
                                                                                       
                                                                                    1. Create a new text file and write some data in it - \"Hello World!\". After the device has been written to run sync, unmount eMMC p1 partition and verify the unmount was successful.
                                                                                      2.  
                                                                                     
                                                                                    $ sync\n$ umount /dev/mmcblk0p1\n$ df\n
                                                                                     
                                                                                    "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#layersyaml-reference","title":"layers.yaml Reference","text":"
                                                                                    machine: agilex\nimage: n6000\ntarget: n6000-image-minimal\nfit: true\nuboot-dtb: [u-boot-nodtb.bin, u-boot.dtb]\nlinux-binary: [bl31.bin, linux.dtb, rootfs.cpio, Image]\nroot-public-qky: userkey_root_public.qky\nroot-private-pem: userkey_root_private.pem\nroot-public-pem: userkey_root_public.pem\n\nrepos:\n  - name: poky\n    url: https://git.yoctoproject.org/git/poky.git\n    branch: hardknott\n  - name: meta-intel-fpga\n    url: https://git.yoctoproject.org/git/meta-intel-fpga.git\n    branch: hardknott\n    add_layers: true\n  - name: meta-intel-fpga-refdes\n    url: https://github.com/altera-opensource/meta-intel-fpga-refdes.git\n    #url__: https://github.com/intel-innersource/os.linux.yocto.reference-design.meta-intel-fpga-refdes.git\n    #url: git@github.com:intel-innersource/os.linux.yocto.reference-design.meta-intel-fpga-refdes.git\n    #branch: rrojo/n6000\n    branch: hardknott\n    patch: refdes-n6000-ppp.patch\n    keep: true\n    add_layers: true\n  - name: meta-openembedded\n    url: https://github.com/openembedded/meta-openembedded.git\n    branch: hardknott\n    add_layers:\n      - meta-oe\n      - meta-networking\n      - meta-python\n  - name: fcs_prepare\n    url: https://github.com/altera-opensource/fcs_apps.git\n    branch: fcs_prepare\n\ningredients:\n  linux:\n    name: linux-socfpga\n    version: '5.10.100'\n    branch: socfpga-5.10.100-lts\n    url: https://github.com/altera-opensource/linux-socfpga.git\n  uboot:\n    name: u-boot-socfpga\n    version: '2021.07'\n    branch: socfpga_v2021.07\n    url: https://github.com/altera-opensource/u-boot-socfpga.git\n  atf:\n    disabled: true\n    version: '2.4.1'\n    branch: socfpga_v2.4.1\n    url: https://github.com/altera-opensource/arm-trusted-firmware.git\nlocal:\n  remove:\n    - MACHINE\n    - UBOOT_CONFIG\n    - IMAGE\n    - SRC_URI\n  values:\n    MACHINE: $machine\n    DL_DIR: $build_dir/downloads\n    DISTRO_FEATURES_append: \" systemd\"\n    VIRTUAL-RUNTIME_init_manager: systemd\n    IMAGE_TYPE: $image\n    IMAGE_FSTYPES: \"+=cpio tar.gz\"\n    PREFERRED_PROVIDER_virtual/kernel: linux-socfpga-lts\n    PREFERRED_VERSION_linux-socfpga-lts: 5.10%\n    UBOOT_CONFIG: agilex-n6000\n    PREFERRED_PROVIDER_virtual/bootloader: u-boot-socfpga\n    PREFERRED_VERSION_u-boot-socfpga: v2021.07%\n
                                                                                    "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#faqs","title":"FAQs","text":"
                                                                                    Below are the Frequently Asked Questions:
                                                                                     
                                                                                       
                                                                                    1.  
                                                                                      How will you get the software stack for HPS (FSBL, U-Boot, Kernel)? Or will there be a package available to them on Git, Intel RDC?
                                                                                       
                                                                                      Answer : HPS software has been around for quite a long time. Support for the OFS and the N6000-PL FPGA SmartNIC Platform will be upstreamed and available from rocketboards.com, just like any other HPS based project.
                                                                                       
                                                                                      2.  
                                                                                    2.  
                                                                                      What are the recommended steps for building the binaries and where will those be located?
                                                                                       
                                                                                      Answer: There are many documents on building the binaries at rocketboards.com. Any reference binaries can be stored at rocketboards.com as well.
                                                                                       
                                                                                      3.  
                                                                                    3.  
                                                                                      What are the host side commands used to put the binaries to Copy Engine and from there to HPS?
                                                                                       
                                                                                      Answer: There is a single command, hps to download the single binary through the Copy Engine to the HPS.
                                                                                       
                                                                                      4.  
                                                                                    4.  
                                                                                      What are the host side commands used to reset the HPS from Host side?
                                                                                       
                                                                                      Answer: This functionality is planned to be added to the hps command.
                                                                                       
                                                                                      5.  
                                                                                    5.  
                                                                                      What is the procedure used to debug the HPS from Host side?
                                                                                       
                                                                                      Answer: Debugging the HPS from the host side is standard HPS debugging. The primary debug tool is UART on the HPS and the Arm DS debugger.
                                                                                       
                                                                                      6.  
                                                                                    6.  
                                                                                      Do we have performance metrics about HPS, like whether any bench marking information available with any sample application is available?
                                                                                       
                                                                                      Answer: Any performance metrics on the HPS would be available on rocketboards.com.
                                                                                       
                                                                                      7.  
                                                                                    7.  
                                                                                      What is the PXE boot flow and what is required to enable the same?
                                                                                       
                                                                                      Answer: On some configurations, HPS is treated as a fully-fledged SoC and can PXE boot itself. But you must add this functionality.
                                                                                       
                                                                                      8.  
                                                                                    "},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#further-links","title":"Further Links","text":"Description Link OFS Getting Started User Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs https://ofs.github.io/hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/ Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs https://ofs.github.io/hw/n6001/dev_guides/fim_dev/ug_dev_fim_ofs_n6001/ Open FPGA Stack Technical Reference Manual for Intel Agilex FPGA PCIe Attach https://ofs.github.io/hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/ AFU Development Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs https://ofs.github.io/hw/N6001/dev_guides/afu_dev/ug_dev_afu_n6001/ Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs https://ofs.github.io/hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/ FPGA Device Feature List (DFL) Framework Overview https://github.com/OFS/linux-dfl/blob/fpga-ofs-dev/Documentation/fpga/dfl.rst#fpga-device-feature-list-dfl-framework-overview ofs-platform-afu-bbb https://github.com/OFS/ofs-platform-afu-bbb Connecting an AFU to a Platform using PIM https://github.com/OFS/ofs-platform-afu-bbb/blob/master/plat_if_develop/ofs_plat_if/docs/PIM_AFU_interface.md example AFUs https://github.com/OFS/examples-afu.git PIM Tutorial https://github.com/OFS/examples-afu/tree/main/tutorial Non-PIM AFU Development https://github.com/OFS/examples-afu/tree/main/tutorial Intel FPGA IP Subsystem for PCI Express IP User Guide https://github.com/OFS/ofs.github.io/docs/hw/common/user_guides/ug_qs_pcie_ss.pdf Memory Subsystem Intel FPGA IP User Guide https://github.com/OFS/ofs.github.io/docs/hw/common/user_guides/ug_qs_mem_ss.pdf OPAE.io https://opae.github.io/latest/docs/fpga_tools/opae.io/opae.io.html OPAE GitHub https://github.com/OFS/opae-sdk README_ofs_n6001_eval.txt https://github.com/OFS/ofs-n6001/blob/release/ofs-2023.1/eval_scripts/README_ofs_n6001_eval.txt FIM MMIO Regions https://ofs.github.io/hw/d5005/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#mmio_regions evaluation script https://github.com/OFS/ofs-n6001/tree/release/ofs-2023.1/eval_scripts OFS https://github.com/OFS OFS GitHub page https://ofs.github.io DFL Wiki https://github.com/OPAE/linux-dfl/wiki"},{"location":"hw/n6001/dev_guides/hps_dev/hps_developer_ug/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                                    Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                                     
                                                                                    OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                                    "},{"location":"hw/n6001/doc_modules/Glossary/","title":"Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel Max10 or Intel Cyclone10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implemented on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Management Component Transport Protocol MCTP A standardized model for communication with management controllers. Defines the transport protocol carrying PLDM messages through the BMC. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE-SDK The OPAE-SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Level Data Model PLDM A specification for reporting telemetry data to the host, such as board temperature, voltage, and current. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/doc_modules/Notices_%26_Disclaimers/","title":"Notices & Disclaimers","text":""},{"location":"hw/n6001/doc_modules/Notices_%26_Disclaimers/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                                    Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                                     
                                                                                    OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                                    "},{"location":"hw/n6001/doc_modules/n6001_wt_program_fpga_via_jtag/","title":"N6001 wt program fpga via jtag","text":"
                                                                                    This walkthrough describes the steps to program the Agilex FPGA on the Intel\u00ae FPGA SmartNIC N6001-PL with a SOF image via JTAG.
                                                                                     
                                                                                    Pre-Requisites:
                                                                                     
                                                                                       
                                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the [OFS Agilex PCIe Attach Getting Started Guide] for instructions on setting up a deployment environment.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • This walkthrough requires a SOF image which will be programmed to the Agilex FPGA. Refer to the [Walkthrough: Compile OFS FIM] Section for step-by-step instructions for generating a SOF image.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • This walkthrough requires a JTAG connection to the n6001. Refer to the [Walkthrough: Set up JTAG] section for step-by-step instructions.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel\u00ae FPGA SmartNIC N6001-PL is connected via JTAG.
                                                                                      •  
                                                                                     
                                                                                    Steps:
                                                                                     
                                                                                       
                                                                                    1.  
                                                                                      Start in your deployment environment.
                                                                                       
                                                                                      2.  
                                                                                    2.  
                                                                                      If the card is already programmed with an OFS enabled design, determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                                                                                       
                                                                                      fpgainfo fme\n
                                                                                       
                                                                                      Example output:
                                                                                       
                                                                                      Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                                                                       
                                                                                      3.  
                                                                                    3.  
                                                                                      Remove the card from PCIe prior to programming. This will disable AER for the PCIe root port to prevent a surprise link-down event during programming.
                                                                                       
                                                                                      sudo pci_device b1:00.0 unplug\n
                                                                                       
                                                                                      4.  
                                                                                    4.  
                                                                                      Switch to the machine with JTAG connection to the n6001, if different than your deployment machine.
                                                                                       
                                                                                      5.  
                                                                                    5.  
                                                                                      Open the Quartus programmer GUI
                                                                                       
                                                                                      quartus_pgmw\n
                                                                                       
                                                                                       
                                                                                      6.  
                                                                                    6.  
                                                                                      Click Hardware Setup to open the Hardware Setup dialog window.
                                                                                       
                                                                                         
                                                                                      1.  
                                                                                        In the Currently selected hardware field select the n6001.
                                                                                         
                                                                                        2.  
                                                                                      2.  
                                                                                        In the Hardware frequency field enter 16000000 Hz
                                                                                         
                                                                                         
                                                                                        3.  
                                                                                      3.  
                                                                                        Click Close
                                                                                         
                                                                                        4.  
                                                                                       
                                                                                      7.  
                                                                                    7.  
                                                                                      In the Quartus Prime Programmer window, click Auto Detect.
                                                                                       
                                                                                      8.  
                                                                                    8.  
                                                                                      If prompted, select the AGFB014R24A2E2V device. The JTAG chain should show the divice.
                                                                                       
                                                                                       
                                                                                      9.  
                                                                                    9.  
                                                                                      Right click the AGFB014R24A2E2V row and selct Change File.
                                                                                       
                                                                                       
                                                                                      10.  
                                                                                    10.  
                                                                                      In the Select New Programming File window that opens, select the .sof image you wish to program and click Open.
                                                                                       
                                                                                      11.  
                                                                                    11.  
                                                                                      Check the Program/Configure box for the AGFB014R24A2E2V row, then click Start. Wait for the Progress bar to show 100% (Success).
                                                                                       
                                                                                      12.  
                                                                                    12.  
                                                                                      Close the Quartus Programmer GUI. You can answer 'No' if a dialog pops up asking to save the 'Chain.cdf' file
                                                                                       
                                                                                      13.  
                                                                                    13.  
                                                                                      Switch to the deployment environment, if different than the JTAG connected machine.
                                                                                       
                                                                                      14.  
                                                                                    14.  
                                                                                      Replug the board PCIe
                                                                                       
                                                                                      sudo pci_device b1:00.0 plug\n
                                                                                       
                                                                                      15.  
                                                                                    15.  
                                                                                      Run fpgainfo fme to verify communication with the board, and to check the PR Interface ID.
                                                                                       
                                                                                      Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                                                                       
                                                                                      16.  
                                                                                    "},{"location":"hw/n6001/doc_modules/n6001_wt_program_fpga_via_rsu/","title":"N6001 wt program fpga via rsu","text":"
                                                                                    This walkthrough describes the steps to program the Agilex FPGA on the Intel\u00ae FPGA SmartNIC N6001-PL with a BIN image via JTAG.
                                                                                     
                                                                                    Pre-Requisites:
                                                                                     
                                                                                       
                                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the [OFS Agilex PCIe Attach Getting Started Guide] for instructions on setting up a deployment environment.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • This walkthrough requires a BIN image which will be programmed to the Agilex FPGA. Refer to the [Walkthrough: Compile OFS FIM] Section for step-by-step instructions for generating a BIN image. The image used for programming must be formatted with PACsign before programming. This is done automatically by the build script.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • This walkthrough requires a JTAG connection to the n6001. Refer to the [Walkthrough: Set up JTAG] section for step-by-step instructions.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • This walkthrough requires a Full Quartus Installation or Standalone Quartus Prime Programmer & Tools running on the machine where the Intel\u00ae FPGA SmartNIC N6001-PL is connected via JTAG.
                                                                                      •  
                                                                                     
                                                                                    Steps:
                                                                                     
                                                                                       
                                                                                    1.  
                                                                                      Start in your deployment environment.
                                                                                       
                                                                                      2.  
                                                                                    2.  
                                                                                      Determine the PCIe B:D.F of the card using OPAE command fpgainfo fme. In this example, the PCIe B:D.F is B1:00.0.
                                                                                       
                                                                                      fpgainfo fme\n
                                                                                       
                                                                                      Example output:
                                                                                       
                                                                                      Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\nPBA: B#FB2CG1@AGF14-A0P2\nMMID: 217000\nSN: Q171211700050\n//****** FME ******//\nInterface                        : DFL\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:98:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : 9035190d637c383453173deb5de25fdd\nUser1 Image Info                 : 893e691edfccfd0aecb1c332ad69551b\nUser2 Image Info                 : 8cd2ae8073e194525bcd682f50935fc7\n
                                                                                       
                                                                                      3.  
                                                                                    3.  
                                                                                      Use the OPAE fpgasupdate command to program a PACsign signed image to flash. The flash slot which will be programmed is determined by the PACsign header.
                                                                                       
                                                                                      sudo fpgasupdate <IMAGE> <PCIe B:D.F>\n
                                                                                       
                                                                                         
                                                                                      •  
                                                                                        Example: update User Image 1 in flash
                                                                                         
                                                                                        sudo fpgasupdate ofs_top_page1_unsigned_user1.bin 98:00.0\n
                                                                                         
                                                                                        •  
                                                                                       
                                                                                         
                                                                                      •  
                                                                                        Example: update User Image 2 in flash
                                                                                         
                                                                                        sudo fpgasupdate ofs_top_page2_unsigned_user2.bin 98:00.0\n
                                                                                         
                                                                                        •  
                                                                                       
                                                                                         
                                                                                      •  
                                                                                        Example: update Factory Image in flash
                                                                                         
                                                                                        sudo fpgasupdate ofs_top_page0_unsigned_factory.bin 98:00.0\n
                                                                                         
                                                                                        •  
                                                                                       
                                                                                      4.  
                                                                                    4.  
                                                                                      Use the OPAE rsu command to reconfigure the FPGA with the new image. You may select which image to configure from (User 1, User 2, Factory).
                                                                                       
                                                                                      sudo rsu fpga --page <PAGE> <PCIe B:D.F>\n
                                                                                       
                                                                                         
                                                                                      •  
                                                                                        Example: configure FPGA with User 1 Image
                                                                                         
                                                                                        sudo rsu fpga --page user1 98:00.0\n
                                                                                         
                                                                                        •  
                                                                                       
                                                                                         
                                                                                      •  
                                                                                        Example: configure FPGA with User 2 Image
                                                                                         
                                                                                        sudo rsu fpga --page user2 98:00.0\n
                                                                                         
                                                                                        •  
                                                                                       
                                                                                         
                                                                                      •  
                                                                                        Example: configure FPGA with Factory Image
                                                                                         
                                                                                        sudo rsu fpga --page factory 98:00.0\n
                                                                                         
                                                                                        •  
                                                                                       
                                                                                      5.  
                                                                                    "},{"location":"hw/n6001/doc_modules/n6001_wt_set_up_jtag/","title":"N6001 wt set up jtag","text":"
                                                                                    Perform the following steps to set up a JTAG connection to the Intel\u00ae FPGA SmartNIC N6001-PL.
                                                                                     
                                                                                    Pre-requisites:
                                                                                     
                                                                                       
                                                                                    • This walkthrough requires an OFS Agilex PCIe Attach deployment environment. Refer to the [OFS Agilex PCIe Attach Getting Started Guide] for instructions on setting up a deployment environment.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • This walkthrough requires a workstation with Quartus Prime Pro Version 23.3 tools installed, specifically the jtagconfig tool.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • This walkthrough requires an [Intel FPGA Download Cable II].
                                                                                      •  
                                                                                     
                                                                                    Steps:
                                                                                     
                                                                                       
                                                                                    1.  
                                                                                      Set the board switches to dynamically select either the Intel\u00ae Agilex\u00ae 7 FPGA or MAX\u00ae 10 device on the JTAG chain.
                                                                                       
                                                                                         
                                                                                      1. Set SW1.1=ON as shown in the next image. The switches are located at the back of the Intel\u00ae FPGA SmartNIC N6001-PL.
                                                                                        2.  
                                                                                       
                                                                                       
                                                                                      2.  
                                                                                    2.  
                                                                                      The Intel\u00ae FPGA SmartNIC N6001-PL has a 10 pin JTAG header on the top side of the board. Connect an Intel\u00ae FPGA Download II Cable to the JTAG header of the Intel\u00ae FPGA SmartNIC N6001-PL as shown in picture below. This picture shows the Intel\u00ae FPGA SmartNIC N6001-PL card installed in the middle bay, top slot of a SuperMicro\u00ae SYS-220HE-FTNR server where the lower slot does not have card installed allowing the Intel\u00ae Download II cables to pass through removed the slot access. 
                                                                                       
                                                                                       
                                                                                      Note: If using the Intel FGPA download Cable on Linux, add the udev rule as described in [Intel FPGA Download Cable Driver for Linux].
                                                                                       
                                                                                      3.  
                                                                                    3.  
                                                                                      Set the JTAG chain to select the Intel\u00ae Agilex\u00ae 7 FPGA as the target by writing to the JTAG enable register in the BMC (Register 0x378). This is done via PMCI registers 0x2040C and 0x20400.
                                                                                       
                                                                                      Note: The commands below are targeted to a board with PCIe B:D.F of 98:00.0. Use the correct PCIe B:D.F of your card.
                                                                                       
                                                                                      sudo opae.io init -d 0000:98:00.0 $USER\nsudo opae.io -d 0000:98:00.0 -r 0 poke 0x2040c 0x100000000\nsudo opae.io -d 0000:98:00.0 -r 0 poke 0x20400 0x37800000002\nsudo opae.io release -d 0000:98:00.0\n
                                                                                       
                                                                                      Note: To later re-direct the JTAG back to the MAX 10 device, execute the following commands.
                                                                                       
                                                                                      sudo opae.io init -d 0000:b1:00.0 $USER\nsudo opae.io -d 0000:b1:00.0 -r 0 poke 0x2040c 0x000000000\nsudo opae.io -d 0000:b1:00.0 -r 0 poke 0x20400 0x37800000002\nsudo opae.io release -d 0000:b1:00.0\n
                                                                                       
                                                                                      Optionally, rather than dynamically commanding Intel\u00ae Agilex\u00ae 7 FPGA/MAX10 selection with the PMCI register settings, you can fix the selection with the following switch settings shown in the table below:
                                                                                       SW1.1 SW2 JTAG Target OFF OFF Intel\u00ae Agilex\u00ae 7 FPGA OFF ON MAX\u00ae 10 FPGA ON X Intel\u00ae Agilex\u00ae 7 FPGA if BMC register 0x378=0x1 ON X MAX\u00ae 10 FPGA if BMC register 0x378=0x0 
                                                                                      4.  
                                                                                    4.  
                                                                                      Use the jtagconfig tool to check that the JTAG chain contains the AGFB014R24A2E2V device.
                                                                                       
                                                                                      <QUARTUS_INSTALL_DIR>/23.3/quartus/bin/jtagconfig\n
                                                                                       
                                                                                      Example expected output:
                                                                                       
                                                                                      TBD\n
                                                                                       
                                                                                      5.  
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/","title":"FPGA Interface Manager Technical Reference Manual for Intel Agilex PCIe Attach: Open FPGA Stack","text":""},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1-overview","title":"1 Overview","text":""},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#11-about-this-document","title":"1.1 About this Document","text":"
                                                                                    This document describes the hardware architecture for the PCIe attach reference FIM of the Open FPGA Stack (OFS) targeting the Intel\u00ae Agilex\u00ae FPGA.  After reviewing this document you should understand the features and functions of the components that comprise the FPGA Interface Manager (FIM), also known as the \"shell.\"
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#12-glossary","title":"1.2 Glossary","text":"
                                                                                    This table defines some of the common terms used when discussing OFS.
                                                                                     
                                                                                    Table 1-1 Glossary Table
                                                                                     Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel\u00ae Max\u00ae 10 or Intel Cyclone\u00ae 10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implanted on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE SDK The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#13-introduction-to-open-fpga-stack","title":"1.3 Introduction to Open FPGA Stack","text":"
                                                                                    The Open FPGA Stack (OFS) is a modular infrastructure of hardware platform components, open source upstreamed software, and broad ecosystem support that enables an efficient path to develop a custom FPGA platform.  OFS provides a framework of FPGA synthesizable code, simulation environment and synthesis/simulation scripts.  The key components of OFS include:
                                                                                     
                                                                                       
                                                                                    • Target development platforms such as Intel-branded Programmable Acceleration Cards (PACs), Acceleration Development Platforms (ADPs) and third-party platforms.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • Board Management Controller RTL and firmware that supports telemetry monitoring and capability for remote configuration updates.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • Source accessible, modular FPGA Interface  manager (FIM) RTL with a UVM infrastructure unit tests that can be leveraged for your own custom FIM design. The FIM can be thought of as the FPGA shell that provides the I/O ring and timing closed management components for the FPGA.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • Basic building blocks for interconnect and PF/VF translation and arbitration; Platform Interface Manager (PIM) which provides Avalon\u00ae bus compliant interfaces.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • AFU examples both in the git repository and workload examples provided by 3rd party vendors.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • A OneAPI acceleration support package (oneapi-asp) that provides a bridge layer that is used by OneAPI runtime to communicate with the kernel.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • Unit level simulation test suite
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • System level simulation through a unified verification methodology (UVM)
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • OPAE software development kit (APIs, upstreamed Linux drivers and software tools)
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • Support for other frameworks to be built on top of the OPAE such as DPDK 
                                                                                      •  
                                                                                     
                                                                                    These components are available under the https://github.com/OFS site.
                                                                                     
                                                                                    The OFS hardware repository supports hardware development and simulation.  Repositories for OFS high level design support and board management controller RTL and firmware source code are also provided.  These repositories can be found in the Intel Opensource Technology GitHub location, which requires entitlement access.  To request access, please contact your local Intel sales representative.
                                                                                     
                                                                                    Table 1-2 OFS Hardware Repositories
                                                                                     Repository Contains ofs-n6001 Contains FIM or shell RTL, automated compilation scripts, and unit tests and UVM framework. oneapi-asp Contains the hardware and software components you need to develop your own OneAPI board support package ofs-platform-afu-bbb Contains the files and scripts to build the platform interface manager. ofs-examples-afu Contains AFU examples you can use. ofs-bmc Provides the OFS Board Management Controller RTL, firmware, scripts and collateral targeting the Intel\u00ae FPGA SmartNIC N6001-PL which can be leveraged for your own OFS design. 
                                                                                    Table 1-3 OFS Software Repositories
                                                                                     OPAE Git Repository Folder Contains opae-sdk Contains the files for building and installing OPAE SDK from source. linux-dfl Contains OFS Linux drivers that are being upstreamed to the Linux kernel. opae-sim Contains the files for an AFU developer to build the Accelerator Functional Unit Simulation Environment (ASE) for workload development. 
                                                                                    Providing the hardware and software source code and supporting test frameworks in a GitHub repository allows you to customize your designs with the latest versions easily.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#14-ofs-features","title":"1.4 OFS Features","text":"
                                                                                    The OFS architecture within the FPGA comprises two partitions:
                                                                                     
                                                                                       
                                                                                    • FPGA Interface Manager (FIM)
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • Accelerator Functional Unit (AFU)
                                                                                      •  
                                                                                     
                                                                                    The FIM or shell provides platform management functionality, clocks, resets and interface access to the host and peripheral features of the acceleration platform. The FIM architecture along with the supporting OPAE software supports features such as partial reconfiguration and virtualization. The FIM provides a standard Arm\u00ae AMBA\u00ae 4 AXI4 datapath interface. The FIM resides in the static region of the FPGA.
                                                                                     
                                                                                    The AFU partition is provided for custom acceleration workloads and may contain both static and partial reconfiguration regions.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#141-fpga-interface-manager-fim","title":"1.4.1 FPGA Interface Manager (FIM)","text":"
                                                                                    The primary components of the FPGA Interface Manager or shell of this reference design are: 
                                                                                     
                                                                                       
                                                                                    • PCIe Subsystem
                                                                                      •  
                                                                                    • HSSI Subsystem
                                                                                      •  
                                                                                    • Memory Subsystem
                                                                                      •  
                                                                                    • Hard Processor System
                                                                                      •  
                                                                                    • Reset Controller
                                                                                      •  
                                                                                    • FPGA Management Engine
                                                                                      •  
                                                                                    • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                                                                                      •  
                                                                                    • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                                                                                      •  
                                                                                    • Platform Management Controller Interface (PMCI) to the board management controller
                                                                                      •  
                                                                                     
                                                                                    The AFU Region provides design space for custom workloads and contains both static and partial reconfiguration regions.  Partial reconfiguration allows you to update your specific logic blocks or entire workload while the rest of your static design is still in operation.
                                                                                     
                                                                                    Note that the BMC RTL and firmware that works with this OFS design provided in a separate entitled repository.  Please email ofs.github@intel.com if you would like to use our BMC code for your own design.
                                                                                     
                                                                                    Figure 1-2 OFS for OFS FIM for Agilex OFS Block Diagram
                                                                                     
                                                                                     
                                                                                    The table provides an overview of the OFS features targeting the Intel\u00ae Agilex\u00ae 7 FPGA.  This reference FIM (shell) is a starting point for your custom FPGA design.  With this initial starting point, you can add or subtract interfaces or ports to different Agilex devices.
                                                                                     
                                                                                    Table 1-4 OFS FIM for Intel\u00ae Agilex\u00ae 7 FPGA Features
                                                                                     Key Feature Description PCIe P-tile PCIe* Gen4x16 Virtualization Includes support for 5 physical functions/4 virtual functions with ability to expand to the P-tile PCIe Hard IP maximum (8PFs, 2K VFs per each Endpoint). Memory 5 DDR Channels:* One HPS DDR4 bank, x40 (x32 Data and x8 ECC), 1200 MHz, 1GB each* Four Fabric DDR4 banks, x32 (no ECC), 1200 MHz, 4GB Ethernet Eight Arm\u00ae AMBA\u00ae 4 AXI4-Stream channels of 25G Ethernet interfacing to an E-tile Ethernet Subsystem. Hard Processor System 64-bit quad core Arm\u00ae Cortex\u00ae-A53 MPCore with integrated peripherals. Configuration and Board Manageability * FPGA Management Engine that provides general control of common FPGA tasks (ex. error reporting, partial reconfiguration)* Platform Controller Management Interface (PMCI) Module contained within the Agilex FPGA that interfaces through Avalon-Streaming x8 QuadSPI and SPI to a Board Management Controller Partial Reconfiguration Partial Reconfiguration region supported in hardware and software OneAPI OneAPI Acceleration Support Package (ASP) provided with compiled FIM to support OneAPI Runtime Software Support * Linux DFL drivers targeting OFS FIMs* OPAE Software Development Kit* OPAE Tools"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#subsystem-interfaces","title":"Subsystem Interfaces","text":"
                                                                                    The PCIe, Memory and Ethernet interfaces in this design use a new flexible subsystem design that provides a standard Arm\u00ae AMBA\u00ae 4 AXI4 interface.  To access these Intel FPGA IP Subsystem documents. Please go to the links below: * Intel FPGA IP Subsystem for PCI Express IP User Guide * Memory Subsystem Intel FPGA IP User Guide * Ethernet Subsystem Intel FPGA IP User Guide (public document)
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#hard-processor-system-hps","title":"Hard Processor System (HPS)","text":"
                                                                                    The HPS SoC contains a 64-bit quad core ARM\u00ae Cortex\u00ae-A53 MPCore with a variety of integrated modules such as on-chip RAM, Ethernet, USB, UARTs and SPI controllers and memory controllers.  For more information about the Intel Agilex HPS, please refer to the Intel Agilex Hard Processor System Technical Reference Manual.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#fpga-management-engine-fme","title":"FPGA Management Engine (FME)","text":"
                                                                                    The FIM contains only one FME, regardless of the number of host interfaces to the FIM. The FME provides management features for the platform and controls reset and loading of the AFU into the partial reconfiguration region of the FPGA.
                                                                                     
                                                                                    Any feature, such as a memory interface or global error control that you want to control through FME, must expose its capability to host software drivers.  New features are exposed to the FME by adding a device feature header (DFH) register at the beginning of the feature's control status register (CSR) space. The FME CSR maps to physical function 0 (PF0) Base address register 0 (BAR0) so that software can access it through a single PCIe link.  For more information about DFHs, refer to the FPGA Device Feature List Framework Overview.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#streaming-datapath","title":"Streaming Datapath","text":"
                                                                                    The FIM implements an AXI4-Stream bus protocol for data transfer in the FIM. AXI4-Stream channels send data packets to and from the host channel IP without data abstraction. Memory-mapped I/O (MMIO) CSR accesses are routed to the ST2MM module, which converts the AXI4-Stream to an AXI4 memory-mapped protocol.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#virtualization","title":"Virtualization","text":"
                                                                                    This design supports virtualization by making use of the virtualization functionality in the PCIe Hard IP and mapping packets to the appropriate physical or virtual function through a PF/VF multiplexer.  
                                                                                     
                                                                                    This reference FIM example supports 5 PFs and 4 VFs; however, you may extend your configuration to whatever the PCIe Hard IP can support or your application requires.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#142-afu","title":"1.4.2 AFU","text":"
                                                                                    An AFU is an acceleration workload that interfaces with the FIM. The AFU boundary in this design comprises both static and partial reconfiguration (PR) regions.  You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region.  A port gasket within the design provides all the PR specific modules and logic required for partial reconfiguration. Only one partial reconfiguration region is supported in this design.
                                                                                     
                                                                                    Similar to the FME, the port gasket exposes its capability to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space.  
                                                                                     
                                                                                    In this design, the AFU region is comprised of:
                                                                                     
                                                                                       
                                                                                    • AFU Interface handler to verify transactions coming from AFU region.
                                                                                      •  
                                                                                    • PF/VF Mux to route transactions to and from corresponding AFU components: 
                                                                                         
                                                                                      • ST2MM module.
                                                                                        •  
                                                                                      • Null Host exerciser (HE_NULL) stub.
                                                                                        •  
                                                                                      • PCIe loopback host exerciser (HE-LB).
                                                                                        •  
                                                                                      • HSSI host exerciser (HE-HSSI).
                                                                                        •  
                                                                                      • Memory Host Exerciser (HE-MEM).
                                                                                        •  
                                                                                      • Traffic Generator to memory (HE-MEM-TG).      Port Gasket (PRG). 
                                                                                        •  
                                                                                      • HPS Copy Engine.
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    • Arm\u00ae AMBA\u00ae 4 AXI4 Streaming to Memory Map (ST2MM) Module that routes MMIO CSR accesses to FME and board peripherals.
                                                                                      •  
                                                                                    • Host exercisers to test PCIe, memory and HSSI interfaces (these can be removed from the AFU region after your FIM design is complete to provide more resource area for workloads)
                                                                                      •  
                                                                                    • HPS Copy Engine to copy second-stage bootloader and Linux OS image from Host DDR to HPS DDR.  
                                                                                      •  
                                                                                    • Port gasket and partial reconfiguration support.
                                                                                      •  
                                                                                    • Component for handling PLDM over MCTP over PCIe Vendor Defined Messages (VDM)
                                                                                      •  
                                                                                     
                                                                                    For this design the PF/VF Mux provides the following mappings (found in src/afu_top/mux/top_cfg_pkg.sv):
                                                                                     
                                                                                    Table 1-5 PF/VF Mapping
                                                                                     Module PF/VF Mapping AXI4 Stream to Memory Mapped Module (ST2MM) PF0 Memory Host Exerciser (HE_MEM) PF0-VF0 HSSI Host Exerciser (HE_HSSI) PF0-VF1 Memory Traffic Generator (HE_MEM_TG) PF0-VF2 Null Host exerciser (HE_NULL) stub PF1-VF0 PCIe Loopback (HE_LB) PF2 Null Host exerciser (HE_NULL) stub PF3 HPS Copy Engine Module PF4 
                                                                                    Figure 1-3 AFU Diagram 
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#143-platform-interface-manager","title":"1.4.3 Platform Interface Manager","text":"
                                                                                    The PIM provides a way to abstract the AXI4-Stream interface to the AFU by providing a library of shims that convert the host channel native packet into other protocols such as AXI4 memory-mapped, Avalon\u00ae streaming (Avalon-ST) or Avalon\u00ae memory-mapped (Avalon-MM). 
                                                                                     
                                                                                    The FPGA or AFU developer implements these interface abstractions in the AFU region (afu_main) of the design.  
                                                                                     
                                                                                    For more information, refer to [AFU Development Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs].
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#144-platform-feature-discovery","title":"1.4.4 Platform Feature Discovery","text":"
                                                                                    This reference design comes with specific Intel FPGA drivers that are upstreamed to linux-dfl.  These drivers abstract the hardware and operating system specific details of the platform to the host. 
                                                                                     
                                                                                    The FIM implements a device feature header (DFH) at the beginning of each host-discoverable feature in a linked list format that allows an FPGA platform driver running on the host to discover FME, port, and AFU features. 
                                                                                     
                                                                                    You must implement a 64-bit DFH Device Feature Header register at the beginning (first 8B aligned address) of the feature CSR space for a new feature to be discovered or enumerated by a driver.  
                                                                                     
                                                                                    During host discovery, the driver traverses the DFH of the first feature from the first address on PF0 BAR0. Based on the information in the DFH, a driver can determine the CSR address range of the feature and other associated details. The end of the DFH contains a \"next DFH offset\" field that points the driver to the DFH of the next feature. 
                                                                                     
                                                                                    The software must continue traversing the linked list until it sees the EOL (End-Of-List) bit set to 1 in the \"next DFH offset\" field it inspects.  A 1 indicates this is the last feature in the feature set. The figure below gives a simple illustration of the feature discovery by traversing the DFH registers. This model is similar to how PCIe enumeration occurs.
                                                                                     
                                                                                    Figure 1-4 Device Feature Header Linked List Traversal
                                                                                     
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#145-ofs-reference-design","title":"1.4.5 OFS Reference Design","text":"
                                                                                    OFS provides FIM designs you can use as a starting point for your own custom design. These designs target a specific programmable acceleration card or development kit and exercise key FPGA device interfaces. 
                                                                                     
                                                                                    The Intel Agilex\u00ae code line for OFS targets the Intel N6001-PL FPGA SmartNIC Platform.  FIM designs are released to https://github.com/OFS/ofs-agx7-pcie-attach for evaluation and use. 
                                                                                     
                                                                                    In addition to the OFS FIM for Agilex that targets the Intel N6001-PL FPGA SmartNIC Platform, vertical market FIMs are available for the Intel N6000-PL SmartNIC Platform. 
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#146-fim-simulation","title":"1.4.6 FIM Simulation","text":"
                                                                                    OFS provides unit tests and a UVM environment for the FIM and a framework for new feature verification. UVM provides a modular, reusable, and scalable testbench structure by providing an API framework that can be deployed across multiple projects. 
                                                                                     
                                                                                    The FIM testbench is UVM compliant and integrates third-party verification IPs from Synopsys that require a license. 
                                                                                     
                                                                                    Verification components include:
                                                                                     
                                                                                       
                                                                                    • FIM monitor to detect correct design behavior
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • FIM assertions for signal level integrity testing
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • Arm\u00ae AMBA\u00ae 4 AXI4 scoreboards to check data integrity
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • FIM coverage to collect functional data
                                                                                      •  
                                                                                     
                                                                                    The verification infrastructure can be found here for evaluation and use. Please refer to the [Simulation User Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs] for more information.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#2-ofs-high-level-architecture","title":"2 OFS High Level Architecture","text":"
                                                                                    OFS provides distinct data paths that simplify the design and integration process for adding or removing interface modules:
                                                                                     
                                                                                       
                                                                                    • High Bandwidth data path for AFU-attached high-performance peripherals (HSSI, Memory, HPS, workload).
                                                                                      •  
                                                                                    • Low Bandwidth data path for OFS management and slow peripheral components (JTAG, I2C, SMBus).
                                                                                      •  
                                                                                    • AFU Peripheral Fabric (APF) to Board Peripheral Fabric (BPF) path to communicate with interface control and status registers (CSRs) and board components.
                                                                                      •  
                                                                                    • Peer-to-peer datapath between AFU components.
                                                                                      •  
                                                                                    • Peer-to-peer datapath between BPF components.
                                                                                      •  
                                                                                     
                                                                                    Depending on your design goals, you can present peripherals to software as:
                                                                                     
                                                                                       
                                                                                    • OFS managed peripherals with a device feature header that is part of a device feature list.
                                                                                      •  
                                                                                    • Native driver managed peripherals that are exposed through an independent physical function or virtual function.
                                                                                      •  
                                                                                     
                                                                                    Figure 2-1 OFS Datapath Structure
                                                                                     
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#3-pcie-subsystem","title":"3 PCIe Subsystem","text":"
                                                                                    The FIM's PCIe Subsystem is a hierarchical design that targets the P-tile PCIe* hard IP and is configured to support Gen4 speeds and Arm\u00ae AMBA\u00ae 4 AXI4 Data Mover functional mode.  The IP supports SR-IOV and is configured to provide 5 PFs and 4 VFs.  Native PCIe TLP packets are sent through the PCIe using Arm\u00ae AMBA\u00ae 4 AXI4 Stream Protocol.  Before they reach the AFU, the packets go through an adapter in the subsystem that converts any headers to a data mover format.  Tag allocation and management for packets sent from the application to the host are done by the PF/VF Mux that is part of the AFU region.
                                                                                     
                                                                                    Figure 3-1 OFS FIM RX-TX Datapath
                                                                                     
                                                                                     
                                                                                    Some key features of the PCIe interface are:
                                                                                     Feature OFS for Intel Agilex FPGA PCIe Subsystem Configuration Mode PCIe Gen4x16 Port Mode Native Endpoint SR-IOV 5PFs, 4VFs MSI-X Support Yes Functional Mode Data Mover Profile Basic TLP Bypass No Header Packing Scheme Simple Data Width 512-bit (64-byte) AXI-ST Clock Frequency 400 MHz Tags Supported 128 Reordering No reordering of requests, no completion reordering Maximum Payload Size 512 Bytes Memory Requests Supported 1CL, 2CL, 4CL MMIO transaction Size 4B, 8B Control Shadow Interface Enabled Completion Timeout Interface Enabled 
                                                                                    The PCIe PF, VF and Base Address Register (BAR) configuration can be modified in the PCIe Subsystem Platform Designer GUI interface.  The current implementation for the OFS FIM for Agilex FPGA is as follows:
                                                                                     
                                                                                    Table 3-1 Function and BAR Table for OFS for Agilex FPGA
                                                                                     PF VF Feature BAR BAR Size PF0 - OFS Managed Peripherals (PCIe, Memory, Ethernet) BAR0 512 KB PF0 - AFU Peripherals BAR0 256 KB PF0 - Board Peripherals BAR0 256 KB PF0 - MSI-X BAR4 16 KB PF0 VF0 Memory Host Exerciser (HE-MEM) BAR0 256 KB PF0 VF1 HSSI Host Exerciser (HE-HSSI) BAR0 256 KB PF0 VF2 Memory Traffic Generator (HE-MEM-TG) BAR0 256 KB PF1 VF0 Null Host exerciser (HE_NULL) BAR0 4 KB PF2 PCIe Loopback (HE-LB) BAR0 256 KB PF3 Null Host exerciser (HE_NULL) BAR0 4 KB PF4 HPS Copy Engine BAR0 4 KB"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#31-pcie-subsystem-header-format","title":"3.1 PCIe Subsystem Header Format","text":"
                                                                                    The first 32 bytes of the TLP from the PCIe subsystem denotes the PCIe header. There are two types of header format supported \u2013 Power User Mode header and Data Mover mode header. The tuser_vendor[0] bit on the AXI4-Stream channel indicates the header format of the TLP; tuser_vendor[0] =0 indicates Power User Mode header and tuser_vendor[0] =1 indicates Data Mover Mode header.
                                                                                     
                                                                                    The OFS FIM for Intel Agilex FPGA implements the Data Mover Functional Mode.  With this implementation, the application has the flexibility to use either mode for PCIe transaction, as shown in the following table. For more detailed information about the PCIe Subsystem, see the PCIe Subsystem Intel FPGA User Guide. 
                                                                                     
                                                                                    Table 3-2 PCIe Subsystem Header Format Support for OFS for Agilex FPGA
                                                                                     Direction Type Power User Data Mover Host to Endpoint MWr, MRd Yes No Host to Endpoint CPL/CPLd Yes Yes Host to Endpoint Msg Yes No Endpoint to Host MWr, MRd Yes Yes Endpoint to Host Intr Yes (MWr) Yes Endpoint to Host CPL/CPLd Yes Yes Endpoint to Host Msg Yes Yes"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#311-power-user-header-format","title":"3.1.1 Power User Header Format","text":"
                                                                                    The Power User Format provides user complete control over PCIe Hard IP. The user can implement functionality of interest with finer control over PCIe Transaction Layer Packet (TLP), credit handling and various mode provided by HIP. 
                                                                                     
                                                                                    The lower 16 bytes of the Power User Format are standard PCIe header as defined by PCIe specification, and the upper 16 bytes are specific to the PCIe Subsystem Intel FPGA IP. 
                                                                                     
                                                                                    Table 3-3 Power User Header Format
                                                                                     
                                                                                     
                                                                                    The mapping of the PCIe defined header to the lower 16 bytes of the AXI4-Stream data channel is shown in the figure below. Each double word (DW) or 4 bytes in the PCIe header is mapped from big-endian to little-endian on AXI-S data channel.  
                                                                                     
                                                                                    Figure 3-2 Power User Header Mapping to Arm\u00ae AMBA\u00ae 4 AXI4 Channel
                                                                                     
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#312-data-mover-header-format","title":"3.1.2 Data Mover Header Format","text":"
                                                                                    The data mover mode allows high bandwidth data transfers to and from Host memory. It hides the complexity of handling PCIe TLPs. This format provides a simple interface for reading and writing to Host Memory. The data mover checks link partner credits before transmitting packets. It also provides MSI-X interrupt generation capability. 
                                                                                     
                                                                                    In Data Mover Mode, the lower 16 bytes are data mover specific and do not follow a PCIe standard header.  
                                                                                     
                                                                                    Table 3-4 Data Mover Header Format
                                                                                     
                                                                                     
                                                                                    The mapping of the data mover header to the lower 16 bytes of the AXI-S data channel is shown below. Each byte in the data mover header is mapped directly to the AXI-S data channel.  
                                                                                     
                                                                                    Figure 3-3 Data Mover Header Mapping to Arm\u00ae AMBA\u00ae 4 AXI4 Channel
                                                                                     
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#32-pcie-subsystem-interface-module","title":"3.2 PCIe Subsystem Interface Module","text":"
                                                                                    The PCIe Subsystem Interface module (/ipss/pcie/rtl/pcie_ss_if.sv), provides the supporting interface between software and the PCIe subsystem.  
                                                                                     
                                                                                    The interface module provides the following:
                                                                                     
                                                                                       
                                                                                    • Device Feature Header Registers
                                                                                      •  
                                                                                    • Control and Status Registers
                                                                                      •  
                                                                                    • Indirect access to PCIe subsystem CSR registers through a CSR mailbox in the PCIe Subsystem Interface.
                                                                                      •  
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#33-data-mover-request-cycles","title":"3.3 Data Mover Request Cycles","text":"
                                                                                    For Host read request cycles using the OFS FIM for Intel Agilex FPGA:
                                                                                     
                                                                                       
                                                                                    • All requests in the RX direction will be MMIO. 
                                                                                      •  
                                                                                    • Requester ID from the request does get sent to the AFU. It is the AFU's responsibility to send back a completion to the host with the correct completer ID. 
                                                                                      •  
                                                                                    • Prefix is not supported. 
                                                                                      •  
                                                                                    • Memory Mapped (MM) Mode is not supported.
                                                                                      •  
                                                                                    • Slot Number is 0.
                                                                                      •  
                                                                                    • Base address is not sent to the AFU.
                                                                                      •  
                                                                                    • Local Address field is not used.
                                                                                      •  
                                                                                     
                                                                                    For AFU/application request cycles using the OFS FIM for Intel Agilex FPGA:
                                                                                     
                                                                                       
                                                                                    • All requests in the TX direction will be Memory Read/Write. 
                                                                                      •  
                                                                                    • The tag must be generated by the AFU/application.
                                                                                      •  
                                                                                    • Prefix is not supported. 
                                                                                      •  
                                                                                    • MM Mode is not supported.
                                                                                      •  
                                                                                    • Slot Number is 0 (non-0 only for switch)
                                                                                      •  
                                                                                    • VF Active, VF number and PF number are obtained from Data Mover Header Packet.
                                                                                      •  
                                                                                     
                                                                                    Figure 3-4 Data Mover Request Cycles
                                                                                     
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#34-data-mover-completion-cycles","title":"3.4 Data Mover Completion Cycles","text":"
                                                                                    For Host completion cycles using the OFS FIM for Intel Agilex FPGA:
                                                                                     
                                                                                       
                                                                                    • All completions in the RX direction will be Data Completions. 
                                                                                      •  
                                                                                    • Prefix is not supported.
                                                                                      •  
                                                                                    • MM Mode is not supported.
                                                                                      •  
                                                                                    • Slot Number is 0.
                                                                                      •  
                                                                                    • Data packet responses (for Memory Read requests from AFU) from the PCIe SS may come out of order when the size is >64B.
                                                                                      •  
                                                                                     
                                                                                    For AFU/application completion cycles using the OFS FIM for Intel Agilex FPGA: * All requests in the TX direction will be Memory Read/Write.  * Requester ID is generated within the FIM. * That tag must be generated by the AFU/application. * Prefix is not supported.  * MM Mode is not supported. * Slot Number is 0. * VF Active, VF Number and PF number are obtained from the Data Mover Header Packet. 
                                                                                     
                                                                                    Figure 3-5 Data Mover Completion Cycles
                                                                                     
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#4-platform-interface-manager","title":"4 Platform Interface Manager","text":"
                                                                                    The FIM interfaces to an application in the AFU region through AXI4-Stream channels.  This format allows the AFU to access the host channel's raw interface without any translation. 
                                                                                     
                                                                                    As a FIM developer, you have the option to provide the raw data format associated with the host interface channel to the workload or AFU developer or you can provide an intermediate protocol using Platform Interface Manager Components or your own custom interface.
                                                                                     
                                                                                    If you expose the raw AXI4-Stream interface of the FIM, workload developers also have the option to convert to a desired protocol using the PIM resources as well.  
                                                                                     
                                                                                    Refer to the AFU Developer Guide and the FPGA Interface Manager Developer Guide for more information on using the PIM in your design.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#5-interconnect-fabric","title":"5 Interconnect Fabric","text":"
                                                                                    There are three types of interconnect fabric in the OFS FIM design:  * AXI4-Stream PF/VF mux/demux fabric  * AFU Peripheral Fabric (APF)  * Board Peripheral Fabric (BPF)
                                                                                     
                                                                                    Figure 5-1 Interonnect Fabric Diagram
                                                                                     
                                                                                     
                                                                                    TLP packets sent from upstream PCIe Subsystem on AXI4-Stream channel are demultiplexed in the AXI4-Stream PF/VF mux/demux fabric and routed to the respective PF/VF function based on the PF/VF information in the TLP header, such as vf_active or the PF/VF number. In the opposite direction, TLP packets from downstream PF/VF function are muxed in the fabric and sent to PCIe subsystem over AXI4-Stream channel.
                                                                                     
                                                                                    All host MMIO requests targeting PF0 BAR0 are routed to the ST2MM module. The ST2MM converts MMIO TLP packets into AXI-Lite memory requests and places the requests onto AFU Peripheral Fabric (APF). AFU peripherals, such as OFS managed AFU features and ST2MM, and Board Peripheral Fabric (BPF) are interconnected by APF. The BPF is the interconnect fabric one hierarchy below APF which connects all the board peripherals. Both APF and BPF allow multiple AXI4-Lite master and slave interconnect topology.
                                                                                     
                                                                                    If you are modifying the APF or BPF connections, you must use Platform Designer to generate the fabrics directly.  Please refer to the FPGA Interface Manager Developer Guide for directions on what files must be modified and how to generate the interconnect.
                                                                                     
                                                                                    For modifying the PF/VF mux you must update the tools/pfvf_config_tool/pcie_host.ofss file and run the ofs-fim-common/pfvf_config_tool/gen_ofs_settings.py script to initiate the PCIe SS and PF/VF mux parameters to be regenerated before running the FIm build script.  
                                                                                     
                                                                                    For details on these modifications, please refer to the [Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs].  
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#51-afu-peripheral-fabric-apf","title":"5.1   AFU Peripheral Fabric (APF)","text":"
                                                                                    The AFU Peripheral Fabric (APF) is a 64-bit Arm AXI4-lite compliant interconnect fabric that connects AFU peripheral modules to board peripheral modules through the Board Peripheral Fabric (BPF). 
                                                                                     
                                                                                    The fabric is clocked by clk_csr and has a read allowance and write allowance of 1, i.e. only 1 active write/read is allowed in the fabric at any single time. 
                                                                                     
                                                                                    The address mapping for components interconnected by the APF is listed below. All components are mapped to PF0 BAR0 and implement AXI-lite slave interface. Note that none of the features in the APF mapping are designed to act as a master.
                                                                                     
                                                                                    Table 5-1 APF Address Mapping
                                                                                     Address Size (Byte) Feature 0x00000\u20130x3FFFF 256K Board Peripherals(See BPF address mapping) AFU Peripherals 0x40000 \u2013 0x4FFFF 64K ST2MM 0x50000 \u2013 0x5FFFF 64K Reserved 0x60000 \u2013 0x60FFF 4K UART 0x61000 \u2013 0x6FFFF 4K Reserved 0x70000 \u2013 0x7FFFF 56K PR Gasket:4K= PR Gasket DFH, control and status4K= Port DFH4K=User Clock52K=Remote STP 0x80000 \u2013 0x80FFF 4K AFU Error Reporting"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#52-board-peripheral-fabric-bpf","title":"5.2   Board Peripheral Fabric (BPF)","text":"
                                                                                    The Board Peripheral Fabric is the 64-bit AXI4-Lite compliant interconnect fabric that connects board peripheral modules to APF. The fabric is clocked by clk_csr and has a read allowance and write allowance of 1, i.e. only 1 active write/read is allowed in the fabric at any single time.
                                                                                     
                                                                                    The address mapping for components interconnected by BPF is listed below. All components are mapped to PF0 BAR0 and implement AXI4-lite slave interface. The Master column indicates if a component also implements AXI4-lite master interface which can send requests to the BPF.
                                                                                     
                                                                                    Table 5-2 BPF Address Mapping
                                                                                     Address Size (Byte) Feature Master 0x00000 \u2013 0x0FFFF 64K FME (FME, Error, etc) No 0x10000 \u2013 0x10FFF 4K PCIe Interface No 0x11000 \u2013 0x11FFF 4K Reserved - 0x12000 \u2013 0x12FFF 4K QSFP Controller 0 No 0x13000 \u2013 0x13FFF 4K QSFP Controller 1 No 0x14000 \u2013 0x14FFF 4K Ethernet Subsystem No 0x20000 \u2013 0x3FFFF 128K PMCI Controller Yes"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#53-arm-amba-4-axi4-stream-pfvf-muxdemux","title":"5.3 Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF Mux/Demux","text":"
                                                                                    The Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF Mux/Demux routes the PCIe TLP packets from the PCIe subsystem AXI4-Stream RX channel to downstream PF/VF based on the pf_num and vf_num information in the PCIe TLP header.
                                                                                     
                                                                                    The Arm\u00ae AMBA\u00ae 4 AXI4-Stream PF/VF mux arbitrates PCIe TLP packets from downstream PF/VF to the PCIe SS AXI-S TX channel. The PF/VF Mux/Demux is an M X N switch that allows any M port to target any N port, and any N port to target any M port, where M is the number of host/upstream ports, and N is the numbers functions/downstream ports.   
                                                                                     
                                                                                    The fpga top package file, /src/afu_top/mux/top_cfg_pkg.sv, contains the PF/VF parameters and mappings.
                                                                                     
                                                                                    Figure 5-2 PF/VF Mux
                                                                                     
                                                                                     
                                                                                    The PF/VF mux integration is part of afu_top (/src/afu_top/mux/top_cfg_pkg.sv).  There are two independent TX PF/VF MUX trees, labeled \"A\" and \"B\".
                                                                                     
                                                                                    Both an A and a B port are passed to each AFU component with a unique PF/VF. You can design your AFU components to send all requests to the primary A port or partition requests across both A and B ports. A typical high-performance AFU sends read requests to the B port and everything else to the A port, giving the arbiter freedom to keep both the host TX and RX channels busy.
                                                                                     
                                                                                    In the reference FIM provided for Intel Agilex OFS, the A and B TX trees have been multiplexed down to a single channel for A and another for B. The A/B multiplexer merges them into a single TX stream that will be passed to the tag remapper.
                                                                                     
                                                                                    The tag remapper provides unique tags as required by the PCIe specification.  Tags are not provided by the PCIe Subsystem FPGA IP.  When creating your own AFU you can leverage this module to generate unique tags.
                                                                                     
                                                                                    Note that the primary PF/VF Mux A supports RX and TX ports.  For the secondary PF/VF Mux B only TX ports are supported and the RX input to the Mux is tied off.  
                                                                                     
                                                                                    The default mapping is shown below:
                                                                                     
                                                                                    Table 5-3 PF/VF Mapping
                                                                                     Module PF/VF Mapping AXI4 Stream to Memory Mapped Module (ST2MM) PF0 Memory Host Exerciser (HE_MEM) PF0-VF0 HSSI Host Exerciser (HE_HSSI) PF0-VF1 Memory Traffic Generator (HE_MEM_TG) PF0-VF2 Virtio Loopback Stub (Virtio_LB) PF1-VF0 PCIe Loopback (HE_LB) PF2 Virtio Loopback Stub (Virtio_LB) PF3 HPS Copy Engine Module PF4 
                                                                                    For information on how to modify the PF/VF mapping for your own design, refer to the [Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs].
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#54-afu-interface-handler","title":"5.4 AFU Interface Handler","text":"
                                                                                    The AFU Interface Handler resides inline between the PCIe AXI4-Stream Adapter and the AXI4-Stream PF/VF Demux/Mux logic. Its main function is to provide: * Unique PCIe tags \u2013 Each PCIe transaction shares the 512 tags across all VFs in the AFU region * AFU error logging for all VFs in the AFU region
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#541-unified-tag-remapping","title":"5.4.1 Unified Tag Remapping","text":"
                                                                                    When a FPGA function sends out a read cycle, it allocates a unique tag which is subsequently used to identify the read completion.  The tag is considered busy; it cannot be assigned to another read cycle until read completion.  While a tag may be unique within a unit, two different units could unknowingly send out two read cycles of the same tag.  The PCIe subsystem requires unique tags for all read cycles irrespective of their origins.  Therefore, a mechanism is needed to uniquify tag globally across different units.
                                                                                     
                                                                                    OFS contains a tag remapper (/ofs-fim-common/src/common/tag_remap/tag_remap.sv) that intercepts the read cycle, finds a globally unique tag, and replaces the original tag value.  It also restores the original tag value when returning completion to the read requester.  tag_remap is placed between the AXI4-Stream interface of the PCIE subsystem and the PF/VF Mux/Demux.
                                                                                     
                                                                                    The logic is described as follows:
                                                                                     
                                                                                       
                                                                                    1. A sub-module (/ofs-fim-common/src/common/tag_remap/ofs_fim_tag_pool.sv) maintains a pool of available tags.
                                                                                      2.  
                                                                                    2. TX read requests are held until a tag is available from the pool by setting tvalid=0 to the host, and tready=0 to the PF/VF Mux/Demux.
                                                                                      3.  
                                                                                    3. When a TX read is dispatched, the tag is marked busy in the pool.
                                                                                      4.  
                                                                                    4. The original tag is stored in tag_reg, so it can be recovered when returning a completion to the unit/function.
                                                                                      5.  
                                                                                    5. Because completion to a read request can split into multiple smaller transfer sizes, responses are monitored and the final completion is detected using PCIe TLP rules.
                                                                                      6.  
                                                                                    6. Tags are released in the pool only when all requested data are transferred.
                                                                                      7.  
                                                                                    7. When the completion returns, the original tag is restored from tag_reg.
                                                                                      8.  
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#542-afu-error-handling","title":"5.4.2 AFU Error Handling","text":"
                                                                                    In this OFS design, the AFU Interface Handler handles error logging for all VFs in the AFU. Errors handled are as follows
                                                                                     
                                                                                    Table 5-4 AFU Error Descriptions
                                                                                     Checker Field Description AFU protocol checker (PCIe TLP) Malformed TLP AFU PCIe TLP contains unsupported format type AFU protocol checker (PCIe TLP) MaxPayloadError AFU memory write payload size exceeds max_payload_length limit AFU protocol checker (PCIe TLP) MaxReadReqSizeError AFU memory read payload size exceeds max_read_request_size limit AFU protocol checker (PCIe TLP) MaxTagError AFU memory read request tag value exceeds the maximum supported tag count AFU protocol checker (PCIe TLP) UnalignedAddrErr The address field in AFU memory write/read request TLP is not DW-aligned. AFU protocol checker (PCIe TLP) UnexpMMIOResp AFU is sending a MMIO read response with no matching MMIO read request. AFU protocol checker (PCIe TLP) MMIOTimedOut AFU is not responding to a MMIO read request within the pre-defined response timeout period. AFU protocol checker (PCIe TLP) MMIODataPayloadOverrun The number of data payload sent by AFU for a MMIO response (cplD) is more than the data length specified in the response. AFU protocol checker (PCIe TLP) MMIOInsufficientData The number of data payload sent by AFU for a memory write request is more than the data length specified in the request. AFU protocol checker (PCIe TLP) TxMWrDataPayloadOverrun The number of data payload sent by AFU for a memory write request is less than the data length specified in the request. AFU protocol checker (PCIe TLP) TxMWrInsufficientData The number of data payload sent by AFU for a memory write request is less than the data length specified in the request. AFU Protocol Checker (Arm\u00ae AMBA\u00ae 4 AXI4) TxValidViolation Three checkers are implemented in the FIM to catch errors and protocol violations."},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#55-tlp-to-axi4-lite-memory-mapped-bridge-st2mm","title":"5.5 TLP to AXI4-Lite Memory Mapped Bridge (ST2MM)","text":"
                                                                                    ST2MM implements the following key features:
                                                                                     
                                                                                       
                                                                                    • Host MMIO bridge     * Maps MMIO TLP packets received from the PCIe Subsystem over streaming interface to AXI4-Lite memory-mapped request. The memory-mapped request is sent to AFU or Board peripherals over APF and BPF.     * Maps AXI4-lite MM response received from AFU or Board peripherals to TLP packets and send the packets over ST streaming channel to host HIA subsystem.
                                                                                      •  
                                                                                    • Sends MMIO response of all 0\u2019s for MMIO read to unused BAR region.
                                                                                      •  
                                                                                    • Interrupt     * Sends interrupt packets to the PCIe subsystem when interrupt requests are received from the peripherals. Interrupts can be requested by a peripheral through a memory write to interrupt CSR registers in the ST2MM.
                                                                                      •  
                                                                                     
                                                                                    Figure 5-3 ST2MM Module
                                                                                     
                                                                                     
                                                                                    ST2MM implements both AXI4-lite master and slave interfaces that are connected to the designated slave and master port on APF. Host memory requests are sent on the ST2MM master interface to AFP where the requests are routed to the targeted peripherals. 
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#6-mmio-regions","title":"6 MMIO Regions","text":"
                                                                                    The FIM and AFU expose their functionalities to the host software through a set of CSR registers that are mapped to an MMIO region (Memory Mapped IO). An MMIO region is an address space within a base address register (BAR) region to which features are memory mapped.  
                                                                                     
                                                                                    For example, when a feature is mapped to an MMIO region, the CSR registers of that feature are located within the address range of that region. There can be multiple MMIO regions within a BAR region. 
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#61-feature-region","title":"6.1 Feature Region","text":"
                                                                                    A group of related CSRs can be categorized as a feature region. For example, a DMA engine has queue management function and quality of service (QoS) function; these are two different features of the DMA engine. A feature region is contained within a single PCIe BAR and cannot span across two BAR region boundaries. 
                                                                                     
                                                                                    A Device Feature Header (DFH) is a block of registers that mark the start of the feature region and sub-feature region, and you must place it at the first address of the region. Each DFH starts at 4KB boundary. A DFH register contains information that OPAE software requires to enumerate the feature. It also has an offset field that points to the next DFH in a feature list. OPAE software traverses the linked list of DFHs in each BAR region to discover all the features implemented on the platform. 
                                                                                     
                                                                                    The EOL field in a DFH Start marks the end of a DFH list and is only set in the DFH of the last feature in the feature list. The feature type field in the DFH is used to differentiate between the different types of feature region. Basic building blocks (BBB) and private features are always a child of an AFU or FPGA Interface Unit (FIU) and must be contained within an AFU or FIU, respectively.  
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#611-device-feature-header-dfh-structure","title":"6.1.1 Device Feature Header (DFH) Structure","text":"
                                                                                    All DFHs must follow a specific structure to be compatible with OPAE software.  Note that only features you want to be exposed to the OPAE software must have a DFH.  For the latest information on DFH structure, please refer to https://github.com/OFS/linux-dfl/blob/fpga-ofs-dev/Documentation/fpga/dfl.rst#device-feature-list-dfl-overview.
                                                                                     
                                                                                    6.2 Control and Status Registers
                                                                                     
                                                                                    All the Control and Status Registers (CSRs) in the FIM are 64-bit registers with the following MMIO write, and MMIO read support.
                                                                                     
                                                                                    Table 6-5: CSR MMIO Read and Write Support
                                                                                     Request Memory Attribute Payload size Memory Ordering MMIO Write UC 4B or 8B Strongly ordered MMIO Read UC 4B or 8B Strongly ordered 
                                                                                    The FIM does not reorder the MMIO requests or responses. For MMIO writes, there is no reordering of requests in FIM, and uncacheable (UC) ordering rules are followed. Similarly, for MMIO reads, there is no re-ordering of requests or responses in the FIM. An AFU may opt to re-order the MMIO read responses but the FIM does not enforce read response ordering. 
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#621-software-access-to-registers","title":"6.2.1 Software Access to Registers","text":"
                                                                                       
                                                                                    • Software accesses 64-bit registers as aligned quadwords. For example, to modify a field (bit or byte) in a 64-bit register, the entire quadword is read, the appropriate field(s) are modified, and the entire quadword is written back.
                                                                                      •  
                                                                                    • When updating registers through multiple accesses (whether in software or due to hardware disassembly), certain registers may have specific requirements on how the accesses must be ordered for proper behavior. These are documented as part of the respective register descriptions.
                                                                                      •  
                                                                                    • For compatibility with future extensions or enhancements, software must assign the last read value to all \u201cReserved and Preserved\u201d (RsvdP) fields when written. In other words, any updates to a register must be read so that the appropriate merge between the RsvdP and updated fields occurs. Also, software must assign a value of zero for \u201cReserved and Zero\u201d (RsvdZ) fields when written.
                                                                                      •  
                                                                                    • PCIe locked operations to FPGA hardware registers are not supported. Software must not issue locked operations to access FPGA hardware registers.
                                                                                      •  
                                                                                     
                                                                                    In the following two cases, the FIM terminates MMIO Read requests by sending a completion with the data (CplD) specified below: * MMIO Timeout:  This occurs when the AFU does not respond within a set timeout. The timeout value is currently configured to 512 pclks (clk_2x). In this case, the FIM returns all 1s.
                                                                                     
                                                                                       
                                                                                    • Illegal MMIO Accesses: This occurs when the read is accessing undefined registers in the FIM or if an AFU access violation.  An example of an access violation is when a PF attempts to access the AFU when it is set to VF mode, or when a VF attempts to access the AFU when it is set to PF mode. In this case, the FME will returns all 0s.
                                                                                      •  
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#622-register-attribute-definition","title":"6.2.2    Register Attribute Definition","text":"
                                                                                    Table 6-6: OFS Register Attribute Definitions
                                                                                     Attribute Expansion Description RW Read/Write This bit can be read or written by software. RO Read Only The bit is set by hardware only. Software can only read this bit. Writes do not have any effect. RW1C Read/ Write 1 to Clear Software can read or clear this bit. The software must write 1 to clear this bit. Writing zero to RW1C bit has no effect. Note that a multi-bit RW1C field may exist. In this case, all bits in the field are cleared if a 1 is written to any of the bits. RW1S Read/ Write 1 to Set Software can read this bit. Writing a 1 to the bit sets it to 1. Writing a 0 has no effect. It is not possible for software to set this bit to 0. The 1 to 0 transition can only be performed by HW. RW1CS Read/Write 1 to Clear Sticky Software can read and clear this bit. Writing a 1 to a bit clears it, while writing a 0 to a bit has no effect. This bit is only reinitialized to its default value by a power-on reset. RWD Read/Write Sticky across Hard Reset The bit can be read or written by SW. This bit is sticky or unchanged by any reset type, including Hard Reset. The bit gets cleared only with power on. *S Sticky across Soft Reset The bit will be sticky or unchanged by soft reset. These bits are only re-initialized to their default value by a power-on reset. *D Sticky across Hard Reset The bit is sticky or unchanged by or unchanged by any reset type, including hard reset. The bit gets cleared only with power on. Rsvd Reserved Reserved for future definitions. Currently don\u2019t care bits. RsvdP Reserved and Protected Reserved for future RW implementations. The software must preserve the value of this bit by read modify write. RsvdZ Reserved and Zero Reserved for future RW1C implementations. The software must write zero to this bit."},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#623-csr-offset-in-bars","title":"6.2.3 CSR Offset in BARs","text":"
                                                                                    The table below captures the FIM and AFU features in the supported BAR regions. The highlighted offset indicates the first DFH in the DFH list of a BAR region where device driver starts the DFH traversal.
                                                                                     
                                                                                    Table 6-7: PF0 BAR0 Features
                                                                                     Offset Feature CSR set 0x00000 FME 0x01000 Thermal Management 0x03000 Global Performance 0x04000 Global Error 0x10000 PCIe 0x12000 QSFP0 0x13000 QSFP1 0x20000 PMCI 0x40000 ST2MM 0x60000 Transceiver CSR Interface 0x61000 Virtual UART 0x62000 EMIF 0x70000 Port Gasket PR logic 0x71000 Port Gasket Port interface 0x73000 Port Gasket Remote Signal Tap 0x80000 AFU Interface Handler- AFU Errors 
                                                                                    Table 6-8: PF0 BAR4 Features
                                                                                     Offset Feature CSR set 0x03000 MSI-X Vector Tables"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#7-clocks","title":"7 Clocks","text":"
                                                                                    The following table provides external clock sources which correspond to pins on the FPGA that drive blocks of internal logic or are used as a reference clock for internal PLLs.  The names in the table are used in the top.sv or are the logical clock names used by their respective functional blocks.  
                                                                                     
                                                                                    Table 7-1: External Clock Source
                                                                                     Clock Frequency Description SYS_RefClk 100 MHz Reference clock to system IOPLL (sys_pll) which provides FIM system clocks. PCIE_REFCLK 100MHz PCIe reference clock 0 PCIE_REFCLK 100MHz PCIe reference clock 1 qsfp_ref_clk 156.25 MHz Ethernet Reference Clock ddr4_x32[0].ref_clk 150MHz Refclk for 32-bit EMIF channel 0 ddr4_x32[1].ref_clk 150 MHz Refclk for 32-bit EMIF channel 1 ddr4_x40[0].ref_clk 150 MHz Refclk for 40-bit EMIF channel 0 ddr4_x40[1].ref_clk 150 MHz Refclk for 40-bit EMIF channel 1 ddr4_hps.ref_clk 150 MHz Refclk for HPS EMIF sdm_config_clk 125 MHz Refclk for HPS EMIF hps_refclk 25 MHz Refclk for HPS EMIF altera_reserved_tck 10 MHz Refclk for HPS EMIF 
                                                                                    Table 7-2: Internal Clocks
                                                                                     
                                                                                    Internal clock generated by the IOPLL as outclk outputs.
                                                                                     outclk# Clock Frequency Description outclk0 clk_sys 470 MHz System clock. Primary clock for PCIe Datapath outclk1 clk_100m 100 MHz For RemoteSTP and user clock, or any logic that requires a 100 MHz clock. outclk2 clk_100m 175 MHz Drives AFU in PR slot outclk3 clk_ptp_slv 155.56 MHz Unused outclk4 clk_50m 50 MHz Drives Virtual UART"},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#8-reset","title":"8 Reset","text":""},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#81-reset-signals","title":"8.1   Reset Signals","text":"
                                                                                    The FIM system reset signals are driven according to their respective requirements derived from their use cases.  The behavioral code defining the reset operation is located in the file rst_ctrl.sv. The FIM System Reset Drivers table below provides a list of the input and feedback signals that compose the various resets.
                                                                                     
                                                                                    Table 8-1: FIM System Reset Drivers
                                                                                     Reset Description nPERST pin Active low PCIe reset pin from the PCIe card edge that may be set by the host machine for cold or warm resets. nCONFIG pin Active low input to the FPGA that causes the FPGA to lose its configuration data, enter a reset state, and tri-state all I/O pins. Host software must reload the FPGA FIM after nCONFIG is activated. ninit_done Active low signal indicating FPGA core configuration is complete and FPGA has entered usermode.  This signal is provided by the configuration monitor block in rst_ctrl.sv. pcie_reset_status Active high reset status from PCIe hard IP. When driven high, this signal indicates that the PCIe IP core is not ready for usermode. pll_locked Active high SYS IOPLL locked signal pcie_cold_rst_ack_n Reset Acknowledge signal from the PCIe subsystem that lets the Reset Controller know that the Cold Reset sequence has been completed. pcie_warm_rst_ack_n Reset Acknowledge signal from the PCIe subsystem that lets the Reset Controller know that the Warm Reset sequence has been completed. 
                                                                                    Upon power-on, the reset module in the FIM holds the FIM in reset until all the reset conditions are de-activated:
                                                                                     
                                                                                       
                                                                                    • nPERST signal is asserted.
                                                                                      •  
                                                                                    • The INIT_DONE pin is driven high to indicate core configuration is complete.
                                                                                      •  
                                                                                    • The SYS IOPLL is locked.
                                                                                      •  
                                                                                    • The reset status from PCIe hard IP is de-asserted indicating the IP is ready for transfer. 
                                                                                      •  
                                                                                     
                                                                                    The reset module places the FIM back into reset if any of these conditions becomes active again. The only way to invoke a system reset to the FIM after power-up is to de-assert the nPERST pin either by performing a warm reboot or through PCIe driver intervention. There are soft reset signals set aside to allow software to reset the Port, AFU and partial reconfiguration IP. 
                                                                                     
                                                                                    THe following table lists the reset outputs from the rst_ctrl.sv block.
                                                                                     
                                                                                    \u200b                                                      Table 8-2: FIM System Reset Outputs
                                                                                     Reset Description rst_n_sys pin System general reset synchronous to clk_sys.  This is a warm reset of the FPGA logic.  Sticky bits in the FME and other CSR registers will not be reset by this signal. rst_n_100m pin System general reset synchronous to clk_100m. ninit_done Active low signal indicating FPGA core configuration is complete and FPGA has entered usermode.  This signal is provided by the configuration monitor block in rst_ctrl.sv. pwr_good_n Hardware reset conditioned by ninit_done and the pll_locked signal.  The signal is generally set when power has been cycled or a physical hardware fault has occurred, requiring a reset of the FPGA logic.   This signal is synchronous to clk_sys. pcie_cold_rst_ack_n Hardware reset conditioned by the pcie_reset_status which is a summary reset driven by the PCIe Hard IP logic tile on the FPGA die.  This signal is synchronous to clk_sys. pcie_warm_rst_ack_n Soft reset conditioned by nPERST when the pcie_reset_status is not asserted, meaning a warm reset is desired.  Cold reset sticky bits in the PCIe subsystem will not be reset by this signal."},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#9-interrupts","title":"9 Interrupts","text":"
                                                                                    The OFS platform supports interrupts through MSI-X feature. The MSI-X Interrupt feature handles FME and AFU interrupts. FME interrupts are primarily used to notify the host of error events happened in the FIM. When any of the bit in the FME error status registers is set, an interrupt request is sent to the MSI-X module. There are FME error status registers for OFS for Agilex FPGA features.  An AFU sends interrupt to the MSI-X module in the PCIE SS on the AXI interrupt request channel. The MSI-X table entries and PBA vectors are implemented in the PCIE SS. The PCIE SS supports upto 4096 vectors in \"Static MSI-X mode.
                                                                                     
                                                                                    Please refer to the Intel FPGA IP Subsystem for PCI Express IP User Guide for more information.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#10-external-memory-interface-emif","title":"10 External Memory Interface (EMIF)","text":"
                                                                                    There are 5 EMIF channels (5 DDR4 banks) on the Intel\u00ae FPGA SmartNIC N6001-PL.  Banks 3D and 2A are dedicated for HPS use, and Banks 3B, 3A, and 2B are connected to the HE-MEM exerciser module in AFU. The provided FIM reference design implements ECC for the HPS external memory bank while ECC is not implemented for the fabric external memory interfaces. The Intel\u00ae FPGA SmartNIC N6001-PL that this design targets supports ECC on all banks except 2B and the user may create a FIM design with ECC support.  Both memory subsystem and HE-MEM implement AXI-MM interface.
                                                                                     
                                                                                    Table 10-1 Memory Subsystem Configuration on the Intel\u00ae FPGA SmartNIC N6001-PL
                                                                                     
                                                                                    This table shows the capabilities of the memory populated on the Intel\u00ae FPGA SmartNIC N6001-PL for reference.
                                                                                     FPGA I/O Bank Width ECC Width ECC Supported Speed Size 3D, HPS x32 x8 1GB 2400 Three 256Mx16 3B x32 x8 4GB 2400 Three 1024Mx16 3A x32 x8 4GB 2400 Three 1024Mx16 2B x32 No ECC 4GB 2400 Two 1024Mx16 2A, HPS x32 x8 1GB 2400 Two 1024Mx16 
                                                                                    Figure 10-1: EMIF Interfaces
                                                                                     
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#101-emif-csr","title":"10.1 EMIF CSR","text":"
                                                                                    The CSR for the EMIF feature is memory mapped to the FME BAR region. The following table captures the EMIF CSR registers.
                                                                                     
                                                                                    Table 10-2: EMIF CSR Registers
                                                                                     EMIF_DFH 0x62000 0x300000001000100F EMIF Management DFH FIELD NAME RANGE ACCESS DEFAULT DESCRIPTION FeatureType [63:60] RO 0x3 Feature Type = Private Feature Reserved40 [59:40] RsvdZ 0x0 Reserved NextDfhByteOffset [39:16] RO 0x1000 Next DFH Byte offset FeatureRev [15:12] RO 0x0 Feature Revision FeatureID [11:0] RO 0x9 Feature Id EMIF_STAT 0x62008 0x0000000000000000 EMIF Status FIELD NAME RANGE ACCESS DEFAULT DESCRIPTION Reserved [63:8] RsvdZ 0x0 Reserved CalFailure [7:4] RO 0x0 EMIF PHY Calibration Failure (1 bit per interface) CalSuccess [3:0] RO 0x0 EMIF PHY Calibration Successful (1 bit per interface) EMIF_CAPABILITY 0x62010 0x000000000000000F EMIF Capability Register FIELD NAME RANGE ACCESS DEFAULT DESCRIPTION Reserved [63:4] RsvdZ 0x0 Reserved EMIFCap [3:0] RO 0xF Attached Memory Capability (1 bit per interface) 
                                                                                    \u200b           
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#11-ethernet-subsystem","title":"11 Ethernet Subsystem","text":""},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#111-ethernet-subsystem-overview","title":"11.1 Ethernet Subsystem Overview","text":"
                                                                                    The Ethernet Subsystem (hssi_ss) provides portability to different Ethernet configurations across platforms and generations and reusability of the hardware framework and software stack. This reference FIM implements an E-tile Ethernet Subsystem IP in a 2x4x25GbE configuration and provides a Linux driver that can be leveraged for customization. Each group of 4x25GbE routes to a QSFP.
                                                                                     
                                                                                    For more information about how to reconfigure the Ethernet Subsystem please refer to the Ethernet Subsystem Intel FPGA IP.
                                                                                     
                                                                                    Table 11-1: Ethernet Configurations for example OFS FIMs for Agilex FPGAs
                                                                                     Parameter Configuration for PCIe Attach Intel Agilex OFS (2x4x25GbE) IP file name hssi_ss_8x25g Number of ports enabled 8 Enabled ports Port 0-7 Port{x} profile 25GbE Port{x} subprofile MAC+PCS Port{x} RSFEC True Port{x} PTP False Enable AN/LT False Enable NPDME True Enable JTAG to Avalon master bridge False Enable Tx packet classifier NA PTP accuracy mode NA Enable iCAL and pCAL recipe at power True TX/RX maximum frame size 1518 Enforce maximum frame size False Link fault generation mode Bidirectional Stop TX traffic when link partner sends PAUSE Yes Bytes to remove from RX frames Remove CRC bytes Forward RX PAUSE requests False Use source address insertion False Enable TX/RX VLAN detection True Enable asynchronous adapter clocks False Enable preamble Passthrough False Enable asynchronous adapter clocks False Enable preamble Passthrough False Enable strict preamble check False Enable strict SFD check False Average IPG 12 Enable adaptation load soft IP True Additional IPG removed as per AM period 0 
                                                                                    To determine which Transceiver Subsystem port maps to the QSFP A and B lanes, please refer to the following table:
                                                                                     
                                                                                    Table 11-2: Transceiver Subsystem Port Mapping 
                                                                                     Port number Configuration for PCIe Attach Intel Agilex OFS (2x4x25GbE) 1 QSFP-A Lane-0 2 QSFP-A Lane-1 3 QSFP-A Lane-2 4 QSFP-A Lane-3 5 QSFP-B Lane-0 6 QSFP-B Lane-1 7 QSFP-B Lane-2 8 QSFP-B Lane-3 9 NA 10 NA 11 NA 12 NA 13 NA 14 NA 15 NA 16 NA 
                                                                                    Figure 11-1: Transceiver Subsystem Block Diagram
                                                                                     
                                                                                     
                                                                                    A host exerciser, named he-hssi, is provided in the pr_slot of the AFU partition. The Ethernet Subsystem interface to the AFU has an Arm\u00ae AMBA\u00ae 4 AXI4 data and sideband interface.  
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#112-parameterization-options","title":"11.2 Parameterization Options","text":"
                                                                                    The Ethernet Subsystem features are highly parameterized to provide the various features/configurations required for the different FIMs. The macro defines can be added to the ofs_top.qsf file and are as follows:
                                                                                     
                                                                                       
                                                                                    • ETH_100G: includes 100G related logic. Used to generate 2x1x100G.
                                                                                      •  
                                                                                    • ETH_10G:  includes 10G related logic. Used to generate 2x4x10G
                                                                                      •  
                                                                                    • ETH_25G: includes 25G related logic (Default behaviour even with no `DEFINES). Used to generate 2x4x25G
                                                                                      •  
                                                                                     
                                                                                    Parameterization is also included in the ofs_fim_eth_plat_defines.svh package file:
                                                                                     
                                                                                    \u2022   INCLUDE_HSSI_PORT_{n}: These are available for every port. If defined for a particular port, that indicates that port is enabled. \u2022  Based on which INCLUDE_HSSI_PORT_{n} is defined in ofs_fim_eth_plat_defines_pkg.svh file, hssi_ss IP needs to be configured manually to have only those ports enabled. 
                                                                                     
                                                                                    Following are parameters defined in ofs_fim_eth_plat_if_pkg.sv: 
                                                                                     
                                                                                    \u2022   MAX_NUM_ETH_CHANNELS: This indicates how many maximum ethernet channels are supported for platfrom \u2022   NUM_QSFP_PORTS: Indicates number of QSFP cages are available on a platform \u2022   NUM_ETH_CHANNELS: Indicates number of channels present for a FIM, i.e. number for ports for Transceiver SS IP  \u2022   NUM_QSFP_LANES: Number of lanes per QSFP port \u2022   NUM_LANES: Number of lanes per HSSI port \u2022   ETH_PACKET_WIDTH: Datawidth of client side AXI-ST interface coming from HSSI SS IP. This is not an user configurabale parameter. User need update this value to reflect HSSI SS IP client side data width for selected configuration.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#113-ofs-ethernet-subsystem-interface-module","title":"11.3 OFS Ethernet Subsystem Interface Module","text":"
                                                                                    A wrapper around the Transceiver Subsystem integrates the following features: * DFH registers  * Control & status registers  * Indirect access to Transceiver SS CSR registers via CSR mailbox in the HSSI SS interface 
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1131-ethernet-subsystem-control-and-status-register-csr-map","title":"11.3.1   Ethernet Subsystem Control and Status Register (CSR) Map","text":"
                                                                                    The Ethernet Subsystem connects to the APF which is memory mapped to PF0 BAR0. The Ethernet Subsystem CSR space in the FIM consists of two parts:
                                                                                     
                                                                                       
                                                                                    • Ethernet Subsystem CSRs assigned from offset 0x000 to 0x7FF
                                                                                      •  
                                                                                    • Additional CSRs are added to FIM at offset 0x800 to 0xFFF
                                                                                      •  
                                                                                     
                                                                                    The PCIe subsystem uses AXI Memory Mapped accesses to read and write Ethernet Subsystem Control and Status Registers in the FIM. The Ethernet Subsystem CSR Map structure is designed to scale according to IP capabilities.
                                                                                     
                                                                                    The Ethernet Subsystem CSR Map can be found ipss/hssi/HSSI_SS_CSR.xls.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#114-ethernet-subsytem-software","title":"11.4 Ethernet Subsytem Software","text":"
                                                                                    There are two pieces of software related to running the Ethernet Subsystem: The Linux* dfl network driver and the user space OPAE Tools.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1141-ethernet-subsystem-linux-driver","title":"11.4.1 Ethernet Subsystem Linux Driver","text":"
                                                                                    The Ethernet subystem is exposed as a feature in the PCIe PF BAR0 region.  It has a Device Feature Header (DFH) specifying the interface. 
                                                                                     
                                                                                    The primary functionality of the driver is to interact with the ethernet MAC and PHY through an indirect register access mailbox implemented by the HSSI_RCFG_CMD0, HSSI_RCFG_DATA0 registers described above. To aid in RTL bringup, the driver provides a debugfs interface directly to the indirect register access mailbox. For each HSSI interface in the system there would be a directory with the following form containing two files, regaddr and regval: /sys/kernel/debug/dfl-fme.X.Y
                                                                                     
                                                                                    To read a register offset in the MAC/PHY write the offset to the regaddr file as a C hex string (e.g. 0x04) and then read the value as string out of regval file. To write a register offset in the MAC/PHY write the offset to the regaddr file as a C hex string (e.g. 0x04) and then write the value as a C hex string to regval file.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1142-ethernet-subsystem-opae-user-space-tool","title":"11.4.2   Ethernet Subsystem OPAE User Space Tool","text":"
                                                                                    User space OPAE Tools that are part of OPAE SDK provide support for the Ethernet Subsystem.  You can use the --help option to print more information about each of these commands: * hssi: Provides a means of interacting with the 10G and 100G HSSI AFUs through the host exerciser. * hssiloopback: Enables and disables Ethernet loopback. * hssistats: Provides MAC statistics.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#12-partial-reconfiguration","title":"12 Partial Reconfiguration","text":"
                                                                                    Partial Reconfiguration (PR) is an Intel FPGA technology that allows users to reconfigure parts of the FPGA device dynamically, while the remainder of the device continues to operate. In a non-partial reconfiguration flow, any change to the design requires full reprogramming of the entire configuration RAM (CRAM) arrays in the device. With partial reconfiguration, you can dynamically reprogram one or more CRAM frames. A partial reconfiguration design has a static region, and a PR regions, which can be modified to implement new logic. The portion of the CRAM on the chip to be reconfigured is contained within a PR region. For the PR flow, your design must be partitioned into static region and reconfigurable region. The static region is the area of your FPGA that is not reconfigured without reprogramming the entire FPGA. An area of the chip that you plan to partially reconfigure is a PR region. 
                                                                                     
                                                                                    The Port Gasket contains all the PR specific modules and logic, such as PR slot reset/freeze control, user clock, remote STP etc. For this reference example only one PR slot is supported.
                                                                                     
                                                                                    The Figure below shows the fundamental concepts required to support PR in OFS platform. There are 4 OFS management DFH \u2013 PR, Port, User Clock and Remote STP in Port Gasket that is exposed to OFS software. These platform capabilities are generally used in conjunction to Partial Reconfiguration. The Figure below also demonstrates the concepts of adding a new interface to the PR region. 
                                                                                     
                                                                                    Figure 12-1 Partial Reconfiguration Gasket
                                                                                     
                                                                                     
                                                                                    The isolation logic is provided on the output signals of the PR region to ensure they don\u2019t glitch and affect the interfacing logic in the Static Region (SR). The isolation logic is controlled by the PR Freeze logic during PR operation.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#user-clock-support","title":"User Clock Support","text":"
                                                                                    The reference platform provides two user configurable clock (uclk_usr, uclk_usr_div2) for use by the AFU. These clocks are generated by a reconfigurable IOPLL. The control and status of these clocks is expose through two pairs of command and status registers (USR_CLK_FREQ_CMD0 / USR_CLK_FREQ_STS0 & USR_CLK_FREQ_CMD1 / USR_CLK_FREQ_STS1). The first pair controls the fPLL reconfiguration functionality. The second pair controls a clock frequency measurement block. 
                                                                                     
                                                                                    The following are the default values of the userclk. 
                                                                                     
                                                                                    uclk_usr: 312.5 MHz 
                                                                                     
                                                                                    uclk_usr_div2: 156.2 MHz 
                                                                                     
                                                                                    Table 12-1 usr_clk_* Acceptable Programming Range
                                                                                     Fabric Frequency Range uclk_usr (H) uclk_usr_div2 (L) Details 0-400 MHz 0-800 MHz 0-400 MHz Clocks set on 2x relationship for L<400 MHz 400-800 MHz 400-800 MHz 400-800 MHz Clks are equal for L>400 MHz 
                                                                                    PLL Reconfiguration 
                                                                                     
                                                                                    The blue bit stream hardware exposes the low level IOPLL reconfiguration interfaces directly to software control. Through the USR_CLK_FREQ_CMD0 register software can select the reference clock, assert the PLL power down pin and issue reads and writes on the IOPLL Avalon-mm reconfiguration bus. Through the USR_CLK_FREQ_STS0 register software can query the IOPLL active reference clock, locked status and readdata returned from the IOPLL AVMM interface for read requests. 
                                                                                     
                                                                                    Clock Frequency Counter 
                                                                                     
                                                                                    The user clocks, generated by the reconfigurable IOPLL, are connected to a frequency measurement circuit. Software selects which of the two clocks to measure by setting the clock select bit in the USER_CLK_FREQ_CMD1 register. After 10ms software can read the frequency, in 10 KHz units, from the USER_CLK_FREQ_STS1 register. Reading this register before 10ms has passed will return undefined data but otherwise has no effect. 
                                                                                     
                                                                                    Configuring User Clocks 
                                                                                     
                                                                                    To program the user clock to the desired frequency, user will set the clock-frequency-low and clock-frequency-high fields in the PR AFU GBS .json file to the desired frequency value. During PR, SW will try to provide the closest possible frequency to the value specified in the .json file. 
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#13-host-exercisers","title":"13 Host Exercisers","text":"
                                                                                    The Host Exerciser (HE) modules are responsible for generating traffic with the intent of exercising the specific interface they are designed to connect to. They are intended to test the interface in simulation and hardware, enable measurement of bandwidth and other performance KPIs and, in come cases, provide an example of data movement between interfaces (PCIe to DDR for e.g.) for adoption into a customer application. 
                                                                                     
                                                                                    ### 13.1 HE-LB and HE-MEM Host Exerciser
                                                                                     
                                                                                    The Host Exerciser Loopback module is a traffic generator that can be attached both to host memory, over PCIe (HE-LB), and to local memory, such as board-attached DDR (HE-MEM). The Host Exerciser (HE) is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth as well as demonstrating data path correctness. The FIM picks up the HE-LB module behind the PIM (Platform Interface Manager).  The PIM converts the AXI with TLP out of the PCIe SS to standard Arm\u00ae AMBA\u00ae 4 AXI4 (MM for completions and Lite for CSR) interfaces. The figure below shows how the HE-LB is instantiated in the FIM.
                                                                                     
                                                                                    Figure 13-1 HE-LB Dataflow Diagram 
                                                                                     
                                                                                    The exerciser has two primary modes: loopback, in which read data is fed back as writes, and read/write mode, in which reads and writes are generated independently. Furthermore, the host_exerciser software provided with OPAE that drives the RTL has two major modes: \"lpbk\" and \"mem\". These modes only select the UUID of the HE AFU, with lpbk selecting a version configured with no local memory (56e203e9-864f-49a7-b94b-12284c31e02b) and mem seeking a version with local memory (8568ab4e-6ba5-4616-bb65-2a578330a8eb). The behavior of each engine with and without local memory is described below.
                                                                                     
                                                                                    Figure 13-2 HE Engine Heirarchy 
                                                                                     
                                                                                    For more details of HE-LB and HE-MEM IP, please refer to ofs-fim-common/src/common/he_lb/README.md
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#132-hssi-host-exerciser-he-hssi","title":"13.2 HSSI Host Exerciser (HE-HSSI)","text":"
                                                                                    HE-HSSI is an Ethernet AFU that handles client side ethernet traffic. The reference HE-HSSI has following features:
                                                                                     
                                                                                       
                                                                                    • HE-HSSI provides an E-tile compatible interface with the Transceiver Subsystem.
                                                                                      •  
                                                                                    • Includes a traffic generator and checker or monitor
                                                                                      •  
                                                                                    • Provides pause signals to the Transceiver Subsystem for XON and XOFF generation
                                                                                      •  
                                                                                    • Generates traffic or incoming traffic that can be looped back into transmit path by enabling loopback mode, which will bypass traffic generator
                                                                                      •  
                                                                                    • At the HE-HSSI interface boundary the Ethernet data interface is AXI4-Stream with 64-bit data at eth_clk clock
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • The Traffic generator and checker modules have a 64-bit data interface at eth_clk clock.
                                                                                      •  
                                                                                    • The traffic generator supports the following modes:     * Fixed length or Random Length     * Incremental pattern or Random pattern
                                                                                      •  
                                                                                    • Traffic checker does a 32-bit crc check in 10/25G. In the 100G configuration, there is no data integrity check, only a packet counter. 
                                                                                      •  
                                                                                    • The CSR of this AFU is accessible through AXI4-Stream PCIe TLP interface
                                                                                      •  
                                                                                    • The PCIe TLP to CSR Interface Conversion module converts PCIe TLPs into simple CSR interface
                                                                                      •  
                                                                                    • The CSR space of the traffic generator and checker modules are accessed in an indirect way using mailbox registers
                                                                                      •  
                                                                                    • If used for more than one channel, each channel has a separate traffic generator and traffic checker with separate CSR space.
                                                                                      •  
                                                                                    • Reads and Writes to individual traffic controller CSR spaces can be done by selecting that particular channel using channel select register.
                                                                                      •  
                                                                                     
                                                                                    The HE-HSSI Ethernet block diagram is below. 
                                                                                     
                                                                                    Figure 13-3: HE-HSSI Block Diagram Block Diagram
                                                                                     
                                                                                     
                                                                                    The diagram below shows the path followed depending on if you enable loopback mode in HE-HSSI or not.  In loopback mode, traffic is looped back into the transmit path which will bypass traffic.  Alternatively, the traffic generates traffic.
                                                                                     
                                                                                    Figure 13-4: HE-HSSI Operation Mode Traffic Patterns
                                                                                     
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1322-he-hssi-csr-map","title":"13.2.2 HE-HSSI CSR Map","text":"
                                                                                    The reference HSSI AFU contains the following registers and a similar arrangement of register space can be implemented for other usecase specific HSSI AFUs. * AFU DFH Register: Device feature header for the AFU (AFU_DFH) * AFU ID Registers: 128-bit UUID for the AFU which occupies two 64-bit registers (AFU_ID_L, AFU_ID_H) * Mailbox Registers: Command and Data register for traffic controller register access. It follows the standard access method defined for OFS. Access method and implementation is same as Reconfiguration Interface defined for the HSSI FIM. (TRAFFIC_CTRL_CMD, TRAFFIC_CTRL_DATA) * Channel Select Register: Channel select register for traffic controller mailbox access. It is used in cases where more than one channel is in the AFU, else it defaults to zero, meaning channel-0 is selected. (TRAFFIC_CTRL_PORT_SEL) * Scratchpad Register: Scratchpad register for CSR access checking. (AFU_SCRATCHPAD) 
                                                                                     
                                                                                    The CSR excel for HE-HSSI module can be found at ofs-common/src/common/he_hssi/HE_HSSI_CSR.xls.
                                                                                     
                                                                                    13.3 HE-Null Overview
                                                                                     
                                                                                    This module is a simple stub that is used to replace various HE and other blocks in the FIM whenever they are bypassed using the qsf compiler directive such as null_he_lb, null_he_hssi, null_he_mem and null_he_mem_tg.  To find out more about these compiler directives, refer to the [Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs].
                                                                                     
                                                                                    Table 13-1  HE-Null DFH
                                                                                     REGISTER NAME ADDRESS ACCESS DEFAULT DESCRIPTION DFH 0x0000 RO 0x0000_0000_1000_0000 DFH register GUID_L 0x0008 RO 0xaa31f54a3e403501 Lower 64b of GUID GUID_H 0x0010 RO 0x3e7b60a0df2d4850 Upper 64b of GUID SCRATCHPAD 0x0018 RW 0x0 Scratchpad 
                                                                                    Figure 13-5: HE-Null Block Diagram
                                                                                     
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#14-reliability-accessibility-serviceability-ras-and-error-handling","title":"14 Reliability, Accessibility, Serviceability (RAS) and Error Handling","text":"
                                                                                       
                                                                                    1. Downstream AFU checker: Identifies AFU violations.  For example, these checker flags violations of the interface specification. 
                                                                                      2.  
                                                                                    2. Upstream software or PCIe link checker: Identifies invalid traffic from PCIe that violates FIM or PCIe specifications. For example, this checker flags an application sending traffic if it violates the FIM specification or creates a PCIe link issue by causing completion timeout or malformed TLP. 
                                                                                      3.  
                                                                                    3. FIM - Checks for bugs in the FIM fabric.
                                                                                      4.  
                                                                                     
                                                                                    Errors reported by the checker are logged in either the FME error registers or Port error registers, or both, as shown in the table below.  For more details on each of the registers, please refer to ofs-fim-common/src/common/fme/xls/n6000/FME_CSR.xls or the SystemVerilog file: ofs-common/src/common/fme/fme_csr.sv.
                                                                                     
                                                                                    Table 14-1: Error Registers
                                                                                     MMIO Region Area Register Description FME CoreFIM FME_ERROR FME Error Status Register 0.  Registers parity errors, underflow or overflow errors and access mismatches. FME CoreFIM FME_ERROR0_MASK FME Error Mask Register 0.  Write a 0 to mask errors in the FME Error Status Register 0. FME External PCIE0_ERROR PCIe0 Error Status Register. FME External PCIE0_ERROR_MASK PCIe0 Error Mask Register 0.  Write a 0 to mask errors in the PCIe0 Error Status Register 0. FME CoreFIM FME_FIRST_ERROR First FME Error Register. FME CoreFIM FME_NEXT_ERROR FME Next Error Register. FME CoreFIM RAS_NOFAT_ERR_STAT Reliability/Accessibility/Serviceability (RAS) Non-Fatal Error Status Register. FME CoreFIM RAS_NOFAT_ERR_MASK RAS Non-Fatal Error Mask Register. Write 0 to mask error fields in RAS_NOFAT_ERR_STAT Register. FME CoreFIM RAS_CATFAT_ERR_STAT RAS Catastrophic and Fatal Errors Status Register. FME CoreFIM RAS_CATFAT_ERR_MASK RAS Catastrophic and Fatal Errors Mask Register. Write 0 to mask error fields in the RAS_CATFAT_ERR_STAT Register. FME CoreFIM RAS_ERROR_INJ RAS error Injection Register. PORT CoreFIM PORT_ERROR Port Error Status Register. PORT CoreFIM PORT_FIRST_ERROR Port First Error Register . PORT CoreFIM PORT_MALFORMED_REQ0 Port Malformed Request Register 0.  Provides malformed request header LSBs. PORT CoreFIM PORT_MALFORMED_REQ1 Port Malformed Request Register 1.  Provides malformed request header MSBs."},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#141-fme-errors","title":"14.1 FME Errors","text":""},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1411-fme_error0","title":"14.1.1   FME_ERROR0","text":"
                                                                                    The FME_ERROR0 register flags CoreFIM FME errors in the Global Error (GLBL_ERROR) private feature. The error bits in this register are sticky bits. You can only clear these bits through software or through a system reset. Writing a 1 to the error field in the register clears the corresponding error bit. Writing a 1 to the corresponding bit in FME_ERROR0_MASK register masks the error.
                                                                                     
                                                                                    Table 14-2: FME Error Types
                                                                                     Error Type Description Fabric errors FIFO underflow/overflow condition in CoreFIM. These errors only occur if you have introduced bugs into the FIM or during very rare single event upset (SEU) or SEU-like events. Invalid port access A port can either be mapped to a PF or a VF, and not both. The checker reads the AfuAccessCtrl field in the FME CSR PORT0_OFFSET register to determine the access type of the Port. If it finds a PF is trying to access a port that is mapped to a VF or vice-versa, an error will be reported. Invalid AFU access An AFU can either be mapped to a PF or a VF, and not both. The checker reads the AfuAccessCtrl field in the FME CSR PORT0_OFFSET register to determine the access type of the AFU associated with the Port. If it finds a PF is trying to access an AFU that is mapped to a VF or vice-versa, an error is reported and a fake response is sent back to the requester to avoid a completion timeout on the host."},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1412-pcie0_error","title":"14.1.2 PCIE0_ERROR","text":"
                                                                                    The PCIe Avalon-ST to AXI4-Stream bridge monitors the PCIe link for errors and logs any such errors in the PCIE0_ERROR register (in PCIE0 feature region) and PCIE0_ERROR register in the GLBL_ERR private feature.    The error bits in the PCIE0_ERROR register are sticky bits that you can only clear through software or through a system reset. Writing a 1 to the error field in the register clears the corresponding error bit. Writing a 1 to the corresponding bit in PCIE0_ERROR0_MASK masks the error.  
                                                                                     
                                                                                    If you have other external FME features, you can add similar _ERROR registers to this space. Please refer to the following spreadsheet in the release branch found at: /ipss/pcie/rtl/PCIE_CSR.xls or the SystemVerilog file: ipss/pcie/rtl/pcie_csr.sv for more details on this register.  
                                                                                    NOTE
                                                                                     
                                                                                    The PCIE0_ERROR register is located in both the Global Error external feature memory space and a separate PCIe external feature memory space.  OPAE software supports the PCIe external feature memory space beginning at offset 0x40000 for OFS EA and going forward.  PCIe registers beginning at 0x4000 in the Global Error external feature memory space is there for backward compatibility to the Intel FPGA PAC D5005 v2.0.1 Acceleration Stack.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1413-fme_first_error-fme_next_error","title":"14.1.3   FME_FIRST_ERROR, FME_NEXT_ERROR","text":"
                                                                                    The FME_FIRST_ERROR register flags which of the FME error reporting registers, such as FME_ERROR0, PCIE0_ERROR0, has reported the first error occurrence. The error fields of the first error register are then continuously logged into the FME_FIRST_ERROR register until a system reset or software clears all the errors in that first error register. Likewise, the FME_NEXT_ERROR indicates which of the FME error reporting registers (except the first error register) has reported the next occurrence of error after the first error register. The error fields of the next error register are continuously logged into the FME_NEXT_ERROR register until a system reset or software clears all the errors in the second error register.
                                                                                     
                                                                                    Please refer to the file in the ofs-fim-common repository folder: src/common/fme/fme_csr.sv for individual register field descriptions or the SystemVerilog file src/common/fme/fme_csr.sv.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1414-reliability-accessibility-serviceability-ras-error-status","title":"14.1.4   Reliability, Accessibility, Serviceability (RAS) Error Status","text":"
                                                                                    The RAS feature in CoreFIM labels errors as non-fatal, fatal or catastrophic based on their impact to the system.  * A non-fatal error usually originates from software or an AFU.  With a non-fatal error, the user application may still be able to recover from the error by performing a soft reset on the AFU, fixing the user application software if necessary, and clearing the error. On the other hand, a fatal or catastrophic error is non-recoverable and requires the platform to be reset. * Non-fatal errors are logged in the RAS_NOFAT_ERR_STAT register and fatal or catastrophic errors are logged in the RAS_CATFAT_ERR_STAT register.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#14141-non-fatal-errors","title":"14.1.4.1    Non-Fatal Errors","text":"
                                                                                    The RAS_NOFAT_ERR_STAT is a read-only status register that is specifically added for the RAS feature. It logs the high-level status of non-fatal errors in the hardware.  Unlike the error bits in the PCIE0_ERROR and FME_ERROR0 registers which are RW1C (software can write a 1 to clear the error), the error bits in this register are read-only and can only be cleared by system reset. Software has an option to mask the error using RAS_NOFAT_ERR_MASK. Please refer to the following file in the ofs-fim-common resository: src/common/fme/xls/n6000/FME_CSR.xls for individual register field descriptions or the SystemVerilog file: src/common/fme/fme_csr.sv.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#14142-catastrophic-fatal-errors","title":"14.1.4.2    Catastrophic & Fatal Errors","text":"
                                                                                    The RAS_CATFAT_ERR_STAT is a read-only status register that is specifically added for the RAS feature. It captures the high-level status of errors that can only be recovered with a system reset. Therefore, the error bits in the RAS_CATFAT_ERR_STAT register are read-only and can only be cleared by system reset or masked through RAS_CATFAT_ERR_MASK. Please refer to the following file in the ofs-fim-common resository: src/common/fme/xls/n6000/FME_CSR.xls for individual register field descriptions or the SystemVerilog file: src/common/fme/fme_csr.sv.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1415-ras-error-injection","title":"14.1.5   RAS Error Injection","text":"
                                                                                    For software testing purposes, you can inject non-fatal, fatal and catastrophic errors into the platform through the RAS_ERROR_INJ register.  These errors are reported in the RAS_CATFAT_ERR_STAT and RAS_NOFAT_ERR_STAT registers. Please refer to the following file in the ofs-fim-common resository: src/common/fme/xls/n6000/FME_CSR.xls for individual register field descriptions or the SystemVerilog file: src/common/fme/fme_csr.sv.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1416-fme-error-interrupts","title":"14.1.6   FME Error Interrupts","text":"
                                                                                    In an event of an FME error, the MSI-X module in the FIM generates an interrupt so the host can decide on the next course of action. The FIM does not stall upstream and downstream traffic after the FME error. However, a port error triggered by invalid request from a user AFU stalls all the traffic going from AFU to PCIe. The interrupt capability is discoverable by querying the NumbSuppInterrupt field of the PORT_CAPABILITY register in the Port private feature.  The MSI-X vector number is recorded in the InterruptVectorNumber field of the GLBL_ERROR_CAPABILITY register of the Global Error external feature.
                                                                                     
                                                                                    An FME error interrupt is generated in response to the FME_ERROR0, PCIE0_ERROR0, RAS_NOFAT_ERR_STAT or RAS_CATFAT_ERR_STAT registers recording a new, unmasked, error per the rules defined by CoreFIM interrupt feature.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1417-fme-error-handling","title":"14.1.7   FME Error Handling","text":"
                                                                                    When the host receives an FME error interrupt, it must take the recommended actions described below to bring the system back to its normal operation.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#14171-catastrophicfatal-error","title":"14.1.7.1    Catastrophic/Fatal Error","text":"
                                                                                    A system reset is mandatory for any catastrophic or fatal error.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#14172-non-fatal-error","title":"14.1.7.2    Non-Fatal Error","text":"
                                                                                    When software receives a non-fatal error interrupt which does not require a system reset, it can take the following steps to clear the error after software handles the error: 1.  Set the *_ERROR_MASK register to all 1\u2019s to mask all errors 2. Clear the *_FIRST_ERROR register 3. Clear the *_ERROR register 4. Set *_ERROR_MASK register to all 0\u2019s to enable all errors
                                                                                     
                                                                                       
                                                                                    • Result: The *_ERROR & *_FIRST_ERROR registers begin capturing new errors.
                                                                                      •  
                                                                                     
                                                                                    NOTE
                                                                                     
                                                                                    A system reset can only clear RAS Error status registers.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#142-mmio-requests","title":"14.2 MMIO Requests","text":"
                                                                                    The FIM is designed to gracefully handle MMIO request scenarios.  
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1421-unsupported-functions-and-bars","title":"14.2.1 Unsupported Functions and BARs","text":"
                                                                                    The PCIe hard IP in the FIM guarantees that only TLP packets for the functions and BARs supported by the FIM (as configured in PCIe HIP IP instantiation) are sent to the FIM.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1422-mmio-request-decoding","title":"14.2.2 MMIO Request Decoding","text":"
                                                                                    The packet router and memory decoder in the FIM ensure that only legal MMIO requests are forwarded to the targeted MMIO region. Full address and BAR decoding is done both in the packet router and the memory decoder to ensure the requests are forwarded to the designated CSR region as defined in the MMIO Regions chapter.  Any unsolicited/illegal MMIO request is dropped, and an error is reported back to host through the FME error register.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1423-unused-fmeport-csr-regions","title":"14.2.3 Unused FME/Port CSR Regions","text":"
                                                                                    All the CSR slaves in FIM which are mapped to the FME or Port CSR regions must always respond to MMIO read requests targeting its associated CSR region. A CSR slave must return all 0s for MMIO reads to its unused CSR region such as a reserved space or a region where no CSR register is implemented for the address. The FIM ensures MMIO reads to FME or Port CSR regions that are not mapped to any CSR slave always gets a response of all 0s. The memory decoder module and fake responder module in the FIM provide this guaranteed response.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1424-unsupported-mmio-request","title":"14.2.4 Unsupported MMIO Request","text":"
                                                                                    Any MMIO request targeting FME or Port CSR regions with a length or address alignment that are not  supported by the FIM is dropped, and an error is logged in PCIE0_ERROR register. The MMIO checker module in the FIM guarantees this response. When an unsupported MMIO read request to the FIM CSR region is detected, the FIM sends back a CPL (completion without data) with error status (UR) back to host.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1425-afu-access-violation","title":"14.2.5 AFU Access Violation","text":"
                                                                                    AFU access violations refer to the scenarios where a PF is attempting to access the MMIO region of an AFU bound to a VF (virtualization enabled), or when a VF is trying to access the MMIO region of an AFU bound to a PF (virtualization disabled). When such a violation is detected, the FIM drops the request and logs the error in the FME_ERROR0 register. If the request is an MMIO read request, the FIM returns a fake response to the host.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#1426-afu-mmio-response-timeout","title":"14.2.6    AFU MMIO Response Timeout","text":"
                                                                                    An AFU MMIO Response timeout functions in the same manner described in the MMIO Response Timeout section.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#15-ofs-design-hierarchy","title":"15 OFS Design Hierarchy","text":"
                                                                                    Files for design, build and unit test simulation are found at https://github.com/OFS/ofs-agx7-pcie-attach, release branch ${{ N6001_Release }}.
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#151-design-guidance","title":"15.1 Design Guidance","text":"
                                                                                    The OFS FIM is designed with configurability and scalability in mind.  At a high level, these are the necessary steps for a user to customize the design.  Please refer to the F[Intel\u00ae FPGA Interface Manager Developer Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs] for detaaled design guidance. 
                                                                                    "},{"location":"hw/n6001/reference_manuals/ofs_fim/mnl_fim_ofs_n6001/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                                    Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                                     
                                                                                    OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122.  \u00a0
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/","title":"Platform Evaluation Script: Open FPGA Stack for Intel Agilex FPGA","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel Max10 or Intel Cyclone10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implemented on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Management Component Transport Protocol MCTP A standardized model for communication with management controllers. Defines the transport protocol carrying PLDM messages through the BMC. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE-SDK The OPAE-SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Level Data Model PLDM A specification for reporting telemetry data to the host, such as board temperature, voltage, and current. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#1-overview","title":"1 Overview","text":""},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#11-about-this-document","title":"1.1 About this Document","text":"
                                                                                    This document serves as a set-up and user guide for the checkout and evaluation of an Intel\u00ae FPGA SmartNIC N6001-PL development platform using Open FPGA Stack (OFS). After reviewing the document, you will be able to:
                                                                                     
                                                                                       
                                                                                    • Set-up and modify the script to your environment
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • Compile and simulate an OFS reference design
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • Run hardware and software tests to evaluate the complete OFS flow
                                                                                      •  
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#table-1-2-software-version-summary","title":"Table 1-2: Software Version Summary","text":"Component Version Description FPGA Platform Intel\u00ae FPGA SmartNIC N6001-PL Intel platform you can use for your custom board development OFS FIM Source Code Branch: https://github.com/OFS/ofs-agx7-pcie-attach, Tag: ofs-2023.3-2 OFS Shell RTL for Intel Agilex FPGA (targeting Intel\u00ae FPGA SmartNIC N6001-PL) OFS FIM Common Branch: https://github.com/OFS/ofs-fim-common/releases/tag/ofs-2023.3-2, Tag: https://github.com/OFS/ofs-fim-common/releases/tag/ofs-2023.3-2 Common RTL across all OFS-based platforms AFU Examples Branch: examples-afu , Tag:ofs-examples-https://github.com/OFS/examples-afu/releases/tag/ofs-2023.3-2 Tutorials and simple examples for the Accelerator Functional Unit region (workload region) OPAE SDK Branch: 2.10.0-1, Tag: 2.10.0-1 Open Programmable Acceleration Engine Software Development Kit Kernel Drivers Branch: ofs-2023.3-6.1-3, Tag: ofs-2023.3-6.1-3 OFS specific kernel drivers OPAE Simulation Branch: opae-sim, Tag: 2.10.0-1 Accelerator Simulation Environment for hardware/software co-simulation of your AFU (workload) Intel Quartus Prime Pro Edition Design Software 23.3 [Intel\u00ae Quartus\u00ae Prime Pro Edition Linux] Software tool for Intel FPGA Development Operating System RHEL 8.6 Operating system on which this script has been tested 
                                                                                    A download page containing the release and already-compiled FIM binary artifacts that you can use for immediate evaluation on the Intel\u00ae FPGA SmartNIC N6001-PL can be found on the OFS ofs-2023.3-2 official release drop on GitHub.
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#2-introduction-to-ofs-evaluation-script","title":"2 Introduction to OFS Evaluation Script","text":"
                                                                                    By following the setup steps and using the OFS evaluation script you can quickly evaluate many features that the OFS framework provides and also leverage this script for your own development.  
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#21-pre-requisites","title":"2.1 Pre-Requisites","text":"
                                                                                    This script uses the following set of software tools which should be installed using the directory structure below. Tool versions can vary.
                                                                                     
                                                                                       
                                                                                    • Intel Quartus\u00ae Prime Pro Software
                                                                                      •  
                                                                                    • Synopsys\u00ae VCS Simulator
                                                                                      •  
                                                                                    • Siemens\u00ae Questa\u00ae Simulator
                                                                                      •  
                                                                                     
                                                                                    Figure 2-1 Folder Hierarchy for Software Tools
                                                                                     
                                                                                     
                                                                                       
                                                                                    1.  
                                                                                      You must create a directory named \"ofs-X.X.X\" where the X represents the current release number, for example ofs-2023.3-2. 
                                                                                       
                                                                                      2.  
                                                                                    2.  
                                                                                      You must clone the required OFS repositories as per Figure 2-2.  Please refer to the BKC table for locations. When cloning the FIM repository, please follow the instructions in section 1.3.2 of the FPGA Interface Manager Developer Guide for Open FPGA Stack: Intel\u00ae FPGA SmartNIC N6001-PL PCIe Attach.  Additionally, please go to FPGA Developer Journey Guide: Open FPGA Stack for the instructions for the BKC installation.
                                                                                       
                                                                                      3.  
                                                                                    3.  
                                                                                      Once the repositories are cloned, download the evaluation script folder (ofs-agx7-pcie-attach_eval.tar.gz) which is available in the OFS release page and copy the evaluation script ofs-agx7-pcie-attach_eval.sh in the location as shown in the example below:
                                                                                       
                                                                                      4.  
                                                                                     
                                                                                    Figure 2-2 Directory Structure for OFS Project
                                                                                     
                                                                                    ## ofs-2023.3-2\n##  -> examples-afu\n##  -> linux-dfl\n##  -> ofs-n6001\n##  -> oneapi-asp\n##  -> oneAPI-samples\n##  -> opae-sdk\n##  -> opae-sim\n##  -> ofs_n6001_eval.sh\n
                                                                                     
                                                                                       
                                                                                    1. Open the README file named (README_ofs-agx7-pcie-attach_eval.txt) which is in  the evaluation script folder(ofs-agx7-pcie-attach_eval.tar.gz) which informs the user which sections to modify in the script prior to building the FIM and running hardware, software and simulation tests.
                                                                                      2.  
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#22-n6001-evaluation-script-modification","title":"2.2 n6001 Evaluation Script modification","text":"
                                                                                    To adapt this script to the user environment please follow the instructions below which explains which line numbers to change in the ofs-agx7-pcie-attach_eval.sh script 
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#user-directory-creation","title":"User Directory Creation","text":"
                                                                                    The user must create the top-level source directory and then clone the OFS repositories
                                                                                     
                                                                                    mkdir ofs-2023.3-2\n
                                                                                     
                                                                                    In the example above we have used ofs-2023.3-2 as the directory name
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#set-up-proxy-server-lines-67-69","title":"Set-Up Proxy Server (lines 67-69)","text":"
                                                                                    Please enter the location of your proxy server to allow access to external internet to build software packages.
                                                                                     
                                                                                    Note: Failing to add proxy server will prevent cloning of repositories and the user will be unable to build the OFS framework.
                                                                                     
                                                                                    export http_proxy=<user_proxy>\nexport https_proxy=<user_proxy>\nexport no_proxy=<user_proxy>\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#license-files-lines-72-74","title":"License Files (lines 72-74)","text":"
                                                                                    Please enter the the license file locations for the following tool variables
                                                                                     
                                                                                    export LM_LICENSE_FILE=<user_license>\nexport DW_LICENSE_FILE=<user_license>\nexport SNPSLMD_LICENSE_FILE=<user_license>\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#tools-location-line-87-88-89-90","title":"Tools Location (line 87, 88, 89, 90)","text":"
                                                                                    Set Location of Quartus, Synopsys, Questasim and oneAPI Tools
                                                                                     
                                                                                    export QUARTUS_TOOLS_LOCATION=/home\nexport SYNOPSYS_TOOLS_LOCATION=/home\nexport QUESTASIM_TOOLS_LOCATION=/home\nexport ONEAPI_TOOLS_LOCATION=/opt\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#quartus-tools-version-line-95","title":"Quartus Tools Version (line 95)","text":"
                                                                                    Set version of Quartus
                                                                                     
                                                                                    export QUARTUS_VERSION=23.3\n
                                                                                     
                                                                                    In the example above \"23.3\" is used as the Quartus tools version
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#opae-tools-line-108","title":"OPAE Tools (line 108)","text":"
                                                                                    change OPAE SDK VERSION
                                                                                     
                                                                                    export OPAE_SDK_VERSION=2.10.0-1\n
                                                                                     
                                                                                    In the example above \"2.10.0-1\" is used as the OPAE SDK tools version
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#ofs-platform-line-173","title":"OFS Platform (line 173)","text":"
                                                                                    Select the oFS platform the evaluation script needs to be run on. It could be one of these : n6000, n6001, fseries-dk, iseries-dk, custom_board.
                                                                                     
                                                                                    export OFS_PLATFORM=n6001\n
                                                                                     
                                                                                    In the example above, n6001 is the platform selected to be tested.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#pcie-bus-number","title":"PCIe (Bus Number)","text":"
                                                                                    The Bus number must be entered by the user after installing the hardware in the chosen server, in the example below \"b1\" is the Bus Number for a single card as defined in the evaluation script.
                                                                                     
                                                                                    export OFS_CARD0_BUS_NUMBER=b1\n
                                                                                     
                                                                                    The evaluation script uses the bus number as an identifier to interrogate the card. The command below will identify the accelerater card plugged into a server. 
                                                                                     
                                                                                    lspci | grep acc\n\nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)<br>\nb1:00.1 Processing accelerators: Intel Corporation Device bcce<br>\nb1:00.2 Processing accelerators: Intel Corporation Device bcce<br>\nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device<br>\nb1:00.4 Processing accelerators: Intel Corporation Device bcce<br>\n
                                                                                     
                                                                                    The result identifies the card as being assigned \"b1\" as the bus number so the entry in the script changes to
                                                                                     
                                                                                    export OFS_CARD0_BUS_NUMBER=b1\n
                                                                                     
                                                                                    The user can also run the following command on the ofs-agx7-pcie-attach_eval.sh script to automatically change the bus number to b1 in the ofs-agx7-pcie-attach_eval.sh script.
                                                                                     
                                                                                    grep -rli 'b1' * | xargs -i@ sed -i 'b1' @
                                                                                     
                                                                                    if the bus number is 85 for example 
                                                                                     
                                                                                    85:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)<br>\n85:00.1 Processing accelerators: Intel Corporation Device bcce<br>\n85:00.2 Processing accelerators: Intel Corporation Device bcce<br>\n85:00.3 Processing accelerators: Red Hat, Inc. Virtio network device<br>\n85:00.4 Processing accelerators: Intel Corporation Device bcce<br>\n
                                                                                     
                                                                                    the command to change to 85 in the evaluation script would be
                                                                                     
                                                                                    grep -rli 'b1' * | xargs -i@ sed -i '85' @
                                                                                     
                                                                                    The ofs-agx7-pcie-attach_eval.sh script has now been modified to the server set-up and the user can proceed to build, compile and simulate the OFS stack
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#3-n6001-evaluation-script","title":"3 n6001 Evaluation Script","text":""},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#31-overview","title":"3.1 Overview","text":"
                                                                                    The figure below shows a snapshot of the full evaluation script menu showing all 62 options and each one one of 11 sub-menus which focus on different areas of evaluation. Each of these menu options are described in the next section.
                                                                                     
                                                                                    Figure 3-1 ofs-agx7-pcie-attach_eval.sh Evaluation Menu
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#311-adp-tools-menu","title":"3.1.1 ADP TOOLS MENU","text":"
                                                                                    By selecting \"List of Documentation for ADP n6001 Project,\" a list of links to the latest OFS documentation appears. Note that these links will take you to documentation for the most recent release which may not correspond to the release version you are evaluating. To find the documentation specific to your release, ensure you clone the intel-ofs-docs tag that corresponds to your OFS version.
                                                                                     
                                                                                    By selecting \"Check Versions of Operating System and Quartus Premier Design Suite\", the tool verifies correct Operating System, Quartus version, kernel parameters, license files and paths to installed software tools.
                                                                                     
                                                                                     Menu Option Example Output 1 - List of Documentation for ADP n6001 Project  Open FPGA Stack Overview                  Guides you through the setup and build steps to evaluate the OFS solution                 https://ofs.github.io 2 - Check versions of Operating System and Quartus Premier Design Suite (QPDS) Checking Linux release                 Linux version 5.15.52-dfl (guest@hw-rae-svr4-l) (gcc (GCC) 8.5.0 20210514 (Red Hat 8.5.0-4), GNU ld version 2.30-79.el8) #1 SMP Fri Sep 23 17:19:37 BST 2022                  Checking RedHat release                 CentOS Linux release 8.3.2011                  Checking Ubuntu release                 cat: /etc/lsb-release: No such file or directory                  Checking Kernel parameters                 BOOT_IMAGE=(hd0,gpt2)/vmlinuz-5.15.52-dfl root=/dev/mapper/cl-root ro crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200                  Checking Licenses                 LM_LICENSE_FILE is set to port@socket number:port@socket number                 DW_LICENSE_FILE is set to port@socket number:port@socket number                 SNPSLMD_LICENSE_FILE is set to port@socket number:port@socket number                  Checking Tool versions                 QUARTUS_HOME is set to /home/intelFPGA_pro/23.3/quartus                 QUARTUS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus                 IMPORT_IP_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../ip                 QSYS_ROOTDIR is set to /home/intelFPGA_pro/23.3/quartus/../qsys/bin                  Checking QPDS Patches                 Quartus Prime Shell                 Version 23.3 Build XXX XX/XX/XXXX Patches X.XX SC Pro Edition                 Copyright (C) XXXX  Intel Corporation. All rights reserved."},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#312-adp-hardware-menu","title":"3.1.2 ADP HARDWARE MENU","text":"
                                                                                    Identifies card by PCIe number, checks power, temperature and current firmware configuration. 
                                                                                     
                                                                                     Menu Option Example Output 3 - Identify Acceleration Development Platform (ADP) n6001 Hardware via PCIe PCIe card detected as                 b1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01)                 b1:00.1 Processing accelerators: Intel Corporation Device bcce                 b1:00.2 Processing accelerators: Intel Corporation Device bcce                 b1:00.4 Processing accelerators: Intel Corporation Device bcce                 Host Server is connected to SINGLE card configuration 4 - Identify the Board Management Controller (BMC) Version and check BMC sensors Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.15.0                 Board Management Controller Build version: 3.15.0                 //****** BMC SENSORS ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2 5 - Identify the FPGA Management Engine (FME) Version Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.15.0                 Management Controller Build version: 3.15.0                 //****** FME ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                 Boot Page                        : user1                 Factory Image Info               : a7c6879683182ce61084c420e51f50b6                 User1 Image Info                 : 8a7440ddff52e0e27dbb989d5eb954f4                 User2 Image Info                 : a7c6879683182ce61084c420e51f50b6 6 - Check Board Power and Temperature Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.15.0                 Board Management Controller Build version: 3.15.0                 //****** POWER ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                 ( 1) VCCRT_GXER_0V9 Voltage      : 0.91 Volts                 etc ......................                  Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.15.0                 Board Management Controller Build version: 3.15.0                 //****** TEMP ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                 ( 1) FPGA E-Tile Temperature [Remote] : 33.50 Celsius                 etc ...................... 7 - Check Accelerator Port status //****** PORT ******//                 Object Id                        : 0xED00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00 8 - Check MAC and PHY status Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.15.0                 Board Management Controller Build version: 3.15.0                 //****** MAC ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                 Number of MACs                   : 255                 mac info is not supported                  Intel Acceleration Development Platform N6001                 Board Management Controller NIOS FW version: 3.2.0                 Board Management Controller Build version: 3.2.0                 //****** PHY ******//                 Object Id                        : 0xEE00000                 PCIe s:b:d.f                     : 0000:B1:00.0                 Vendor Id                        : 0x8086                 Device Id                        : 0xBCCE                 SubVendor Id                     : 0x8086                 SubDevice Id                     : 0x1771                 Socket Id                        : 0x00                 Ports Num                        : 01                 Bitstream Id                     : 0x50102027135A894                 Bitstream Version                : 5.0.1                 Pr Interface Id                  : 7dbb989d-5eb9-54f4-8a74-40ddff52e0e2                  //****** HSSI information ******//                 HSSI version                     : 1.0                 Number of ports                  : 8                 Port0                            :25GbE        DOWN                 Port1                            :25GbE        DOWN                 Port2                            :25GbE        DOWN                 Port3                            :25GbE        DOWN                 Port4                            :25GbE        DOWN                 Port5                            :25GbE        DOWN                 Port6                            :25GbE        DOWN                 Port7                            :25GbE        DOWN"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#313-adp-pfvf-mux-menu","title":"3.1.3 ADP PF/VF MUX MENU","text":"
                                                                                    This menu reports the number of PF/VF functions in the reference example and also allows you to reduce the number to 1PF and 1VF to reduce resource utilisation and create a larger area for your workload development. This selection is optional and if the user wants to implement the default number of PF's and VF's then option 9, 10 and 11 should not be used.  Additionally the code used to make the PF/VF modification can be leveraged to increase or modify the number of PF/VFs in the existing design within the limits that the PCIe Subsystem supports (8PF/2K VFs)
                                                                                     
                                                                                     Menu Option Description 9 - Check PF/VF Mux Configuration  This menu selection displays the current configuration of the pcie_host.ofss file which is located at the following directory $OFS_ROOTDIR/tools/pfvf_config_tool                  [ProjectSettings]                 platform = n6001                 family = Agilex                 fim = base_x16                 Part = AGFB014R24A2E2V                 IpDeployFile = pcie_ss.sh                 IpFile = pcie_ss.ip                 OutputName = pcie_ss                 ComponentName = pcie_ss                 is_host = True                  [pf0]                 num_vfs = 3                 pg_enable = True                  [pf1]                  [pf2]                  [pf3]                  [pf4] 10 - Modify PF/VF Mux Configuration As an example this menu selection modifies the pcie_host.ofss file to 1 PF located in the following directory $OFS_ROOTDIR/tools/pfvf_config_tool             This option also displays the the modified pcie_host.ofss file 11 - Build PF/VF Mux Configuration If option 10 is not used then then the default number of PF's and VF's is used to build the FIM, if option 10 is selected then only 1 VF is built to reduce logic utilisation"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#314-adp-fimpr-build-menu","title":"3.1.4 ADP FIM/PR BUILD MENU","text":"
                                                                                    Builds FIM, Partial Reconfiguration Region and Remote Signal Tap
                                                                                     
                                                                                     Menu Option Description 12 - Check ADP software versions for ADP n6001 Project OFS_ROOTDIR is set to /home/user_area/ofs-X.X.X/ofs-n6001                 OPAE_SDK_REPO_BRANCH is set to release/X.X.X                 OPAE_SDK_ROOT is set to /home/user_area/ofs-X.X.X/ofs-n6001/../opae-sdk                 LD_LIBRARY_PATH is set to /home/user_area/ofs-X.X.X/ofs-n6001/../opae-sdk/lib64: 13 - Build FIM for n6001 Hardware This option builds the FIM based on the setting for the $ADP_PLATFORM, $FIM_SHELL environment variable. Check this variable in the following file ofs_n6001_eval.sh 14 - Check FIM Identification of FIM for n6001 Hardware The FIM is identified by the following file fme-ifc-id.txt located at $OFS_ROOTDIR/$FIM_WORKDIR/syn/syn_top/ 15 - Build Partial Reconfiguration Tree for n6001 Hardware This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development and also for the oneAPI build flow  16 - Build Base FIM Identification(ID) into PR Build Tree template This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU and oneAPI workloads 17 - Build Partial Reconfiguration Tree for n6001 Hardware with Remote Signal Tap This option builds the Partial Reconfiguration Tree which is needed for AFU testing/development and also for the oneAPI build flow for the Remote Signal Tap flow 18 - Build Base FIM Identification(ID) into PR Build Tree template with Remote Signal Tap This option copies the contents of the fme-ifc-id.txt into the Partial Reconfiguration Tree for Remote Signal Tap to allow the FIM amd Partial Reconfiguration Tree to match and hence allow subsequent insertion of AFU and oneAPI workloads"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#315-adp-hardware-programmingdiagnostic-menu","title":"3.1.5 ADP HARDWARE PROGRAMMING/DIAGNOSTIC MENU","text":"
                                                                                    The following submenu allows you to: * Program and check flash  * Perform a remote system update (RSU) of the FPGA image into the FPGA * Bind virtual functions to VFIO PCIe driver  * Run host exerciser (HE) commands such as loopback to test interfaces VFIO PCI driver binding * Read the control and status registers (CSRs) for bound modules that are part of the OFS reference design.
                                                                                     
                                                                                     Menu Option Description 19 - Program BMC Image into n6001 Hardware The user must place a new BMC flash file in the following directory $OFS_ROOTDIR/bmc_flash_files. Once the user executes this option a new BMC image will be programmed. A remote system upgrade command is initiated to store the new BMC image 20 - Check Boot Area Flash Image from n6001 Hardware This option checks which location area in FLASH the image will boot from, the default is user1              Boot Page : user1 21 - Program FIM Image into user1 area for n6001 Hardware This option programs the FIM image \"ofs_top_page1_unsigned_user1.bin\" into user1 area in flash 22 - Initiate Remote System Upgrade (RSU) from user1 Flash Image into n6001 Hardware This option initiates a Remote System Upgrade and soft reboots the server and re-scans the PCIe bus for the new image to be loaded              2022-11-10 11:26:24,307 - [[pci_address(0000:b1:00.0), pci_id(0x8086, 0xbcce)]] performing RSU operation             2022-11-10 11:26:24,310 - [[pci_address(0000:b0:02.0), pci_id(0x8086, 0x347a)]] removing device from PCIe bus             2022-11-10 11:26:24,357 - waiting 10 seconds for boot             2022-11-10 11:26:34,368 - rescanning PCIe bus: /sys/devices/pci0000:b0/pci_bus/0000:b0             2022-11-10 11:26:35,965 - RSU operation complete 23 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 24 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 25 - Create Virtual Functions (VF) and bind driver to vfio-pci n6001 Hardware This option creates vfio-pci driver binding for the PF's and VF's             Once the VF's have been bound to the driver the user can select menu option 23 to check that the new drivers are bound 26 - Verify FME Interrupts with hello_events The hello_events utility is used to verify FME interrupts. This tool injects FME errors and waits for error interrupts, then clears the errors 27 - Run HE-LB Test This option runs 5 tests              1) checks and generates traffic with the intention of exercising the path from the AFU to the Host at full bandwidth             2) run a loopback throughput test using one cacheline per request             3) run a loopback read test using four cachelines per request             4) run a loopback write test using four cachelines per request             5) run a loopback throughput test using four cachelines per request 28 - Run HE-MEM Test This option runs 2 tests              1) Checking and generating traffic with the intention of exercising the path from FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host             2) run a loopback throughput test using one cacheline per request 29 - Run HE-HSSI Test This option runs 1 test              HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs, and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic              1) Send traffic through the 10G AFU              30 - Run Traffic Generator AFU Test This option runs 3 tests              TG AFU has an OPAE application to access & exercise traffic, targeting a specific bank              1) Run the preconfigured write/read traffic test on channel 0             2) Target channel 1 with a 1MB single-word write only test for 1000 iterations             3) Target channel 2 with 4MB write/read test of max burst length for 10 iterations 31 - Read from CSR (Command and Status Registers) for n6001 Hardware This option reads from the following CSR's             HE-LB Command and Status Register Default Definitions             HE-MEM Command and Status Register Default Definitions             HE-HSSI Command and Status Register Default Definitions"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#316-adp-hardware-afu-testing-menu","title":"3.1.6  ADP HARDWARE AFU TESTING MENU","text":"
                                                                                    This submenu tests partial reconfiguration by building and loading an memory-mapped I/O example AFU/workload, executes software from host, and tests remote signal tap.
                                                                                     
                                                                                     Menu Option Description 32 - Build and Compile host_chan_mmio example This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS (Green Bit Stream) ready for hardware programming 33 - Execute host_chan_mmio example This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test 34 - Modify host_chan_mmio example to insert Remote Signal Tap This option inserts a pre-defined host_chan_mmio.stp Signal Tap file into the OFS code to allow a user to debug the host_chan_mmio AFU example 35 - Build and Compile host_chan_mmio example with Remote Signal Tap This option builds the host_chan_mmio example from the following repo $OFS_PLATFORM_AFU_BBB/plat_if_tests/$AFU_TEST_NAME, where AFU_TEST_NAME=host_chan_mmio. This produces a GBS(Green Bit Stream) ready for hardware programming with Remote Signal tap enabled 36 - Execute host_chan_mmio example with Remote Signal Tap This option builds the host code for host_chan_mmio example and programs the GBS file and then executes the test. The user must open the Signal Tap window when running the host code to see the transactions in the Signal Tap window"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#317-adp-hardware-afu-bbb-testing-menu","title":"3.1.7 ADP HARDWARE AFU BBB TESTING MENU","text":"
                                                                                    This submenu tests partial reconfiguration using a hello_world example AFU/workload, executes sw from host
                                                                                     
                                                                                     Menu Option Description 37 - Build and Compile hello_world example This option builds the hello_ world example from the following repo $FPGA_BBB_CCI_SRC/samples/tutorial/afu_types/01_pim_ifc/$AFU_BBB_TEST_NAME, where AFU_BBB_NAME=hello_world. This produces a GBS(Green Bit Stream) ready for hardware programming 38 - Execute hello_world example This option builds the host code for hello_world example and programs the GBS file and then executes the test"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#318-adp-oneapi-project-menu","title":"3.1.8 ADP ONEAPI PROJECT MENU","text":"
                                                                                    Builds oneAPI kernel, executes sw from host and runs diagnostic tests
                                                                                     
                                                                                     Menu Option Result 39 - Check oneAPI software versions for n6001 Project This option checks the setup of the oneAPI software and adds the relevant oneAPI environment variables to the terminal. This option also informs the user to match the oneAPI software version to the oneAPI-samples version 40 - Build and clone shim libraries required by oneAPI host This option builds the oneAPI directory structure 41 - Install oneAPI Host Driver This option Installs the oneAPI Host driver at the following location /opt/Intel/OpenCLFPGA/oneAPI/Boards/, and requires sudo permission 42 - Uninstall oneAPI Host Driver This option Uninstall's the oneAPI Host driver, and requires sudo permissions 43 - Diagnose oneAPI Hardware This option Checks ICD (Intel Client Driver) and FCD (FPGA Client Driver), oneAPI library locations and detects whether oneAPI BSP is loaded into the FPGA 44 - Build oneAPI BSP ofs_n6001 Default Kernel (hello_world) This option Builds the oneAPI BSP using hello_world kernel 45 - Build oneAPI MakeFile Environment This option Builds the oneAPI environment using a Makefile for kernel insertion  46 - Compile oneAPI Sample Application (board_test) for Emulation This option compiles the board_test kernel for Emulation 47 - Run oneAPI Sample Application (board_test) for Emulation This option executes the board_test kernel for Emulation 48 - Generate oneAPI Optimization report for (board_test) This option generates an optimization report for the board_test kernel 49 - Check PF/VF Mapping Table, vfio-pci driver binding and accelerator port status This option checks the current vfio-pci driver binding for the PF's and VF's 50 - Unbind vfio-pci driver This option unbinds the vfio-pci driver for the PF's and VF's 51 - Create Virtual Function (VF) and bind driver to vfio-pci n6001 Hardware This option creates vfio-pci driver binding for the PF's and VF's             Once the VF's have been bound to the driver the user can select menu option 45 to check that the new drivers are bound 52 - Program OpenCL BSP ofs_n6001 Default Kernel (hello_world) This option programs the FPGA with a aocf file based on the hello_world kernel 53 - Compile oneAPI Sample Application (board_test) for Hardware This option compiles the board_test kernel for Hardware 54 - Run oneAPI Sample Application (board_test) for Hardware This option builds the host code for board_test kernel and executes the program running through kernel and host bandwidth tests"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#319-adp-unit-test-project-menu","title":"3.1.9 ADP UNIT TEST PROJECT MENU","text":"
                                                                                    Builds, compiles and runs standalone simulation block tests.  More unit test examples are found at the following location ofs_n6001/sim/unit_test 
                                                                                     
                                                                                     Menu Option Result 55 - Generate Simulation files for Unit Test This option builds the simulation file set for running a unit test simulation 56 - Simulate Unit Test dfh_walker and log waveform This option runs the dfh_walker based on the environment variable \"UNIT_TEST_NAME=dfh_walker\" in the evaluation script. A user can change the test being run by modifying this variable"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#3110-adp-uvm-project-menu","title":"3.1.10 ADP UVM PROJECT MENU","text":"
                                                                                    Builds, compiles and runs full chip simulation tests.  The user should execute the options sequentially ie 68,69, 70 and 71
                                                                                     
                                                                                     Menu Option Description 57 - Check UVM software versions for n6001 Project DESIGNWARE_HOME is set to /home/synopsys/vip_common/vip_Q-2020.03A                 UVM_HOME  is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm                 VCS_HOME is set to /home/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel                 VERDIR is set to /home/user_area/ofs-X.X.X/ofs-n6001/verification                 VIPDIR is set to /home/user_area/ofs-X.X.X/ofs-n6001/verification 58 - Compile UVM IP This option cmpiles the UVM IP  59 - Compile UVM RTL and Testbench This option compiles the UVM RTL and Testbench 60 - Simulate UVM dfh_walking_test and log waveform This option runs the dfh_walking test based on the environment variable \"UVM_TEST_NAME=dfh_walking_test\" in the evaluation script. A user can change the test being run by modifying this variable 61 - Simulate all UVM test cases (Regression Mode) This option runs the n6001 regression mode, cycling through all UVM tests defined in verification/tests/test_pkg.svh file"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#3111-adp-build-all-project-menu","title":"3.1.11 ADP BUILD ALL PROJECT MENU","text":"
                                                                                    Builds the complete OFS flow, good for regression testing and overnight builds
                                                                                     
                                                                                    For this menu a user can run a sequence of tests (compilation, build and simulation) and executes them sequentially. After the script is successfully executed, a set of binary files is produced which a you can use to evaluate your hardware. Log files are also produced which checks whether the tests passed.
                                                                                     
                                                                                    A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 62 from the main menu the script will execute 24 tests ie (main menu options 2, 9, 12, 13, 14, 15, 16, 17, 18, 32, 34, 35, 37, 39, 40, 44, 45, 53, 55, 56, 57, 58, 59 and 60. These 24 menu options are chosen to build the complete OFS flow covering build, compile and simulation.
                                                                                     
                                                                                     Menu Option Result 62 - Build and Simulate Complete n6001 Project Generating Log File with date and timestamp                 Log file written to /home/guest/ofs-2.3.1/log_files/n6001_log_2022_11_10-093649/ofs_n6001_eval.log"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#definition-of-multi-test-set-up","title":"Definition of Multi-Test Set-up","text":"
                                                                                    Menu Option 62 above in the evaluation script can be refined to tailor it to the users need and is principally defined by the variable below
                                                                                     
                                                                                    MULTI_TEST[A,B]=C
                                                                                     
                                                                                    where
                                                                                     
                                                                                    A= Total Number of menu options in script B= Can be changed to a number to select the test order C= Menu Option in Script
                                                                                     
                                                                                    Example 1 MULTI_TEST[62,0]=2
                                                                                     
                                                                                    A= 62 is the total number of options in the script B= 0 indicates that this is the first test to be run in the script C= Menu option in Script ie 2- List of Documentation for ADP n6001 Project
                                                                                     
                                                                                    Example 2 MULTI_TEST[62,0]=2 MULTI_TEST[62,1]=9
                                                                                     
                                                                                    In the example above two tests are run in order ie 0, and 1 and the following menu options are executed ie 2- List of Documentation for ADP n6001 Project and 9 - Check ADP software versions for ADP n6001 Project
                                                                                     
                                                                                    The user can also modify the build time by de-selecting options they do not wish to use, see below for a couple of use-case scenarios.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#default-user-case","title":"Default User Case","text":"
                                                                                    A user can run a sequence of tests and execute them sequentially. In the example below when the user selects option 62 from the main menu the script will execute 24 tests ie (main menu options 2, 9, 12, 13, 14, 15, 16, 17, 18, 32, 34, 35, 37, 39, 40, 44, 45, 53, 55, 56, 57, 58, 59 and 60. All other tests with an \"X\" indicates do not run that test.
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#user-case-for-adp-fimpr-build-menu","title":"User Case for ADP FIM/PR BUILD MENU","text":"
                                                                                    In the example below when the user selects option 62 from the main menu the script will only run options from the ADP FIM/PR BUILD MENU (7 options, main menu options 12, 13, 14, 15, 16, 17 and 18). All other tests with an \"X\" indicates do not run that test.
                                                                                     
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#4-n6001-common-test-scenarios","title":"4 n6001 Common Test Scenarios","text":"
                                                                                    This section will describe the most common compile build scenarios if a user wanted to evaluate an acceleration card on their server. The Pre-requisite column indcates the menu comamnds that must be run befere executing the test eg To run Test 5 then a user needs to have run option 13, 15 and 16 before running options 23, 24, 25, 32 and 33.
                                                                                     Test Test Scenario Pre-Requisite Menu Option Menu Option Test 1 FIM Build - 13 Test 2 Partial Reconfiguration Build 13 15, 16 Test 3 Program FIM and perform Remote System Upgrade 13 21, 22 Test 4 Bind PF and VF to vfio-pci drivers - 23, 24, 25 Test 5 Build, compile and test AFU on hardware 13, 15, 16 23, 24, 25, 32, 33 Test 6 Build, compile and test AFU Basic Building Blocks on hardware 13, 15, 16 23, 24, 25, 37, 38 Test 7 Build, compile and test oneAPI on hardware 13, 15, 16 39, 40, 41, 44, 45, 49, 50, 51, 52, 53, 54 Test 8 Build and Simulate Unit Tests - 55, 56 Test 9 Build and Simulate UVM Tests - 57, 58, 59, 60"},{"location":"hw/n6001/user_guides/ug_eval_ofs_n6001/ug_eval_script_ofs_n6001/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                                    Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                                     
                                                                                    OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/","title":"Getting Started Guide: Open FPGA Stack for Intel\u00ae Agilex FPGAs Targeting the Intel\u00ae FPGA SmartNIC N6001-PL","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#10-introduction","title":"1.0 Introduction","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#11-about-this-document","title":"1.1 About This Document","text":"
                                                                                    The purpose of this document is to help users get started in evaluating the 2023.3-2  version of the Open FPGA Stack (OFS) for the Intel Agilex FPGA targeting the Intel N6001-PL FPGA SmartNIC Platform. After reviewing the document a user shall be able to:
                                                                                     
                                                                                       
                                                                                    • Set up their server environment according to the Best Known Configuration (BKC)
                                                                                      •  
                                                                                    • Build and install the OFS Linux Kernel drivers
                                                                                      •  
                                                                                    • Build and install the Open Programmable Acceleration Engine Software Development Kit (OPAE SDK) software on top of the OFS Linux kernel drivers
                                                                                      •  
                                                                                    • Load and Verify the Firmware and FIM versions loaded on their boards
                                                                                      •  
                                                                                    • Verify the full stack functionality offered by the OFS solution
                                                                                      •  
                                                                                    • Know where to find additional information on other OFS ingredients
                                                                                      •  
                                                                                     
                                                                                    The following flow charts show a high level overview of the initial bring up process, split into three sequential diagrams. Detailed instructions for each installation process are shown in their respective sections.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#diagram-1-installing-the-opae-sdk","title":"Diagram 1: Installing the OPAE SDK","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#diagram-2-installing-the-linux-dfl-drivers","title":"Diagram 2: Installing the Linux DFL Drivers","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#diagram-3-bringing-up-the-intel-fpga-smartnic-n6001-pl","title":"Diagram 3: Bringing up the Intel\u00ae FPGA SmartNIC N6001-PL","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#12-terminology","title":"1.2 Terminology","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-1-glossary","title":"Table 1: Glossary","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel Max10 or Intel Cyclone10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implemented on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Management Component Transport Protocol MCTP A standardized model for communication with management controllers. Defines the transport protocol carrying PLDM messages through the BMC. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE-SDK The OPAE-SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Level Data Model PLDM A specification for reporting telemetry data to the host, such as board temperature, voltage, and current. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#13-introduction-to-ofs","title":"1.3 Introduction to OFS","text":"
                                                                                    Each OFS FIM targets a specific platform, but the modular hardware, software, simulation and test infrastructure allow users to modify each part of the design and test environment for their own custom acceleration platform card. This OFS release targets the Intel\u00ae FPGA SmartNIC N6001-PL.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#14-intended-audience","title":"1.4 Intended Audience","text":"
                                                                                    The information in this document is intended for customers evaluating the Intel\u00ae FPGA SmartNIC N6001-PL. The card is an acceleration development platform (ADP) intended to be used as a starting point for evaluation and development. This document will cover key topics related to initial bring up and development, with links for deeper dives on the topics discussed therein.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#15-reference-documents","title":"1.5 Reference Documents","text":"
                                                                                    Documentation for N6001 is collected on GitHub. The document list is as follows:
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-2-reference-documents","title":"Table 2: Reference Documents","text":"Document Purpose Getting Started Guide: Intel Open FPGA Stack for Intel Agilex FPGA Guides you through the setup and build steps to evaluate the OFS solution targeting an Intel N6001-PL FPGA SmartNIC Platform FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA Describes the OFS FIM architecture and features. Software Reference Manual: Intel Open FPGA Stack Describes the Open Programmable Acceleration Engine (OPAE) Software Development Kit, the OPAE C++ and Python API and management interfaces.  This document also covers building the OPAE SDK, how to add a new PCIe device, and debugging the software stack. FPGA Interface Manager Developer Guide: Intel Open Stack for Intel Agilex FPGA Provides guidance on developing an FPGA Interface Manager (FIM) for a custom FPGA acceleration board. Hard Processor System Software Developer Guide: Intel OFS for Intel Agilex FPGAs Describes the HPS configuration and software package for the Intel OFS release targeting Intel Agilex OFS and guides you through the steps to build your own Yocto application image for HPS. Accelerator Functional Unit Developer Guide: Intel Open FPGA Stack Provides guidance on how to build and test an AFU when designing to an OFS-based FPGA Interface Manager Simulation User Guide: Intel Open FPGA Stack for Intel Agilex FPGA Provides steps for setting up the UVM verification tool suite and running UVM unit tests Security User Guide: Intel Open FPGA Stack (request access by emailing ofs.github@intel.com) Describes how to create keys and sign bitstreams for your custom design; includes new VAB feature oneAPI Accelerator Support Package (ASP): Getting Started User Guide Describes how to get started using OneAPI with the OFS FIM BSP. oneAPI Accelerator Support Package(ASP) Reference Manual: Open FPGA Stack Describes how to use the provided shim for OFS for creating your custom OneAPI board support package. Platform Evaluation Script: Open FPGA Stack for Intel Agilex FPGA Serves as a set-up and user guide for the checkout and evaluation of an Intel FPGA SmartNIC N6001 Platform using Open FPGA Stack (OFS)."},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#16-component-version-summary","title":"1.6 Component Version Summary","text":"
                                                                                    The OFS 2023.3-2 Release targeting the Intel\u00ae FPGA SmartNIC N6001-PL is built upon tightly coupled software and firmware versions. Use this section as a general reference for the versions which compose this release.
                                                                                     
                                                                                    The following table highlights the hardware which makes up the Best Known Configuration (BKC) for the OFS 2023.3-2 release.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-3-hardware-bkc","title":"Table 3: Hardware BKC","text":"Component Version 1 x Intel\u00ae FPGA SmartNIC N6001-PL, SKU2 1 x Supermicro Server SYS-220HE 1 x Intel FPGA Download Cable II (Only Required for manual flashing) 1 x 2x5 Extension header - Samtech Part No: ESQ-105-13-L-D (Only Required for manual flashing) 
                                                                                    The following table highlights the versions of the software which compose the OFS stack. The installation of the OPAE SDK on top of the Linux DFL drivers will be discussed in their relevant sections in this document.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-4-software-version-summary","title":"Table 4: Software Version Summary","text":"Component Version FPGA Platform Intel\u00ae FPGA SmartNIC N6001-PL, release notes: https://github.com/OFS/ofs-n6001/releases/tag/ofs-2023.3-2 under \"Known Issues\" OPAE SDK 2.10.0-1 Kernel Drivers ofs-2023.3-6.1-3 OneAPI-ASP ofs-2023.3-2 OFS FIM Source Code for N6001 ofs-2023.3-2 OFS FIM Common Resources Tag: ofs-fim-common-1.1.0-rc2 OFS Platform AFU BBB ofs-2023.3-2 Intel Quartus Prime Pro Edition Design Software* Quartus Prime Pro Version 23.3 for Linux Operating System RedHatEnterprise Linux\u00ae (RHEL) 8.6 
                                                                                    The following table highlights the differences between N6000/1 PL FPGA SmartNIC platforms (SKU1/SKU2). Use this table to identify which version of the N6000/1-PL FPGA SmartNIC platform you have. The board identification printed by the fpgainfo fme commands depends on both the OPAE SDK and Linux DFL drivers from sections 3.0 OFS DFL Kernel Drivers through 4.0 OPAE Software Development Kit to be installed before it can be run.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-5-intel-n60001-pl-fpga-smartnic-platform-sku-mapping","title":"Table 5: Intel N6000/1-PL FPGA SmartNIC Platform SKU Mapping","text":"SKU Mapping SKU Value Primary Difference fpgainfo Identification N6000 Q1613314XXXXX PCIe Gen 4 1x16 mechanical bifurcated 2x8 logical to host, with one PCIe Gen 4x8 endpoint reserved for Intel E810-C-CAM2 NIC, the other reserved for FIM \"Intel Acceleration Development Platform N6000\" N6001 Q0216514XXXXX PCIe Gen 4 1x16 mechanical and logical connection between host and FIM \"Intel Acceleration Development Platform N6001\" 
                                                                                    The following table highlights the programmable firmware versions that are supported on the Intel N6001-PL FPGA SmartNIC Platform in the OFS 2023.3-2 release. Programming and verifying these components is discussed in their respective sections.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-6-intel-fpga-smartnic-n6001-pl-programmable-component-version-summary","title":"Table 6: Intel\u00ae FPGA SmartNIC N6001-PL Programmable Component Version Summary","text":"Component Version PR Interface ID 1d6beb4e-86d7-5442-a763-043701fb75b7 Bitstream ID 00x50102023508A422 BMC RTL 3.15.0 BMC NIOS FW 3.15.0"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#17-initial-server-setup","title":"1.7 Initial Server Setup","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#171-server-bmc-and-bios-updates","title":"1.7.1 Server BMC and BIOS Updates","text":"
                                                                                    Both the server BIOS and BMC need to match the versions listed below in Table 7: Supermicro Server BMC BKC. These updates only apply for this specific Best Known Configuration (BKC) - other server manufacturers may require different BIOS updates. Please consult your server's user guide and release notes for update information.
                                                                                     
                                                                                    Information about the server\u2019s currently loaded firmware can be found on the BMC web portal dashboard. Accessing this page requires an Ethernet cable to be attached to an open port on the server labelled \u201cIPMI\u201d. During boot the BMC\u2019s login IP will be presented on the screen.
                                                                                     
                                                                                    Open this IP address in a browser and enter your login credentials. The default username is ADMIN, and the default password has been printed on the service tag that pulls out from the front of the server case. It is recommended the user change their BMC\u2019s default username as soon as they are able
                                                                                     
                                                                                    After logging in you should be able to review information about the BMC and BIOS by referring to the System box, visible upon initial loading of the page. Double check that the values match those in Table 7 Supermicro Server BMC BKC. If they do not, you may download the appropriate versions from the Supermicro product page by selecting the BIOS option and downloading the most recent \u201cBundled Software File Name\u201d. Follow the BMC and BIOS update instructions included in the Supermicro manuals page in the document X12/H12 BMC Manual in Appendix A.2 Updating Firmware Using BMC Web GUI.
                                                                                     
                                                                                    If using a different server model, refer to that server\u2019s user guide for instructions on remote system management.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-7-supermicro-server-bmc-bkc","title":"Table 7: Supermicro Server BMC BKC","text":"Component Version BIOS Version American Megatrends International, LLC(1.4)"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#172-server-fan-speed","title":"1.7.2 Server Fan Speed","text":"
                                                                                    The recommended fan speed setting is to use the 100% preset. If using a different server model, refer to that server\u2019s user guide for instructions on changing fan speed. The following steps will help users on the Supermicro platform.
                                                                                     
                                                                                       
                                                                                    1. Log in to the Supermicro server BMC. (This requires an Ethernet cable to be attached to an open port on the server labelled \u201cIPMI\u201d.)
                                                                                      2.  
                                                                                    2. During boot the BMC\u2019s login IP will be presented on the screen. Open this IP address in a browser and enter your login credentials. The default username is ADMIN, and the default password has been printed on the service tag that pulls out from the front of the server case.
                                                                                      3.  
                                                                                    3. On the left menu select System -> Component Info, select the Fan tab, under Advanced Settings click the circle next to Full Speed.
                                                                                      4.  
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#173-server-bios-configuration-changes","title":"1.7.3 Server BIOS Configuration Changes","text":"
                                                                                    To enter the Supermicro server\u2019s BIOS setup page, reboot, and press \\<Delete> when prompted. You can browse the tabs / options with a combination of arrow keys along with \\<Escape> and \\<Enter>.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#1731-enabling-intel-vt-d-technology","title":"1.7.3.1 Enabling Intel VT-d Technology","text":"
                                                                                    Navigate right to the Advanced tab, then select the following menu options: Chipset Configuration -> North Bridge -> IIO Configuration -> Intel VT for Directed I/O (VT-d). If not already, enable the option Intel VT for Directed I/O (VT-d).
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#1732-pcie-slot-mapping","title":"1.7.3.2 PCie Slot Mapping","text":"
                                                                                    The Intel N6001-PL FPGA SmartNIC Platform is officially verified in the upper middle PCIe x16 slot (Slot 3). If using a different slot, refer to the information in Table 8 PCIe Slot Mapping for which port to manually change in the server BIOS.
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-8-pcie-slot-mapping","title":"Table 8: PCIe Slot Mapping","text":"CPU Number Port Number (in BIOS) PCIe Slot CPU1 Port 2 5 and 6 CPU1 Port 4 7 and 8 CPU2 Port 2 1 and 2 CPU2 Port 4 3 and 4"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#174-intel-fpga-smartnic-n6001-pl-installation-procedure","title":"1.7.4 Intel\u00ae FPGA SmartNIC N6001-PL Installation Procedure","text":"
                                                                                    The following instructions will help to ensure safe installation of the Intel\u00ae FPGA SmartNIC N6001-PL into a supported server. Refer to Table 8 PCIe Slot Mapping for a breakdown of the available PCIe slots on a SuperMicro SYS-220HE.
                                                                                     
                                                                                       
                                                                                    1. Position the board over the selected connector on the motherboard.
                                                                                      2.  
                                                                                    2. Press down gently and firmly to seat the card in the PCIe slot, and then secure the bracket to the system chassis with the retention screw.
                                                                                      3.  
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-9-intel-fpga-smartnic-n6001-pl-installation-procedure","title":"Table 9: Intel\u00ae FPGA SmartNIC N6001-PL Installation Procedure","text":"Callout Description 1 Retention screw 2 Press down here gently 3 Press down here gently 4 Motherboard 
                                                                                    Do not bend the card while inserting into a slot. Do not apply much pressure in regions 2 or 3 while inserting.
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#175-intel-fpga-smartnic-n6001-pl-removal-procedure","title":"1.7.5 Intel\u00ae FPGA SmartNIC N6001-PL Removal Procedure","text":"
                                                                                    The following instructions will help to ensure safe removal of the platform from a supported server.
                                                                                     
                                                                                       
                                                                                    1. Disconnect all power cords from the server power supply(s).
                                                                                      2.  
                                                                                    2. Remove the retention bracket screw.
                                                                                      3.  
                                                                                    3. Carefully lift the card out of the PCIe slot.
                                                                                      4.  
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-10-intel-fpga-smartnic-n6001-pl-removal-procedure","title":"Table 10: Intel\u00ae FPGA SmartNIC N6001-PL Removal Procedure","text":"Callout Description 1 Retention screw 2 Pull up here gently 3 Motherboard 
                                                                                    Do not bend the card while removing it from the slot.
                                                                                     
                                                                                     
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#20-ofs-stack-architecture-overview-for-reference-platform","title":"2.0 OFS Stack Architecture Overview for Reference Platform","text":""},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#21-hardware-components","title":"2.1 Hardware Components","text":"
                                                                                    The OFS hardware architecture decomposes all designs into a standard set of modules, interfaces, and capabilities. Although the OFS infrastructure provides a standard set of functionality and capability, the user is responsible for making the customizations to their specific design in compliance with the specifications outlined in the FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA.
                                                                                     
                                                                                    OFS is a hardware and software infrastructure that provides an efficient approach to developing a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#211-fpga-interface-manager","title":"2.1.1 FPGA Interface Manager","text":"
                                                                                    The FPGA Interface Manager (FIM), or shell of the FPGA provides platform management functionality, clocks, resets, and interface access to the host and peripheral features on the acceleration platform. The OFS architecture for Intel Agilex FPGA provides modularity, configurability and scalability.  The primary components of the FPGA Interface Manager or shell of the reference design are:
                                                                                     
                                                                                       
                                                                                    • PCIe Subsystem - a hierarchical design that targets the P-tile PCIe hard IP and is configured to support Gen4 speeds and Arm AXI4-Stream Data Mover functional mode.
                                                                                      •  
                                                                                    • Ethernet Subsystem - provides portability to different Ethernet configurations across platforms and generations and reusability of the hardware framework and software stack.
                                                                                      •  
                                                                                    • Memory Subsystem - composed of 5 DDR4 channels; one HPS DDR4 bank, x40 (x32 Data and x8 ECC), 1200 MHz, 1GB each, and four Fabric DDR4 banks, x32 (no ECC), 1200 MHz, 4GB
                                                                                      •  
                                                                                    • Hard Processor System - 64-bit quad core ARM\u00ae Cortex*-A53 MPCore with integrated peripherals.
                                                                                      •  
                                                                                    • Reset Controller
                                                                                      •  
                                                                                    • FPGA Management Engine - Provides a way to manage the platform and enable acceleration functions on the platform.
                                                                                      •  
                                                                                    • AFU Peripheral Fabric for AFU accesses to other interface peripherals
                                                                                      •  
                                                                                    • Board Peripheral Fabric for master to slave CSR accesses from Host or AFU
                                                                                      •  
                                                                                    • Platform Management Controller Interface (PMCI) to the board management controller
                                                                                      •  
                                                                                     
                                                                                    The FPGA Management Engine (FME) provides management features for the platform and the loading/unloading of accelerators through partial reconfiguration. Each feature of the FME exposes itself to the kernel-level OFS drivers on the host through a Device Feature Header (DFH) register that is placed at the beginning of Control Status Register (CSR) space. Only one PCIe link can access the FME register space in a multi-host channel design architecture at a time.
                                                                                     
                                                                                    Note: For more information on the FIM and its external connections, refer to the FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#212-afu","title":"2.1.2 AFU","text":"
                                                                                    An AFU is an acceleration workload that interfaces to the FIM. The AFU boundary in this reference design comprises both static and partial reconfiguration (PR) regions.  You can decide how you want to partition these two areas or if you want your AFU region to only be a partial reconfiguration region. A port gasket within the design provides all the PR specific modules and logic required to support partial reconfiguration. Only one partial reconfiguration region is supported in this design.
                                                                                     
                                                                                    Similar to the FME, the port gasket exposes its capability to the host software driver through a DFH register placed at the beginning of the port gasket CSR space. In addition, only one PCIe link can access the port register space.  
                                                                                     
                                                                                    You can compile your design in one of the following ways:
                                                                                     
                                                                                       
                                                                                    • Your entire AFU resides in a partial reconfiguration region of the FPGA.
                                                                                      •  
                                                                                    • The AFU is part of the static region and is compiled as a flat design.
                                                                                      •  
                                                                                    • Your AFU contains both static and PR regions.
                                                                                      •  
                                                                                     
                                                                                    In this design, the AFU region is comprised of:
                                                                                     
                                                                                       
                                                                                    • AFU Interface handler to verify transactions coming from AFU region.
                                                                                      •  
                                                                                    • PF/VF Mux to route transactions to and from corresponding AFU components: ST2MM module, Virtio LB stub, PCIe loopback host exerciser (HE-LB), HSSI host exerciser (HE-HSSI), Memory Host Exerciser (HE-MEM), Traffic Generator to memory (HE-MEM-TG), Port Gasket (PRG) and HPS Copy Engine.
                                                                                      •  
                                                                                    • AXI4 Streaming to Memory Map (ST2MM) Module that routes MMIO CSR accesses to FME and board peripherals.
                                                                                      •  
                                                                                    • Host exercisers to test PCIe, memory and HSSI interfaces (these can be removed from the AFU region after your FIM design is complete to provide more resource area for workloads)
                                                                                      •  
                                                                                    • Basic HPS Copy Engine to copy second-stage bootloader and Linux OS image from Host DDR to HPS DDR.  
                                                                                      •  
                                                                                    • Port gasket and partial reconfiguration support.
                                                                                      •  
                                                                                    • Component for handling PLDM over MCTP over PCIe Vendor Defined Messages (VDM)
                                                                                      •  
                                                                                     
                                                                                    The AFU has the option to consume native packets from the host or interface channels or to instantiate a shim provided by the Platform Interface Manager (PIM) to translate between protocols.
                                                                                     
                                                                                    Note: For more information on the Platform Interface Manager and AFU development and testing, refer to the AFU Development Guide: OFS for Intel\u00ae Agilex\u00ae PCIe Attach FPGAs.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#22-ofs-software-overview","title":"2.2 OFS Software Overview","text":"
                                                                                    The responsibility of the OFS kernel drivers is to act as the lowest software layer in the FPGA software stack, providing a minimalist driver implementation between the host software and functionality that has been implemented on the development platform. This leaves the implementation of IP-specific software in user-land, not the kernel. The OFS software stack also provides a mechanism for interface and feature discovery of FPGA platforms.
                                                                                     
                                                                                    The OPAE SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space, and can be found on the OPAE SDK Github.
                                                                                     
                                                                                    The OFS drivers decompose implemented functionality, including external FIM features such as HSSI, EMIF and SPI, into sets of individual Device Features. Each Device Feature has its associated Device Feature Header (DFH), which enables a uniform discovery mechanism by software. A set of Device Features are exposed through the host interface in a Device Feature List (DFL). The OFS drivers discover and \"walk\" the Device Features in a Device Feature List and associate each Device Feature with its matching kernel driver.
                                                                                     
                                                                                    In this way the OFS software provides a clean and extensible framework for the creation and integration of additional functionalities and their features.
                                                                                     
                                                                                    *Note: A deeper dive on available SW APIs and programming model is available in the Software Reference Manual: Intel\u00ae Open FPGA Stack and on kernel.org.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#30-ofs-dfl-kernel-drivers","title":"3.0 OFS DFL Kernel Drivers","text":"
                                                                                    OFS DFL driver software provides the bottom-most API to FPGA platforms. Libraries such as OPAE and frameworks like DPDK are consumers of the APIs provided by OFS. Applications may be built on top of these frameworks and libraries. The OFS software does not cover any out-of-band management interfaces. OFS driver software is designed to be extendable, flexible, and provide for bare-metal and virtualized functionality. An in depth look at the various aspects of the driver architecture such as the API, an explanation of the DFL framework, and instructions on how to port DFL driver patches to other kernel distributions can be found on https://github.com/OPAE/linux-dfl/wiki.
                                                                                     
                                                                                    An in-depth review of the Linux device driver architecture can be found on opae.github.io.
                                                                                     
                                                                                    The DFL driver suite can be automatically installed using a supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3-2 Release Page.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#31-ofs-dfl-kernel-driver-environment-setup","title":"3.1 OFS DFL Kernel Driver Environment Setup","text":"
                                                                                    All OFS DFL kernel driver code resides in the Linux DFL GitHub repository. This repository is open source and should not require any permissions to access. It includes a snapshot of the Linux kernel with the OFS driver included in /drivers/fpga/*. Downloading, configuration, and compilation will be discussed in this section. The only operating systems supported out of the box by the Linux DFL kernel drivers is RHEL 8.6.
                                                                                     
                                                                                    This installation process assumes the user has access to an internet connection in order to clone specific GitHub repositories, and to satisfy package dependencies.
                                                                                     
                                                                                    1. Make the following changes on your installation machine to satisfy all dependencies:
                                                                                     
                                                                                    $ subscription-manager release --set=8.6\n$ sudo dnf update\n
                                                                                     
                                                                                    If you wish to install the pre-built linux-dfl package available on the OFS 2023.3-2 Release Page skip to section 3.3 Installing the OFS DFL Kernel Drivers from Pre-Built Packages.
                                                                                     
                                                                                    2. You must satisfy the following package dependencies if building and installing the drivers from source. Double check that all packages have been found and installed. The following section assumes you require use of a proxy to download from remote repositories. If you do not, then you may safely ignore all references to proxies in the following code block.
                                                                                     
                                                                                    # If you require the use of a proxy, add it to DNF using by editing the following file\n$ sudo nano /etc/dnf/dnf.conf\n# Include your proxy by adding the following line, replacing the URL with your proxy's URL\n# proxy=http://proxy.server.com:port``\n$ sudo dnf update\n$ subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms\n$ sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n\n$ sudo dnf install -y python3 python3-pip python3-devel \\\ngdb vim git gcc gcc-c++ make cmake libuuid-devel rpm-build systemd-devel nmap \\\npython3-jsonschema json-c-devel tbb-devel rpmdevtools libcap-devel \\\npython3-pyyaml hwloc-devel libedit-devel git kernel-headers kernel-devel elfutils-libelf-devel ncurses-devel openssl-devel bison flex cli11-devel spdlog-devel numactl-devel\n\n$ python3 -m pip install --user jsonschema virtualenv pudb pyyaml\n\n$ sudo pip3 uninstall setuptools\n\n$ sudo pip3 install Pybind11==2.10.0 --proxy http://yourproxy:xxx\n\n$ sudo pip3 install setuptools==59.6.0 --prefix=/usr --proxy http://yourproxy:xxx\n\n$ wget http://ftp.pbone.net/mirror/archive.fedoraproject.org/epel/8.4/Everything/x86_64/Packages/p/pybind11-devel-2.4.3-2.el8.x86_64.rpm\n\n$ wget http://ftp.pbone.net/mirror/archive.fedoraproject.org/epel/8.4/Everything/x86_64/Packages/p/python3-pybind11-2.4.3-2.el8.x86_64.rpm\n\n$ sudo dnf localinstall ./python3-pybind11-2.4.3-2.el8.x86_64.rpm ./pybind11-devel-2.4.3-2.el8.x86_64.rpm -y\n
                                                                                     
                                                                                    It is recommended you create an empty top level directory for your OFS related repositories to keep the working environment clean. All steps in this installation will use a generic top-level directory at /home/OFS/. If you have created a different top-level directory, replace this path with your custom path.
                                                                                     
                                                                                    3. Initialize an empty git repository and clone the LTS tagged DFL driver source code:
                                                                                     
                                                                                    $ mkdir /home/OFS/\n$ cd /home/OFS/\n$ git init\n$ git clone https://github.com/OFS/linux-dfl\n$ cd /home/OFS/linux-dfl\n$ git checkout tags/ofs-2023.3-6.1-3\n
                                                                                     
                                                                                    4. Verify that the correct tag/branch have been checkout out.
                                                                                     
                                                                                    $ git describe --tags\nofs-2023.3-6.1-3\n
                                                                                     
                                                                                    Note: If two different tagged releases are tied to the same commit, running git describe tags may report the other release's tag. This is why the match is made explicit.
                                                                                     
                                                                                    Note: The linux-dfl repository is roughly 5 GB in size.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#32-building-and-installing-the-ofs-dfl-kernel-drivers-from-source","title":"3.2 Building and Installing the OFS DFL Kernel Drivers from Source","text":"
                                                                                    1. The following set of instructions walk you through copying an existing kernel configuration file on your machine and changing the minimal required configuration settings.:
                                                                                     
                                                                                    $ cd /home/OFS/linux-dfl\n$ cp /boot/config-`uname -r` .config\n$ cat configs/dfl-config >> .config\n$ echo 'CONFIG_LOCALVERSION=\"-dfl\"' >> .config\n$ echo 'CONFIG_LOCALVERSION_AUTO=y' >> .config\n$ sed -i -r 's/CONFIG_SYSTEM_TRUSTED_KEYS=.*/CONFIG_SYSTEM_TRUSTED_KEYS=\"\"/' .config\n$ sed -i '/^CONFIG_DEBUG_INFO_BTF/ s/./#&/' .config\n$ echo 'CONFIG_DEBUG_ATOMIC_SLEEP=y' >> .config\n$ export LOCALVERSION=\n$ make olddefconfig\n
                                                                                     
                                                                                    1.1. The above command may report errors resembling symbol value 'm' invalid for CHELSIO_IPSEC_INLINE. These errors indicate that the nature of the config has changed between the currently executing kernel and the kernel being built. The option \"m\" for a particular kernel module is no longer a valid option, and the default behavior is to simply turn the option off. However the option can likely be turned back on by setting it to 'y'. If the user wants to turn the option back on, change it to 'y' and re-run \"make olddefconfig\":
                                                                                     
                                                                                    $ cd /home/OFS/linux-dfl\n$ echo 'CONFIG_CHELSIO_IPSEC_INLINE=y' >> .config\n$ make olddefconfig\n
                                                                                     
                                                                                    (Optional) To use the built-in GUI menu for editing kernel configuration parameters, you can opt to run make menuconfig.
                                                                                     
                                                                                    2. Linux kernel builds take advantage of multiple processors to parallelize the build process. Display how many processors are available with the nproc command, and then specify how many make threads to utilize with the -j option. Note that number of threads can exceed the number of processors. In this case, the number of threads are set to the number of processors in the system.
                                                                                     
                                                                                    $ cd /home/OFS/linux-dfl\n$ make -j $(nproc)\n
                                                                                     
                                                                                    3. The following options are available to build a set of packages. The package options for this flow as as follows:
                                                                                     
                                                                                       
                                                                                    • rpm-pkg: Build both source and binary RPM kernel packages
                                                                                      •  
                                                                                    • binrpm-pkg: Build only the binary kernel RPM package
                                                                                      •  
                                                                                    • deb-pkg: Build both source and binary deb kernel packages
                                                                                      •  
                                                                                    • bindeb-pkg: Build only the binary kernel deb package
                                                                                      •  
                                                                                     
                                                                                    If you are concerned about the size of the resulting package and binaries, they can significantly reduce the size of the package and object files by using the make variable INSTALL_MOD_STRIP. If this is not a concern, feel free to skip this step. The below instructions will build a set of binary RPM packages:
                                                                                     
                                                                                    cd /home/OFS/linux-dfl\nmake INSTALL_MOD_STRIP=1 binrpm-pkg\n
                                                                                     
                                                                                    4. By default a directory is created in your home directory called rpmbuild. This directory will house all of the kernel packages which have been built. You need to navigate to the newly built kernel packages and install them. The following files were generated using the build command executed in the previous step:
                                                                                     
                                                                                    $ cd ~/rpmbuild/RPMS/x86_64\n$ ls\nkernel-6.1.41_dfl.x86_64.rpm  kernel-headers-6.1.41_dfl.x86_64.rpm\n$ sudo dnf localinstall kernel*.rpm\n
                                                                                     
                                                                                    5. The system will need to be rebooted in order for changes to take affect. After a reboot, select the newly built kernel as the boot target. This can be done pre-boot using the command grub2-reboot, which removes the requirement for user intervention. After boot, verify that the currently running kernel matches expectation.
                                                                                     
                                                                                    $ uname -r\n6.1.41-dfl\n
                                                                                     
                                                                                    6. Verify the DFL drivers have been successfully installed by reading version information directly from /lib/modules. Recall that the name of the kernel built as apart of this section is 6.1.41-dfl. If the user set a different name for their kernel, change this path as needed:
                                                                                     
                                                                                    $ cd /usr/lib/modules/6.1.41-dfl/kernel/drivers/fpga\n$ ls\ndfl-afu.ko     dfl-fme.ko      dfl-fme-region.ko  dfl.ko             dfl-pci.ko      fpga-mgr.ko     intel-m10-bmc-sec-update.ko\ndfl-fme-br.ko  dfl-fme-mgr.ko  dfl-hssi.ko        dfl-n3000-nios.ko  fpga-bridge.ko  fpga-region.ko\n
                                                                                     
                                                                                    If an N6000/1-PL FPGA SmartNIC Platform is present on the server, you can also double check driver versions using the lsmod command:
                                                                                     
                                                                                    $ lsmod | grep dfl\nuio_dfl                20480  0\ndfl_emif               16384  0\nuio                    20480  1 uio_dfl\nptp_dfl_tod            16384  0\ndfl_intel_s10_iopll    20480  0\n8250_dfl               20480  0\ndfl_fme_region         20480  0\ndfl_fme_br             16384  0\ndfl_fme_mgr            20480  2\ndfl_fme                49152  0\ndfl_afu                36864  0\ndfl_pci                20480  0\ndfl                    40960  11 dfl_pci,uio_dfl,dfl_fme,intel_m10_bmc_pmci,dfl_fme_br,8250_dfl,qsfp_mem,ptp_dfl_tod,dfl_afu,dfl_intel_s10_iopll,dfl_emif\nfpga_region            20480  3 dfl_fme_region,dfl_fme,dfl\nfpga_bridge            20480  4 dfl_fme_region,fpga_region,dfl_fme,dfl_fme_br\nfpga_mgr               20480  4 dfl_fme_region,fpga_region,dfl_fme_mgr,dfl_fme\n
                                                                                     
                                                                                    Refer to the following table for a brief description of each driver:
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-11-dfl-driver-modules","title":"Table 11: DFL Driver Modules","text":"Module Name Short Description uio_dfl Generic DFL driver for Userspace I/O devices dfl_intel_s10_iopll DFL Intel S10 IOPLL driver uio UIO Driver dfl_fme_region FPGA Region for DFL FPGA Management Engine ptp_dfl_tod DFL ToD driver dfl_emif DFL EMIF driver dfl_fme_br FPGA Bridge for DFL FPGA Management Engine 8250_dfl DFL Intel UART driver dfl_fme_mgr FPGA Manager for DFL FPGA Management Engine dfl_fme FPGA Management Engine driver dfl_afu FPGA Accelerated Function Unit driver dfl_pci FPGA DFL PCIe Device Driver dfl FPGA Device Feature List (DFL) Support fpga_region FPGA Region Driver fpga_bridge FPGA Bridge Driver fpga_mgr FPGA manager framework 
                                                                                    7. Two kernel parameters must be added to the boot commandline for the newly installed kernel. First, open the file grub:
                                                                                     
                                                                                    $ sudo vim /etc/default/grub\n
                                                                                     
                                                                                    8. In the variable GRUB_CMDLINE_LINUX add the following parameters in bold: GRUB_CMDLINE_LINUX=\"crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200\"
                                                                                     
                                                                                    Note: If you wish to instead set hugepages on a per session bassis, you can perform the following steps. These settings will be lost on reboot.
                                                                                     
                                                                                    $ mkdir -p /mnt/huge \n$ mount -t hugetlbfs nodev /mnt/huge \n$ echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages \n$ echo 2048 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages \n
                                                                                     
                                                                                    9. Save your edits, then apply them to the GRUB2 configuration file.
                                                                                     
                                                                                    $ sudo grub2-mkconfig\n
                                                                                     
                                                                                    10. Warm reboot. Your kernel parameter changes should have taken affect.
                                                                                     
                                                                                    $ cat /proc/cmdline\nBOOT_IMAGE=(hd1,gpt2)/vmlinuz-6.1.41-dfl root=/dev/mapper/cl-root ro crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap intel_iommu=on pcie=realloc hugepagesz=2M hugepages=200 rhgb quiet\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#33-installing-the-ofs-dfl-kernel-drivers-from-pre-built-packages","title":"3.3 Installing the OFS DFL Kernel Drivers from Pre-Built Packages","text":"
                                                                                    1. Make the following changes on your installation machine to satisfy all dependencies:
                                                                                     
                                                                                    $ subscription-manager release --set=8.6\n$ sudo dnf update\n
                                                                                     
                                                                                    2. To use the pre-built Linux DFL packages, the user will need to download the files from the OFS 2023.3-2 Release Page. You can choose to either install using the SRC RPMs, or to use the pre-built RPM packages targeting the official supported release platform.
                                                                                     
                                                                                    $ tar xf kernel-6.1.41_dfl-1.x86_64-<<version>>.tar.gz\n$ sudo dnf localinstall kernel-6.1.41_dfl_<<version>>.x86_64.rpm \\\nkernel-devel-6.1.41_dfl_<<version>>.x86_64.rpm \\\nkernel-headers-6.1.41_dfl_<<version>>.x86_64.rpm\n### OR\n$ sudo dnf localinstall kernel-6.1.41_dfl_<<version>>.src.rpm\n
                                                                                     
                                                                                    3. After installation has completed you should continue with steps 4-9 in previous section 3.2 Building and Installing the OFS DFL Kernel Drivers from Source.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#40-opae-software-development-kit","title":"4.0 OPAE Software Development Kit","text":"
                                                                                    The OPAE SDK software stack sits in user space on top of the OFS kernel drivers. It is a common software infrastructure layer that simplifies and streamlines integration of programmable accelerators such as FPGAs into software applications and environments. OPAE consists of a set of drivers, user-space libraries, and tools to discover, enumerate, share, query, access, manipulate, and reconfigure programmable accelerators. OPAE is designed to support a layered, common programming model across different platforms and devices. To learn more about OPAE, its documentation, code samples, an explanation of the available tools, and an overview of the software architecture, visit the opae reference page.
                                                                                     
                                                                                    The OPAE SDK source code is contained within a single GitHub repository hosted at the OPAE Github. This repository is open source and does not require any permissions to access. You have two options to install OPAE as discussed below - using pre-built packages offered by Intel, or building the source code locally.
                                                                                     
                                                                                    The OPAE SDK can be automatically installed using a supplied Python 3 installation script. This script ships with a README detailing execution instructions on the OFS 2023.3-2 Release Page.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#41-opae-sdk-build-environment-setup","title":"4.1 OPAE SDK Build Environment Setup","text":"
                                                                                    This installation process assumes the you have access to an internet connection in order to pull specific GitHub repositories, and to satisfy package dependencies.
                                                                                     
                                                                                    1. Before OPAE SDK installation you must remove any prior OPAE frameworks.  To remove these packages:
                                                                                     
                                                                                    sudo dnf remove opae*\n
                                                                                     
                                                                                    2. The following repository changes must be enabled in order to install all dependencies:
                                                                                     
                                                                                    $ subscription-manager release --set=8.6\n$ sudo dnf update\n$ subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms\n$ sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n
                                                                                     
                                                                                    It is recommended you create an empty top level directory for their OFS related repositories to keep the working environment clean. All steps in this installation will use a generic top-level directory at /home/OFS/. If the you have created a different top-level directory, replace this path with your custom path.
                                                                                     
                                                                                    3. Initialize an empty git repository and clone the tagged OPAE SDK source code:
                                                                                     
                                                                                    $ cd /home/OFS/\n$ git init\n$ git clone https://github.com/OFS/opae-sdk opae-sdk\n$ cd /home/OFS/opae-sdk\n$ git checkout tags/2.10.0-1\n
                                                                                     
                                                                                    5. Verify that the correct tag/branch have been checkout out.
                                                                                     
                                                                                    $ git describe --tags\n2.10.0-1\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#42-installing-the-opae-sdk-with-pre-built-packages","title":"4.2 Installing the OPAE SDK with Pre-Built Packages","text":"
                                                                                    You can skip the entire build process and use a set of pre-built binaries supplied by Intel. Visit the OFS ofs-2023.3-2 Release Page and navigate to the bottom of the page, under the Assets tab you will see a file named opae-2.10.0-1.x86_64-<>_<>.tar.gz. Download this package and extract its contents: 
                                                                                    $ dnf install --enablerepo=codeready-builder-for-rhel-8-x86_64-rpms -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel\n\n$ pip3 install --upgrade --prefix=/usr pip setuptools pybind11\n\n$ tar xf opae-2.10.0-1.x86_64-<<date>>_<<build>>.tar.gz\n
                                                                                     
                                                                                    For a fast installation you can delete the source RPM as it isn't necessary, and install all remaining OPAE RPMs:
                                                                                     
                                                                                    $ rm opae-*.src.rpm\n$ sudo dnf localinstall opae*.rpm\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#43-building-and-installing-the-opae-sdk-by-script","title":"4.3 Building and Installing the OPAE SDK by Script","text":"
                                                                                    1. Build the OPAE SDK source code, and pack it into several local RPM packages. Building the code into packages allows for easier installation and removal. This build script can take advantage of multiple processors to parallelize the build process. Display how many processors are available with the $(nproc) command, and then specify how many make threads to utilize with the -j option. Note that number of threads can exceed the number of processors. In this case, the number of threads are set to the number of processors in the system.
                                                                                     
                                                                                    cd /home/OFS/\n\n$ podman pull registry.access.redhat.com/ubi8:8.6\n$ podman run -ti -v \"$PWD\":/src:Z -w /src registry.access.redhat.com/ubi8:8.6\n\n# Everything after runs within container:\n\n# Enable EPEL\n$ dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n\n$ dnf install --enablerepo=codeready-builder-for-rhel-8-x86_64-rpms -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel doxygen python3-sphinx pandoc rpm-build rpmdevtools python3-virtualenv yaml-cpp-devel libudev-devel libcap-devel numactl-devel\n\n$ pip3 install --upgrade --prefix=/usr pip setuptools pybind11\n\n$ ./opae-sdk/packaging/opae/rpm/create unrestricted\n\n$ exit\n
                                                                                     
                                                                                    2. After a successful compile there should be packages generated:
                                                                                     
                                                                                    The below table lists a short description for each package:
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-12-opae-package-description","title":"Table 12: OPAE Package Description","text":"Package Name Description opae OPAE SDK is a collection of libraries and tools to facilitate the development of software applications and accelerators using OPAE. It provides a library implementing the OPAE C API for presenting a streamlined and easy-to-use interface for software applications to discover, access, and manage FPGA devices and accelerators using the OPAE software stack. opae-debuginfo This package provides debug information for package opae. Debug information is useful when developing applications that use this package or when debugging this package. opae-debugsource This package provides debug sources for package opae. Debug sources are useful when developing applications that use this package or when debugging this package. opae-devel OPAE headers, tools, sample source, and documentation opae-devel-debuginfo This package provides debug information for package opae-devel. Debug information is useful when developing applications that use this package or when debugging this package. opae-tools This package contains OPAE base tools binaries opae-extra-tools Additional OPAE tools opae-extra-tools-debuginfo This package provides debug information for package opae-extra-tools. Debug information is useful when developing applications that use this package or when debugging this package. 
                                                                                    3. Install the OPAE SDK packages:
                                                                                     
                                                                                    $ cd /home/OFS/opae-sdk/packaging/opae/rpm\n$ rm -rf opae-2.10.0-1.el8.src.rpm \n$ sudo dnf localinstall -y opae*.rpm\n
                                                                                     
                                                                                    4. Check that all packages have been installed:
                                                                                     
                                                                                    $ rpm -qa | grep opae\nopae-packager-2.10.0-1.x86_64\nopae-devel-2.10.0-1.x86_64\nopae-PACSign-2.10.0-1.x86_64\nopae-tools-extra-2.10.0-1.x86_64\nopae-2.10.0-1.x86_64\nopae-tools-2.10.0-1.x86_64\nopae-libs-2.10.0-1.x86_64\nopae-opae.admin-2.10.0-1.x86_64\nopae-tests-2.10.0-1.x86_64\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#44-fpga-device-access-permissions","title":"4.4 FPGA Device Access Permissions","text":"
                                                                                    Access to FPGA accelerators and devices is controlled using file access permissions on the Intel\u00ae FPGA device files, /dev/dfl-fme.* and /dev/dfl-port.*, as well as to the files reachable through /sys/class/fpga_region/.
                                                                                     
                                                                                    In order to allow regular (non-root) users to access accelerators, you need to grant them read and write permissions on /dev/dfl-port.* (with * denoting the respective socket, i.e. 0 or 1). E.g.:
                                                                                     
                                                                                    sudo chmod a+rw /dev/dfl-port.0\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#45-memlock-limit","title":"4.5 Memlock limit","text":"
                                                                                    Depending on the requirements of your application, you may also want to increase the maximum amount of memory a user process is allowed to lock. The exact way to do this depends on your Linux distribution.
                                                                                     
                                                                                    You can check the current memlock limit using
                                                                                     
                                                                                    ulimit -l\n
                                                                                     
                                                                                    A way to permanently remove the limit for locked memory for a regular user is to add the following lines to your /etc/security/limits.conf:
                                                                                     
                                                                                    user1    hard   memlock           unlimited\nuser1    soft   memlock           unlimited\n
                                                                                     
                                                                                    This removes the limit on locked memory for user user1. To remove it for all users, you can replace user1 with *:
                                                                                     
                                                                                    *    hard   memlock           unlimited\n*    soft   memlock           unlimited\n
                                                                                     
                                                                                    Note that settings in the /etc/security/limits.conf file don't apply to services. To increase the locked memory limit for a service you need to modify the application's systemd service file and add the line:
                                                                                     
                                                                                    [Service]\nLimitMEMLOCK=infinity\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#46-opae-tools-overview","title":"4.6 OPAE Tools Overview","text":"
                                                                                    The following section offers a brief introduction including expected output values for the utilities included with OPAE. A full explanation of each command with a description of its syntax is available in the opae-sdk GitHub repo.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#461-board-management-with-fpgainfo","title":"4.6.1 Board Management with fpgainfo","text":"
                                                                                    The fpgainfo utility displays FPGA information derived from sysfs files.
                                                                                     
                                                                                    Displays FPGA information derived from sysfs files. The command argument is one of the following: errors, power, temp, port, fme, bmc, phy or mac, security. Some commands may also have other arguments or options that control their behavior.
                                                                                     
                                                                                    For systems with multiple FPGA devices, you can specify the BDF to limit the output to the FPGA resource with the corresponding PCIe configuration. If not specified, information displays for all resources for the given command.
                                                                                     
                                                                                    Note: Your BItstream ID and PR Interface Id may not match the below examples.
                                                                                     
                                                                                    The following examples walk through sample outputs generated by fpgainfo.
                                                                                     
                                                                                    $ sudo fpgainfo fme\n\nIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : a2b5fd0e7afca4ee6d7048f926e75ac2\nUser1 Image Info                 : 1d6beb4e-86d7-5442-a763-043701fb75b7\nUser2 Image Info                 : 1d6beb4e-86d7-5442-a763-043701fb75b7\n
                                                                                     
                                                                                    $ sudo fpgainfo bmc\n\nIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\n( 1) VCCRT_GXER_0V9 Voltage                             : 0.91 Volts\n( 2) FPGA VCCIO_1V2 Voltage                             : 1.21 Volts\n( 3) Inlet 12V Aux Rail Current                         : 0.87 Amps\n( 4) FPGA E-Tile Temperature [Remote]                   : 47.00 Celsius\n( 5) AVDD_ETH_0V9_CVL Voltage                           : 1.48 Volts\n( 6) FPGA E-TILE Temperature #3                         : 51.00 Celsius\n...\n(77) FPGA FABRIC Remote Digital Temperature#3           : 47.00 Celsius\n(78) MAX10 & Board CLK PWR 3V3 Inlet Current            : 0.97 Amps\n(79) CVL Non Core Rails Inlet Current                   : 0.01 Amps\n(80) FPGA Core Voltage Phase 0 VR Temperature           : 49.50 Celsius\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#462-sensor-monitoring-with-fpgad","title":"4.6.2 Sensor Monitoring with fpgad","text":"
                                                                                    The fpgad is a service that can help you protect the server from crashing when the hardware reaches an upper non-recoverable or lower non-recoverable sensor threshold (also called as fatal threshold). The fpgad is capable of monitoring each of the 80 sensors reported by the Board Management Controller. This service is only available once the installation instructions in sections 3.2 Building and Installing the OFS DFL Kernel Drivers and 4.1 OPAE SDK Build Environment Setup have been completed . Note: Qualified OEM server systems should provide the required cooling for your workloads. Therefore, using fpgad may be optional.
                                                                                     
                                                                                    When the opae-tools-extra-2.10.0-1.x86_64  package is installed, fpgad is placed in the OPAE binaries directory (default: /usr/bin). The configuration file fpgad.cfg is located at /etc/opae. The log file fpgad.log which monitors fpgad actions is located at /var/lib/opae/. The fpgad periodically reads the sensor values and if the values exceed the warning threshold stated in the fpgad.conf or the hardware defined warning threshold, it masks the PCIe Advanced Error Reporting (AER) registers for the Intel N6000/1-PL FPGA SmartNIC Platform to avoid system reset. Use the following command to start the fpgad service:
                                                                                     
                                                                                    Use the following command to start the fpgad service:
                                                                                     
                                                                                    $ sudo systemctl start fpgad\n
                                                                                     The configuration file only includes the threshold setting for critical sensor 12V Aux Rail Voltage (sensor 29). This sensor does not have a hardware defined warning threshold and hence fpgad relies on the configuration file. The fpgad uses information contained within this file to mask the PCIe AER register when the sensor reaches the warning threshold.
                                                                                     
                                                                                    You may create another entry below the 12V Aux Voltage entry for any other sensors on the board. The updated configuration file includes a new entry for (18) Board Front Side Temperature with arbitrary values:
                                                                                     
                                                                                    {\n\"configurations\": {\n    \"fpgad-xfpga\": {\n        \"configuration\": {\n        },\n        \"enabled\": true,\n        \"plugin\": \"libfpgad-xfpga.so\",\n        \"devices\": [\n            [ \"0x8086\", \"0xbcc0\" ],\n            [ \"0x8086\", \"0xbcc1\" ]\n        ]\n    },\n    \"fpgad-vc\": {\n        \"configuration\": {\n            \"cool-down\": 30,\n            \"get-aer\": [\n                \"setpci -s %s ECAP_AER+0x08.L\",\n                \"setpci -s %s ECAP_AER+0x14.L\"\n            ],\n            \"disable-aer\": [\n                \"setpci -s %s ECAP_AER+0x08.L=0xffffffff\",\n                \"setpci -s %s ECAP_AER+0x14.L=0xffffffff\"\n            ],\n            \"set-aer\": [\n                \"setpci -s %s ECAP_AER+0x08.L=0x%08x\",\n                \"setpci -s %s ECAP_AER+0x14.L=0x%08x\"\n            ],\n            \"config-sensors-enabled\": true,\n            \"sensors\": [\n                {\n                    \"name\": \"12V AUX Voltage\",\n                    \"low-warn\": 11.40,\n                    \"low-fatal\": 10.56\n                },\n                {\n                    \u201cname\u201d: \u201c3V3 VR Temperature\u201d,\n                    \u201clow-warn\u201d: 50.00,\n                    \u201clow-fatal\u201d: 100.00\n\n                }\n            ]\n        },\n        \"enabled\": true,\n        \"plugin\": \"libfpgad-vc.so\",\n        \"devices\": [\n            [ \"0x8086\", \"0x0b30\" ],\n            [ \"0x8086\", \"0x0b31\" ],\n            [ \"0x8086\", \"0xaf00\" ],\n            [ \"0x8086\", \"0xbcce\" ]\n            ]\n    }\n},\n\"plugins\": [\n    \"fpgad-xfpga\",\n    \"fpgad-vc\"\n]\n}\n
                                                                                     
                                                                                    You can monitor the log file to see if upper or lower warning threshold levels are hit. For example:
                                                                                     
                                                                                    $ tail -f /var/lib/opae/fpgad.log | grep \u201csensor.*warning\u201d\nfpgad-vc: sensor ' Columbiaville Die Temperature ' warning\n
                                                                                     
                                                                                    You must take appropriate action to recover from this warning before the sensor value reaches upper or lower fatal limits. On reaching the warning threshold limit, the daemon masks the AER registers and the log file will indicate that the sensor is tripped. Sample output: Warning message when the 'CVL Core0 Voltage VR Temperature' exceeds the upper warning threshold limit
                                                                                     
                                                                                    $ tail -f /var/lib/opae/fpgad.log \nfpgad-vc: sensor 'CVL Core Voltage VR Temperature' warning.\nfpgad-vc: saving previous ECAP_AER+0x08 value 0x057ff030 for 0000:b0:02.0\nfpgad-vc: saving previous ECAP_AER+0x14 value 0x0000f1c1 for 0000:b0:02.0\nfpgad-vc: sensor 'CVL Core Voltage VR Temperature' still tripped.\n
                                                                                     
                                                                                    If the upper or lower fatal threshold limit is reached, then a power cycle of server is required to recover the Intel N6001-PL SmartNIC FPGA Platform. AER is unmasked by the fpgad after the sensor values are within the normal range which is above the lower warning or below the upper warning threshold.
                                                                                     
                                                                                    To stop fpgad:
                                                                                     
                                                                                    $ sudo systemctl stop fpgad.service\n
                                                                                     
                                                                                    To check status of fpgad:
                                                                                     
                                                                                    $ sudo systemctl status fpgad.service\n
                                                                                     
                                                                                    Optional: To enable fpgad to re-start on boot, execute
                                                                                     
                                                                                    $ sudo systemctl enable fpgad.service\n
                                                                                     
                                                                                    For a full list of systemctl commands, run the following command:
                                                                                     
                                                                                    $ systemctl -h\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#463-updating-with-fpgasupdate","title":"4.6.3 Updating with fpgasupdate","text":"
                                                                                    The fpgasupdate tool updates the Intel Max10 Board Management Controller (BMC) image and firmware (FW), root entry hash, and FPGA Static Region (SR) and user image (PR). The fpgasupdate tool only accepts images that have been formatted using PACsign. If a root entry hash has been programmed onto the board, then you must also sign the image using the correct keys. Refer to the Security User Guide: Intel Open FPGA Stack for information on created signed images and on programming and managing the root entry hash.  
                                                                                     
                                                                                    The Intel\u00ae FPGA SmartNIC N6001-PL ships with a factory, user1, and user2 programmed image for both the FIM and BMC FW and RTL on all cards. The platform ships with a single FIM image that can be programmed into either user1 or user2, depending in the image selected.
                                                                                     
                                                                                    Use the following chart for information on the Bitstream ID and Pr Interface ID, two unique values reported by fpgainfo which can be used to identify the loaded FIM.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-13-fim-version-summary-for-ofs-20233-2-release","title":"Table 13: FIM Version Summary for OFS 2023.3-2 Release","text":"FIM Version Bitstream ID Pr Interface ID File Name Download Location ofs-2023.3-2 00x50102023508A422 1d6beb4e-86d7-5442-a763-043701fb75b7 ofs_top_page[\u00bd]_unsigned_user[\u00bd].bin ofs-2023.3-2 Release Page ofs-n6001-0.9.0-rc2 0x50102025AD3DD11 92ec8960-2f2f-5544-9804-075d2e8a71a1 ofs_top_page[\u00bd]_unsigned_user[\u00bd].bin ofs-2.3.0 Release Page OFS-2.3.0 0x50102022267A9ED f59830f7-e716-5369-a8b0-e7ea897cbf82 ofs_top_page[\u00bd]_unsigned_user[\u00bd].bin ofs-2.3.0 Release Page OFS-2.2.0 0x501020295B081F0 8c157a52-1cf2-5d37-9514-944af0a060da ofs_top_page[\u00bd]_unsigned_user[\u00bd].bin ofs-2.2.0-beta Release Page"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-14-bmc-version-summary-for-ofs-20233-2-release","title":"Table 14: BMC Version Summary for OFS 2023.3-2 Release","text":"BMC FW and RTL Version File Name Download Location 3.15.0 AC_BMC_RSU_user_retail_3.15.0_unsigned.rsu n/a 
                                                                                       
                                                                                    1.  
                                                                                      Example loading a new version of the BMC RTL and FW.
                                                                                       
                                                                                      $ sudo fpgasupdate AC_BMC_RSU_user_retail_3.11.0_unsigned.rsu <PCI ADDRESS>\n[2022-04-14 16:32:47.93] [WARNING ] Update starting. Please do not interrupt.                                           \n[2022-04-14 16:32:47.93] [INFO    ] updating from file /home/user/AC_BMC_RSU_user_retail_3.11.0_unsigned.rsu with size 904064                                   \n[2022-04-14 16:32:47.94] [INFO    ] waiting for idle                                                                    \n[2022-04-14 16:32:47.94] [INFO    ] preparing image file                                                                \n[2022-04-14 16:33:26.98] [INFO    ] writing image file                                                                  \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [904064/904064 bytes][Elapsed Time: 0:00:00.00]                                           \n[2022-04-14 16:33:26.98] [INFO    ] programming image file                                                              \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [Elapsed Time: 0:00:26.02]                                                                 \n[2022-04-14 16:33:53.01] [INFO    ] update of 0000:b1:00.0 complete                                                     \n[2022-04-14 16:33:53.01] [INFO    ] Secure update OK                                                                    \n[2022-04-14 16:33:53.01] [INFO    ] Total time: 0:01:05.07\nsudo rsu bmcimg\n
                                                                                       
                                                                                      2.  
                                                                                    2.  
                                                                                      Example for loading a Static Region (SR) update image. This process will take up to 20 minutes.
                                                                                       
                                                                                      3.  
                                                                                     
                                                                                    $ sudo fpgasupdate ofs_top_page1_unsigned_user1.bin <PCI ADDRESS>\n[2022-04-14 16:42:31.58] [WARNING ] Update starting. Please do not interrupt.                                           \n[2022-04-14 16:42:31.58] [INFO    ] updating from file ofs_top_page1_pacsign_user1.bin with size 19928064               \n[2022-04-14 16:42:31.60] [INFO    ] waiting for idle                                                                    \n[2022-04-14 16:42:31.60] [INFO    ] preparing image file                                                                \n[2022-04-14 16:42:38.61] [INFO    ] writing image file                                                                  \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [19928064/19928064 bytes][Elapsed Time: 0:00:16.01]                                       \n[2022-04-14 16:42:54.63] [INFO    ] programming image file                                                              \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588][Elapsed Time: 0:06:16.40]                                                                 \n[2022-04-14 16:49:11.03] [INFO    ] update of 0000:b1:00.0 complete                                                     \n[2022-04-14 16:49:11.03] [INFO    ] Secure update OK                                                                    \n[2022-04-14 16:49:11.03] [INFO    ] Total time: 0:06:39.45\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#464-signing-images-with-pacsign","title":"4.6.4 Signing Images with PACSign","text":"
                                                                                    PACSign is an OPAE utility which allows users to insert authentication markers into bitstreams targeted for the Intel\u00ae FPGA SmartNIC N6001-PL. PACSign also allows users updating their Static Region (SR) to designate which partition of flash (user1, user2, factory) to overwrite given a specific FIM binary image. All binary images must be signed using PACSign before fpgasupdate can use them for an update. Assuming no Root Entry Hash (REH) has been programmed on the device, the following examples demonstrate how to prepend the required secury authentication data, and specifiy which region of flash to update. More information, including charts detailing the different certification types and their required options, are fully described in the PACsign README on GitHub.
                                                                                     
                                                                                    For more information on PACSign and on general security practices surrounding the Intel N6001-PL FPGA SmartNIC device, visit the Security User Guide: Intel Open FPGA Stack.
                                                                                     
                                                                                    PACSign can be run on images that have previously been signed. It will overwrite any existing authentication data.
                                                                                     
                                                                                    The following example creates an unsigned SR image from an existing signed SR binary update image, targeting the user1 partition in flash.
                                                                                     
                                                                                    $ PACSign SR -t UPDATE -s 0 -H openssl_manager -i ofs_top_page1_pacsign_user1.bin -o new_image.bin\nNo root key specified.  Generate unsigned bitstream? Y = yes, N = no: y\nNo CSK specified.  Generate unsigned bitstream? Y = yes, N = no: y\nNo root entry hash bitstream specified.  Verification will not be done.  Continue? Y = yes, N = no: y\n2021-10-18 14:42:54,490 - PACSign.log - WARNING - Bitstream is already signed - removing signature blocks\n
                                                                                     
                                                                                    --->
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#465-loading-images-with-rsu","title":"4.6.5 Loading Images with rsu","text":"
                                                                                    The rsu performs a Remote System Update operation on a N6000/1-PL device, given its PCIe address. An rsu operation sends an instruction to the device to trigger a power cycle of the card and forces reconfiguration from flash for either the BMC or FPGA image.
                                                                                     
                                                                                    The Intel\u00ae FPGA SmartNIC N6001-PL contains two regions of flash you may store FIM images. These locations are referred to as user1 and user2. After an image has been programmed with fpgasupdate in either of these regions you may choose to perform an rsu to switch. This operation indicates to the BMC which region to configure the FPGA device from after power-on.
                                                                                     
                                                                                    If the factory image has been updated, Intel strongly recommends the user to immediately RSU to the factory image to ensure the image is functional.
                                                                                     
                                                                                    The user can determine which region of flash was used to configure their FPGA device using the command fpgainfo fme and referring to the row labelled Boot Page.
                                                                                     
                                                                                    $ sudo fpgainfo fme | grep Boot\nBoot Page                        : user1\n
                                                                                     
                                                                                    Swapping between user1 and user2 skips load times that are created when using fpgasupdate to flash a new FIM image.
                                                                                     
                                                                                    rsu Overview
                                                                                     
                                                                                    Mode 1: RSU
                                                                                     
                                                                                    Perform RSU (remote system update) operation on a development platform given its PCIe address. An RSU operation sends an instruction to the device to trigger a power cycle of the card only. This will force reconfiguration from flash for either BMC, Retimer, SDM, (on devices that support these) or the FPGA.
                                                                                     
                                                                                    Mode 2: Default FPGA Image
                                                                                     
                                                                                    Set the default FPGA boot sequence. The --page option determines the primary FPGA boot image. The --fallback option allows a comma-separated list of values to specify fallback images.
                                                                                     
                                                                                    The following example will load an image stored in user2.
                                                                                     
                                                                                    $ sudo rsu fpga --page=user2 0000:b1:00.0\n2022-04-15 09:25:22,951 - [[pci_address(0000:b1:00.0), pci_id(0x8086, 0xbcce)]] performing RSU operation\n2022-04-15 09:25:22,955 - [[pci_address(0000:b0:02.0), pci_id(0x8086, 0x347a)]] removing device from PCIe bus\n2022-04-15 09:25:22,998 - waiting 10 seconds for boot\n2022-04-15 09:25:33,009 - rescanning PCIe bus: /sys/devices/pci0000:b0/pci_bus/0000:b0\n2022-04-15 09:25:34,630 - RSU operation complete\n
                                                                                     
                                                                                    Note: As a result of using the rsu command, the host rescans the PCI bus and may assign a different Bus/Device/Function (B/D/F) value than the originally assigned value.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#466-verify-fme-interrupts-with-hello_events","title":"4.6.6 Verify FME Interrupts with hello_events","text":"
                                                                                    The hello_events utility is used to verify FME interrupts. This tool injects FME errors and waits for error interrupts, then clears the errors.
                                                                                     
                                                                                    Sample output from sudo hello_events.
                                                                                     
                                                                                    $ sudo hello_events\nWaiting for interrupts now...\ninjecting error\nFME Interrupt occurred\nSuccessfully tested Register/Unregister for FME events!\nclearing error\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#467-host-exercisor-modules","title":"4.6.7 Host Exercisor Modules","text":"
                                                                                    The reference FIM and unchanged FIM compilations contain Host Exerciser Modules (HEMs). These are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. There are three HEMs present in the Intel OFS Reference FIM - HE-LPBK, HE-MEM, and HE-HSSI. These exercisers are tied to three different VFs that must be enabled before they can be used. Execution of these exercisers requires you bind specific VF endpoint to vfio-pci. The host-side software looks for these endpoints to grab the correct FPGA resource.
                                                                                     
                                                                                    Refer to the Intel FPGA Interface Manager Technical Reference Manual: Intel Open FPGA Stack for Intel Agilex FPGA for a full description of these modules.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-15-module-pfvf-mappings","title":"Table 15: Module PF/VF Mappings","text":"Module PF/VF ST2MM PF0 HE-MEM PF0-VF0 HE-HSSI PF0-VF1 HE-MEM_TG PF0-VF2 HE-LB Stub PF1-VF0 HE-LB PF2 VirtIO LB Stub PF3 HPS Copy Engine PF4"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#4671-he-mem-he-lb","title":"4.6.7.1 HE-MEM / HE-LB","text":"
                                                                                    The host exerciser used to exercise and characterize the various host-FPGA interactions eg. MMIO, Data transfer from host to FPGA , PR, host to FPGA memory etc. Host Exerciser Loopback (HE-LBK) AFU can move data between host memory and FPGA.
                                                                                     
                                                                                    HE-LBK supports: - Latency (AFU to Host memory read) - MMIO latency (Write+Read) - MMIO BW (64B MMIO writes) - BW (Read/Write, Read only, Wr only)
                                                                                     
                                                                                    Host Exerciser Loopback Memory (HE-MEM) AFU is used to exercise use of FPGA connected DDR, data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host.
                                                                                     
                                                                                    HE-LB is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth. HE-MEM is used to exercise use of FPGA connected DDR; data read from the host is written to DDR, and the same data is read from DDR before sending it back to the host. HE-MEM uses external DDR memory (i.e. EMIF) to store data. It has a customized version of the AVMM interface to communicate with the EMIF memory controller. Both exercisers rely on the user-space tool host_exerciser. When using the Intel N6001-PL FPGA SmartNIC Platform, optimal performance requires the exercisers be run at 400 MHz.
                                                                                     
                                                                                    Execution of these exercisers requires you to bind specific VF endpoint to vfio-pci. The following commands will bind the correct endpoint for a device with B/D/F 0000:b1:00.0 and run through a basic loopback test.
                                                                                     
                                                                                    Note: While running the opae.io init command listed below, if no output is present after completion then the command has failed. Double check that Intel VT-D and IOMMU have been enabled in the kernel as discussed in step 12 in section 3.1 Intel OFS DFL Kernel Driver Environment Setup.
                                                                                     
                                                                                    $ sudo pci_device  0000:b1:00.0 vf 3\n$ sudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci                                                                  \nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci \niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188                                                                  \nAssigning /dev/vfio/188 to DCPsupport                                                                                  \nChanging permissions for /dev/vfio/188 to rw-rw----\nsudo host_exerciser --clock-mhz 400 lpbk\n    starting test run, count of 1\nAPI version: 1\nAFU clock from command line: 400 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 1024\n    Host Exerciser numWrites: 1025\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Number of clocks: 5224\n    Total number of Reads sent: 1024\n    Total number of Writes sent: 1022\n    Bandwidth: 5.018 GB/s\n    Test lpbk(1): PASS\n
                                                                                     
                                                                                    The following example will run a loopback throughput test using one cacheline per request.
                                                                                     
                                                                                    $ sudo pci_device  0000:b1:00.0 vf 3\n$ sudo opae.io init -d 0000:b1:00.2 user:user\n$ sudo host_exerciser --clock-mhz 400 --mode trput --cls cl_1 lpbk\n    starting test run, count of 1\nAPI version: 1\nAFU clock from command line: 400 MHz\nAllocate SRC Buffer\nAllocate DST Buffer\nAllocate DSM Buffer\n    Host Exerciser Performance Counter:\n    Host Exerciser numReads: 512\n    Host Exerciser numWrites: 513\n    Host Exerciser numPendReads: 0\n    Host Exerciser numPendWrites: 0\n    Number of clocks: 3517\n    Total number of Reads sent: 512\n    Total number of Writes sent: 512\n    Bandwidth: 7.454 GB/s\n    Test lpbk(1): PASS\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#4672-traffic-generator-afu-test-application","title":"4.6.7.2 Traffic Generator AFU Test Application","text":"
                                                                                    Beginning in OPAE version 2.0.11-1+ the TG AFU has an OPAE application to access & exercise traffic, targeting a specific bank. The supported arguments for test configuration are:
                                                                                     
                                                                                       
                                                                                    • Number of test loops: --loops
                                                                                      •  
                                                                                    • Number of read transfers per test loop: -r,--read
                                                                                      •  
                                                                                    • Number of write transfers per test loop: -w,--write
                                                                                      •  
                                                                                    • Burst size of each transfer: -b,--bls
                                                                                      •  
                                                                                    • Address stride between each transfer: --stride
                                                                                      •  
                                                                                    • Target memory TG: -m,--mem-channel
                                                                                      •  
                                                                                     
                                                                                    Below are some example commands for how to execute the test application. To run the preconfigured write/read traffic test on channel 0:
                                                                                     
                                                                                    $ mem_tg tg_test\n
                                                                                     
                                                                                    Target channel 1 with a 1MB single-word write only test for 1000 iterations
                                                                                     
                                                                                    $ mem_tg --loops 1000 -r 0 -w 2000 -m 1 tg_test\n
                                                                                     
                                                                                    Target channel 2 with 4MB write/read test of max burst length for 10 iterations
                                                                                     
                                                                                    $ mem_tg --loops 10 -r 8 -w 8 --bls 255 -m 2 tg_test\n
                                                                                     
                                                                                    $ sudo mem_tg --loops 1000 -r 2000 -w 2000 --stride 2 --bls 2  -m 1 tg_test\n[2022-07-15 00:13:16.349] [tg_test] [info] starting test run, count of 1\nMemory channel clock frequency unknown. Assuming 300 MHz.\nTG PASS\nMem Clock Cycles: 17565035\nWrite BW: 4.37232 GB/s\nRead BW: 4.37232 GB/s\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#4673-he-hssi","title":"4.6.7.3 HE-HSSI","text":"
                                                                                    HE-HSSI is responsible for handling client-side ethernet traffic. It wraps the 10G and 100G HSSI AFUs, and includes a traffic generator and checker. The user-space tool hssi exports a control interface to the HE-HSSI's AFU's packet generator logic.
                                                                                     
                                                                                    The hssi application provides a means of interacting with the 10G and with the 100G HSSI AFUs. In both 10G and 100G operating modes, the application initializes the AFU, completes the desired transfer as described by the mode- specific options, and displays the ethernet statistics by invoking ethtool --statistics INTERFACE.
                                                                                     
                                                                                    The following example walks through the process of binding the VF corresponding with the HE-HSSI exerciser to vfio-pci, sending traffic, and verifying that traffic was received.
                                                                                     
                                                                                    1. Create 3 VFs in the PR region.
                                                                                     
                                                                                    $ sudo pci_device b1:00.0 vf 3 \n
                                                                                     
                                                                                    2. Verify all 3 VFs were created.
                                                                                     
                                                                                    $ lspci -s b1:00 \nb1:00.0 Processing accelerators: Intel Corporation Device bcce (rev 01) \nb1:00.1 Processing accelerators: Intel Corporation Device bcce \nb1:00.2 Processing accelerators: Intel Corporation Device bcce \nb1:00.3 Processing accelerators: Red Hat, Inc. Virtio network device \nb1:00.4 Processing accelerators: Intel Corporation Device bcce \nb1:00.5 Processing accelerators: Intel Corporation Device bccf \nb1:00.6 Processing accelerators: Intel Corporation Device bccf \nb1:00.7 Processing accelerators: Intel Corporation Device bccf \n
                                                                                     
                                                                                    3. Bind all of the PF/VF endpoints to the vfio-pci driver.
                                                                                     
                                                                                    $ sudo opae.io init -d 0000:b1:00.1 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.1 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.1 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.1 is 187\nAssigning /dev/vfio/187 to DCPsupport\nChanging permissions for /dev/vfio/187 to rw-rw----\n\n$ sudo opae.io init -d 0000:b1:00.2 user:user\nUnbinding (0x8086,0xbcce) at 0000:b1:00.2 from dfl-pci\nBinding (0x8086,0xbcce) at 0000:b1:00.2 to vfio-pci\niommu group for (0x8086,0xbcce) at 0000:b1:00.2 is 188\nAssigning /dev/vfio/188 to DCPsupport\nChanging permissions for /dev/vfio/188 to rw-rw----\n\n...\n\n$ sudo opae.io init -d 0000:b1:00.7 user:user\nBinding (0x8086,0xbccf) at 0000:b1:00.7 to vfio-pci\niommu group for (0x8086,0xbccf) at 0000:b1:00.7 is 319\nAssigning /dev/vfio/319 to DCPsupport\nChanging permissions for /dev/vfio/319 to rw-rw----\n
                                                                                     
                                                                                    4. Check that the accelerators are present using fpgainfo. Note your port configuration may differ from the below.
                                                                                     
                                                                                    $ sudo fpgainfo port \n//****** PORT ******//\nObject Id                        : 0xEC00000\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\n//****** PORT ******//\nObject Id                        : 0xE0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.7\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 4dadea34-2c78-48cb-a3dc-5b831f5cecbb\n//****** PORT ******//\nObject Id                        : 0xC0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.6\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 823c334c-98bf-11ea-bb37-0242ac130002\n//****** PORT ******//\nObject Id                        : 0xA0B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.5\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCF\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 8568ab4e-6ba5-4616-bb65-2a578330a8eb\n//****** PORT ******//\nObject Id                        : 0x80B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.4\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 44bfc10d-b42a-44e5-bd42-57dc93ea7f91\n//****** PORT ******//\nObject Id                        : 0x40B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.2\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n//****** PORT ******//\nObject Id                        : 0x20B1000000000000\nPCIe s:b:d.f                     : 0000:B1:00.1\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x01\nAccelerator GUID                 : 3e7b60a0-df2d-4850-aa31-f54a3e403501\n
                                                                                     
                                                                                    The following table contains a mapping between each VF, Accelerator GUID, and component.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-16-accelerator-pfvf-and-guid-mappings","title":"Table 16: Accelerator PF/VF and GUID Mappings","text":"Component VF Accelerator GUID Intel N6001-PL FPGA SmartNIC Platform base PF XXXX:XX:XX.0 N/A VirtIO Stub XXXX:XX:XX.1 3e7b60a0-df2d-4850-aa31-f54a3e403501 HE-MEM Stub XXXX:XX:XX.2 56e203e9-864f-49a7-b94b-12284c31e02b Copy Engine XXXX:XX:XX.4 44bfc10d-b42a-44e5-bd42-57dc93ea7f91 HE-MEM XXXX:XX:XX.5 8568ab4e-6ba5-4616-bb65-2a578330a8eb HE-HSSI XXXX:XX:XX.6 823c334c-98bf-11ea-bb37-0242ac130002 MEM-TG XXXX:XX:XX.7 4dadea34-2c78-48cb-a3dc-5b831f5cecbb 
                                                                                    5. Check Ethernet PHY settings with fpgainfo.
                                                                                     
                                                                                    $ sudo fpgainfo phy -B 0xb1 \nIIntel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\n//****** HSSI information ******//\nHSSI version                     : 1.0\nNumber of ports                  : 8\nPort0                            :25GbE        DOWN\nPort1                            :25GbE        DOWN\nPort2                            :25GbE        DOWN\nPort3                            :25GbE        DOWN\nPort4                            :25GbE        DOWN\nPort5                            :25GbE        DOWN\nPort6                            :25GbE        DOWN\nPort7                            :25GbE        DOWN\n
                                                                                     
                                                                                    6. Set loopback mode.
                                                                                     
                                                                                    $ sudo hssiloopback --loopback enable  --pcie-address 0000:b1:00.0 \nargs Namespace(loopback='enable', pcie_address='0000:b1:00.0', port=0)\nsbdf: 0000:b1:00.0\nFPGA dev: {'segment': 0, 'bus': 177, 'dev': 0, 'func': 0, 'path': '/sys/class/fpga_region/region0', 'pcie_address': '0000:b1:00.0'}\nargs.hssi_grps{0: ['dfl_dev.6', ['/sys/bus/pci/devices/0000:b1:00.0/fpga_region/region0/dfl-fme.0/dfl_dev.6/uio/uio0']]}\nfpga uio dev:dfl_dev.6\n\n--------HSSI INFO START-------\nDFH                     :0x3000000010002015\nHSSI ID                 :0x15\nDFHv                    :0.5\nguidl                   :0x99a078ad18418b9d\nguidh                   :0x4118a7cbd9db4a9b\nHSSI version            :1.0\nFirmware Version        :1\nHSSI num ports          :8\nPort0                   :25GbE\nPort1                   :25GbE\nPort2                   :25GbE\nPort3                   :25GbE\nPort4                   :25GbE\nPort5                   :25GbE\nPort6                   :25GbE\nPort7                   :25GbE\n--------HSSI INFO END-------\n\nhssi loopback enabled to port0\n
                                                                                     
                                                                                    7. Send traffic through the 10G AFU.
                                                                                     
                                                                                    $ sudo hssi --pci-address b1:00.6 hssi_10g --num-packets 100       \n10G loopback test\n  port: 0\n  eth_loopback: on\n  he_loopback: none\n  num_packets: 100\n  packet_length: 64\n  src_address: 11:22:33:44:55:66\n    (bits): 0x665544332211\n  dest_address: 77:88:99:aa:bb:cc\n    (bits): 0xccbbaa998877\n  random_length: fixed\n  random_payload: incremental\n  rnd_seed0: 5eed0000\n  rnd_seed1: 5eed0001\n  rnd_seed2: 25eed\n  eth:\n\nNo eth interface, so not honoring --eth-loopback.\n0x40000           ETH_AFU_DFH: 0x1000010000001000\n0x40008          ETH_AFU_ID_L: 0xbb370242ac130002\n0x40010          ETH_AFU_ID_H: 0x823c334c98bf11ea\n0x40030      TRAFFIC_CTRL_CMD: 0x0000000000000000\n0x40038     TRAFFIC_CTRL_DATA: 0x0000000100000000\n0x40040 TRAFFIC_CTRL_PORT_SEL: 0x0000000000000000\n0x40048        AFU_SCRATCHPAD: 0x0000000045324511\n\n0x3c00         number_packets: 0x00000064\n0x3c01          random_length: 0x00000000\n0x3c02         random_payload: 0x00000000\n0x3c03                  start: 0x00000000\n0x3c04                   stop: 0x00000000\n0x3c05           source_addr0: 0x44332211\n0x3c06           source_addr1: 0x00006655\n0x3c07             dest_addr0: 0xaa998877\n0x3c08             dest_addr1: 0x0000ccbb\n0x3c09        packet_tx_count: 0x00000064\n0x3c0a              rnd_seed0: 0x5eed0000\n0x3c0b              rnd_seed1: 0x5eed0001\n0x3c0c              rnd_seed2: 0x00025eed\n0x3c0d             pkt_length: 0x00000040\n0x3cf4          tx_end_tstamp: 0x000003d2\n0x3d00                num_pkt: 0xffffffff\n0x3d01               pkt_good: 0x00000064\n0x3d02                pkt_bad: 0x00000000\n0x3d07            avst_rx_err: 0x00000000\n0x3d0b          rx_sta_tstamp: 0x00000103\n0x3d0c          rx_end_tstamp: 0x0000053b\n0x3e00               mac_loop: 0x00000000\n\nHSSI performance:\n        Selected clock frequency : 402.832 MHz\n        Latency minimum : 642.948 ns\n        Latency maximum : 896.155 ns\n        Achieved Tx throughput : 18.4528 GB/s\n        Achieved Rx throughput : 16.7101 GB/s\n\nNo eth interface, so not showing stats.\n
                                                                                     
                                                                                    The hssi_loopback utility works in conjunction with a packet generator accelerator function unit (AFU) to test high-speed serial interface (HSSI) cards. The hssi_loopback utility tests both external and internal loopbacks.
                                                                                     
                                                                                    The hssistats tool provides the MAC statistics.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#50-upgrading-the-intel-fpga-smartnic-n6001-pl-with-20233-2-version-of-the-bmc-and-fim","title":"5.0 Upgrading the Intel\u00ae FPGA SmartNIC N6001-PL with 2023.3-2 Version of the BMC and FIM","text":"
                                                                                    If your Intel\u00ae FPGA SmartNIC N6001-PL does not have the 2022.3.1 version of the FIM and BMC, use this section to begin your upgrade process. The upgrade process depends on both the OPAE SDK and kernel drivers, which were installed in sections 3.0 Intel OFS DFL Kernel Drivers and 4.0 OPAE Software Development Kit. Use the output of fpgainfo and compare against the table below to determine if an upgade is necessary.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-17-fim-version-summary-for-intel-ofs-20233-2-release","title":"Table 17: FIM Version Summary for Intel OFS 2023.3-2 Release","text":"FIM Version Bitstream ID Pr Interface ID File Name Download Location 1 00x50102023508A422 1d6beb4e-86d7-5442-a763-043701fb75b7 ofs_top_page[1 / 2]_unsigned_user[1 / 2].bin ofs-2023.3-2 Release Page"},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#table-18-bmc-version-summary-for-intel-ofs-20233-2-release","title":"Table 18: BMC Version Summary for Intel OFS 2023.3-2 Release","text":"BMC FW and RTL Version File Name Download Location 3.15.0 AC_BMC_RSU_user_retail_3.15.0_unsigned.rsu n/a 
                                                                                    Sample output of fpgainfo with matching values:
                                                                                     
                                                                                    Intel Acceleration Development Platform N6001\nBoard Management Controller NIOS FW version: 3.15.0\nBoard Management Controller Build version: 3.15.0\n//****** FME ******//\nObject Id                        : 0xED00001\nPCIe s:b:d.f                     : 0000:B1:00.0\nVendor Id                        : 0x8086\nDevice Id                        : 0xBCCE\nSubVendor Id                     : 0x8086\nSubDevice Id                     : 0x1771\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 00x50102023508A422\nBitstream Version                : 5.0.1\nPr Interface Id                  : 1d6beb4e-86d7-5442-a763-043701fb75b7\nBoot Page                        : user1\nFactory Image Info               : a2b5fd0e7afca4ee6d7048f926e75ac2\nUser1 Image Info                 : 1d6beb4e-86d7-5442-a763-043701fb75b7\nUser2 Image Info                 : 1d6beb4e-86d7-5442-a763-043701fb75b7\n
                                                                                     
                                                                                       
                                                                                    1. If your output does not match the table above, download the appropriate FIM image from the Intel OFS 2023.3-2 (Intel Agilex) release page. Once downloaded transfer the file over to the server and use the fpgasupdate utility to perform an upgrade of the BMC.
                                                                                      2.  
                                                                                     
                                                                                    $ sudo fpgasupdate AC_BMC_RSU_user_retail_3.15.0_unsigned.rsu\n[2022-04-14 16:32:47.93] [WARNING ] Update starting. Please do not interrupt.                                           \n[2022-04-14 16:32:47.93] [INFO    ] updating from file /home/user/AC_BMC_RSU_user_retail_3.15.0_unsigned.rsu with size 904064                                   \n[2022-04-14 16:32:47.94] [INFO    ] waiting for idle                                                                    \n[2022-04-14 16:32:47.94] [INFO    ] preparing image file                                                                \n[2022-04-14 16:33:26.98] [INFO    ] writing image file                                                                  \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [904064/904064 bytes][Elapsed Time: 0:00:00.00]                                           \n[2022-04-14 16:33:26.98] [INFO    ] programming image file                                                              \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [Elapsed Time: 0:00:26.02]                                                                 \n[2022-04-14 16:33:53.01] [INFO    ] update of 0000:b1:00.0 complete                                                     \n[2022-04-14 16:33:53.01] [INFO    ] Secure update OK                                                                    \n[2022-04-14 16:33:53.01] [INFO    ] Total time: 0:01:05.07\nsudo rsu bmcimg\n
                                                                                     2. Load the new FIM image.
                                                                                     
                                                                                    ```bash\n$ sudo fpgasupdate ofs_top_page1_unsigned_user1.bin <PCI ADDRESS>\n[2022-04-14 16:42:31.58] [WARNING ] Update starting. Please do not interrupt.                                           \n[2022-04-14 16:42:31.58] [INFO    ] updating from file ofs_top_page1_pacsign_user1.bin with size 19928064               \n[2022-04-14 16:42:31.60] [INFO    ] waiting for idle                                                                    \n[2022-04-14 16:42:31.60] [INFO    ] preparing image file                                                                \n[2022-04-14 16:42:38.61] [INFO    ] writing image file                                                                  \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] [19928064/19928064 bytes][Elapsed Time: 0:00:16.01]                                       \n[2022-04-14 16:42:54.63] [INFO    ] programming image file                                                              \n(100%) [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588][Elapsed Time: 0:06:16.40]                                                                 \n[2022-04-14 16:49:11.03] [INFO    ] update of 0000:b1:00.0 complete                                                     \n[2022-04-14 16:49:11.03] [INFO    ] Secure update OK                                                                    \n[2022-04-14 16:49:11.03] [INFO    ] Total time: 0:06:39.45\nsudo rsu fpga --page=user1 <PCI ADDRESS>\n```\n
                                                                                     
                                                                                       
                                                                                    1. Verify output of fpgainfo matches the table above.
                                                                                      2.  
                                                                                    "},{"location":"hw/n6001/user_guides/ug_qs_ofs_n6001/ug_qs_ofs_n6001/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                                    Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                                     
                                                                                    OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/","title":"Simulation User Guide: Open FPGA Stack for Intel Intel\u00ae Agilex\u00ae 7 FPGA","text":"Term Abbreviation Description Advanced\u00a0Error\u00a0Reporting AER The PCIe AER driver is the extended PCI Express error reporting capability providing more robust error reporting. (link) Accelerator\u00a0Functional\u00a0Unit AFU Hardware Accelerator implemented in FPGA logic which offloads a computational operation for an application from the CPU to improve performance.\u00a0Note: An AFU region is the part of the design where an AFU may reside. This AFU may or may not be a partial reconfiguration region. Basic Building Block BBB Features within an AFU or part of an FPGA interface that can be reused across designs. These building blocks do not have stringent interface requirements like the FIM's AFU and host interface requires. All BBBs must have a (globally unique identifier) GUID. Best\u00a0Known\u00a0Configuration BKC The software and hardware configuration Intel uses to verify the solution. Board\u00a0Management\u00a0Controller BMC Supports features such as board power managment, flash management, configuration management, and board telemetry monitoring and protection. The majority of the BMC logic is in a separate component, such as an Intel Max10 or Intel Cyclone10 device; a small portion of the BMC known as the PMCI resides in the main Agilex FPGA. Configuration and\u00a0Status\u00a0Register CSR The generic name for a register space which is accessed in order to interface with the module it resides in (e.g. AFU, BMC, various sub-systems and modules). Data Parallel C++ DPC++ DPC++ is Intel\u2019s implementation of the SYCL standard. It supports additional attributes and language extensions which ensure DCP++ (SYCL) is efficiently implemented on Intel hardware. Device\u00a0Feature\u00a0List DFL The DFL, which is implemented in RTL, consists of a self-describing data structure in PCI BAR space that allows the DFL driver to automatically load the drivers required for a given FPGA configuration.\u00a0This concept is the foundation for the OFS software framework. (link) FPGA\u00a0Interface\u00a0Manager FIM Provides platform management, functionality, clocks, resets and standard interfaces to host and AFUs. The FIM resides in the static region of the FPGA and contains the FPGA Management Engine (FME) and I/O ring. FPGA\u00a0Management\u00a0Engine FME Performs reconfiguration and other FPGA management functions. Each FPGA device only has one FME which is accessed through PF0. Host\u00a0Exerciser\u00a0Module HEM Host exercisers are used to exercise and characterize the various host-FPGA interactions, including Memory Mapped Input/Output (MMIO), data transfer from host to FPGA, PR, host to FPGA memory, etc. Input/Output Control IOCTL System calls used to manipulate underlying device parameters of special files. Intel\u00a0Virtualization\u00a0Technology for\u00a0Directed I/O Intel VT-d Extension of the VT-x and VT-I processor virtualization technologies which adds new support for I/O device virtualization. Joint Test Action Group JTAG Refers to the IEEE 1149.1 JTAG standard; Another FPGA configuration methodology. Management Component Transport Protocol MCTP A standardized model for communication with management controllers. Defines the transport protocol carrying PLDM messages through the BMC. Memory\u00a0Mapped\u00a0Input/Output MMIO The memory space users may map and access both control registers and system memory buffers with accelerators. oneAPI Accelerator Support Package oneAPI-asp A collection of hardware and software components that enable oneAPI kernel to communicate with oneAPI runtime and OFS shell components. oneAPI ASP hardware components and oneAPI kernel form the AFU region of a oneAPI system in OFS. Open\u00a0FPGA\u00a0Stack OFS OFS is a software and hardware infrastructure providing an efficient approach to develop a custom FPGA-based platform or workload using an Intel, 3rd party, or custom board. Open\u00a0Programmable\u00a0Acceleration\u00a0Engine Software Development Kit OPAE-SDK The OPAE-SDK is a software framework for managing and accessing programmable accelerators (FPGAs). It consists of a collection of libraries and tools to facilitate the development of software applications and accelerators. The OPAE SDK resides exclusively in user-space. Platform\u00a0Interface\u00a0Manager PIM An interface manager that comprises two components: a configurable platform specific interface for board developers and a collection of shims that AFU developers can use to handle clock crossing, response sorting, buffering and different protocols. Platform Level Data Model PLDM A specification for reporting telemetry data to the host, such as board temperature, voltage, and current. Platform Management Controller Interface PMCI The portion of the BMC that resides in the Agilex FPGA and allows the FPGA to communicate with the primary BMC component on the board. Partial Reconfiguration PR The ability to dynamically reconfigure a portion of an FPGA while the remaining FPGA design continues to function. For OFS designs, the PR region is referred to as the pr_slot. Port N/A When used in the context of the fpgainfo port command it represents the interfaces between the static FPGA fabric and the PR region containing the AFU. Remote System Update RSU The process by which the host can remotely update images stored in flash through PCIe. This is done with the OPAE software command \"fpgasupdate\". Secure Device Manager SDM The SDM is the point of entry to the FPGA for JTAG commands and interfaces, as well as for device configuration data (from flash, SD card, or through PCI Express* hard IP). Static Region SR The portion of the FPGA design that cannot be dynamically reconfigured during run-time. Single-Root\u00a0Input-Output\u00a0Virtualization SR-IOV Allows the isolation of PCI Express resources for manageability and performance. SYCL SYCL SYCL (pronounced \"sickle\") is a royalty-free, cross-platform abstraction layer that enables code for heterogeneous and offload processors to be written using modern ISO C++ (at least C++ 17). It provides several features that make it well-suited for programming heterogeneous systems, allowing the same code to be used for CPUs, GPUs, FPGAs or any other hardware accelerator. SYCL was developed by the Khronos Group, a non-profit organization that develops open standards (including OpenCL) for graphics, compute, vision, and multimedia. SYCL is being used by a growing number of developers in a variety of industries, including automotive, aerospace, and consumer electronics. Test Bench TB Testbench or Verification Environment is used to check the functional correctness of the Design Under Test (DUT) by generating and driving a predefined input sequence to a design, capturing the design output and comparing with-respect-to expected output. Universal Verification Methodology UVM A modular, reusable, and scalable testbench structure via an API framework.  In the context of OFS, the UVM enviroment provides a system level simulation environment for your design. Virtual\u00a0Function\u00a0Input/Output VFIO An Input-Output Memory Management Unit (IOMMU)/device agnostic framework for exposing direct device access to userspace.\u00a0(link)"},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#1-overview","title":"1 Overview","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#11-about-this-document","title":"1.1 About this Document","text":"
                                                                                    This document serves as a set-up and user guide for the UVM simulation tool using OFS. After reviewing the document, you will be able to:
                                                                                     
                                                                                       
                                                                                    • Set-up the UVM verification tool suite
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • Run pre-existing UVM unit tests and also create new UVM tests for your design
                                                                                      •  
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#2-introduction-to-uvm","title":"2 Introduction to UVM","text":"
                                                                                    The Functional Verification Environment for OFS is UVM (Universal Verification Methodology) compliant and provides configurable setup for verifying various FIM features in simulation.
                                                                                     
                                                                                    The purpose of this document is to demonstrate a full chip level and unit level UVM based verification environment for the current base shell FIM architecture as well as providing examples to extend the setup for different OFS variants by reusing the existing architecture and infrastructure for UVM based verification.
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#3-universal-testbench-architecture","title":"3 Universal Testbench Architecture","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#31-overview","title":"3.1 Overview","text":"
                                                                                    The main idea behind UVM is to develop modular, reusable, and a scalable testbench structure by providing an API framework that can be deployed across multiple projects.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#32-core-verification-concepts","title":"3.2 Core Verification Concepts","text":"
                                                                                    The following is the list of verification components that will be used to design a UVM testbench architecture:
                                                                                     
                                                                                    \u2022   Sequencer \u2022   Driver \u2022   Monitor \u2022   Scoreboard \u2022   Virtual Sequencer \u2022   Interface \u2022   Verification Environment \u2022   TOP Testbench
                                                                                     
                                                                                    Figure 1 provides the general UVM Testbench and the verification components involved in the top-level architecture.
                                                                                     
                                                                                     
                                                                                    Figure 1 Typical UVM Testbench
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4-ofs-testbench-architecture","title":"4 OFS Testbench Architecture","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#41-overview","title":"4.1 Overview","text":"
                                                                                    OFS (Open FPGA Stack) provides a UVM (Universal Verification Methodology) environment for the FIM with a modular, reusable, and scalable testbench structure via an API framework.
                                                                                     
                                                                                    The framework consists of a FIM Testbench which is UVM compliant and integrates third party VIPs from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this Testbench. UVM RAL(Register Abstraction Level) is used for CSR (Command and Status Registers) verification. 
                                                                                     
                                                                                    The qualified verification IPs will help to detect incorrect protocol behavior, help to focus on FIM features and accelerate the verification process.
                                                                                     
                                                                                    Verification components include:
                                                                                     
                                                                                    \u2022 FIM monitor to detect correct design behavior\n\u2022 FIM assertions for signal level integrity testing\n\u2022 Arm AMBA Arm\u00ae AMBA\u00ae 4 AXI4 scoreboards to check data integrity\n\u2022 FIM coverage to collect functional data\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#42-base-fim-dut","title":"4.2 Base FIM DUT","text":"
                                                                                    The hardware architecture of an Agilex FIM is based on the OFS hardware architecture. The following is the list of features and subsystems supported in the base shell.
                                                                                     
                                                                                    \u2022   PCIe Subsystem\n\u2022   HSSI Subsystem\n\u2022   Memory Subsystem\n\u2022   HPS Subsystem\n\u2022   FME\n\u2022   AFU with PR support\n\u2022   QSFP Controllers\n\u2022   PMCI Controller, MCTP\n
                                                                                     
                                                                                     
                                                                                    Figure 2 DUT Base Shell Diagram
                                                                                     
                                                                                    Figure 2 shows the high level architecture of an Agilex Base Shell. It has a Gen4x16, 100G Ethernet Datapath in a 2x4x25G configuration. The Agilex Base Shell is a shell that will enable a user to build other shell variants for a custom configuration. For the N6001 board there is one shell variant
                                                                                     
                                                                                    base_x16
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#43-full-chip-level-verification-archiecture-for-fim","title":"4.3 Full Chip Level Verification Archiecture for FIM","text":"
                                                                                    Figure 3 shows a graphical representation a full chip testbench that includes major RTL blocks depicted in a OFS Agilex based UVM environment 
                                                                                     
                                                                                     
                                                                                    Figure 3 OFS FIM Testbench
                                                                                     
                                                                                    The major connection is the interface between the Xeon CPU and the FPGA where the PCIe Verification IP is connected to PCIe Subsystem. Therefore, as a full chip simulation environment, PCIe host VIP is the sole VIP/BFM used. PCIe host VIP connects to PCIe device which resides in FPGA in serial mode.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#431-testbench-components","title":"4.3.1 Testbench components","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4311-tb-top","title":"4.3.1.1 TB TOP","text":"
                                                                                    TOP is the top level testbench and consists of a FIM DUT instance and top-level UVM Verification Environment instance. It binds RTL inputs with the verification environmnet interfaces to drive stimulus. It also has clock generation and reset driving logic.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4312-fim-verification-environment","title":"4.3.1.2 FIM Verification Environment","text":"
                                                                                    This is the top most verification environment class and consists of the protocol specific PCI Express and AXI UVM environment VIP instances, Monitors, Scoreboards, Assertions, Functional coverage Modules and other verification components. It instantiates Virtual sequencers to control stimuli for FIM traffic from different sequencers of the VIPs.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4313-synopsys-vips","title":"4.3.1.3 Synopsys VIPs","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#43131-pci-vip-as-host","title":"4.3.1.3.1 PCI VIP as Host","text":"
                                                                                    This is Synopsys Licensed PCI Express Gen4 VIP and acts as Root Port. The PCI Express link is connected to the DUT using TX-RX lanes. Agent is an active component and includes a sequencer to generate TLPs, Driver to drive it on a PCI Express link and Monitor to check the protocol correctness.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#43132-axi-streaming-vip-monitors","title":"4.3.1.3.2 AXI-Streaming VIP Monitors","text":"
                                                                                    This is Synopsys Licensed AXI streaming interface Verification IP used as a Passive Agent to monitor AXI-ST links at various points. Please refer to Figure 3 to see all the AXI-ST monitors connected to different modules.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#43133-axi4-memory-mapped-vip-monitors","title":"4.3.1.3.3 AXI4-Memory Mapped VIP Monitors","text":"
                                                                                    This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 memory mapped interface Verification IP used in passive mode to observe memory requests. For each master-slave pair, the verification environment has a VIP instantiated in passive mode for monitoring protocol violations and performing data integrity checks.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#43134-axi4-lite-vip-monitors","title":"4.3.1.3.4 AXI4-Lite VIP Monitors","text":"
                                                                                    This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 Lite interface Verification IP used in passive mode to observe MMIO requests. For each master-slave pair, the verification environment has a VIP instantiated in passive mode for monitoring protocol violations and perfoming data integrity checks. Please refer to Figure 3 to see all the Arm\u00ae AMBA\u00ae 4 AXI4-Lite monitors connected to different modules.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#43135-axi4-lite-vip-as-pmci-master","title":"4.3.1.3.5 AXI4-Lite VIP as PMCI Master","text":"
                                                                                    This is Synopsys Licensed Arm\u00ae AMBA\u00ae 4 AXI4 Lite interface Verification IP used to drive and observe MMIO requests as PMCI master. For each master-slave pair, the verification environment has a VIP instantiated in active mode and includes a sequencer to generate MMIO requests, driver to drive these requests on AXI-4 Lite interface to BPF and a monitor for observing protocol violations and data integrity checks. 
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4314-axi4-s-scoreboard","title":"4.3.1.4 AXI4-S Scoreboard","text":"
                                                                                    The Arm\u00ae AMBA\u00ae 4 AXI4-S scoreboard checks data integrity of source and sink components. It has input transactions from VIP monitors and a TLP to AXI converter for PCIe TLP packets. It makes sure the valid TLPs or AXI transactions are not missed while traversing from Host to AFU and reverse. The scoreboard will be disabled for error testing especially for invalid TLP requests and UR responses.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4315-virtual-sequencer","title":"4.3.1.5 Virtual Sequencer","text":"
                                                                                    The virtual sequencer is the top-level sequencer which controls Enumeration, MMIO Requests, downstream and Upstream traffic as well as HSSI and Memory transactions. It makes sure that the transactions are ordered correctly as per the design specification while running a UVM test simulation.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4316-fim-monitor","title":"4.3.1.6 FIM Monitor","text":"
                                                                                    The FIM monitor is used for checking the correctness of a specific FIM feature. As an example, a user can add interrupt related checks, error generation and clearing checks, Protocol checker impacts etc. in this component. 
                                                                                     
                                                                                    This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4317-fim-assertions","title":"4.3.1.7 FIM Assertions","text":"
                                                                                    The assertion component is used for checking the signal level integrity of a specific FIM feature or behavior. As an example, we can add interrupt signal triggering and clear assertions, Error detection to error register bit assertions, protocol checker and clearing logic, FLR behavior assertion etc. in this top-level module. There are alos assertions to make sure the inputs are driven correctly to catch errors in a users simulation. 
                                                                                     
                                                                                    This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#4318-fim-functional-coverage","title":"4.3.1.8 FIM Functional Coverage","text":"
                                                                                    The FIM functional coverage component will have the functional coverage of each feature like interrupts, CSR's, MMIO requests, Byte align transactions, error reporting etc. to demonstrate a variety of FIM features. This will help us to find holes in the design as well as wide verification coverage to make sure thorough testing of a design. It will also provide a user what the FIM is capable of and how much code coverage they are testing.
                                                                                     
                                                                                    This would be an independent plug-n-play component and can be reused with any user developed UVM FIM TB.
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#5-uvm-verification-set-up","title":"5 UVM Verification set-up","text":"
                                                                                    To run the tutorial steps in this guide requires the following development environment:
                                                                                     Item Version Intel Quartus Prime Pro Intel Quartus Prime Pro 23.3 Simulator Synopsys VCS P-2019.06-SP2-5 or newer for UVM simulation of top level FIM Simulator (Questasim) Questasim 2023.2 or newer for UVM simulation of top level FIM"},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#51-uvm-prerequisite","title":"5.1 UVM Prerequisite","text":"
                                                                                    Retrieve OFS repositories.
                                                                                     
                                                                                    The OFS FIM source code is included in the OTCShare GitHub repository. Create a new directory to use as a clean starting point to store the retrieved files.  The following is a short description of each repository, followed by the git commands for cloning.  The instructions section uses the HTTPS git method for cloning repositories.  Cloning the repo using the HTTPS method requires a personal access token.  Please see this blog post for information about obtaining a personal access token Token authentication requirements for Git operations.
                                                                                     
                                                                                    Navigate to location for storage of OFS source, create the top-level source directory and clone OFS repositories.
                                                                                     
                                                                                    $ mkdir ofs-2023.3-2\n$ cd ofs-2023.3-2\n$ export OFS_BUILD_ROOT=$PWD\n$ git clone --branch --recurse-submodules https://github.com/ofs-n6001.git\n\nCloning into 'ofs-n6001'...'\nUsername for 'https://github.com': <<Enter your git hub username>>\nPassword for 'https://<<Your username>>': <<Enter your personal access token>>\nremote: Enumerating objects:  ....\n...\n...\nResolving deltas  ..., done.\n\n$ cd ofs-n6001\n$ git checkout tags/ofs-2023.3-2\n
                                                                                     
                                                                                    Verify that the correct tag/branch have been checked out        
                                                                                     
                                                                                    $ git describe --tags\n\n$ ofs-2023.3-2\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#52-license-requirements","title":"5.2 License Requirements","text":"
                                                                                    The FIM Testbench is UVM compliant and integrates third party VIP's from Synopsys for PCI Express, Arm\u00ae AMBA\u00ae 4 AXI4Arm\u00ae AMBA\u00ae 4 AXI4-Streaming interface and Arm\u00ae AMBA\u00ae 4 AXI4-Memory Mapped interface for comprehensive verification. The user will need to acquire licenses for these VIPs to use this TB. UVM RAL (Register Abstraction Layer) is used for CSR Verification.
                                                                                     
                                                                                    The Qualified Verification IPs will help to detect incorrect protocol behavior easily, help to focus on FIM features and accelerate the verification process.
                                                                                     
                                                                                    \u2022 VCS & DVE\n\u2022 SNPS-Assertions\n\u2022 Verdi\n\u2022 VerdiCoverage\n\u2022 VerdiSimDB\n\u2022 VerdiTransactionDebugUltra\n\u2022 VIP-AMBA-AXI-SVT\n\u2022 VIP-AMBA-STREAM-SVT\n\u2022 VIP-PCIE-SVT\n\u2022 VIP-PCIE-TS-SVT\n\u2022 VIP-PCIE-G3-OPT-SVT\n\u2022 VIP-Ethernet-SVT\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#53-software-tools-requirements","title":"5.3 Software Tools Requirements","text":"
                                                                                    The following tools are required for successful UVM set-up
                                                                                     
                                                                                       
                                                                                    • Python 3.6.8
                                                                                      •  
                                                                                    • Synopsys PCIE and AMBA AXI UVM VIP Q-2020.03A License
                                                                                      •  
                                                                                    • Synopsys Verdi R-2020.12-SP2 License      Note: Makefile can be modified to use DVE instead of Verdi
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • VCS R-2020.12-SP2 License 
                                                                                      •  
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#54-creating-a-software-tools-script","title":"5.4 Creating a Software Tools Script","text":"
                                                                                    The UVM tool set-up is best done by creating a simple set-up script so all applicable tools are sourced before running the tests.
                                                                                     
                                                                                    The following environment variables can be pasted into a script and used prior to running the UVM verification environment
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#license-files","title":"License Files","text":"
                                                                                    export LM_LICENSE_FILE=\nexport SNPSLMD_LICENSE_FILE=\n
                                                                                     
                                                                                    The license environment variables LM_LICENSE_FILE and SNPSLMD_LICENSE_FILE can point to a server license on your system.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#general-environment-variables","title":"General Environment Variables","text":"
                                                                                    export IOFS_BUILD_ROOT=$PWD\nexport OFS_ROOTDIR=<user_path>/ofs-n6001\nexport WORKDIR=$OFS_ROOTDIR\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#quartus-tools","title":"Quartus Tools","text":"
                                                                                    export QUARTUS_HOME=<user_path>/intelFPGA_pro/23.3/quartus\nexport QUARTUS_ROOTDIR=$QUARTUS_HOME\nexport QUARTUS_INSTALL_DIR=$QUARTUS_ROOTDIR\nexport QUARTUS_ROOTDIR_OVERRIDE=$QUARTUS_ROOTDIR\nexport IMPORT_IP_ROOTDIR=$QUARTUS_ROOTDIR/../ip\nexport QSYS_ROOTDIR=$QUARTUS_ROOTDIR/../qsys/bin\nexport PATH=$QUARTUS_HOME/bin:$QUARTUS_HOME/qsys/bin:$QUARTUS_HOME/sopc_builder/bin/:$PATH\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-verification-tools","title":"Synopsys Verification Tools","text":"
                                                                                    export DESIGNWARE_HOME=<user_path>/synopsys/vip_common/vip_Q-2020.03A\nexport PATH=$DESIGNWARE_HOME/bin:$PATH\nexport UVM_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel/etc/uvm\nexport VCS_HOME=<user_path>/synopsys/vcsmx/S-2021.09-SP1/linux64/rhel\nexport PATH=$VCS_HOME/bin:$PATH\nexport VERDIR=$OFS_ROOTDIR/verification\nexport VIPDIR=$VERDIR\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-verification-tools","title":"QuestaSIM Verification Tools","text":"
                                                                                    export MTI_HOME=<user_path>/mentor/questasim/2023.2/linux64\nexport PATH=$MTI_HOME/linux_x86_64/:$MTI_HOME/bin/:$PATH\nexport QUESTA_HOME=$MTI_HOME\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#6-running-a-uvm-simulation-test-and-analysing-results","title":"6 Running a UVM Simulation Test and Analysing Results","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#61-simulation","title":"6.1 Simulation","text":"
                                                                                    The default simulator used in the simulation script is Synopsys VCS-MX. Users can refer to the options and adopt the options for other simulators. The script is a makefile that calls vlogan, vcs and simv for compilation, elaboration and simulation, respectively.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#62-file-structure","title":"6.2 File Structure","text":"
                                                                                    After cloning the repo, the verification and ofs-common directories contain all UVM verification related files. The directory structure is shown in Figure 4 below.
                                                                                     
                                                                                     
                                                                                    Figure 4 UVM Verification Directory File Structure
                                                                                     
                                                                                    ofs-n6001/verification/testbench has a testbench, uvm env, virtual sequencer, RAL etc. 
                                                                                     
                                                                                    ofs-n6001/tests contains all uvm tests and sequences. 
                                                                                     
                                                                                    Users can run the simulation under \"ofs-n6001/verification/scripts\" directory and the simulation result is outputted to a \"sim\" directory for Synopsys VCS or sim_msim for Questasim.
                                                                                     
                                                                                    The simulation result folder is named after the test name with increasing suffix number. If user runs the same test multiple times, the suffix is incremented by 1 each time.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#63-uvm-test-suite","title":"6.3 UVM Test Suite","text":"
                                                                                    The UVM environment contains a variety of tests that have been developed to test out the FIM portion of OFS.
                                                                                     
                                                                                    The table below has four columns which describe the \"Test Name\", \"DUT Scope\", \"Test Scenario\" and the \"Checking Criteria\".
                                                                                     
                                                                                    Tests are located at ofs-n6001/ofs-common/verification/fpga_family/agilex/tests
                                                                                     Test Name DUT Scope Test Scenario Checking Criteria afu_mmio_flr_pf0_test PF0 FLR Reset Apply FLR Reset for PF0 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF0 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf0_vf0_test PF0_VF0_FLR Reset Apply FLR Reset for PF0_VF0 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR       access are working fine afu_mmio_flr_pf0_vf1_test PF0_VF1_FLR Reset Apply FLR Reset for PF0_VF1 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf0_vf2_test PF0_VF2_FLR Reset Apply FLR Reset for PF0_VF2 and deassert Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf2_test PF2 FLR Reset Apply FLR Reset for PF2 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF2 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf3_test PF3 FLR Reset Apply FLR Reset for PF3 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF3 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_mmio_flr_pf4_test PF4 FLR Reset Apply FLR Reset for PF4 and deassert.                 Initiate MMIO transactions for all PFs. Make sure all completions are sent/received and no pending transactions are seen. Apply FLR Reset for PF4 and deassert. Initiate mmio access and ensure all PFs CSR access are working fine Initiate mmio access before and after FLR Reset and ensure all PF/VFs CSR access are working fine afu_stress_5bit_tag_test AFU-Stress To check the AFU Stress by sending traffic from all PF/VF. i.e Send traffic on HE-LPK/HE-MEM and initiating MMIO access to other PF/VFs Data checking afu_stress_8bit_tag_test AFU-Stress To check the AFU Stress by sending traffic from all PF/VF. i.e Send traffic on HE-LPK/HE-MEM and initiating MMIO access to other PF/VFs Data checking afu_stress_test AFU-Stress 1. Initiate transactions to all the supported PF/VF from PCIE VIP and ensure that traffic is sent to all blocks of the AFU.                 2. Ensure that CE/HE-LB/HE-MEM/HSSI/BPF/FME are seeing traffic.                 3. Ensure that HE-LB/HE-MEM/CE sends DMWR/DMRD requests to PCIE VIP.                 4. Ensure the Mux/DeMux blocks is able to handle the traffic based on the PF's/VF's and proper muxing/demuxing happens. Data checking bar_32b_test PCIe MMIO Path Set the BAR values to 32bit and test mmio access BAR address, Register Base Offset bar_64b_test PCIe MMIO Path Set the BAR values to 64bit and test mmio access BAR address, Register Base Offset dfh_walking_test DFH DFH walking offset checking, eol checking -> tb emif_csr_test EMIF CSR access data checking fme_csr_test FME CSR CSR accesses data checking fme_err_intr_test Interrupt FME Interrupt request using FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read fme_intr_test Interrupt FME interrupt request using RAS ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read fme_multi_err_intr_test Interrupt FME interrupt request using RAS ERROR and FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests generated from FME and RAS ERROR bits he_hssi_csr_test HE-HSSI CSR accesses data checking he_hssi_rx_lpbk_25G_10G_test HE-HSSI Sending back to back ethernet data traffic with 25G speed on HSSI RX Port0-7 lanes using Ethernet VIPs Enable the loopback mode in HE-HSSI and compare the pkts recived on HSSI TX Port(DATA CHECKING) he_hssi_tx_lpbk_P0_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port0  lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P1_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port1 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P2_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port2 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P3_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port3 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P4_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port4 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P5_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port5 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P6_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port6 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_hssi_tx_lpbk_P7_test HE-HSSI Send back to back traffic with 25G speed on HSSI TX Port7 lanes using Traffic generator registers Check the CRC errors on loopback packets on Traffic Monitor Registers he_lpbk_cont_test HE-LPBK Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking he_lpbk_csr_test HE-LPBK CSR accesses data checking he_lpbk_flr_rst_test HE-LPBK Set HE_LPK in  all the modes randomly and iterate the transactions in loop. At the end of every loop assert the Soft reset in the middle of the transactions data checking, counter checking he_lpbk_long_rst_test HE-LPBK Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_lpbk_long_test HE-LPBK Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_lpbk_multi_user_intr_test HE-LPBK generate user HE_LB interrupt interrupt checking he_lpbk_rd_cont_test HE-LPBK Read only mode/ Continuous mode. Randomize num_lines, addresses, req_len data checking he_lpbk_rd_test HE-LPBK Read only mode. Randomize num_lines, addresses, req_len counter checking he_lpbk_reqlen16_test HE-LPBK To check the behavior of HE_LPK block when req_length 16 and num_lines set to 1024 Cache lines in Loopback mode data checking, counter checking he_lpbk_reqlen1_test HE-LPBK Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_lpbk_reqlen2_test HE-LPBK Loopback mode. 128 CLs, req_len = 2CL, random addresses data checking, counter checking he_lpbk_reqlen4_test HE-LPBK Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_lpbk_reqlen8_test HE-LPBK Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_lpbk_rst_in_middle_test PCIe MMIO Path Set HE_LPK in  all the modes randomly and iterate the transactions in loop. At the end of every loop assert the Soft reset in the middle of the transactions Register Base Offset he_lpbk_test HE-LPBK Loopback mode.  Randomize num_lines, addresses, req_len data checking, counter checking he_lpbk_thruput_contmode_test HE-LPBK Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking he_lpbk_thruput_test HE-LPBK generate user HE_LB interrupt counter checking he_lpbk_user_intr_test HE-LPBK Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_lpbk_wr_cont_test HE-LPBK Write only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking he_lpbk_wr_test HE-LPBK Write only mode. Randomize num_lines, addresses, req_len counter checking he_mem_cont_test HE-MEM Continuous mode/LPBK mode, random num_lines, addresses, req_len data checking, counter checking he_mem_csr_test HE-MEM CSR accesses data checking he_mem_lpbk_long_rst_test HE-MEM Multiple iterations of he_lpb_seq with soft reset HE-LB in middle data checking, counter checking he_mem_lpbk_long_test HE-MEM Multiple iterations of he_lpb_seq with STOP HE-LB in middle data checking, counter checking he_mem_lpbk_reqlen16_test HE-MEM To check the behavior of HE_MEM block when req_length 16 and num_lines set to 1024 Cache lines in Loopback mode data checking, counter checking he_mem_lpbk_reqlen1_test HE-MEM Loopback mode. 128 CLs, req_len = 1CL, random addresses data checking, counter checking he_mem_lpbk_reqlen2_test HE-MEM Loopback mode. 128 CLs, req_len = 2CL, random addresses data checking, counter checking he_mem_lpbk_reqlen4_test HE-MEM Loopback mode. 128 CLs, req_len = 4CL, random addresses data checking, counter checking he_mem_lpbk_reqlen8_test HE-MEM Loopback mode. 128 CLs, req_len = 8CL, random addresses data checking, counter checking he_mem_lpbk_test HE-MEM Loopback mode.  Randomize num_lines, addresses, req_len data checking, counter checking he_mem_multi_user_intr_test Interrupt Back to back multiple User interrupt request from HE MEM Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read for multiple back to back request he_mem_rd_cont_test HE-MEM Read only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_rd_test HE-MEM Read only mode. Randomize num_lines, addresses, req_len counter checking he_mem_thruput_contmode_directed_test HE-MEM Set HE_LPK in thruput mode and send traffic with req len 1 and num_lines set to 40 data checking, counter checking he_mem_thruput_contmode_test HE-MEM Continuous mode, Read/Write mode. 50/50. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_thruput_test HE-MEM Read/Write mode. 50/50. Randomize num_lines, addresses, req_len counter checking he_mem_user_intr_test Interrupt FME interrupt request using RAS ERROR and FME ERROR assertion Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests generated from FME and RAS ERROR bits he_mem_wr_cont_test HE-MEM Write only mode/Continuous mode. Randomize num_lines, addresses, req_len data checking, counter checking he_mem_wr_test HE-MEM Write only mode. Randomize num_lines, addresses, req_len counter checkingt he_random_test All HEs Enable all HEs and randomize modes data checking if in lpbk mode, counter checking helb_rd_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Read only mode Measure the performance helb_rd_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Read only mode Measure the performance helb_rd_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Read only mode Measure the performance helb_thruput_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Thruput mode Measure the performance helb_thruput_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Thruput mode Measure the performance helb_thruput_4cl_5bit_tag_test Performance Set HE_LPK in thruput mode and send traffic with req len 4 and num_lines set to 1024. Measure the Read/Write performance Measure the performance helb_thruput_4cl_8bit_tag_test Performance Set HE_LPK in thruput mode and send traffic with req len 4 and num_lines set to 1024. Measure the Read/Write performance Measure the performance helb_thruput_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Thruput mode Measure the performance helb_wr_1cl_test Performance HE-LB; ReqLen = 1CL; 1024 CLs; Write only mode Measure the performance helb_wr_2cl_test Performance HE-LB; ReqLen = 2CL; 1024 CLs; Write only mode Measure the performance helb_wr_4cl_test Performance HE-LB; ReqLen = 4CL; 1024 CLs; Write only mode Measure the performance hssi_ss_test HSSI CSR accesses data checking malformedtlp_pcie_rst_test Protocol Checker generate malformed TLP protocol error and initiate pcie reset to clear the error Check for error malformedtlp_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MaxTagError_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transactions are completing. mem_tg_csr_test MEM-TG CSR access data checking mem_tg_traffic_gen_test MEM-TG This test checks the MEM_TG traffic generator flow for 1 bank data checking mini_smoke_test All HEs shorter simpler version of random test for turn-in sanity check data checking if in lpbk mode, counter checking mix_intr_test Interrupt Mix interrupt testcase to send multiple FME and User interrupt request simultaneously Test checks for interrupt assertion, deassertion, mask feature, PBA bits and MSIX host memory data integrity through backdoor memory read plus verifying interrupt requests through different sources - FME and HE-MEM modules mmio_pcie_mrrs_128B_mps_128B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_128B_mps_256B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_256B_mps_128B_test PCIe - Max Payload/Max Read Req Size Random length mmio Write Checking valid possible combination of MPS & MRRS mmio_pcie_mrrs_256B_mps_256B_test PCIe - Max Payload/Max Read Req Size Random length mmio Read Checking valid possible combination of MPS & MRRS mmio_stress_nonblocking_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux with non-blocking MMIO reads data checking mmio_stress_test PF/VF Mux/Demux Stressing MMIO on PF/VF Mux/Demux data checking mmio_test PCIe MMIO Path MMIO targeting PF0(ST2MM, FME, PMCI, QSFP, HSSI SS), PF1, PF2,PF3, PF4, PF1.VF1, PF1.VF2 data checking mmio_unimp_test PCIe MMIO Path MMIO acccess to unimplemented addresses data checking MMIODataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MMIOInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. MMIOTimedout_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR  CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. pcie_csr_test PCIESS Earlier msix registers were in fme block but now it has moved from fme to pciess. Hence coded a seperate test for msix data checking pcie_pmci_mctp_multi_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM multiple packets received from PMCI controller over APF and BPF to PCIe HIA subsystem pcie_pmci_mctp_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM (Vendor Defined Message) single packet received from PCIe HIA subsystem to PMCI controller over APF and BPF via AXI4-lite memory write request pmci_csr_test PMCI CSR CSR accesses data checking pmci_fme_csr_test PMCI FME CSR CSR accesses data checking pmci_pcie_mctp_multi_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM multiple packets received from PMCI controller over APF and BPF to PCIe HIA subsystem pmci_pcie_mctp_vdm_test MCTP Vendor specific messaging capability MCTP PCIe VDM single packet received from PMCI controller over APF and BPF to PCIe HIA subsystem pmci_pciess_csr_test PMCI PCIESS CSR CSR accesses data checking pmci_qsfp_csr_test PMCI QSFP CSR CSR accesses data checking port_gasket_csr_test PORT GASKET CSR accesses data checking qsfp_csr_test QSFP CSR accesses data checking TxMWrDataPayloadOverrun_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. TxMWrInsufficientData_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. uart_intr_test UART Checking Generates UART interrupt Check interrupt UnexpMMIORspErr_test Protocol Checker Checks for different PCIe Protocol error generation (upstram/downstream) and check the clear mechanism of AFU for these detected protocol errors.                 1. Apply the error.                 2. Wait 5us 1us                 3. Set the port reset bit true and then false again (within 1us) (You will not be able to perform a read-modify-write because F's will be returned on reads. Write a 0x5 to set and a 0x4 to clear).                 4. Wait 124us 135us (or 7.5us 28us if MMIO_TIMEOUT_CYCLES is 512.)                 5. Read bit 31 of the AFU_INTF_ERROR, BlockingTraffic. If it is set it means that we did not wait long enough in step 4.                 6. Read the AFU_INTF_ERROR  register, Be sure only the expected error(s) are set.                 7. Read the AFU_INTF_FIRST_ERROR CSR. Be sure only one error bit is set and it is the expected one.                 8. Clear the errors in the AFU_INTF_ERROR CSR by writing one to the bits that are set.                 9. Read the AFU_INTF_FIRST_ERROR CSR. Be sure all bits are cleared 1.Check AFU_INTF_ERROR and AFU_INTF_FIRST_ERROR register is getting set with correct error vector.                 2.After clearing the error register ,check if normal transcation are completing. vdm_err_vid_test Vendor ID check generate a packet with undefined Vendor-ID from Host to PMCI_SS ID check 
                                                                                     
                                                                                    The next section describes how to compile and build the UVM environment prior to running each UVM test and analyinsg the results in the log files 
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#64-ip-compile","title":"6.4 IP Compile","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-vcs","title":"Synopsys VCS","text":"
                                                                                    To compile all IPs for the Synopsys VCS simulater:
                                                                                     
                                                                                        cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk cmplib_adp\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-tbd","title":"Questasim (TBD)","text":"
                                                                                    To compile all IPs for the Questasim simulater:
                                                                                     
                                                                                        cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk cmplib_adp\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#65-rtl-test-bench-compile","title":"6.5 RTL & Test Bench Compile","text":"
                                                                                    The RTL file list for compilation is located here: verification/scripts/rtl_comb.f
                                                                                     
                                                                                    The TB file list for compilation is located here: verification/scripts/ver_list.f
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-vcs_1","title":"Synopsys VCS","text":"
                                                                                    To compile RTL and Testbench for the Synopsys VCS simulater
                                                                                     
                                                                                        cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_adp DUMP=1\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-tbd_1","title":"Questasim (TBD)","text":"
                                                                                    To compile RTL and Testbench for the Questasim simulater
                                                                                     
                                                                                        cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_adp DUMP=1\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#66-ip-and-rtl-test-bench-compile","title":"6.6 IP and RTL & Test Bench Compile","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-vcs_2","title":"Synopsys VCS","text":"
                                                                                    If the user wants to compile all IPs and RTL Testbench in one command for Synopsys VCS then follow the procedure below
                                                                                     
                                                                                        cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk build_all DUMP=1\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-tbd_2","title":"Questasim (TBD)","text":"
                                                                                    If the user wants to compile all IPs and RTL Testbench in one command for Questasim then follow the procedure below
                                                                                     
                                                                                        cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk build_all DUMP=1\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-vcs_3","title":"Synopsys VCS","text":"
                                                                                    To run a simulation for Synopsys VCS:
                                                                                     
                                                                                        cd $VERDIR/scripts\n\n    gmake -f Makefile_VCS.mk run TESTNAME=ofs_mmio_test DUMP=1\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-tbd_3","title":"Questasim (TBD)","text":"
                                                                                    To run a simulation for Questasim:
                                                                                     
                                                                                        cd $VERDIR/scripts\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=ofs_mmio_test DUMP=1\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-vcs_4","title":"Synopsys VCS","text":"
                                                                                    To dump the waveform, \"DUMP=1\" must be added into the command line for Synopsys VCS build and simulation. 
                                                                                     
                                                                                        gmake -f Makefile_VCS.mk build_adp DUMP=1\n\n    gmake -f Makefile_VCS.mk run TESTNAME=<test_case_name> DUMP=1\n
                                                                                     
                                                                                    Or
                                                                                     
                                                                                        gmake -f Makefile_VCS.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-tbd_4","title":"Questasim (TBD)","text":"
                                                                                    To dump the waveform, \"DUMP=1\" must be added into the command line for Questasim build and simulation. 
                                                                                     
                                                                                        gmake -f Makefile_MSIM.mk build_adp DUMP=1\n\n    gmake -f Makefile_MSIM.mk run TESTNAME=<test_case_name> DUMP=1\n
                                                                                     
                                                                                    Or
                                                                                     
                                                                                        gmake -f Makefile_MSIM.mk build_run TESTNAME=<test_case_name> DUMP=1\n
                                                                                     
                                                                                    There are some optimizations in the Table below for convenience if you want to bypass some commands for both Synopsys VCS and Questasim:
                                                                                     Command (Synopsys VCS) Command (Questasim) Details gmake -f Makefile_VCS.mk build_all DUMP=1 gmake -f Makefile_MSIM.mk build_all DUMP=1 compile IP + compile RTL gmake -f Makefile_VCS.mk build_run TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk build_run TESTNAME= DUMP=1 compile RTL + run test gmake -f Makefile_VCS.mk do_it_all TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk do_it_all TESTNAME= DUMP=1 compile IP, RTL and run test gmake -f Makefile_VCS.mk rundb TESTNAME= DUMP=1 gmake -f Makefile_MSIM.mk rundb TESTNAME= DUMP=1 run test in sim dir + over-writes content"},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#67-uvm-regression-test","title":"6.7 UVM Regression Test","text":"
                                                                                    cd $VERDIR/scripts\n\npython uvm_regress.py -l -n 8 -p adp -k top_pkg -s vcs -c none\n\nFor Regression in VCS with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -p adp -k top_pkg -s vcs -c none\n\nResults are created in a sim directory ($VERDIR/sim) with individual testcase log dir\n\nFor Regression in MSIM with top/test package, execute the following command \n    python uvm_regress.py -l -n 8 -p adp -k top_pkg -s msim -c none\n
                                                                                     
                                                                                    Results are created in a sim directory ($VERDIR/sim_msim) with individual testcase log dir
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#68-uvm-waveform-and-transcript-analysis","title":"6.8 UVM Waveform and Transcript Analysis","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#synopsys-vcs_5","title":"Synopsys VCS","text":"
                                                                                    Running Synopsys VCS UVM tests will generate a ofs-n6001/verification/sim directory
                                                                                     
                                                                                    \u2022 All build time logs are located at ofs-n6001/verification/sim\n\n\u2022 Each testcase will have separate directory inside sim ofs-n6001/verification/sim/<test_case_name>\n
                                                                                     
                                                                                    There are two tracker or log files that are available: runsim.log and trans.log. 
                                                                                     
                                                                                    runsim.log is the simulation log file generated from Synopsys VCS. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 5
                                                                                     
                                                                                     
                                                                                    Figure 5 runsim.log
                                                                                     
                                                                                    trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 6
                                                                                     
                                                                                     
                                                                                    Figure 6 trans.log
                                                                                     
                                                                                    The waveform generated is named as \"inter.vpd\". To open the waveform, go to simulation result directory and run 
                                                                                     
                                                                                        dve -full64 -vpd inter.vpd &\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#questasim-tbd_5","title":"Questasim (TBD)","text":"
                                                                                    Running Questasim UVM tests will generate a ofs-n6001/verification/sim_msim directory
                                                                                     
                                                                                    \u2022 All build time logs are at ofs-n6001/verification/sim_msim\n\n\u2022 Each testcase will have separate directory inside sim ofs-n6001/verification/sim_msim/<test_case_name>\n
                                                                                     
                                                                                    There are two tracker or log files that are available: runsim.log and trans.log. 
                                                                                     
                                                                                    runsim.log is the simulation log file generated from Questasim. The test sequence prints useful information for debugging purpose, such as the base address for each function or block. For HE-LB and HE-MEM, key information such as SRC_ADDR, DST_ADDR, NUM_LINES, mode, req_len etc is printed out as shown in Figure 7
                                                                                     
                                                                                     
                                                                                    Figure 7 runsim.log
                                                                                     
                                                                                    trans.log is generated from PCIe host VIP. trans.log records all transaction information coming in or going out of the VIP. Users can find traffic direction(DIR), TLP type, Tag, Address or BDF, 3 or 4 dword header of the TLP as shown in Figure 8
                                                                                     
                                                                                     
                                                                                    Figure 8 trans.log
                                                                                     
                                                                                    The waveform generated is named as \"vsim.wlf\". To open the waveform, go to simulation result directory and run 
                                                                                     
                                                                                        vsim -view vsim.wlf &\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#69-uvm-coverage-analysis","title":"6.9 UVM Coverage Analysis","text":"
                                                                                    The following command allows to run a single testcase with coverage enabled
                                                                                     
                                                                                        gmake -f Makefile_VCS.mk cmplib_adp && gmake -f Makefile_VCS.mk build_adp DUMP=1 DEBUG=1 COV_FUNCTIONAL=1&& gmake -f Makefile_VCS.mk run TESTNAME=<TESTCASE-NAME> DUMP=1 DEBUG=1 COV_FUNCTIONAL=1 &\n
                                                                                     
                                                                                    The following command shows how to merge and generate the coverage report
                                                                                     
                                                                                        urg -dir <$VERDIR/sim/simv.vdb> <$VERDIR/sim/regression.vdb> -format both -dbname <regression_database_name>\n
                                                                                     
                                                                                    This will generate both urgreport directory and .vdb file Multiple regression.vdb from different regressions can be merged with the same command. 
                                                                                        e.g \"urg -dir <path1_till_simv.vdb> <path1_till_regression.vdb> <path2_till_regression.vdb> -report <dir> -format both -dbname <dirname>\"\n
                                                                                     
                                                                                    The following commands shows how to launch DVE and check the coverage reports
                                                                                     
                                                                                    To open DVE of a single regression or testcase, execute:\n\n    dve -full64 -cov -covdir simv.vdb regression.vdb &\n\nTo open DVE of a merged regression test, execute:\n\n    dve -full64 -cov -covdir <dirname.vdb> &\n
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#7-csr-verification-using-uvm-ral","title":"7 CSR Verification using UVM RAL","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#71-overview","title":"7.1 Overview","text":"
                                                                                    The UVM Register Layer provides a standard base class library that enable users to implement the object-oriented model to access the DUT registers and memories. The UVM Register Layer is also referred to as UVM Register Abstraction Layer (UVM RAL). Design registers can be accessed independently of the physical bus interface. i.e. by calling read/write methods. This is shown in Figure 9 below.
                                                                                     
                                                                                     
                                                                                    Figure 9 RAL UVM Testbench
                                                                                     
                                                                                    The RAL register models for different CSR's mimics the design registers. All RAL files are located here.
                                                                                     
                                                                                        ofs-n6001/verification/testbench/ral\n
                                                                                     
                                                                                    The RAL model is generated through the Synopsys RALGEN tool and is used for CSR verification.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#72-ral-integration","title":"7.2 RAL Integration","text":"
                                                                                    For UVM RAL model integration to the environment, adapters for each CSR is implemented and integrated into the Testbench Environment. It is used to convert the PCIe bus sequence items into uvm_reg_bus_op and vice versa. The CSR test cases pick up all the registers from the respective CSR blocks and perform a default value, wr/rd check on them. 
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#73-ral-model-generation","title":"7.3 RAL Model Generation","text":"
                                                                                    Steps for RAL model generation
                                                                                     
                                                                                    Excel(xls) file containing the registers is required. Make sure there are separate xls files for each CSR and the worksheet name must contain the name \"reg_fields\".  
                                                                                     
                                                                                    Excel sheet snapshot example below for EMIF_CSR.xls located at /ipss/mem/rtl/adp
                                                                                     
                                                                                     
                                                                                    \u2022 Navigate to ofs-n6001/ofs-common/verification/common/scripts/ral\n\u2022 Copy the excel file (xls) to the above area\n\u2022 In the bash terminal run mk_ral.sh <Excel sheet name without extension > <output *.sv file name without ral_  prepended >\n\u2022 The above steps generate two ral *.sv files. File with _cov suffix is a coverage enabled ral model. \n\u2022 Copy *.sv files to ofs-n6001/verification/testbench/ral\n
                                                                                     
                                                                                     
                                                                                    \u2022 As an example to generate ral_ac_ce.sv from AC_CE_CSR.xls file the command is\n\n    mk_ral.sh AC_CE_CSR ac_ce\n
                                                                                     
                                                                                    This generates two ral models (ral_ac_ce.sv and ral_ac_ce_cov.sv)
                                                                                     
                                                                                    To add new registers
                                                                                     
                                                                                    \u2022 To create new registers, copy existing ones and modify the cells in the xls. Make sure the last line is also a copied blank line\n\u2022 Follow all the steps of RAL model generation\n
                                                                                     
                                                                                    To Generate a RAL model when a new xls sheet is created for a new component
                                                                                     
                                                                                    \u2022 Copy the relevant xls sheet to ofs-n6001/ofs-common/verification/common/scripts/ral\n\u2022 Follow all the steps of RAL model generation\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#74-top-level-verification-architecture-for-csr-testing","title":"7.4 Top Level Verification Architecture for CSR testing","text":""},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#741-testbench-components","title":"7.4.1 Testbench components","text":"
                                                                                    The testbench components for RAL are defined below
                                                                                     
                                                                                    \u2022 ral_reg_iofs_* (uvm_reg) generated by the steps as mentioned in section 5.3\n
                                                                                     
                                                                                    The uvm register class is written by extending the uvm_reg. A register represents a set of fields that are accessible as a single entity Each register contains any number of fields, which mirror the values of the corresponding elements in hardware
                                                                                     
                                                                                    \u2022 ral_block_iofs_* (uvm_block) generated in the same register file\n
                                                                                     
                                                                                    A register model is an instance of a register block, which may contain any number of registers, register files, memories, and other blocks
                                                                                     
                                                                                    \u2022 ral_block_ofs (uvm_block) \u2013 Contains all the CSR block instantiations\n\u2022 Reg2vip_*_adapter (uvm_reg_adapter) \u2013 This class defines an interface for converting between uvm_reg_bus_op and a specific bus transaction. For each CSR a respective adapter is present\n
                                                                                     
                                                                                    All the components are defined in ofs-n6001/ofs-common/verification/testbench
                                                                                     
                                                                                    Integration of components in testbench
                                                                                     
                                                                                    \u2022 The RAL blocks and adapters for each CSR is instantiated in tb_env\n\u2022 The bar range for each CSR is also set in the tb_env\n
                                                                                     
                                                                                    Sample Environment Integration snippets 
                                                                                     
                                                                                     
                                                                                     
                                                                                     
                                                                                     
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#8-modifying-uvm-testbench","title":"8 Modifying UVM Testbench","text":"
                                                                                    The next sections describe what needs to be considered when modifying the UVM, adding a new interface to the testbench and creating a new UVM test for a customised OFS Accelerator platform.
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#81-modifying-uvm-environment-for-new-shell-variant","title":"8.1 Modifying UVM environment for new Shell Variant","text":"
                                                                                    OFS n6001 comprises a shell based on PCIe Gen4x16 and is named base_x16
                                                                                     
                                                                                    This base_x16 shell is described by an RTL file list, IP File lists and setup scripts to complete the build flow
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#82-modifying-uvm-environment-and-setting-up-compile-and-run-flow","title":"8.2 Modifying UVM environment and setting up Compile and Run flow","text":"
                                                                                    All the variants can mostly reuse the existing UVM infrastructure to setup the build and run flow
                                                                                     
                                                                                    \u2022 Create directory under $OFS_BUILD_ROOT new variant e.g ofs-n9000\n\u2022 Change directory to $OFS_BUILD_ROOT/ofs-n9000/verification/scripts\n\u2022 modify Makefile it to point to the new RTL, IP and script files.\n
                                                                                     
                                                                                    Following these three steps above will enable the build and sim flow to run the existing UVM TB and tests with new IOFS n9000 variant.
                                                                                     
                                                                                    Adding a new interface requires signal connections in the testbench. An additional BFM or verification IP is needed to drive the new interface. The main testbench file tb_top.sv is found at the following location
                                                                                     
                                                                                        ofs-n6001/verification/testbench\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#83-adding-a-new-ral-directory","title":"8.3 Adding a new RAL directory","text":"
                                                                                    In case the design has many register changes and the user decides to generate all the new RAL models instead of reusing from existing base RAL models, the following steps will help to create and integrate a new RALDIR in the UVM environment.
                                                                                     
                                                                                    \u2022 Generate the new RAL files in desired directory. Preferably under the \"ofs-n6001/verification/common/testbench\" \n\u2022 By default, the Makefile points to base FIM RAL so set the RALDIR path in the Makefile to the new generated RAL file directory\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#84-modifying-tbtest-files-for-new-variant","title":"8.4 Modifying TB/Test files for new Variant","text":"
                                                                                    Create a define for the new variant. e.g 'define FIM_NEW. If you are modifying common files then add new logic under this define so other projects will not get affected with variant specific change.
                                                                                     
                                                                                    If there are more changes, please create separate \"testbench\" and \"test\" folders under this new variant. 
                                                                                     
                                                                                    \u2022 Extend variant specific env file from base env file to add variant specific changes.\n\u2022 Create new test/seq lib files in \"tests\" folder.\n\u2022 Create new variant package, add new TB/Tests/Sequence lib files and also import the base package files.\n
                                                                                     
                                                                                    If you are adding new files then make sure it's included in Makefile for the build+run flow. 
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#85-uvm-pcie-drivers","title":"8.5 UVM PCIe Drivers","text":"
                                                                                    The \"svt_pcie_driver_app_transaction_base_sequence\" is part of Synopsys PCIe VIP library. You can find the sequence definition in the following two directories
                                                                                     
                                                                                    \u2022 Navigate to \"$DESIGNWARE_HOME/vip/svt/pcie_svt/Q-2020.03/sverilog/src/vcs/svt_pcie_driver_app_transaction_sequence_collection.svp\" file. All the base and PCIe sequences are defined in this file.\n\n\u2022 When the OFS UVM build command is executed, it creates \"vip\" directory under \"$OFS_BUILD_ROOT/ofs-n6001/verification\". You can also find the same sequence file at \"$IOFS_BUILD_ROOT/ofs-n6001/verification/vip/pcie_vip/src/sverilog/vcs/svt_pcie_driver_app_transaction_sequence_collection.sv\"\n
                                                                                    "},{"location":"hw/n6001/user_guides/ug_sim_ofs_n6001/ug_sim_ofs_n6001/#notices-disclaimers","title":"Notices & Disclaimers","text":"
                                                                                    Intel\u00ae technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure.  Performance varies by use, configuration and other factors. Your costs and results may vary.  You may not use or facilitate the use of this document in connection with any infringement or other legal analysis concerning Intel products described herein. You agree to grant Intel a non-exclusive, royalty-free license to any patent claim thereafter drafted which includes subject matter disclosed herein. No license (express or implied, by estoppel or otherwise) to any intellectual property rights is granted by this document, with the sole exception that you may publish an unmodified copy. You may create software implementations based on this document and in compliance with the foregoing that are intended to execute on the Intel product(s) referenced in this document. No rights are granted to create modifications or derivatives of this document. The products described may contain design defects or errors known as errata which may cause the product to deviate from published specifications.  Current characterized errata are available on request. Intel disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. You are responsible for safety of the overall system, including compliance with applicable safety-related requirements or standards.  \u00a9 Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. 
                                                                                     
                                                                                    OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission of the Khronos Group\u2122. 
                                                                                    "},{"location":"sw/build_chain/fpga_api/api_build/","title":"Building OPAE SDK Artifacts","text":""},{"location":"sw/build_chain/fpga_api/api_build/#steps","title":"Steps","text":"
                                                                                       
                                                                                    1. Fetch the OPAE SDK source tree
                                                                                      2.  
                                                                                    2. Configure the OPAE SDK CMake project
                                                                                      3.  
                                                                                    3. Build OPAE SDK targets
                                                                                      4.  
                                                                                     
                                                                                    The example below lists commands that can be used to fetch and build OPAE SDK.
                                                                                     
                                                                                    # fetch the source\ngit clone https://github.com/OPAE/opae-sdk.git\ncd opae-sdk\n# configure CMake\ncmake ..\n# build\nmake\n
                                                                                     
                                                                                    For a list of targets that can be built, type make help from the build directory.
                                                                                     
                                                                                    CMake options that may be set during the configuration include the following:
                                                                                     
                                                                                    |----------------------------|-----------------------|-------------------------------------|---------------------------------------|----------------|\n| cmake flag                 | Optional or Mandatory | Purpose                             | Valid values                          | Default value  |\n|----------------------------|-----------------------|-------------------------------------|---------------------------------------|----------------|\n| -DCMAKE_BUILD_TYPE         | Optional              | Set compiler flags                  | Debug/Release/Coverage/RelWithDebInfo | RelWithDebInfo |\n| -DOPAE_BUILD_LEGACY        | Optional              | Enable/disable opae-legacy.git      | ON/OFF                                | OFF            |\n| -DOPAE_BUILD_SPHINX_DOC    | Optional              | Enable/disable documentation build  | ON/OFF                                | OFF            |\n| -DOPAE_BUILD_TESTS         | Optional              | Enable/disable building unit tests  | ON/OFF                                | OFF            |\n| -DOPAE_INSTALL_RPATH       | Optional              | Enable/disable rpath for install    | ON/OFF                                | OFF            |\n| -DOPAE_BUILD_LIBOPAE_CXX   | Optional              | Enable/disable OPAE C++ bindings    | ON/OFF                                | ON             | \n| -DOPAE_WITH_PYBIND11       | Optional              | Enable/disable pybind11 binaries    | ON/OFF                                | ON             |\n| -DOPAE_BUILD_PYTHON_DIST   | Optional              | Enable/disable Python Distribution  | ON/OFF                                | OFF            |\n| -DOPAE_ENABLE_MOCK         | Optional              | Enable/disable mocks for unit tests | ON/OFF                                | OFF            |\n| -DOPAE_BUILD_SIM           | Optional              | Enable/disable opae-sim.git         | ON/OFF                                | OFF            |\n
                                                                                    "},{"location":"sw/build_chain/fpga_driver/driver_build/","title":"Building the OPAE Intel FPGA driver (out-of-tree)","text":"
                                                                                    The Intel FPGA driver included with OPAE SDK releases is packaged as an RPM or DEB package as well as a source tarball. Starting with OPAE SDK release of 1.4, the driver can be built from source out-of-tree but requires the following packages:
                                                                                     
                                                                                    For RPM package managers (Red Hat, CentOS, Fedora, etc.) * kernel-headers * kernel-devel * gcc * make
                                                                                     
                                                                                    For DEB package managers (Debian, Ubuntu, etc.) * kernel-headers-generic * gcc * make
                                                                                     
                                                                                    After installation of necessary distribution packages, follow the steps in the example below to build the Intel Kernel driver. NOTE The example below references Intel FPGA Kernel driver version 2.0.2. but can be applied to later versions.
                                                                                     
                                                                                    tar zxf opae-intel-fpga-driver-2.0.2-1.tar.gz\ncd opae-intel-fpga-driver-2.0.2\nmake\n
                                                                                    "},{"location":"sw/drv_arch/drv_arch/","title":"Open Programmable Accelerator Engine (OPAE) Linux Device Driver Architecture","text":"
                                                                                    The OPAE FPGA Linux Device Driver provides interfaces for user-space applications to configure, enumerate, open, and access FPGA accelerators on platforms equipped with Intel FPGA solutions. The OPAE FPGA driver also enables system-level management functions such as FPGA reconfiguration and virtualization.
                                                                                    "},{"location":"sw/drv_arch/drv_arch/#hardware-architecture","title":"Hardware Architecture","text":"
                                                                                    The Linux Operating System treats the FPGA hardware as a PCIe* device. A predefined data structure, Device Feature List (DFL), allows for dynamic feature discovery in an Intel FPGA solution.
                                                                                     
                                                                                     
                                                                                    The Linux Device Driver implements PCIe Single Root I/O Virtualization (SR-IOV) for the creation of Virtual Functions (VFs). The device driver can release individual accelerators for assignment to virtual machines (VMs).
                                                                                     
                                                                                    "},{"location":"sw/drv_arch/drv_arch/#fpga-management-engine-fme","title":"FPGA Management Engine (FME)","text":"
                                                                                    The FPGA Management Engine provides error reporting, reconfiguration, performance reporting, and other infrastructure functions. Each FPGA has one FME which is always accessed through the Physical Function (PF). The Intel Xeon\u00ae Processor with Integrated FPGA also performs power and thermal management. These functions are not available on the Intel Programmable Acceleration Card (PAC).
                                                                                     
                                                                                    User-space applications can acquire exclusive access to the FME using open(), and release it using close(). Device access may be managed by standard Linux interfaces and tools.
                                                                                     
                                                                                    If an application terminates without freeing the FME or Port resources, Linux closes all file descriptors owned by the terminating process, freeing those resources.
                                                                                    "},{"location":"sw/drv_arch/drv_arch/#port","title":"Port","text":"
                                                                                    A Port represents the interface between two components: * The FPGA Interface Manager (FIM) which is part of the static FPGA fabric * The Accelerator Function Unit (AFU) which is the partially reconfigurable region
                                                                                     
                                                                                    The Port controls the communication from software to the AFU and makes features such as reset and debug available.
                                                                                    "},{"location":"sw/drv_arch/drv_arch/#accelerator-function-unit-afu","title":"Accelerator Function Unit (AFU)","text":"
                                                                                    An AFU attaches to a Port. The AFU provides a 256 KB memory mapped I/O (MMIO) region for accelerator-specific control registers.
                                                                                     
                                                                                       
                                                                                    • Use open() on the Port device to acquire access to an AFU associated with the Port device.
                                                                                      •  
                                                                                    • Use close()on the Port device to release the AFU associated with the Port device.
                                                                                      •  
                                                                                    • Use mmap() on the Port device to map accelerator MMIO regions.
                                                                                      •  
                                                                                    "},{"location":"sw/drv_arch/drv_arch/#partial-reconfiguration-pr","title":"Partial Reconfiguration (PR)","text":"
                                                                                    Use PR to reconfigure an AFU from a bitstream file. Successful reconfiguration has the following requirement:
                                                                                     
                                                                                       
                                                                                    • You must generate the reconfiguration AFU for the exact FIM. The AFU and FIM are compatible if their interface IDs match. You can verify this match by comparing the interface ID in the bitstream header against the interface ID that is exported by the driver in sysfs.
                                                                                      •  
                                                                                     
                                                                                    In all other cases PR fails and may cause system instability.
                                                                                     
                                                                                    Platforms that support 512-bit Partial Reconfiguration require binutils >= version 2.25.
                                                                                     
                                                                                    Close any software programs accessing the FPGA, including those running in a virtualized host before initiating PR. For virtualized environments, the recommended sequence is as follows:
                                                                                     
                                                                                       
                                                                                    1. Unload the driver from the guest
                                                                                      2.  
                                                                                    2. Release the VF from the guest
                                                                                      3.  
                                                                                     
                                                                                    Releasing the VF from the guest while an application on the guest is still accessing its resources may lead to VM instabilities. We recommend closing all applications accessing the VF in the guest before releasing the VF.
                                                                                     
                                                                                       
                                                                                    1. Disable SR-IOV
                                                                                      2.  
                                                                                    2. Perform PR
                                                                                      3.  
                                                                                    3. Enable SR-IOV
                                                                                      4.  
                                                                                    4. Assign the VF to the guest
                                                                                      5.  
                                                                                    5. Load the driver in the guest
                                                                                      6.  
                                                                                    "},{"location":"sw/drv_arch/drv_arch/#fpga-virtualization","title":"FPGA Virtualization","text":"
                                                                                    To enable accelerator access from applications running on a VM, create a VF for the port using the following process:
                                                                                     
                                                                                       
                                                                                    1.  
                                                                                      Release the Port from the PF using the associated ioctl on the FME device.
                                                                                       
                                                                                      2.  
                                                                                    2.  
                                                                                      Use the following command to enable SR-IOV and VFs. Each VF can own a single Port with an AFU. In the following command, N is the number of Port released from the PF.
                                                                                       
                                                                                      3.  
                                                                                     
                                                                                        echo N > $PCI_DEVICE_PATH/sriov_numvfs\n
                                                                                     
                                                                                    The number, 'N', cannot be greater than the number of supported VFs. This can be read from $PCI_DEVICE_PATH/sriov_totalvfs.
                                                                                     
                                                                                       
                                                                                    1.  
                                                                                      Pass the VFs through to VMs using hypervisor interfaces.
                                                                                       
                                                                                      2.  
                                                                                    2.  
                                                                                      Access the AFU on a VF from applications running on the VM using the same driver inside the VM.
                                                                                       
                                                                                      3.  
                                                                                     
                                                                                    Creating VFs is only supported for port devices. Consequently, PR and other management functions are only available through the PF.    
                                                                                     
                                                                                     
                                                                                    If assigning multiple devices to the same VM on a guest IOMMU, you may need to increase the hard_limit option in order to avoid hitting a limit of pinned memory. The hard limit should be more than (VM memory size x Number of PCIe devices).
                                                                                    "},{"location":"sw/drv_arch/drv_arch/#driver-organization","title":"Driver Organization","text":""},{"location":"sw/drv_arch/drv_arch/#pcie-module-device-driver","title":"PCIe Module Device Driver","text":"
                                                                                    FPGA devices appear as a PCIe devices. Once enumeration detects a PCIe PF or VF, the Linux OS loads the FPGA PCIe device driver. The device driver performs the following functions:
                                                                                     
                                                                                       
                                                                                    1. Walks through the Device Feature List in PCIe device base address register (BAR) memory to discover features and their sub-features and creates necessary platform devices.
                                                                                      2.  
                                                                                    2. Enables SR-IOV.
                                                                                      3.  
                                                                                    3. Introduces the feature device infrastructure, which abstracts operations for sub-features and provides common functions to feature device drivers.
                                                                                      4.  
                                                                                    "},{"location":"sw/drv_arch/drv_arch/#pcie-module-device-driver-functions","title":"PCIe Module Device Driver Functions","text":"
                                                                                    The PCIe Module Device Driver performs the following functions:
                                                                                     
                                                                                       
                                                                                    1. PCIe discovery, device enumeration, and feature discovery.
                                                                                      2.  
                                                                                    2. Creates sysfs directories for the device, FME, and Port.
                                                                                      3.  
                                                                                    3. Creates the platform driver instances, causing the Linux kernel to load their respective drivers.
                                                                                      4.  
                                                                                    "},{"location":"sw/drv_arch/drv_arch/#fme-platform-module-device-driver","title":"FME Platform Module Device Driver","text":"
                                                                                    The FME Platform Module Device Driver loads automatically after the PCIe driver creates the FME Platform Module. It provides the following features for FPGA management:
                                                                                     
                                                                                       
                                                                                    1.  
                                                                                      Power and thermal management, error reporting, performance reporting, and other infrastructure functions. You can access these functions via sysfs interfaces the FME driver provides.
                                                                                       
                                                                                      2.  
                                                                                    2.  
                                                                                      Partial Reconfiguration. During PR sub-feature initialization, the FME driver registers the FPGA Manager framework to support PR. When the FME receives the relevant ioctl request from user-space, it invokes the common interface function from the FPGA Manager to reconfigure the AFU using PR.
                                                                                       
                                                                                      3.  
                                                                                    3.  
                                                                                      Port management for virtualization (releasing/assigning port device).
                                                                                       
                                                                                      4.  
                                                                                     
                                                                                    After a port device is released, you can use the PCIe driver SR-IOV interfaces to create/destroy VFs.
                                                                                     
                                                                                    For more information, refer to \"FPGA Virtualization\".
                                                                                    "},{"location":"sw/drv_arch/drv_arch/#fme-platform-module-device-driver-functions","title":"FME Platform Module Device Driver Functions","text":"
                                                                                    The FME Platform Module Device Driver performs the the following functions:
                                                                                     
                                                                                       
                                                                                    • Creates the FME character device node.
                                                                                      •  
                                                                                    • Creates the FME sysfs files and implements the FME sysfs file accessors.
                                                                                      •  
                                                                                    • Implements the FME private feature sub-drivers.
                                                                                      •  
                                                                                    • FME private feature sub-drivers:     * FME Header     * Partial Reconfiguration     * Global Error     * Global Performance
                                                                                      •  
                                                                                    "},{"location":"sw/drv_arch/drv_arch/#port-platform-module-device-driver","title":"Port Platform Module Device Driver","text":"
                                                                                    After the PCIe Module Device Driver creates the Port Platform Module device, the FPGA Port and AFU driver are loaded.  This module provides an interface for user-space applications to access the individual accelerators, including basic reset control on the Port, AFU MMIO region export, DMA buffer mapping service, and remote debug functions.
                                                                                    "},{"location":"sw/drv_arch/drv_arch/#port-platform-module-device-driver-functions","title":"Port Platform Module Device Driver Functions","text":"
                                                                                    The Port Platform Module Device Driver performs the the following functions:
                                                                                     
                                                                                       
                                                                                    • Creates the Port character device node.
                                                                                      •  
                                                                                    • Creates the Port sysfs files and implements the Port sysfs file accessors.
                                                                                      •  
                                                                                    • Implements the following Port private feature sub-drivers.     * Port Header     * AFU     * Port Error     * Signal Tap
                                                                                      •  
                                                                                    "},{"location":"sw/drv_arch/drv_arch/#opae-fpga-driver-interface","title":"OPAE FPGA Driver Interface","text":"
                                                                                    The user-space interface consists of a sysfs hierarchy and ioctl requests. Most kernel attributes can be accessed/modified via sysfs nodes in this hierarchy. More complex I/O operations are controlled via ioctl requests. The OPAE API implementation, libopae-c, has been designed to use this interface to interact with the OPAE FPGA kernel drivers.
                                                                                    "},{"location":"sw/fpga_api/fpga_api/","title":"OPAE C API Reference","text":"
                                                                                    The reference documentation for the OPAE C API is grouped into the following sections:
                                                                                     
                                                                                       
                                                                                    • Types
                                                                                         
                                                                                      • types.h
                                                                                        •  
                                                                                      • types_enum.h
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    • Enumeration API
                                                                                         
                                                                                      • enum.h
                                                                                        •  
                                                                                      • properties.h
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    • Access API
                                                                                         
                                                                                      • access.h
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    • Event API
                                                                                         
                                                                                      • event.h
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    • MMIO and Shared Memory APIs
                                                                                         
                                                                                      • mmio.h
                                                                                        •  
                                                                                      • buffer.h
                                                                                        •  
                                                                                      • umsg.h
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    • Management API
                                                                                         
                                                                                      • manage.h
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    • Metrics API
                                                                                         
                                                                                      • metrics.h
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    • SysObject
                                                                                         
                                                                                      • sysobject.h
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    • Utilities
                                                                                         
                                                                                      • utils.h
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    • Samples
                                                                                         
                                                                                      • hello_fpga.c
                                                                                        •  
                                                                                      • hello_events.c
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#types","title":"Types","text":"
                                                                                    The OPAE C API defines a number of types; most prominent are the types fpga_token, fpga_handle, and fpga_properties. All regular types are defined in types.h, while the values of enumeration types are defined in types_enum.h.
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#typesh","title":"types.h","text":"
                                                                                    types.h
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#types_enumh","title":"types_enum.h","text":"
                                                                                    types_enum.h
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#enumeration-api","title":"Enumeration API","text":"
                                                                                    The OPAE enumeration API allows selective discovery of FPGA resources. When enumerating resources, a list of filter criteria can be passed to the respective function to select a subset of all resources in the system. The fpgaEnumerate() function itself then returns a list of fpga_tokens denoting resources, which can be used in subsequent API calls.
                                                                                     
                                                                                    Filter criteria are specified using one or more fpga_properties object. These objects need to be created using fpgaGetProperties() (defined in ) before being passed to fpgaEnumerate(). Individual attributes of an fpga_properties object are set using specific accessors, which are also defined in ."},{"location":"sw/fpga_api/fpga_api/#enumh","title":"enum.h","text":"
                                                                                    enum.h
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#propertiesh","title":"properties.h","text":"
                                                                                    properties.h
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#access-api","title":"Access API","text":"
                                                                                    The access API provides functions for opening and closing FPGA resources. Opening a resource yields an fpga_handle, which denotes ownership and can be used in subsequent API calls to interact with a specific resource. Ownership can be exclusive or shared.
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#accessh","title":"access.h","text":"
                                                                                    access.h
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#event-api","title":"Event API","text":"
                                                                                    The event API provides functions and types for handling asynchronous events such as errors or accelerator interrupts.
                                                                                     
                                                                                    To natively support asynchronous event, the driver for the FPGA platform needs to support events natively (in which case the OPAE C library will register the event directly with the driver). For some platforms that do not support interrupt-driven event delivery, you need to run the FPGA Daemon (fpgad) to enable asynchronous OPAE events. fpgad will act as a proxy for the application and deliver asynchronous notifications for registered events.
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#eventh","title":"event.h","text":"
                                                                                    event.h
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#mmio-and-shared-memory-apis","title":"MMIO and Shared Memory APIs","text":"
                                                                                    These APIs feature functions for mapping and accessing control registers through memory-mapped IO (mmio.h), allocating and sharing system memory buffers with an accelerator (buffer.h), and using low-latency notifications (umsg.h).
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#mmioh","title":"mmio.h","text":"
                                                                                    mmio.h
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#bufferh","title":"buffer.h","text":"
                                                                                    buffer.h
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#umsgh","title":"umsg.h","text":"
                                                                                    umsg.h
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#management-api","title":"Management API","text":"
                                                                                    The management APIs define functions for reconfiguring an FPGA (writing new partial bitstreams) as well as assigning accelerators to host interfaces.
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#manageh","title":"manage.h","text":"
                                                                                    manage.h
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#metrics-api","title":"Metrics API","text":"
                                                                                    The metrics APIs define functions for discovery/enumeration of metrics information and reading metrics values.
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#metricsh","title":"metrics.h","text":"
                                                                                    metrics.h
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#sysobject","title":"SysObject","text":"
                                                                                    The SysObject API can be used to get system objects by name. Names used with the SysObject API are driver-specific and may not be compatible across plugins and/or drivers. For example, SysObject names used with the xfpga plugin will apply to the OPAE Linux Kernel driver and refer to sysfs nodes under the sysfs tree for the resource used with the SysObject API.
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#sysobjecth","title":"sysobject.h","text":"
                                                                                    sysobject.h
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#utilities","title":"Utilities","text":"
                                                                                    Functions for mapping fpga_result values to meaningful error strings are provided by the utilities API.
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#utilsh","title":"utils.h","text":"
                                                                                    utils.h
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#samples","title":"Samples","text":"
                                                                                    Code samples demonstrate how to use OPAE C API.
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#hello_fpgac","title":"hello_fpga.c","text":"
                                                                                    hello_fpga.c
                                                                                    "},{"location":"sw/fpga_api/fpga_api/#hello_eventsc","title":"hello_events.c","text":"
                                                                                    hello_events.c
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/","title":"OPAE C++ Core API Reference","text":"
                                                                                    The reference documentation for the OPAE C++ Core API is grouped into the following sections:
                                                                                     
                                                                                       
                                                                                    • Overview
                                                                                      •  
                                                                                    • Goals
                                                                                         
                                                                                      • Simplicity
                                                                                        •  
                                                                                      • Extensibility and Interoperability
                                                                                        •  
                                                                                      • Modern C++ Coding Practices
                                                                                        •  
                                                                                      • Error Handling
                                                                                        •  
                                                                                      • Coding Style
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    • Fundamental Types
                                                                                         
                                                                                      • Properties
                                                                                        •  
                                                                                      • pvalue.h
                                                                                        •  
                                                                                      • properties.h
                                                                                        •  
                                                                                      • Resource Classes
                                                                                        •  
                                                                                      • token.h
                                                                                        •  
                                                                                      • handle.h
                                                                                        •  
                                                                                      • shared_buffer.h
                                                                                        •  
                                                                                      • errors.h
                                                                                        •  
                                                                                      • events.h
                                                                                        •  
                                                                                      • sysobject.h
                                                                                        •  
                                                                                      • Exceptions
                                                                                        •  
                                                                                      • except.h
                                                                                        •  
                                                                                      • Misc
                                                                                        •  
                                                                                      • version.h
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#overview","title":"Overview","text":"
                                                                                    The OPAE C++ API enables C++ developers with the means to use FPGA resources by integrating the OPAE software stack into C++ applications.
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#goals","title":"Goals","text":""},{"location":"sw/fpga_api/fpga_cxx_api/#simplicity","title":"Simplicity","text":"
                                                                                    Keep the API as small and lightweight as possible. Although features such as system validation and orchestration are beyond the scope of this API, using this API for their development should be relatively easy.
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#extensibility-and-interoperability","title":"Extensibility and Interoperability","text":"
                                                                                    While keeping to the goal of simplicity, the OPAE C++ API is designed to allow for better reuse by either extending the API or by integrating with other languages.
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#modern-c-coding-practices","title":"Modern C++ Coding Practices","text":"
                                                                                    The OPAE C++ API uses the C++ 11 standard library and makes use of its features whenever practical. The OPAE C++ API is also designed to require the minimum number of third-party libraries/dependencies.
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#error-handling","title":"Error Handling","text":"
                                                                                    The OPAE C++ API is designed to throw exceptions when appropriate. The structure of OPAE C++ exceptions is similar to the error codes in the OPAE C API. This gives users of the API more freedom on error handling while providing better debug information in cases of failure.
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#coding-style","title":"Coding Style","text":"
                                                                                    For formatting of the OPAE C++ API complies with most of the recommendations of the Google C++ style. For example, the OPAE C++ API uses:
                                                                                     
                                                                                       
                                                                                    • opening braces on the same line as their scope definition
                                                                                      •  
                                                                                    • spaces instead of tabs for indentation
                                                                                      •  
                                                                                    • indentation of two spaces
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#fundamental-types","title":"Fundamental Types","text":"
                                                                                    Basic types for the OPAE C++ API are found in the opae::fpga::types namespace. They serve as an adapter layer between the OPAE C API and the OPAE C++ layer. Aside from providing a C++ binding to the C fundamental types, these types also:
                                                                                     
                                                                                       
                                                                                    • manage the lifetime and scope of the corresponding C struct.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • For example a C++ destructor will take care of calling the appropriate C function to release the data structure being wrapped.
                                                                                      •  
                                                                                     
                                                                                       
                                                                                    • provide a friendly syntax for using the OPAE C type.
                                                                                      •  
                                                                                     
                                                                                    Most classes in this namespace have a c_type() method that returns the C data structure being wrapped, making it easy to use the OPAE C++ type with the OPAE C API. Alternatively, most classes in this namespace have implicit conversion operators that enable interoperability with the OPAE C API.
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#properties","title":"Properties","text":"
                                                                                    C++ class properties wraps fpga_properties and uses pvalue and guid_t to get and set properties stored in an instance of an fpga_properties. pvalue and guid_t are designed to call an accessor method in the OPAE C API to either read property values or write them. Most accessor methods in the OPAE C API share a similar signature, so pvalue generalizes them into common operations that translate into calling the corresponding C API function. guid_t follows similar patterns when reading or assigning values.
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#pvalueh","title":"pvalue.h","text":"
                                                                                    pvalue.h
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#propertiesh","title":"properties.h","text":"
                                                                                    properties.h
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#resource-classes","title":"Resource Classes","text":"
                                                                                    The token, handle, and shared_buffer classes are used to enumerate and access FPGA resources. properties are used to narrow the search space for token's. Before enumerating the accelerator resources in the system, applications can produce one or more properties objects whose values are set to the desired characteristics for the resource. For example, an application may search for an accelerator resource based on its guid.
                                                                                     
                                                                                    Once one or more token's have been enumerated, the application must choose which token's to request. The token is then converted to a handle by requesting that a handle object be allocated and opened for it.
                                                                                     
                                                                                    Once a handle has been successfully opened, the application can read and write the associated configuration and status space. Additionally, the application may use the handle to allocate shared_buffer's or to register event's. The shared_buffer and event objects retain a reference to their owning handle so that the handle does not lose scope before freeing the shared_buffer and event objects.
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#tokenh","title":"token.h","text":"
                                                                                    token.h
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#handleh","title":"handle.h","text":"
                                                                                    handle.h
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#shared_bufferh","title":"shared_buffer.h","text":"
                                                                                    shared_buffer.h
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#errorsh","title":"errors.h","text":"
                                                                                    errors.h
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#eventsh","title":"events.h","text":"
                                                                                    events.h
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#sysobjecth","title":"sysobject.h","text":"
                                                                                    sysobject.h
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#exceptions","title":"Exceptions","text":"
                                                                                    When the OPAE C++ API encounters an error from the OPAE C API, it captures the current source code location and the error code into an object of type except, then throws the except. Applications should implement the appropriate catch blocks required to respond to runtime exceptions.
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#excepth","title":"except.h","text":"
                                                                                    except.h
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#misc","title":"Misc","text":"
                                                                                    The version class wraps the OPAE C version API.
                                                                                    "},{"location":"sw/fpga_api/fpga_cxx_api/#versionh","title":"version.h","text":"
                                                                                    version.h
                                                                                    "},{"location":"sw/fpga_api/fpga_python_api/","title":"OPAE Python API Reference","text":"
                                                                                    The reference documentation for the OPAE Python API and is grouped into the following sections:
                                                                                     
                                                                                       
                                                                                    • Module Types, Methods, and Constants
                                                                                      •  
                                                                                    • Fundamental Types
                                                                                         
                                                                                      • Properties
                                                                                        •  
                                                                                      • Token
                                                                                        •  
                                                                                      • Handle
                                                                                        •  
                                                                                      • Event
                                                                                        •  
                                                                                      • Shared Buffer
                                                                                        •  
                                                                                      • Error
                                                                                        •  
                                                                                      • SysObject
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_api/fpga_python_api/#module-types-methods-and-constants","title":"Module Types, Methods, and Constants","text":""},{"location":"sw/fpga_api/fpga_python_api/#fundamental-types","title":"Fundamental Types","text":""},{"location":"sw/fpga_api/fpga_python_api/#properties","title":"Properties","text":""},{"location":"sw/fpga_api/fpga_python_api/#token","title":"Token","text":""},{"location":"sw/fpga_api/fpga_python_api/#handle","title":"Handle","text":""},{"location":"sw/fpga_api/fpga_python_api/#event","title":"Event","text":""},{"location":"sw/fpga_api/fpga_python_api/#shared-buffer","title":"Shared Buffer","text":""},{"location":"sw/fpga_api/fpga_python_api/#error","title":"Error","text":""},{"location":"sw/fpga_api/fpga_python_api/#sysobject","title":"SysObject","text":""},{"location":"sw/fpga_api/plug_guide/readme/","title":"Plugin Developer's Guide","text":""},{"location":"sw/fpga_api/plug_guide/readme/#overview","title":"Overview","text":"
                                                                                    Beginning with OPAE C library version 1.2.0, OPAE implements a plugin-centric model. This guide serves as a reference to define the makeup of an OPAE C API plugin and to describe a sequence of steps that one may follow when constructing an OPAE C API plugin.
                                                                                    "},{"location":"sw/fpga_api/plug_guide/readme/#plugin-required-functions","title":"Plugin Required Functions","text":"
                                                                                    An OPAE C API plugin is a runtime-loadable shared object library, also known as a module. On Linux systems, the dl family of APIs from libdl are used to interact with shared objects. Refer to \"man dlopen\" and \"man dlsym\" for examples of using the libdl API.
                                                                                     
                                                                                    An OPAE C API plugin implements one required function. This function is required to have C linkage, so that its name is not mangled.
                                                                                     
                                                                                        int opae_plugin_configure(opae_api_adapter_table *table, const char *config);\n
                                                                                     
                                                                                    During initialization, the OPAE plugin manager component loads each plugin, searching for its opae_plugin_configure function. If none is found, then the plugin manager rejects that plugin. When it is found, opae_plugin_configure is called passing a pointer to a freshly-created opae_api_adapter_table and a buffer consisting of configuration data for the plugin.
                                                                                     
                                                                                    The job of the opae_plugin_configure function is to populate the given adapter table with each of the plugin's API entry points and to consume and comprehend the given configuration data in preparation for initialization.
                                                                                    "},{"location":"sw/fpga_api/plug_guide/readme/#opae-api-adapter-table","title":"OPAE API Adapter Table","text":"
                                                                                    The adapter table is a data structure that contains function pointer entry points for each of the OPAE APIs implemented by a plugin. In this way, it adapts the plugin-specific behavior to the more general case of a flat C API. Note that OPAE applications are only required to link with opae-c. In other words, the name of the plugin library should not appear on the linker command line. In this way, plugins are truly decoupled from the OPAE C API, and they are required to adapt to the strict API specification by populating the adapter table only. No other linkage is required nor recommended.
                                                                                     
                                                                                    adapter.h contains the definition of the opae_api_adapter_table. An abbreviated version is depicted below, along with supporting type opae_plugin:
                                                                                     
                                                                                        typedef struct _opae_plugin {\n        char *path;\n        void *dl_handle;\n    } opae_plugin;\n\n    typedef struct _opae_api_adapter_table {\n\n        struct _opae_api_adapater_table *next;\n        opae_plugin plugin;\n\n        fpga_result (*fpgaOpen)(fpga_token token, fpga_handle *handle,\n                                int flags);\n\n        fpga_result (*fpgaClose)(fpga_handle handle);\n\n        ...\n\n        fpga_result (*fpgaEnumerate)(const fpga_properties *filters,\n                                     uint32_t num_filters, fpga_token *tokens,\n                                     uint32_t max_tokens,\n                                     uint32_t *num_matches);\n\n        ...\n\n        // configuration functions\n        int (*initialize)(void);\n        int (*finalize)(void);\n\n        // first-level query\n        bool (*supports_device)(const char *device_type);\n        bool (*supports_host)(const char *hostname);\n\n    } opae_api_adapter_table;\n
                                                                                     
                                                                                    Some points worth noting are that the adapter tables are organized in memory by adding them to a linked list data structure. This is the use of the next structure member. (The list management is handled by the plugin manager.) The plugin structure member contains the handle to the shared object instance, as created by dlopen. This handle is used in the plugin's opae_plugin_configure to load plugin entry points. A plugin need only implement the portion of the OPAE C API that a target application needs. Any API entry points that are not supported should be left as NULL pointers (the default) in the adapter table. When an OPAE API that has no associated entry point in the adapter table is called, the result for objects associated with that plugin will be FPGA_NOT_SUPPORTED.
                                                                                     
                                                                                    The following code illustrates a portion of the opae_plugin_configure for a theoretical OPAE C API plugin libfoo.so:
                                                                                     
                                                                                        /* foo_plugin.c */\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        adapter->fpgaOpen = dlsym(adapter->plugin.dl_handle, \"foo_fpgaOpen\");\n        adapter->fpgaClose =\n                dlsym(adapter->plugin.dl_handle, \"foo_fpgaClose\");\n\n        ...\n\n        adapter->fpgaEnumerate =\n                dlsym(adapter->plugin.dl_handle, \"foo_fpgaEnumerate\");\n\n        ...\n\n        return 0;\n    }\n
                                                                                     
                                                                                    Notice that the implementations of the API entry points for plugin libfoo.so are prefixed with foo_. This is the recommended practice to avoid name collisions and to enhance the debugability of the application. Upon successful configuration, opae_plugin_configure returns 0 to indicate success. A non-zero return value indicates failure and causes the plugin manager to reject the plugin from futher consideration.
                                                                                    "},{"location":"sw/fpga_api/plug_guide/readme/#plugin-optional-functions","title":"Plugin Optional Functions","text":"
                                                                                    Once the plugin manager loads and configures each plugin, it uses the adapter table to call back into the plugin so that it can be made ready for runtime. This is the job of the opae_plugin_initialize entry point, whose signature is defined as:
                                                                                     
                                                                                        int opae_plugin_initialize(void);\n
                                                                                     
                                                                                    The function takes no parameters, as the configuration data was already given to the plugin by opae_plugin_configure. opae_plugin_initialize returns 0 if no errors were encountered during initialization. A non-zero return code indicates that plugin initialization failed. A plugin makes its opae_plugin_initialize available to the plugin manager by populating the adapter table's initialize entry point as shown:
                                                                                     
                                                                                        /* foo_plugin.c */\n\n    int foo_plugin_initialize(void)\n    {\n        ...\n\n        return 0; /* success */\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->initialize =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_initialize\");\n\n        ...\n\n        return 0;\n    }\n
                                                                                     
                                                                                    If a plugin does not implement an opae_plugin_initialize entry point, then the initialize member of the adapter table should be left uninitialized. During plugin initialization, if a plugin has no opae_plugin_initialize entry in its adapter table, the plugin initialization step will be skipped, and the plugin will be considered to have initialized successfully.
                                                                                     
                                                                                    Once plugin initialization is complete for all loaded plugins, the system is considered to be running and fully functional.
                                                                                     
                                                                                    During teardown, the plugin manager uses the adapter table to call into each plugin's opae_plugin_finalize entry point, whose signature is defined as:
                                                                                     
                                                                                        int opae_plugin_finalize(void);\n
                                                                                     
                                                                                    opae_plugin_finalize returns 0 if no errors were encountered during teardown. A non-zero return code indicates that plugin teardown failed. A plugin makes its opae_plugin_finalize available to the plugin manager by populating the adapter table's finalize entry point as shown:
                                                                                     
                                                                                        /* foo_plugin.c */\n\n    int foo_plugin_finalize(void)\n    {\n        ...\n\n        return 0; /* success */\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->finalize =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_finalize\");\n\n        ...\n\n        return 0;\n    }\n
                                                                                     
                                                                                    If a plugin does not implement an opae_plugin_finalize entry point, then the finalize member of the adapter table should be left uninitialized. During plugin cleanup, if a plugin has no opae_plugin_finalize entry point in its adapter table, the plugin finalize step will be skipped, and the plugin will be considered to have finalized successfully.
                                                                                     
                                                                                    In addition to initialize and finalize, an OPAE C API plugin has two further optional entry points that relate to device enumeration. During enumeration, when a plugin is being considered for a type of device, the plugin may provide input on that decision by exporting an opae_plugin_supports_device entry point in the adapter table:
                                                                                     
                                                                                        bool opae_plugin_supports_device(const char *device_type);\n
                                                                                     
                                                                                    opae_plugin_supports_device returns true if the given device type is supported and false if it is not. A false return value from opae_plugin_supports_device causes device enumeration to skip the plugin.
                                                                                     
                                                                                    Populating the opae_plugin_supports_device is done as:
                                                                                     
                                                                                        /* foo_plugin.c */\n\n    bool foo_plugin_supports_device(const char *device_type)\n    {\n        if (/* device_type is supported */)\n            return true;\n\n        ...\n\n        return false;\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->supports_device =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_supports_device\");\n\n        ...\n\n        return 0;\n    }\n
                                                                                     
                                                                                    .. note::\n    The `opae_plugin_supports_device` mechanism serves as a placeholder only.\n    It is not implemented in the current version of the OPAE C API.\n
                                                                                     
                                                                                    Similarly to determining whether a plugin supports a type of device, a plugin may also answer questions about network host support by populating an opae_plugin_supports_host entry point in the adapter table:
                                                                                     
                                                                                        bool opae_plugin_supports_host(const char *hostname);\n
                                                                                     
                                                                                    opae_plugin_supports_host returns true if the given hostname is supported and false if it is not. A false return value from opae_plugin_supports_host causes device enumeration to skip the plugin.
                                                                                     
                                                                                    Populating the opae_plugin_supports_host is done as:
                                                                                     
                                                                                        /* foo_plugin.c */\n\n    bool foo_plugin_supports_host(const char *hostname)\n    {\n        if (/* hostname is supported */)\n            return true;\n\n        ...\n\n        return false;\n    }\n\n    int opae_plugin_configure(opae_api_adapter_table *table, const char *config)\n    {\n        ... \n\n        adapter->supports_host =\n                dlsym(adapter->plugin.dl_handle, \"foo_plugin_supports_host\");\n\n        ...\n\n        return 0;\n    }\n
                                                                                     
                                                                                    .. note::\n    The `opae_plugin_supports_host` mechanism serves as a placeholder only.\n    It is not implemented in the current version of the OPAE C API.\n
                                                                                    "},{"location":"sw/fpga_api/plug_guide/readme/#plugin-construction","title":"Plugin Construction","text":"
                                                                                    The steps required to implement an OPAE C API plugin, libfoo.so, are:
                                                                                     
                                                                                       
                                                                                    • Create foo_plugin.c: implements opae_plugin_configure, opae_plugin_initialize, opae_plugin_finalize, opae_plugin_supports_device, and opae_plugin_supports_host as described in the previous sections.
                                                                                      •  
                                                                                    • Create foo_plugin.h: implements function prototypes for each of the plugin-specific OPAE C APIs.
                                                                                      •  
                                                                                     
                                                                                        /* foo_plugin.h */\n\n    fpga_result foo_fpgaOpen(fpga_token token, fpga_handle *handle,\n                             int flags);\n\n    fpga_result foo_fpgaClose(fpga_handle handle);\n\n    ...\n\n    fpga_result foo_fpgaEnumerate(const fpga_properties *filters,\n                                  uint32_t num_filters, fpga_token *tokens,\n                                  uint32_t max_tokens,\n                                  uint32_t *num_matches);\n    ...\n
                                                                                     
                                                                                       
                                                                                    • Create foo_types.h: implements plugin-specific types for opaque data structures.
                                                                                      •  
                                                                                     
                                                                                        /* foo_types.h */\n\n    struct _foo_token {\n        ...\n    };\n\n    struct _foo_handle {\n        ...\n    };\n\n    struct _foo_event_handle {\n        ...\n    };\n\n    struct _foo_object {\n        ...\n    };\n
                                                                                     
                                                                                       
                                                                                    • Create foo_enum.c: implements foo_fpgaEnumerate, foo_fpgaCloneToken, and foo_fpgaDestroyToken.
                                                                                      •  
                                                                                    • Create foo_open.c: implements foo_fpgaOpen.
                                                                                      •  
                                                                                    • Create foo_close.c: implements foo_fpgaClose.
                                                                                      •  
                                                                                    • Create foo_props.c: implements foo_fpgaGetProperties, foo_fpgaGetPropertiesFromHandle, foo_fpgaUpdateProperties
                                                                                      •  
                                                                                    • Create foo_mmio.c: implements foo_fpgaMapMMIO, foo_fpgaUnmapMMIO foo_fpgaWriteMMIO64, foo_fpgaReadMMIO64, foo_fpgaWriteMMIO32, foo_fpgaReadMMIO32.
                                                                                      •  
                                                                                    • Create foo_buff.c: implements foo_fpgaPrepareBuffer, foo_fpgaReleaseBuffer, foo_fpgaGetIOAddress.
                                                                                      •  
                                                                                    • Create foo_error.c: implements foo_fpgaReadError, foo_fpgaClearError, foo_fpgaClearAllErrors, foo_fpgaGetErrorInfo.
                                                                                      •  
                                                                                    • Create foo_event.c: implements foo_fpgaCreateEventHandle, foo_fpgaDestroyEventHandle, foo_fpgaGetOSObjectFromEventHandle, foo_fpgaRegisterEvent, foo_fpgaUnregisterEvent.
                                                                                      •  
                                                                                    • Create foo_reconf.c: implements foo_fpgaReconfigureSlot.
                                                                                      •  
                                                                                    • Create foo_obj.c: implements foo_fpgaTokenGetObject, foo_fpgaHandleGetObject, foo_fpgaObjectGetObject, foo_fpgaDestroyObject, foo_fpgaObjectGetSize, foo_fpgaObjectRead, foo_fpgaObjectRead64, foo_fpgaObjectWrite64.
                                                                                      •  
                                                                                    • Create foo_clk.c: implements foo_fpgaSetUserClock, foo_fpgaGetUserClock.
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/","title":"OPAE C API Programming Guide","text":""},{"location":"sw/fpga_api/prog_guide/readme/#overview","title":"Overview","text":"
                                                                                    The OPAE C library (libopae-c) is a lightweight user-space library that provides abstractions for FPGA resources in a compute environment. The OPAE C library  builds on the driver stack that supports the FPGA device, abstracting  hardware- and OS-specific details. It provides access to the underlying FPGA resources as a set of features available to software programs running on the host. These features include the acceleration logic preconfigured on the FPGA and functions to manage and reconfigure the FPGA. The library enables your applications to transparently and seamlessly benefit from FPGA-based acceleration.
                                                                                     
                                                                                     
                                                                                    By providing a unified C API, the library supports different FPGA integration and deployment models, ranging from single-node systems with one or a few FPGA devices to large-scale FPGA deployments in a data center. At one end of the spectrum, the API supports a simple application using a PCIe link to reconfigure  the FPGA with different accelerator functions. At the other end of the spectrum, resource management and orchestration services in a data center can use this API to discover and select FPGA resources and then allocate them for use by acceleration workloads.
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/#opae-role","title":"OPAE Role","text":"
                                                                                    The OPAE provides a common base layer for a wide range of applications without sacrificing performance or efficiency. The abstraction layer limits  the details of the FPGA hardware that software applications must handle. 
                                                                                     
                                                                                    The OPAE provides consistent interfaces to crucial components of the platform. The  OPAE does not constrain frameworks and applications by making optimizations with limited applicability. When the OPAE does provide convenience functions or optimizations, they are optional.
                                                                                     
                                                                                    For example, the OPAE provides an interface to allocate physically contiguous buffers in system memory that user-space software and an accelerator can share.  This interface enables the most basic feature set of allocating and sharing a large page of memory in one API call. However, it  does not provide a malloc()-like interface backed by a memory pool or slab allocator. Higher layers of the software stack can make such domain-specific optimizations.
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/#intel-accelerator-stack-hardware-terminology","title":"Intel Accelerator Stack Hardware Terminology","text":"
                                                                                    The following terms define the hardware and hardware processes involved in creating an accelerator function. 
                                                                                     
                                                                                       
                                                                                    • FPGA: Field Programmable Gate Array  is a discrete or integrated device connecting to a host CPU via PCIe or other type of interconnects.
                                                                                      •  
                                                                                    • Accelerator Function Unit (AFU): The AFU is the supplied implementation of an accelerator, typically in HDL. AFUs implement a function such as compression, encryption, or mathematical operations. The Quartus Prime Pro software synthesizes the RTL logic into a bitstream. 
                                                                                      •  
                                                                                    • Accelerator Function (AF): The AF is the compiled binary for an AFU. An AF is a raw binary file (.rbf) bitstream. A tool (fpgaconf) reconfigures the FPGA using an AF bitstream.
                                                                                      •  
                                                                                    • Reconfiguration: The process of reprogramming the FPGA with a different AF.
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/#opae-software-concepts-reflected-in-the-c-api","title":"OPAE Software Concepts Reflected in the C API","text":"
                                                                                    The following OPAE data structures and functions integrate AFUs into the OPAE environment.  The OPAE C API models these data structures and functions. For more information on the object  models refer to the Object model section.
                                                                                     
                                                                                       
                                                                                    • Accelerator: An accelerator is an allocable accelerator function implemented in an FPGA.  An accelerator tracks the  ownership of an AFU (or part of it) for a process that uses it. Multiple processes can share an accelerator.
                                                                                      •  
                                                                                    • Device: The OPAE enumerates and models two device types: the FPGA and the AFU.
                                                                                      •  
                                                                                    • Events: Events are asynchronous notifications. The FPGA driver triggers particular events to indicate error conditions. Accelerator logic can also define its own events. User applications can choose to be notified when particular events occur and respond appropriately.
                                                                                      •  
                                                                                    • Shared memory buffers: Software allocates shared memory buffers in user process memory on the host. Shared memory buffers facilitate data transfers between the user process and the  accelerator that it owns.
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/#opae-library","title":"OPAE Library","text":"
                                                                                    Linking with this library is straightforward. Code using the  OPAE library should include the header file fpga.h. Taking the GCC compiler on Linux as an example, here is the simplest compile and link command:
                                                                                     
                                                                                    gcc myprog.c -I</path/to/fpga.h> -L</path/to/libopae-c.so> -lopae-c -luuid -ljson-c -lpthread
                                                                                     
                                                                                    .. note::
                                                                                     
                                                                                    The OPAE library uses the third-party `libuuid` and `libjson-c` libraries that are not distributed with \nthe OPAE library. Make sure to install these libraries.\n
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/#sample-code","title":"Sample Code","text":"
                                                                                    The library source includes two code samples. Use these samples to learn how to call functions in the library. Build and run these samples to determine if your installation and environment are set up properly. 
                                                                                     
                                                                                    Refer to the Running the Hello FPGA Example chapter in the Intel\u00ae Acceleration Stack Quick Start Guide for for Intel Programmable Acceleration Card with Intel Arria\u00ae 10 GX FPGA for more information about using the sample code.  
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/#high-level-directory-structure","title":"High-Level Directory Structure","text":"
                                                                                    Building and installing the OPAE library results in the following directory structure on the Linux OS. Windows and MacOS have similar directories and files.
                                                                                     Directory & Files Contents include/opae Directory containing all header files include/opae/fpga.h Top-level header for user code to include include/opae/access.h Header file for accelerator acquire/release, MMIO, memory management, event handling, and so on include/opae/bitstream.h Header file for bitstream manipulation functions include/opae/common.h Header file for error reporting functions include/opae/enum.h Header file for AFU enumeration functions include/opae/manage.h Header file for FPGA management functions include/opae/types.h Various type definitions lib Directory containing shared library files lib/libopae-c.so The shared dynamic library for linking with the user application doc Directory containing API documentation doc/html Directory for documentation of HTML format doc/latex Directory for documentation of LaTex format doc/man Directory for documentation of Unix man page format"},{"location":"sw/fpga_api/prog_guide/readme/#basic-application-flow","title":"Basic Application Flow","text":"
                                                                                    The figure below shows the basic application flow from the viewpoint of a user-process. 
                                                                                     
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/#api-components","title":"API Components","text":"
                                                                                    The API object model abstracts the physical FPGA device and available functions. It is a generalized model and extends to  describe any FPGA type. 
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/#object-models","title":"Object Models","text":"
                                                                                       
                                                                                    • fpga_objtype: An enum type that represents the type of an FPGA resource, either FPGA_DEVICE or FPGA_ACCELERATOR.  An FPGA_DEVICE object corresponds to a physical FPGA device. Only FPGA_DEVICE objects can invoke management functions. The FPGA_ACCELERATOR represents an instance of an AFU. 
                                                                                      •  
                                                                                    • fpga_token: An opaque type that represents a resource known to, but not necessarily owned by, the calling process. The calling process must own a resource before it can invoke functions of the resource.
                                                                                      •  
                                                                                    • fpga_handle: An opaque type that represents a resource owned by the calling process. The API functions fpgaOpen() and fpgaClose() acquire and release ownership of a resource that an fpga_handle represents. (Refer to the Functions section for more information.)
                                                                                      •  
                                                                                    • fpga_properties: An opaque type for a properties object. Your applications use these properties to query and search for appropriate resources. The  FPGA Resource Properties section documents properties visible to your applications.
                                                                                      •  
                                                                                    • fpga_event_handle: An opaque handle the FPGA driver uses to notify your application about an event. 
                                                                                      •  
                                                                                    • fpga_event_type: An enum type that represents the types of events. The following are valid values:  FPGA_EVENT_INTERRUPT, FPGA_EVENT_ERROR, and FPGA_EVENT_POWER_THERMAL. (The Intel Programmable Acceleration Card (PAC) with Intel Arria 10 GX FPGA does not handle thermal and power events.)
                                                                                      •  
                                                                                    • fpga_result: An enum type to represent the result of an API function. If the function returns successfully the result is FPGA_OK. Otherwise, the result is the appropriate error codes. Function fpgaErrStr() translates an error code into human-readable strings.
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/#functions","title":"Functions","text":"
                                                                                    The table below groups important API calls by their functionality. For more information about each of the functions, refer to the  OPAE C API reference manual.
                                                                                     Functionality API Call FPGA Accelerator Description Enumeration fpgaEnumerate() Yes Yes Query FPGA resources that match certain properties Enumeration: Properties fpga[Get, Update, Clear, Clone, Destroy Properties]() Yes Yes Manage fpga_properties life cycle fpgaPropertiesGet[Prop]() Yes Yes Get the specified property Prop, from the FPGA Resource Properties table fpgaPropertiesSet[Prop]() Yes Yes Set the specified property Prop, from the FPGA Resource Properties table Access: Ownership fpga[Open, Close]() Yes Yes Acquire/release ownership Access: Reset fpgaReset() Yes Yes Reset an accelerator Access: Event handling fpga[Register, Unregister]Event() Yes Yes Register/unregister an event to be notified about fpga[Create, Destroy]EventHandle() Yes Yes Manage fpga_event_handle life cycle Access: MMIO fpgaMapMMIO(), fpgaUnMapMMIO() Yes Yes Map/unmap MMIO space fpgaGetMMIOInfo() Yes Yes Get information about the specified MMIO space fpgaReadMMIO[32, 64]() Yes Yes Read a 32-bit or 64-bit value from MMIO space fpgaWriteMMIO[32, 64]() Yes Yes Write a 32-bit or 64-bit value to MMIO space Memory management: Shared memory fpga[Prepare, Release]Buffer() Yes Yes Manage memory buffer shared between the calling process and an accelerator fpgaGetIOAddress() Yes Yes Return the device I/O address of a shared memory buffer fpgaBindSVA() Yes Yes Bind IOMMU shared virtual addressing Management: Reconfiguration fpgaReconfigureSlot() Yes No Replace an existing AFU with a new one Error report fpgaErrStr() Yes Yes Map an error code to a human readable string 
                                                                                    .. note::
                                                                                     
                                                                                    The UMsg APIs are not supported for the Intel(R) PAC cards. They will be deprecated in a future release.\n
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/#fpga-resource-properties","title":"FPGA Resource Properties","text":"
                                                                                    Applications query resource properties by specifying the property name for Prop in the  fpgaPropertiesGet[Prop]() and fpgaPropertiesSet[Prop]() functions. The FPGA and Accelerator columns state whether or not the Property is available for the FPGA or Accelerator objects.
                                                                                     Property FPGA Accelerator Description Parent No Yes fpga_token of the parent object ObjectType Yes Yes The type of the resource: either FPGA_DEVICE or FPGA_ACCELERATOR Bus Yes Yes The bus number Device Yes Yes The PCI device number Function Yes Yes The PCI function number SocketId Yes Yes The socket ID DeviceId Yes Yes The device ID NumSlots Yes No Number of AFU slots available on an FPGA_DEVICE resource BBSID Yes No The FPGA Interface Manager (FIM) ID of an FPGA_DEVICE resource BBSVersion Yes No The FIM version of an FPGA_DEVICE resource VendorId Yes No The vendor ID of an FPGA_DEVICE resource GUID Yes Yes The GUID of an FPGA_DEVICE or FPGA_ACCELERATOR resource NumMMIO No Yes The number of MMIO space of an FPGA_ACCELERATOR resource NumInterrupts No Yes The number of interrupts of an FPGA_ACCELERATOR resource AcceleratorState No Yes The state of an FPGA_ACCELERATOR resource: either FPGA_ACCELERATOR_ASSIGNED or FPGA_ACCELERATOR_UNASSIGNED"},{"location":"sw/fpga_api/prog_guide/readme/#opae-c-api-return-codes","title":"OPAE C API Return Codes","text":"
                                                                                    The OPAE C library returns a code for every exported public API function.  FPGA_OK indicates successful completion of the requested operation. Any return code other than FPGA_OK indicates an error or unexpected behavior. When using the OPAE C API, always check the API return codes. 
                                                                                     Error Code Description FPGA_OK Operation completed successfully FPGA_INVALID_PARAM Invalid parameter supplied FPGA_BUSY Resource is busy FPGA_EXCEPTION An exception occurred FPGA_NOT_FOUND A required resource was not found FPGA_NO_MEMORY Not enough memory to complete operation FPGA_NOT_SUPPORTED Requested operation is not supported FPGA_NO_DRIVER Driver is not loaded FPGA_NO_DAEMON FPGA Daemon (fpgad) is not running FPGA_NO_ACCESS Insufficient privileges or permissions FPGA_RECONF_ERROR Error while reconfiguring FPGA"},{"location":"sw/fpga_api/prog_guide/readme/#usage-models","title":"Usage Models","text":""},{"location":"sw/fpga_api/prog_guide/readme/#query-and-search-for-a-resource","title":"Query and Search for a Resource","text":"
                                                                                    The user-code first populates an fpga_properties object with the required properties. Then, fpgaEnumerate() searches for matching resources. fpgaEnumerate() may return more than one matching resource.
                                                                                     
                                                                                    #include \"fpga/fpga.h\"\n\nfpga_guid               guid;\nfpga_properties         filter = NULL;\nfpga_result             res;\nfpga_token              tokens[MAX_NUM_TOKENS];\nuint32_t                num_matches = 0;\n\n/* Start with an empty properties object */\nres = fpgaGetProperties(NULL, &filter);\n\n/* Populate the properties object with required values.\n   In this case, search for accelerators that matches \n   the specified GUID.\n*/\nuuid_parse(GUID, guid);\nres = fpgaPropertiesSetObjectType(filter, FPGA_ACCELERATOR);\nres = fpgaPropertiesSetGuid(filter, guid);\n\n/* Query the number of matching resources */\nres = fpgaEnumerate(&filter, 1, NULL, 1, &num_matches);\n\n/* Return tokens for all matching resources */\nres = fpgaEnumerate(&filter, 1, tokens, num_matches, &num_matches);\n\n/* Destroy the properties object */\nres = fpgaDestroyProperties(&filter);\n\n/* More code */\n......\n\n/* Destroy tokens */\nfor (uint32_t i = 0; i < num_matches; ++i) {\n    res = fpgaDestroyToken(tokens[i]);\n}\n
                                                                                     
                                                                                    The fpgaEnumerate() function can take multiple fpga_propertiesobjects in an array. In such cases, the function performs a logical OR of  the properties object and returns resources that match any of  the multiple properties. The  fpga_token objects that fpgaEnumerate() returns, do not signify ownership. To acquire ownership of a resource represented by a token, pass the token to fpgaOpen().
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/#acquire-and-release-a-resource","title":"Acquire and Release a Resource","text":"
                                                                                    Use fpgaOpen() and fpgaClose() to acquire and release ownership of a resource.  The calling process must own the resource before it can initiate MMIO, access share memory buffers,  and use functions offered by the resource.
                                                                                     
                                                                                        #include \"fpga/fpga.h\"\n\n    fpga_handle             handle;\n    fpga_result             res;\n\n    /* Acquire ownership of a resource that \n    `fpgaEnumerate()` previously returned as a token */\n\n    res = fpgaOpen(token, &handle);\n\n    /* More code */\n    ......\n\n    /* Release the ownership */\n    res = fpgaClose(handle);\n
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/#shared-memory-buffer","title":"Shared Memory Buffer","text":"
                                                                                    This code snippet shows how to prepare a memory buffer to be shared between the calling process and an accelerator.
                                                                                     
                                                                                        #include \"fpga/fpga.h\"\n\n    fpga_handle             handle;\n    fpga_result             res;\n\n    /* Hint for the virtual address of the buffer */\n    volatile uint64_t       *addr_hint;\n    /* An ID we can use to reference the buffer later */\n    uint32_t                bufid;\n    /* Flag to indicate whether or not the buffer is preallocated */\n    int                     flag = 0;\n\n    /* Allocate (if necessary), pin, and map a buffer to be accessible\n       by an accelerator\n    */\n    res = fpgaPrepareBuffer(handle, BUF_SIZE, (void **) &addr_hint,\n                            &bufid, flag);\n\n    /* The actual address mapped to the buffer */\n    uint64_t                iova;\n    /* Get the IO virtual address for the buffer */\n    res = fpgaGetIOAddress(handle, bufid, &iova);\n\n    /* Inform the accelerator about the virtual address by writing to its mapped\n       register file\n    */\n    ......\n\n    /* More code */\n    ......\n\n    /* Release the shared buffer */\n    res = fpgaReleaseBuffer(handle, bufid);\n
                                                                                     
                                                                                    .. note::
                                                                                     
                                                                                    The `flag` variable can take a constant `FPGA_BUF_PREALLOCATED` to\nindicate that the calling process has already allocated the address space\nthat `addr_hint` points to.\n
                                                                                    "},{"location":"sw/fpga_api/prog_guide/readme/#mmio","title":"MMIO","text":"
                                                                                    This code snippet shows how to map and unmap the register file of an accelerator into the calling process's virtual memory space.
                                                                                     
                                                                                        #include \"fpga/fpga.h\"\n\n    fpga_handle             handle;\n    fpga_result             res;\n\n    /* Index of the MMIO space. There might be multiple spaces on an accelerator */\n    uint32_t                mmio_num = 0;\n    /* Mapped address */\n    uint64_t                mmio_addr;\n\n    /* Map MMIO */\n    res = fpgaMapMMIO(handle, mmio_num, &mmio_addr);\n\n    /* Write to a 32-bit value to the mapped register file at a certain byte\n       offset.\n\n       CSR_CTL is the offset in the mapped space to where the value will be\n       written. It's defined elsewhere.\n    */\n    res = fpgaWriteMMIO32(handle, mmio_num, CSR_CTL, value);\n\n    /* More code */\n    ......\n\n    /* Unmap MMIO */\n    res = fpgaUnmapMMIO(handle, mmio_num);\n
                                                                                     
                                                                                    .. Note::
                                                                                     
                                                                                    Every AFU has its own register adress space and its own protocol to control operation through \nthe registers. \n
                                                                                    "},{"location":"sw/fpga_api/quick_start/readme/","title":"Quick Start Guide","text":""},{"location":"sw/fpga_api/quick_start/readme/#overview","title":"Overview","text":"
                                                                                    The OPAE C library is a lightweight user-space library that provides an abstraction for FPGA resources in a compute environment. Built on top of the OPAE Intel\u00ae FPGA driver stack that supports Intel\u00ae FPGA platforms, the library abstracts away hardware-specific and OS-specific details and exposes the underlying FPGA resources as a set of features accessible from within software programs running on the host.
                                                                                     
                                                                                    These features include the acceleration logic preconfigured on the device, as well as functions to manage and reconfigure the device. Hence, the library can enable user applications to transparently and seamlessly leverage FPGA-based acceleration.
                                                                                     
                                                                                    In this document, we will explore the initial steps on how to set up the required libraries and utilities to use the FPGA devices.
                                                                                     
                                                                                    If you do not have access to an Intel\u00ae Xeon\u00ae processor with integrated FPGA, or a programmable FPGA acceleration card for Intel\u00ae Xeon\u00ae processors, you will not be able to run the examples below. However, you can still make use of the AFU simulation environment (ASE) to develop and test accelerator RTL with OPAE applications.
                                                                                     
                                                                                    The source for the OPAE SDK Linux device drivers is available at the OPAE Linux DFL drivers repository.
                                                                                    "},{"location":"sw/fpga_api/quick_start/readme/#build-the-opae-linux-device-drivers-from-the-source","title":"Build the OPAE Linux device drivers from the source","text":"
                                                                                    For building the OPAE kernel and kernel driver, the kernel development environment is required. So before you build the kernel, you must install the required packages. Run the following commands (we are using Fedora 32 as an example):
                                                                                     
                                                                                    sudo yum install gcc gcc-c++ make kernel-headers kernel-devel elfutils-libelf-devel ncurses-devel openssl-devel bison flex\n
                                                                                     
                                                                                    Download the OPAE upstream kernel tree from GitHub:
                                                                                     
                                                                                    git clone https://github.com/OPAE/linux-dfl.git -b fpga-upstream-dev-5.8.0\n
                                                                                     
                                                                                    Configure the kernel:
                                                                                     
                                                                                    cd linux-dfl\ncp /boot/config-`uname -r` .config\ncat configs/n3000_d5005_defconfig >> .config \necho 'CONFIG_LOCALVERSION=\"-dfl\"' >> .config\necho 'CONFIG_LOCALVERSION_AUTO=y' >> .config\nmake olddefconfig\n
                                                                                     
                                                                                    Compile and install the new kernel:
                                                                                     
                                                                                    make -j\nsudo make modules_install\nsudo make install\n
                                                                                     
                                                                                    After the installation finishes, reboot your system. Log back into the system, and confirm the correct version for the kernel:
                                                                                     
                                                                                    $ uname -a\nLinux localhost.localdomain 5.8.0-rc1-dfl-g73e16386cda0 #6 SMP Wed Aug 19 08:38:32 EDT 2020 x86_64 x86_64 x86_64 GNU/Linux\n
                                                                                    "},{"location":"sw/fpga_api/quick_start/readme/#building-and-installing-the-opae-sdk-from-the-source","title":"Building and installing the OPAE SDK from the source","text":"
                                                                                    Download the OPAE SDK source package:
                                                                                     
                                                                                       
                                                                                    1. Go to the section corresponding to the desired release on GitHub:
                                                                                      2.  
                                                                                    2. Click the Source code (tar.gz) link under the section's Assets.
                                                                                      3.  
                                                                                    3.  
                                                                                      On the command line, go through the following steps to install it:
                                                                                       
                                                                                      # Unpack\ntar xfvz opae-sdk-<release>.tar.gz\n# Configure\ncd opae-sdk-<release>\nmkdir build\ncd build\n# The default installation prefix is `/usr/local`;\n# You have the option to configure for a different location\ncmake [-DCMAKE_INSTALL_PREFIX=<prefix>] ..\n# Compile\nmake\n# Install: you need system administration privileges (`sudo`)\n# if you have elected to install in the default location\n[sudo] make install\n
                                                                                       
                                                                                      4.  
                                                                                     
                                                                                    The remainder of this guide assumes you installed into /usr/local.
                                                                                    "},{"location":"sw/fpga_api/quick_start/readme/#configuring-the-fpga-loading-an-fpga-afu","title":"Configuring the FPGA (loading an FPGA AFU)","text":"
                                                                                    The fpgaconf tool exercises the AFU reconfiguration functionality. It shows how to read a bitstream from a disk file, check its validity and compatibility, and then injects it into FPGA to be configured as a new AFU, which can then be discovered and used by user applications.
                                                                                     
                                                                                    For this step, you require a valid green bitstream (GBS) file. To reconfigure the FPGA slot, you can issue the following command as system administrator:
                                                                                     
                                                                                    sudo fpgaconf -b 0x5e <filename>.gbs\n
                                                                                     
                                                                                    The -b option to fpgaconf indicates the target bus number of the FPGA slot to be reconfigured. Alternatively, you can also specify the target socket number of the FPGA using the -s option.
                                                                                     
                                                                                    $ sudo fpgaconf --help\nUsage:\n        fpgaconf [-hvn] [-b <bus>] [-d <device>] [-f <function>] [-s <socket>] <gbs>\n\n                -h,--help           Print this help\n                -v,--verbose        Increase verbosity\n                -n,--dry-run        Don't actually perform actions\n                -b,--bus            Set target bus number\n                -d,--device         Set target device number\n                -f,--function       Set target function number\n                -s,--socket         Set target socket number\n
                                                                                     
                                                                                    The sample application on the Building a Sample Application\nsection requires loading of an AFU called \"Native Loopback\nAdapter\" (NLB) on the FPGA. Please refer to the NLB documentation\nfor the location of the NLB's green bitstream. You also can verify\nif the NLB green bitstream has already been loaded into the FPGA\nslot by typing the following command and checking the output\nmatches the following:\n\n```bash\n$ cat /sys/class/fpga_region/region0/dfl-port.0/afu_id\nd8424dc4a4a3c413f89e433683f9040b\n```\n
                                                                                     
                                                                                    The fpgaconf tool is not available for the Intel PAC N3000. The NLB is\nalready included in the AFU.\n
                                                                                    "},{"location":"sw/fpga_api/quick_start/readme/#building-a-sample-application","title":"Building a sample application","text":"
                                                                                    The library source includes code samples. Use these samples to learn how to call functions in the library. Build and run these samples as quick sanity checks to determine if your installation and environment are set up properly.
                                                                                     
                                                                                    In this guide, we will build hello_fpga.c. This is the \"Hello World!\" example of using the library.  This code searches for a predefined and known AFU called \"Native Loopback Adapter\" on the FPGA. If found, it acquires ownership and then interacts with the AFU by sending it a 2MB message and waiting for the message to be echoed back. This code exercises all major components of the API except for AFU reconfiguration: AFU search, enumeration, access, MMIO, and memory management.
                                                                                     
                                                                                    You can also find the source for hello_fpga in the samples directory of the OPAE SDK repository on GitHub.
                                                                                     
                                                                                        int main(int argc, char *argv[])\n    {\n        fpga_properties    filter = NULL;\n        fpga_token         afu_token;\n        fpga_handle        afu_handle;\n        fpga_guid          guid;\n        uint32_t           num_matches;\n\n        volatile uint64_t *dsm_ptr    = NULL;\n        volatile uint64_t *status_ptr = NULL;\n        volatile uint64_t *input_ptr  = NULL;\n        volatile uint64_t *output_ptr = NULL;\n\n        uint64_t        dsm_wsid;\n        uint64_t        input_wsid;\n        uint64_t        output_wsid;\n        fpga_result     res = FPGA_OK;\n\n        if (uuid_parse(NLB0_AFUID, guid) < 0) {\n            fprintf(stderr, \"Error parsing guid '%s'\\n\", NLB0_AFUID);\n            goto out_exit;\n        }\n\n        /* Look for accelerator by its \"afu_id\" */\n        res = fpgaGetProperties(NULL, &filter);\n        ON_ERR_GOTO(res, out_exit, \"creating properties object\");\n\n        res = fpgaPropertiesSetObjectType(filter, FPGA_ACCELERATOR);\n        ON_ERR_GOTO(res, out_destroy_prop, \"setting object type\");\n\n        res = fpgaPropertiesSetGuid(filter, guid);\n        ON_ERR_GOTO(res, out_destroy_prop, \"setting GUID\");\n\n        /* TODO: Add selection via BDF / device ID */\n\n        res = fpgaEnumerate(&filter, 1, &afu_token, 1, &num_matches);\n        ON_ERR_GOTO(res, out_destroy_prop, \"enumerating accelerators\");\n\n        if (num_matches < 1) {\n            fprintf(stderr, \"accelerator not found.\\n\");\n            res = fpgaDestroyProperties(&filter);\n            return FPGA_INVALID_PARAM;\n        }\n\n        /* Open accelerator and map MMIO */\n        res = fpgaOpen(afu_token, &afu_handle, 0);\n        ON_ERR_GOTO(res, out_destroy_tok, \"opening accelerator\");\n\n        res = fpgaMapMMIO(afu_handle, 0, NULL);\n        ON_ERR_GOTO(res, out_close, \"mapping MMIO space\");\n\n        /* Allocate buffers */\n        res = fpgaPrepareBuffer(afu_handle, LPBK1_DSM_SIZE,\n                    (void **)&dsm_ptr, &dsm_wsid, 0);\n        ON_ERR_GOTO(res, out_close, \"allocating DSM buffer\");\n\n        res = fpgaPrepareBuffer(afu_handle, LPBK1_BUFFER_ALLOCATION_SIZE,\n                   (void **)&input_ptr, &input_wsid, 0);\n        ON_ERR_GOTO(res, out_free_dsm, \"allocating input buffer\");\n\n        res = fpgaPrepareBuffer(afu_handle, LPBK1_BUFFER_ALLOCATION_SIZE,\n                   (void **)&output_ptr, &output_wsid, 0);\n        ON_ERR_GOTO(res, out_free_input, \"allocating output buffer\");\n\n        printf(\"Running Test\\n\");\n\n        /* Initialize buffers */\n        memset((void *)dsm_ptr,    0,    LPBK1_DSM_SIZE);\n        memset((void *)input_ptr,  0xAF, LPBK1_BUFFER_SIZE);\n        memset((void *)output_ptr, 0xBE, LPBK1_BUFFER_SIZE);\n\n        cache_line *cl_ptr = (cache_line *)input_ptr;\n        for (uint32_t i = 0; i < LPBK1_BUFFER_SIZE / CL(1); ++i) {\n            cl_ptr[i].uint[15] = i+1; /* set the last uint in every cacheline */\n        }\n\n        /* Reset accelerator */\n        res = fpgaReset(afu_handle);\n        ON_ERR_GOTO(res, out_free_output, \"resetting accelerator\");\n\n        /* Program DMA addresses */\n        uint64_t iova;\n        res = fpgaGetIOAddress(afu_handle, dsm_wsid, &iova);\n        ON_ERR_GOTO(res, out_free_output, \"getting DSM IOVA\");\n\n        res = fpgaWriteMMIO64(afu_handle, 0, CSR_AFU_DSM_BASEL, iova);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_AFU_DSM_BASEL\");\n\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 0);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 1);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        res = fpgaGetIOAddress(afu_handle, input_wsid, &iova);\n        ON_ERR_GOTO(res, out_free_output, \"getting input IOVA\");\n        res = fpgaWriteMMIO64(afu_handle, 0, CSR_SRC_ADDR, CACHELINE_ALIGNED_ADDR(iova));\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_SRC_ADDR\");\n\n        res = fpgaGetIOAddress(afu_handle, output_wsid, &iova);\n        ON_ERR_GOTO(res, out_free_output, \"getting output IOVA\");\n        res = fpgaWriteMMIO64(afu_handle, 0, CSR_DST_ADDR, CACHELINE_ALIGNED_ADDR(iova));\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_DST_ADDR\");\n\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_NUM_LINES, LPBK1_BUFFER_SIZE / CL(1));\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_NUM_LINES\");\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CFG, 0x42000);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        status_ptr = dsm_ptr + DSM_STATUS_TEST_COMPLETE/8;\n\n        /* Start the test */\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 3);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        /* Wait for test completion */\n        while (0 == ((*status_ptr) & 0x1)) {\n            usleep(100);\n        }\n\n        /* Stop the device */\n        res = fpgaWriteMMIO32(afu_handle, 0, CSR_CTL, 7);\n        ON_ERR_GOTO(res, out_free_output, \"writing CSR_CFG\");\n\n        /* Check output buffer contents */\n        for (uint32_t i = 0; i < LPBK1_BUFFER_SIZE; i++) {\n            if (((uint8_t*)output_ptr)[i] != ((uint8_t*)input_ptr)[i]) {\n                fprintf(stderr, \"Output does NOT match input \"\n                    \"at offset %i!\\n\", i);\n                break;\n            }\n        }\n\n        printf(\"Done Running Test\\n\");\n\n        /* Release buffers */\n    out_free_output:\n        res = fpgaReleaseBuffer(afu_handle, output_wsid);\n        ON_ERR_GOTO(res, out_free_input, \"releasing output buffer\");\n    out_free_input:\n        res = fpgaReleaseBuffer(afu_handle, input_wsid);\n        ON_ERR_GOTO(res, out_free_dsm, \"releasing input buffer\");\n    out_free_dsm:\n        res = fpgaReleaseBuffer(afu_handle, dsm_wsid);\n        ON_ERR_GOTO(res, out_unmap, \"releasing DSM buffer\");\n\n        /* Unmap MMIO space */\n    out_unmap:\n        res = fpgaUnmapMMIO(afu_handle, 0);\n        ON_ERR_GOTO(res, out_close, \"unmapping MMIO space\");\n\n        /* Release accelerator */\n    out_close:\n        res = fpgaClose(afu_handle);\n        ON_ERR_GOTO(res, out_destroy_tok, \"closing accelerator\");\n\n        /* Destroy token */\n    out_destroy_tok:\n        res = fpgaDestroyToken(&afu_token);\n        ON_ERR_GOTO(res, out_destroy_prop, \"destroying token\");\n\n        /* Destroy properties object */\n    out_destroy_prop:\n        res = fpgaDestroyProperties(&filter);\n        ON_ERR_GOTO(res, out_exit, \"destroying properties object\");\n\n    out_exit:\n        return res;\n\n    }\n
                                                                                     
                                                                                    Linking with the OPAE library is straightforward.  Code using this library should include the header file fpga.h. Taking the GCC compiler on Linux as an example, the minimalist compile and link line should look like:
                                                                                     
                                                                                    gcc -std=c99 hello_fpga.c -I/usr/local/include -L/usr/local/lib -lopae-c -luuid -ljson-c -lpthread -o hello_fpga\n
                                                                                     
                                                                                    The API uses some features from the C99 language standard. The\n`-std=c99` switch is required if the compiler does not support C99 by\ndefault.\n
                                                                                     
                                                                                    Third-party library dependency: The library internally uses\n`libuuid` and `libjson-c`. But they are not distributed as part of the\nlibrary. Make sure you have these libraries properly installed.\n
                                                                                     
                                                                                    The layout of AFU is different between the N3000 card and Rush Creek/Darby Creek.\nIn the N3000 card, the NLB and DMA are contained in the AFU, so we need to do\nenumeration again in AFU to discover the NLB.\nTo run the hello_fpga application on the N3000 card, it should use the `-c`\noption to support the N3000 card:\n\n```bash\n$ sudo ./hello_fpga -c\nRunning Test\nRunning on bus 0x08.\nAFU NLB0 found @ 28000\nDone Running Test\n```\n
                                                                                     
                                                                                    To run the hello_fpga application; just issue:
                                                                                     
                                                                                    $ sudo ./hello_fpga\nRunning Test\nDone\n
                                                                                    "},{"location":"sw/fpga_api/quick_start/readme/#setup-iofs-release1-bitstream-on-fpga-pcie-card","title":"Setup IOFS Release1 Bitstream on FPGA PCIe card","text":"
                                                                                    Program IOFS Release1 bitstream on the FPGA D5005 or N6000 cards and reboot the system.
                                                                                     
                                                                                    Run this command:
                                                                                     
                                                                                    $ lspci | grep acc\n3b:00.0 Processing accelerators: Intel Corporation Device af00 (rev 01)\n
                                                                                     
                                                                                    Number of virtual functions supported by bitstream:
                                                                                     
                                                                                    $ cat /sys/bus/pci/devices/0000:3b:00.0/sriov_totalvfs \noutput: 3\n
                                                                                     
                                                                                    Enable FPGA virtual functions:
                                                                                     
                                                                                    sudo sh -c \"echo 3 > /sys/bus/pci/devices/0000:3b:00.0/sriov_numvfs\"\n
                                                                                     
                                                                                    List of FPGA PF and VF's:
                                                                                     
                                                                                    Physical Functions (PFs):\n  3b:00.0 Processing accelerators: Intel Corporation Device af00 (rev 01)\n\nVirtual Functions (VFs).\n  3b:00.1 Processing accelerators: Intel Corporation Device af01 (rev 01)\n  3b:00.2 Processing accelerators: Intel Corporation Device af01 (rev 01)\n  3b:00.3 Processing accelerators: Intel Corporation Device af01 (rev 01)\n
                                                                                     
                                                                                    Bind vfio-pcie driver to FPGA virtual functions:
                                                                                     
                                                                                    sudo opaevfio  -i 0000:3b:00.1 -u userid -g groupid\nsudo opaevfio  -i 0000:3b:00.2 -u userid -g groupid\nsudo opaevfio  -i 0000:3b:00.3 -u userid -g groupid\n
                                                                                     
                                                                                    List of fpga accelerators:
                                                                                     
                                                                                    $ fpgainfo port\n\n  //****** PORT ******//\n  Object Id                        : 0x600D000000000000\n  PCIe s:b:d.f                     : 0000:3b:00.3\n  Device Id                        : 0xAF00\n  Socket Id                        : 0xFF\n  Accelerator Id                   : 43425ee6-92b2-4742-b03a-bd8d4a533812\n  Accelerator GUID                 : 43425ee6-92b2-4742-b03a-bd8d4a533812\n  //****** PORT ******//\n  Object Id                        : 0x400D000000000000\n  PCIe s:b:d.f                     : 0000:3b:00.2\n  Device Id                        : 0xAF00\n  Socket Id                        : 0xFF\n  Accelerator Id                   : 8568AB4E-6bA5-4616-BB65-2A578330A8EB\n  Accelerator GUID                 : 8568AB4E-6bA5-4616-BB65-2A578330A8EB\n  //****** PORT ******//\n  Object Id                        : 0x200D000000000000\n  PCIe s:b:d.f                     : 0000:3b:00.1\n  Device Id                        : 0xAF00\n  Socket Id                        : 0xFF\n  Accelerator Id                   : 56e203e9-864f-49a7-b94b-12284c31e02b\n  Accelerator GUID                 : 56e203e9-864f-49a7-b94b-12284c31e02b\n\nFPGA VF1/3b:00.1/Host Exerciser Loopback Accelerator GUID: 56E203E9-864F-49A7-B94B-12284C31E02B\nFPGA VF2/3b:00.2/Host Exerciser Memory Accelerator GUID: 8568AB4E-6bA5-4616-BB65-2A578330A8EB\nFPGA VF3/3b:00.3/Host Exerciser HSSI Accelerator GUID: 43425ee6-92b2-4742-b03a-bd8d4a533812\n
                                                                                     
                                                                                    Unbind pcie-vfio dirver to FPGA virtual functions:
                                                                                     
                                                                                    sudo opaevfio  -r 0000:3b:00.1\n
                                                                                     
                                                                                    Host Exerciser Loopback (HE-LBK) AFU can move data between host memory and FPGA:
                                                                                     
                                                                                    $ host_exerciser lpbk\n\n  [lpbk] [info] starting test run, count of 1\n  Input Config:0\n  Allocate SRC Buffer\n  Allocate DST Buffer\n  Allocate DSM Buffer\n  Start Test\n  Test Completed\n  Host Exerciser swtest msg:0\n  Host Exerciser numReads:32\n  Host Exerciser numWrites:32\n  Host Exerciser numPendReads:0\n  Host Exerciser numPendWrites:0\n  [lpbk] [info] Test lpbk(1): PASS\n
                                                                                     
                                                                                      In order to successfully run hello\\_fpga, the user needs to configure\n  system hugepage to reserve 2M-hugepages.\n  For example, the command below reserves 20 2M-hugepages:\n\n  ```bash\n  echo 20 | sudo tee /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages\n  ```\n\n  For x86_64 architecture CPU, user can use the following command to find out available huge page sizes:\n\n  ```bash\n  $ grep pse /proc/cpuinfo | uniq\n  flags : ... pse ...\n  ```\n\n  If this command returns a non-empty string, 2MB pages are supported:\n\n  ```bash\n  $ grep pse /proc/cpuinfo | uniq\n  flags : ... pdpe1gb ...\n  ```\n\n  If this commands returns a non-empty string, 1GB pages are supported.\n  ````\n\n````{note}\nThe default configuration for many Linux distributions currently sets a\nrelatively low limit for pinned memory allocations per process \n(RLIMIT_MEMLOCK, often set to a default of 64kiB).\n\nTo run an OPAE application that attempts to share more memory than specified\nby this limit between software and an accelerator, you can either:\n\n* Run the application as root, or\n* Increase the limit for locked memory via `ulimit`:\n\n```bash\nulimit -l unlimited\n```\n\nSee the Installation Guide for how to permanently adjust the memlock limit.\n
                                                                                    "},{"location":"sw/fpga_dfl_drv/fpga_dfl_drv/","title":"Enable OPAE on FPGA PCIe drivers","text":"
                                                                                    .. toctree::\n.. highlight:: c\n.. highlight:: console\n
                                                                                     
                                                                                    FPGA PCIe driver for PCIe-based Field-Programmable Gate Array (FPGA) solutions which implement the Device Feature List (DFL). This driver provides interfaces for user space applications to configure, enumerate, open and access FPGA accelerators on the FPGA DFL devices. additionally, it also enables system level management functions such as FPGA partial reconfiguration, power management, virtualization with DFL framework and DFL feature device drivers.
                                                                                     
                                                                                    OPAE 1.4.0 release supports both FPGA Intel Linux driver as well as Linux FPGA DFL driver patch set2. Linux PCIe FPGA DFL driver supports Intel FPGA devices.
                                                                                     
                                                                                    FPGA DFL Linux driver source code patchset2 available  https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers?h=linux-5.4.y
                                                                                     
                                                                                    FPGA DFL Linux driver source code patchset1 available  https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/fpga?h=v4.19.14
                                                                                     
                                                                                    "},{"location":"sw/fpga_tools/readme/","title":"fpga_tools","text":""},{"location":"sw/fpga_tools/readme/#fpgainfo","title":"fpgainfo","text":""},{"location":"sw/fpga_tools/readme/#name","title":"NAME","text":"
                                                                                    fpgainfo - FPGA information tool
                                                                                    "},{"location":"sw/fpga_tools/readme/#synopsis","title":"SYNOPSIS","text":"
                                                                                    fpgainfo [-h | --help] [-s | --socket-id] <command> [<args>]\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#description","title":"DESCRIPTION","text":"
                                                                                    fpgainfo is a tool to show FPGA information derived from sysfs files. The command argument is one of the following: errors, power, temp and is used to specify what type of information to report. Some commands may also have other arguments/options that can be used to control the behavior of that command.
                                                                                    "},{"location":"sw/fpga_tools/readme/#common-options","title":"COMMON OPTIONS","text":"
                                                                                    --help, -h
                                                                                     
                                                                                    Print help information and exit.\n
                                                                                     
                                                                                    --socket-id, -s
                                                                                     
                                                                                    Socket ID encoded in BBS. Default=0\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#fpgainfo-commands","title":"FPGAINFO COMMANDS","text":"
                                                                                    errors
                                                                                     
                                                                                    Show/clear errors of an FPGA resource as specified by the first argument.\nError information is parsed to display in human readable form.\n
                                                                                     
                                                                                    power
                                                                                     
                                                                                    Show total power consumed by the FPGA hardware in watts\n
                                                                                     
                                                                                    temp
                                                                                     
                                                                                    Show FPGA temperature values in degrees Farenheit\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#errors-options","title":"ERRORS OPTIONS","text":"
                                                                                    --clear, -c
                                                                                     
                                                                                    Clear errors for the given FPGA resource\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#errors-arguments","title":"ERRORS ARGUMENTS","text":"
                                                                                    The first argument to the errors command is used to specify what kind of resource to act on. It must be one of the following: fme,port,first_error,pcie0,pcie1,bbs,gbs,all More details on the errors reported for the resource can be found below:
                                                                                    "},{"location":"sw/fpga_tools/readme/#errors-resources","title":"ERRORS RESOURCES","text":"
                                                                                    fme
                                                                                     
                                                                                    Show/clear errors pertaining to the FME\n
                                                                                     
                                                                                    port
                                                                                     
                                                                                    Show/clear errors pertaining to the PORT\n
                                                                                     
                                                                                    first_error
                                                                                     
                                                                                    Show/clear first errors encountered by the FPGA\n
                                                                                     
                                                                                    pcie0
                                                                                     
                                                                                    Show/clear errors pertaining to the PCIE0 lane\n
                                                                                     
                                                                                    pcie1
                                                                                     
                                                                                    Show/clear errors pertaining to the PCIE1 lane\n
                                                                                     
                                                                                    bbs
                                                                                     
                                                                                    Show/clear errors pertaining to the BBS (blue bitstream)\n
                                                                                     
                                                                                    gbs
                                                                                     
                                                                                    Show/clear errors pertaining to the GBS (green bitstream)\n
                                                                                     
                                                                                    all
                                                                                     
                                                                                    Show/clear errors for all resources\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#examples","title":"EXAMPLES","text":"
                                                                                    This command shows the current power consumtion: 
                                                                                    ./fpgainfo power\n
                                                                                     
                                                                                    This command shows the current temperature reading: 
                                                                                    ./fpgainfo temp\n
                                                                                     
                                                                                    This command shows the errors for the FME resource: 
                                                                                    ./fpgainfo errors fme\n
                                                                                     This command clears all the errors on all resources: 
                                                                                    ./fpgainfo errors all -c\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#fpgaconf","title":"fpgaconf","text":""},{"location":"sw/fpga_tools/readme/#name_1","title":"NAME","text":"
                                                                                    fpgadiag - Configure a green bitstream to an FPGA
                                                                                    "},{"location":"sw/fpga_tools/readme/#synopsis_1","title":"SYNOPSIS","text":"
                                                                                    fpgaconf [-hvn] [-b <bus>] [-d <device>] [-f <function>] [-s <socket>] <gbs>
                                                                                    "},{"location":"sw/fpga_tools/readme/#description_1","title":"DESCRIPTION","text":"
                                                                                    fpgaconf writes accelerator configuration bitstreams (also referred to as \"green bitstreams\" to an FPGA device recognized by OPAE. In the process, it also checks the green bitstream file for compatibility with the targeted FPGA and its current infrastructure bitstream (the \"blue bistream\"). fpgaconf takes the following arguments:
                                                                                     
                                                                                    -h, --help
                                                                                     
                                                                                    Print usage information\n
                                                                                     
                                                                                    -v, --verbose
                                                                                     
                                                                                    Print more verbose messages while enumerating and configuring. Can be\ngiven more than once\n
                                                                                     
                                                                                    -n, --dry-run
                                                                                     
                                                                                    Perform enumeration, but skip any operations with side-effects (like the\nactual configuration of the bitstream\n
                                                                                     
                                                                                    -b, --bus
                                                                                     
                                                                                    PCI bus number of the FPGA to target\n
                                                                                     
                                                                                    -d, --device
                                                                                     
                                                                                    PCI device number of the FPGA to target\n
                                                                                     
                                                                                    -f, --function
                                                                                     
                                                                                    PCI function number of the FPGA to target\n
                                                                                     
                                                                                    -s, --socket
                                                                                     
                                                                                    Socket number of the FPGA to target\n
                                                                                     
                                                                                    fpgaconf will enumerate available FPGA devices in the system and select compatible FPGAs for configuration. If there are more than one candidate FPGAs that are compatible with the given green bitstream, fpgaconf will exit and ask you to be more specific in selecting the target FPGAs (e.g. by specifying a socket number, or a PCIe bus/device/function).
                                                                                    "},{"location":"sw/fpga_tools/readme/#examples_1","title":"EXAMPLES","text":"
                                                                                    fpgaconf my_green_bitstream.gbs
                                                                                     
                                                                                    Program \"my_green_bitstream.gbs\" to a compatible FPGA\n
                                                                                     
                                                                                    fpgaconf -v -s 0 my_green_bitstream.gbs
                                                                                     
                                                                                    Program \"my_green_bitstream.gbs\" to the FPGA in socket 0, if compatible,\nwhile printing out slightly more verbose information\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#fpgad","title":"fpgad","text":""},{"location":"sw/fpga_tools/readme/#name_2","title":"NAME","text":"
                                                                                    fpgad - log errors and generate events
                                                                                    "},{"location":"sw/fpga_tools/readme/#synopsis_2","title":"SYNOPSIS","text":"
                                                                                    fpgad --daemon [--directory=<dir>] [--logfile=<file>] [--pidfile=<file>] [--umask=<mode>] [--socket=<sock>] [--null-bitstream=<file>] fpgad [--socket=<sock>] [--null-bitstream=<file>]
                                                                                    "},{"location":"sw/fpga_tools/readme/#description_2","title":"DESCRIPTION","text":"
                                                                                    Periodically monitors/reports the error status reflected in the device driver's error status sysfs files. Establishes the channel by which events are communicated to the OPAE application. Programs a NULL bitstream in response to AP6 event.
                                                                                     
                                                                                    fpgad is required to be running before API calls fpgaRegisterEvent and fpgaUnregisterEvent will succeed.
                                                                                     
                                                                                    Use SIGINT to stop fpgad.
                                                                                     
                                                                                    -d, --daemon
                                                                                     
                                                                                    When given, fpgad executes as a system demon process.\n
                                                                                     
                                                                                    -D, --directory <dir>
                                                                                     
                                                                                    When running in daemon mode, execute from the given directory.\nIf omitted when daemonizing, /tmp is used.\n
                                                                                     
                                                                                    -l, --logfile <file>
                                                                                     
                                                                                    When running in daemon mode, send output to file. When not in daemon mode, the output is sent to stdout.\nIf omitted when daemonizaing, /tmp/fpgad.log is used.\n
                                                                                     
                                                                                    -p, --pidfile <file>
                                                                                     
                                                                                    When running in daemon mode, write the daemon's process id to file.\nIf omitted when daemonizing, /tmp/fpgad.pid is used.\n
                                                                                     
                                                                                    -m, --umask <mode>
                                                                                     
                                                                                    When running in daemon mode, use the mode value as the file mode creation mask passed to umask.\nIf omitted when daemonizing, 0 is used.\n
                                                                                     
                                                                                    -s, --socket <sock>
                                                                                     
                                                                                    Listen for event API registration requests on sock. The default socket value used by the API is\n/tmp/fpga_event_socket.\n
                                                                                     
                                                                                    -n, --null-bitstream <file>
                                                                                     
                                                                                    Specify the NULL bitstream to program when an AP6 event occurs. This option may be given multiple\ntimes. The bitstream, if any, that matches the FPGA's PR interface id will be programmed when AP6\nis detected.\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#troubleshooting","title":"TROUBLESHOOTING","text":"
                                                                                    If any issues are encountered, try the following for additional debug information:
                                                                                     
                                                                                       
                                                                                    1. Examine the log file when in daemon mode.
                                                                                      2.  
                                                                                    2. Run in non-daemon mode and view stdout.
                                                                                      3.  
                                                                                    "},{"location":"sw/fpga_tools/readme/#examples_2","title":"EXAMPLES","text":"
                                                                                    fpgad --daemon --null-bitstream=my_null_bits.gbs
                                                                                    "},{"location":"sw/fpga_tools/readme/#see-also","title":"SEE ALSO","text":"
                                                                                    umask
                                                                                    "},{"location":"sw/fpga_tools/readme/#fpgadiag","title":"fpgadiag","text":""},{"location":"sw/fpga_tools/readme/#name_3","title":"NAME","text":"
                                                                                    fpgadiag - FPGA diagnosis and testing tool.
                                                                                    "},{"location":"sw/fpga_tools/readme/#synopsis_3","title":"SYNOPSIS","text":"
                                                                                    fpgadiag [-m | --mode=] <mode> [-t | --target=] <target> [options]\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#description_3","title":"DESCRIPTION","text":"
                                                                                    fpgadiag includes several tests to diagnose, test and report on the FPGA hardware.
                                                                                     
                                                                                    <mode> chooses which test to run. <target> specifies on what platform to run the test. <target> can be either fpga or ase, where ase stands for \"AFU Simulation Environment\".
                                                                                     
                                                                                    The tests that can be selected by <mode> include:
                                                                                     
                                                                                    lpbk1
                                                                                     
                                                                                    The test performs loopback test on the number of cachelines specified with \nthe `BEGIN` option. _fpgadiag_ sets up source and  destination buffers in \nmain memory. The FPGA then performs a memcpy from a source buffer to the \ndestination buffer, one cacheline at a time.\n\nA cacheline is 64 bytes. When `BEGIN = END`, you perform one iteration. When \n`BEGIN = END + x`, you perform `x` iterations. The first iteration consists \nof copying `BEGIN` cachelines; the second iteration consists of copying \n`BEGIN+1` cache lines; the third iteration consists of copying `BEGIN+3` \ncache lines, etc.\n\nThe latency is shown as the number of clock ticks.\n\nWhen you specify `MULTI-CL`, you copy `MULTI-CL` cache lines at a time.\nThere is always a WrFence. `WR-FENCE` chooses what virtual channel the \nWrFence occurs on.\n\nIf you specify continuous mode with `--cont`, the program runs an iteration\nuntil the timeout specified in `TIMEOUT` completes.\n
                                                                                     
                                                                                    read
                                                                                     
                                                                                    This test performs only a read, not a memcpy. It is used to measure read \nbandwidth.\n
                                                                                     
                                                                                    write 
                                                                                     
                                                                                    This test is used to measure write bandwidth.\n
                                                                                     
                                                                                    trput
                                                                                     
                                                                                    This test measures both read and write bandwidth by performing 50% read and \n50% write tests.\n
                                                                                     
                                                                                    sw
                                                                                     
                                                                                    This is a send-and-respond (ping-pong) test where one side sends data and \nwaits for answer.\n
                                                                                     
                                                                                    Each test requires presence of one of these bitstreams, as documented below. Before running a test, make sure its required bitstream is properly configured on the platform.
                                                                                     
                                                                                       
                                                                                    • nlb mode 0 for the lpbk1 test.
                                                                                      •  
                                                                                    • nlb mode 3 for the trput, read, and write tests.
                                                                                      •  
                                                                                    • nlb mode 7 for the sw test.
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_tools/readme/#options","title":"OPTIONS","text":""},{"location":"sw/fpga_tools/readme/#common-options_1","title":"Common options","text":"
                                                                                    --help, -h
                                                                                     
                                                                                    Print help information and exit.\n
                                                                                     
                                                                                    --target=, -t
                                                                                     
                                                                                    Values accepted for this switch are fpga or ase. Default=fpga\n
                                                                                     
                                                                                    --mode=, -m
                                                                                     
                                                                                    The test to run. Values accepted for this switch are `lpbk1`, `read`,\n`write`, `trput`, `sw`\n
                                                                                     
                                                                                    --config=, -c
                                                                                     
                                                                                    A configuration file in the JSON format that specifies options for a test.\nIf an option is specified both in the configuration file and on the command \nline, the value in the configuration file prevails\n
                                                                                     
                                                                                    --socket-id=, -s
                                                                                     
                                                                                    Socket ID encoded in BBS. Default=0\n
                                                                                     
                                                                                    --bus-number=, -B
                                                                                     
                                                                                    Bus number of the PCIe device. Default=0\n
                                                                                     
                                                                                    --device=, -D
                                                                                     
                                                                                    Device number of the PCIe device. Default=0\n
                                                                                     
                                                                                    --function=, -F
                                                                                     
                                                                                    Function number of the PCIe device. Default=0\n
                                                                                     
                                                                                    --freq=, -T
                                                                                     
                                                                                    Clock frequency in Hz. Default=400 MHz\n
                                                                                     
                                                                                    --suppress-hdr, -S
                                                                                     
                                                                                    Suppress column headers for text output. Default=off\n
                                                                                     
                                                                                    --csv, -V
                                                                                     
                                                                                    Comma separated value format. Default=off\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#lpbk1-test-options","title":"lpbk1 test options","text":"
                                                                                    --guid=, -g
                                                                                     
                                                                                    Accelerator ID to enumerate. Default=D8424DC4-A4A3-C413-F89E-433683F9040B\n
                                                                                     
                                                                                    --begin=B, -b
                                                                                     
                                                                                    1 <= B <= 65535. Default=1, B = number of cache lines\n
                                                                                     
                                                                                    --end=E, -e
                                                                                     
                                                                                    1 <= E <= 65535. Default=B, B and E designate number of cache lines\n
                                                                                     
                                                                                    --multi-cl=M, -U
                                                                                     
                                                                                    M can equal 1, 2, or 4. Default=1\n
                                                                                     
                                                                                    --cont, -L
                                                                                     
                                                                                    Continuous mode. Default=off\n
                                                                                     
                                                                                    --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                                                                     
                                                                                    timeout for --cont mode (microseconds portion default=0; milliseconds \nportion default=0; seconds portion default=1; minutes portion default=0;\nhours portion default=0)\n
                                                                                     
                                                                                    --cache-policy=, -p
                                                                                     
                                                                                    Can be wrline-I, wrline-M, or wrpush-I Default=wrline-M\n
                                                                                     
                                                                                    --cache-hint=, -i
                                                                                     
                                                                                    Can be rdline-I or rdline-S. Default=rdline-I\n
                                                                                     
                                                                                    --read-vc=, -r
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. Default=auto\n
                                                                                     
                                                                                    --write-vc=, -w
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. Default=auto\n
                                                                                     
                                                                                    --wrfence-vc=, -f
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1. Default=auto\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#read-test-options","title":"read test options","text":"
                                                                                    --guid=, -g
                                                                                     
                                                                                    Accelerator ID to enumerate. Default=F7DF405C-BD7A-CF72-22F1-44B0B93ACD18\n
                                                                                     
                                                                                    --begin=B, -b
                                                                                     
                                                                                    1 <= B <= 65535. Default=1, B = number of cache lines\n
                                                                                     
                                                                                    --end=E, -e
                                                                                     
                                                                                    1 <= E <= 65535. Default=B, B and E designate number of cache lines\n
                                                                                     
                                                                                    --multi-cl=M, -u
                                                                                     
                                                                                    M can equal 1, 2, or 4. Default=1\n
                                                                                     
                                                                                    --strided-access=S, -a
                                                                                     
                                                                                    1<= S <= 64. Default=1\n
                                                                                     
                                                                                    --cont, -L
                                                                                     
                                                                                    Continuous mode. Default=off\n
                                                                                     
                                                                                    --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                                                                     
                                                                                    timeout for --cont mode (microseconds portion default=0; milliseconds \nportion default=0; seconds portion default=1; minutes portion default=0;\nhours portion default=0)\n
                                                                                     
                                                                                    --cache-hint=, -i
                                                                                     
                                                                                    Can be rdline-I or rdline-S. Default=rdline-I\n
                                                                                     
                                                                                    --warm-fpga-cache -H; --cool-fpga-cache -M
                                                                                     
                                                                                    Attempt to prime the cache with hits. Default=off, Attempt to prime the \ncache with misses. Default=off\n
                                                                                     
                                                                                    --cool-cpu-cache, -C
                                                                                     
                                                                                    Attempt to prime the cpu cache with misses. Default=off\n
                                                                                     
                                                                                    --read-vc=, -r
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. Default=auto\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#write-test-options","title":"write test options","text":"
                                                                                    --guid=, -g
                                                                                     
                                                                                    Accelerator ID to enumerate. Default=F7DF405C-BD7A-CF72-22F1-44B0B93ACD18\n
                                                                                     
                                                                                    --begin=B, -b
                                                                                     
                                                                                    1 <= E <= 65535. Default=B, B and E designate number of cache lines\n
                                                                                     
                                                                                    --multi-cl=M, -u
                                                                                     
                                                                                    M can equal 1, 2, or 4. Default=1\n
                                                                                     
                                                                                    --strided-access=S, -a
                                                                                     
                                                                                    1<= S <= 64. Default=1\n
                                                                                     
                                                                                    --cont, -L
                                                                                     
                                                                                    Continuous mode. Default=off\n
                                                                                     
                                                                                    --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                                                                     
                                                                                    timeout for --cont mode (microseconds portion default=0; milliseconds \nportion default=0; seconds portion default=1; minutes portion default=0;\nhours portion default=0)\n
                                                                                     
                                                                                    --cache-policy=, -p
                                                                                     
                                                                                    Can be wrline-I, wrline-M, or wrpush-I Default=wrline-M\n
                                                                                     
                                                                                    --warm-fpga-cache -H; --cool-fpga-cache -M
                                                                                     
                                                                                    Attempt to prime the cache with hits. Default=off, Attempt to prime the \ncache with misses. Default=off\n
                                                                                     
                                                                                    --cool-cpu-cache, -C
                                                                                     
                                                                                    Attempt to prime the cpu cache with misses. Default=off\n
                                                                                     
                                                                                    --write-vc=, -w
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. Default=auto\n
                                                                                     
                                                                                    --wrfence-vc=, -f
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. Default=`WRITE-VC`\n
                                                                                     
                                                                                    --alt-wr-pattern, -l
                                                                                     
                                                                                    Alternate Write Pattern. Default=off\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#trput-test-options","title":"trput test options","text":"
                                                                                    --guid=, -g
                                                                                     
                                                                                    Accelerator ID to enumerate. Default=F7DF405C-BD7A-CF72-22F1-44B0B93ACD18\n
                                                                                     
                                                                                    --begin=B, -b
                                                                                     
                                                                                    1 <= B <= 65535. Default=1, B = number of cache lines\n
                                                                                     
                                                                                    --end=E, -e
                                                                                     
                                                                                    1 <= E <= 65535. Default=B, B and E designate number of cache lines\n
                                                                                     
                                                                                    --multi-cl=M, -u
                                                                                     
                                                                                    M can equal 1, 2, or 4. Default=1\n
                                                                                     
                                                                                    --strided-access=S, -a
                                                                                     
                                                                                    1<= S <= 64. Default=1\n
                                                                                     
                                                                                    --cont, -L
                                                                                     
                                                                                    Continuous mode. Default=off\n
                                                                                     
                                                                                    --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                                                                     
                                                                                    timeout for --cont mode (microseconds portion default=0; milliseconds \nportion default=0; seconds portion default=1; minutes portion default=0;\nhours portion default=0)\n
                                                                                     
                                                                                    --cache-policy=, -p
                                                                                     
                                                                                    Can be wrline-I, wrline-M, or wrpush-I Default=wrline-M\n
                                                                                     
                                                                                    --cache-hint=, -i
                                                                                     
                                                                                    Can be rdline-I or rdline-S. Default=rdline-I\n
                                                                                     
                                                                                    --read-vc=, -r
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. Default=auto\n
                                                                                     
                                                                                    --write-vc=, -w
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. Default=auto\n
                                                                                     
                                                                                    --wrfence-vc=, -f
                                                                                     
                                                                                    Can be  auto, vl0, vh0, vh1. Default=`WRITE-VC`\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#sw-test-options","title":"sw test options","text":"
                                                                                    --guid=, -g
                                                                                     
                                                                                    Accelerator ID to enumerate. Default=7BAF4DEA-A57C-E91E-168A-455D9BDA88A3\n
                                                                                     
                                                                                    --begin=B, -b
                                                                                     
                                                                                    1 <= B <= 65535. Default=1, B = number of cache lines\n
                                                                                     
                                                                                    --end=E, -e
                                                                                     
                                                                                    1 <= E <= 65535. Default=B, B and E designate number of cache lines\n
                                                                                     
                                                                                    --multi-cl=M, -u
                                                                                     
                                                                                    M can equal 1, 2, or 4. Default=1\n
                                                                                     
                                                                                    --strided-access=S, -a
                                                                                     
                                                                                    1<= S <= 64. Default=1\n
                                                                                     
                                                                                    --cache-policy=, -p
                                                                                     
                                                                                    Can be wrline-I, wrline-M, or wrpush-I. Default=wrline-M\n
                                                                                     
                                                                                    --cache-hint= -i
                                                                                     
                                                                                    Can be rdline-I or rdline-S. Default=rdline-I\n
                                                                                     
                                                                                    --read-vc=, -r
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random Default=auto\n
                                                                                     
                                                                                    --write-vc=, -w
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random Default=auto\n
                                                                                     
                                                                                    --wrfence-vc=, -f
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1. Default=`WRITE-VC`\n
                                                                                     
                                                                                    --notice=, -N
                                                                                     
                                                                                    Can be poll, csr-write, umsg-data, or umsg-hint. Default=poll\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#examples_3","title":"EXAMPLES","text":"
                                                                                    This command starts an lpbk1 test on the FPGA on bus 0x5e. The test  copies 57535, 57536, 57537, ..., up to 65535 cache lines, one line at a time. The test output is printed in the CSV format with header suppressed. 
                                                                                    ./fpgadiag --mode=lpbk1 --target=fpga -SV --bus-number=0x5e --begin=57535\n--end=65535 --cache-hint=rdline-I --cache-policy=wrpush-I --multi-cl=1\n--write-vc=vl0 --read-vc=vh1 --wrfence-vc=auto\n
                                                                                     
                                                                                    This command starts a read test on the FPGA located on bus 0xbe. The test reads 2045 cache lines in the continuous mode with a 15-second timeout period.  Data is accessed with a strided pattern with a 10-byte stride length. 
                                                                                    ./fpgadiag --mode=read --target=fpga -SV --bus-number=0xbe --begin=2045 --cont\n--timeout-sec=15   --cache-hint=rdline-I --multi-cl=1 -a=10 --write-vc=vh1\n--read-vc=auto --wrfence-vc=auto\n
                                                                                     
                                                                                    This command starts an sw test on the FPGA located on bus 0xbe. The test notifies completion using a CSR write. 
                                                                                    ./fpgadiag --mode=sw --target=fpga -SV --bus-number=0xbe --begin=4 --end=8192\n--cache-hint=rdline-I --cache-policy=wrline-I --notice=csr-write --write-vc=vl0\n--wrfence-vc=auto --read-vc=random \n
                                                                                    "},{"location":"sw/fpga_tools/readme/#troubleshooting_1","title":"TROUBLESHOOTING","text":"
                                                                                    When a test fails to run or gives errors, check the following:
                                                                                     
                                                                                       
                                                                                    • Is Intel FPGA driver properly installed?  See Installation Guide  for driver installation instructions.
                                                                                      •  
                                                                                    • Are FPGA port permissions set properly? Check the permission bits of the port, for example, /dev/intel-fpga-port-0. Users need READ and WRITE permissions to run fpgadiag tests.
                                                                                      •  
                                                                                    • Is hugepage properly configured on the system?  See Installation Guide for hugepage configuration steps.
                                                                                      •  
                                                                                    • Is the required bitstream loaded? See DESCRIPTION for information about what bitstream is required by which test.
                                                                                      •  
                                                                                    • Are --begin and --end values set properly? --end must be no smaller than the --begin. Also, --begin must be a multiple of the --multi-cl value.
                                                                                      •  
                                                                                    • The --warm-fpga-cache and --cool-fpga-cache options in the read and write tests are mutually exclusive.
                                                                                      •  
                                                                                    • The timeout options are only meaningful for the continuous mode  (with the --cont option).
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_tools/readme/#mmlink","title":"mmlink","text":""},{"location":"sw/fpga_tools/readme/#name_4","title":"NAME","text":"
                                                                                    MMLink - Debugging RTL.
                                                                                    "},{"location":"sw/fpga_tools/readme/#synopsis_4","title":"SYNOPSIS","text":"
                                                                                    mmlink [-B <bus>] [-D <device>] [-F <function>] [-S <socket>] [-P <TCP port>] [-I <IP Address>]
                                                                                    "},{"location":"sw/fpga_tools/readme/#description_4","title":"DESCRIPTION","text":"
                                                                                    Remote signaltap is software  tool used for debug RTL (AFU), effectively a signal trace capability that Quartus places into a green bitstream. Remote Signal Tap provides  access the RST part of the Port MMIO space, and then runs the remote protocol on top.
                                                                                    "},{"location":"sw/fpga_tools/readme/#examples_4","title":"EXAMPLES","text":"
                                                                                    ./mmlink  -B 0x5e -P 3333
                                                                                     
                                                                                    MMLink app starts and listens for connection.
                                                                                    "},{"location":"sw/fpga_tools/readme/#options_1","title":"OPTIONS","text":"
                                                                                    -B,--bus FPGA Bus number.
                                                                                     
                                                                                    -D,--device FPGA Device number.
                                                                                     
                                                                                    -F,--functio FPGA function number.
                                                                                     
                                                                                    -S,--socket FPGA socket number.
                                                                                     
                                                                                    -P,--port TCP port number.
                                                                                     
                                                                                    -I,--ip IP address of FPGA system. 
                                                                                    "},{"location":"sw/fpga_tools/readme/#notes","title":"NOTES","text":"
                                                                                    Driver privilege:
                                                                                     
                                                                                    Change AFU driver privilege to user .
                                                                                     
                                                                                    command: chmod 777 /dev/intel-fpga-port.0
                                                                                     
                                                                                    set memlock:    
                                                                                     
                                                                                    command: ulimit -l 10000
                                                                                    "},{"location":"sw/fpga_tools/readme/#coreidle","title":"coreidle","text":""},{"location":"sw/fpga_tools/readme/#name_5","title":"NAME","text":"
                                                                                    coreidle - idles cores for shared TDP sockets to run online cores at maximum capacity.
                                                                                    "},{"location":"sw/fpga_tools/readme/#synopsis_5","title":"SYNOPSIS","text":"
                                                                                    coreidle [-B <bus>] [-D <device>] [-F <function>] [-S <socket>] [-G <GBS path>]
                                                                                    "},{"location":"sw/fpga_tools/readme/#description_5","title":"DESCRIPTION","text":"
                                                                                    This tools parses input GBS, extracts power from metadata ,calculates fpga power, number of online and idle cores. It moves threads from idle cores to online cores.
                                                                                    "},{"location":"sw/fpga_tools/readme/#examples_5","title":"EXAMPLES","text":"
                                                                                    ./coreidle  -B 0x5e -G /home/lab/gbs/mode0.gbs
                                                                                     
                                                                                    Idle cores to run online cores at maximum capacity.
                                                                                    "},{"location":"sw/fpga_tools/readme/#options_2","title":"OPTIONS","text":"
                                                                                    -B,--bus FPGA Bus number.
                                                                                     
                                                                                    -D,--device FPGA Device number.
                                                                                     
                                                                                    -F,--functio FPGA function number.
                                                                                     
                                                                                    -S,--socket FPGA socket number.
                                                                                     
                                                                                    -G,--gbs Green bitstream file path.
                                                                                    "},{"location":"sw/fpga_tools/readme/#fpgamux","title":"fpgamux","text":""},{"location":"sw/fpga_tools/readme/#name_6","title":"NAME","text":"
                                                                                    fpgamux - Software MUX for running multiple AFU (accelerator functional unit) tests in one GBS (green bitsream)\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#synopsis_6","title":"SYNOPSIS","text":"
                                                                                    fpgamux [-h] [-S|--socket-id SOCKET_ID] [-B|--bus-number BUS] [-D|--device DEVICE] [-F|--function FUNCTION]\n          [-G|--guid GUID] -m|--muxfile MUXFILE.json\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#description_6","title":"DESCRIPTION","text":"
                                                                                    fpgamux is a testing tool to interact with multiple AFUs that have been synthesized into one GBS along with the CCIP-MUX BBB (basic building block). The CCIP-MUX uses upper bits in the MMIO addresses to route MMIO reads/writes to the AFU running on the corresponding CCIP-MUX port. fpgamux uses a configuration file that lists the software components and configuration to use.
                                                                                     
                                                                                    .. note::
                                                                                     
                                                                                      Only one (the first) AFU is discoverable by the OPAE driver. Enumerating acceleration on an FPGA will find\n  the accelerator associated with the first AFU only. The first software component in the configuration will\n  be used to determine the GUID to use for enumeration. This can be overridden with the -G|--guid option.\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#options_3","title":"OPTIONS","text":"
                                                                                    -S SOCKET_ID, --socket-id SOCKET_ID\n   socket id of FPGA resource\n\n-B BUS, --bus BUS\n   bus id of FPGA resource\n\n-D DEVICE, --device DEVICE\n   device id of FPGA resource\n\n\n-F FUNCTION, --function FUNCTION\n   function id of FPGA resource\n\n-G, --guid\n   specify what guid to use for the accelerator enumeration\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#configuration","title":"CONFIGURATION","text":"
                                                                                    fpgamux uses a configuration file (in JSON format) to determine what software components to instantiate and how to configure them for interacting with the AFUs in the GBS. This schema for this is listed below:
                                                                                     
                                                                                    [\n    {\n        \"app\" : \"fpga_app\",\n        \"name\" : \"String\",\n        \"config\" : \"Object\"\n    }\n]\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#examples_6","title":"EXAMPLES","text":"
                                                                                    An example configuration with two components is listed below:
                                                                                     
                                                                                    [\n    {\n        \"app\" : \"nlb0\",\n        \"name\" : \"nlb0\",\n        \"config\" :\n        {\n            \"begin\" : 1,\n            \"end\" : 1,\n            \"multi-cl\" : 1,\n            \"cont\" : false,\n            \"cache-policy\" : \"wrline-M\",\n            \"cache-hint\" : \"rdline-I\",\n            \"read-vc\" : \"vh0\",\n            \"write-vc\" : \"vh1\",\n            \"wrfence-vc\" : \"write-vc\",\n            \"timeout-usec\" : 0,\n            \"timeout-msec\" : 0,\n            \"timeout-sec\" : 1,\n            \"timeout-min\" : 0,\n            \"timeout-hour\" : 0,\n            \"freq\" : 400000000\n        }\n    },\n    {\n        \"app\" : \"nlb3\",\n        \"name\" : \"nlb3\",\n        \"config\" :\n        {\n            \"mode\" : \"read\",\n            \"begin\" : 1,\n            \"end\" : 1,\n            \"multi-cl\" : 1,\n            \"strided-access\" : 1,\n            \"cont\" : false,\n            \"warm-fpga-cache\" : false,\n            \"cool-fpga-cache\" : false,\n            \"cool-cpu-cache\" : false,\n            \"cache-policy\" : \"wrline-M\",\n            \"cache-hint\" : \"rdline-I\",\n            \"read-vc\" : \"vh0\",\n            \"write-vc\" : \"vh1\",\n            \"wrfence-vc\" : \"write-vc\",\n            \"alt-wr-pattern\" : false,\n            \"timeout-usec\" : 0,\n            \"timeout-msec\" : 0,\n            \"timeout-sec\" : 1,\n            \"timeout-min\" : 0,\n            \"timeout-hour\" : 0,\n            \"freq\" : 400000000\n        }\n    }\n]\n
                                                                                    "},{"location":"sw/fpga_tools/readme/#userclk","title":"userclk","text":""},{"location":"sw/fpga_tools/readme/#name_7","title":"NAME","text":"
                                                                                    userclk - to set afu high and low clock frequency.
                                                                                    "},{"location":"sw/fpga_tools/readme/#synopsis_7","title":"SYNOPSIS","text":"
                                                                                    userclk [-B <bus>] [-D <device>] [-F <function>] [-S <socket>] [-P <Port id>] [-H <User clock high frequency>] -L <User clock low frequency>]
                                                                                    "},{"location":"sw/fpga_tools/readme/#description_7","title":"DESCRIPTION","text":"
                                                                                    userclk tool used to set high and low clock frequency to acceleration function unit.
                                                                                    "},{"location":"sw/fpga_tools/readme/#examples_7","title":"EXAMPLES","text":"
                                                                                    ./userclk  -B 0x5e -H 400 -L 200
                                                                                     
                                                                                    Sets AFU frequency.
                                                                                    "},{"location":"sw/fpga_tools/readme/#options_4","title":"OPTIONS","text":"
                                                                                    -B,--bus FPGA Bus number.
                                                                                     
                                                                                    -D,--device FPGA Device number.
                                                                                     
                                                                                    -F,--functio FPGA function number.
                                                                                     
                                                                                    -S,--socket FPGA socket number.
                                                                                     
                                                                                    -P,--port Port id.
                                                                                     
                                                                                    -H,--freq-high User clock high frequency. 
                                                                                     
                                                                                    -L,--freq-low User clock low frequency. 
                                                                                    "},{"location":"sw/fpga_tools/coreidle/coreidle/","title":"coreidle","text":""},{"location":"sw/fpga_tools/coreidle/coreidle/#synopsis","title":"SYNOPSIS","text":"
                                                                                    coreidle [-v] [-B <bus>] [-D <device>] [-F <function>] [-S <socket>] [-G <GBS path>]
                                                                                    "},{"location":"sw/fpga_tools/coreidle/coreidle/#description","title":"DESCRIPTION","text":"
                                                                                    coreidle parses the Accelerator Function Unit (AFU) metadata and extracts power information. coreidle calculates the FPGA power and calculates the number of online and idle cores. It moves threads from idle cores to online cores.  coreidle is only available the Integrated FPGA Platform. You cannot run coreidle on the PCIe Accelerator Card (PAC).
                                                                                    "},{"location":"sw/fpga_tools/coreidle/coreidle/#examples","title":"EXAMPLES","text":"
                                                                                    ./coreidle  -B 0x5e -G /home/lab/gbs/mode0.gbs
                                                                                     
                                                                                    Idle cores to run online cores at maximum capacity.
                                                                                    "},{"location":"sw/fpga_tools/coreidle/coreidle/#options","title":"OPTIONS","text":"
                                                                                    -v,--version Prints version information and exit.
                                                                                     
                                                                                    -B,--bus FPGA bus number.
                                                                                     
                                                                                    -D,--device FPGA device number.
                                                                                     
                                                                                    -F,--functio FPGA function number.
                                                                                     
                                                                                    -S,--socket FPGA socket number.
                                                                                     
                                                                                    -G,--gbs Green bitstream file path.
                                                                                    "},{"location":"sw/fpga_tools/fecmode/fecmode/","title":"fecmode (N3000 specific tool)","text":""},{"location":"sw/fpga_tools/fecmode/fecmode/#synopsis","title":"SYNOPSIS","text":"
                                                                                    fecmode [<mode>][<args>]\n
                                                                                    "},{"location":"sw/fpga_tools/fecmode/fecmode/#description","title":"DESCRIPTION","text":"
                                                                                    Fecmode changes FEC mode of external ethernet PHY, this tool only support on N3000 Card.
                                                                                    "},{"location":"sw/fpga_tools/fecmode/fecmode/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                                                                    --segment, -S
                                                                                     
                                                                                    segment number of the PCIe device.
                                                                                     
                                                                                    --bus, -B 
                                                                                     
                                                                                    bus number of the PCIe device. 
                                                                                     
                                                                                    --device, -D 
                                                                                     
                                                                                    device number of the PCIe device.
                                                                                     
                                                                                    --function, -F function number of the PCIe device
                                                                                     
                                                                                    --rsu, -r reboot card only if mode is not configured
                                                                                     
                                                                                    --debug, -d output debug information 
                                                                                    "},{"location":"sw/fpga_tools/fecmode/fecmode/#fec-mode","title":"FEC Mode","text":"
                                                                                    no no FEC.
                                                                                     
                                                                                    kr BaseR FEC (Fire-Code) correction \u2013 4 orders
                                                                                     
                                                                                    rs Reed-Solomon FEC correction \u2013 7 orders
                                                                                    "},{"location":"sw/fpga_tools/fecmode/fecmode/#example","title":"EXAMPLE","text":"
                                                                                    This command change FEC mode to \u201ckr\u201d: 
                                                                                    # fecmode -B 0x25 kr\n
                                                                                     
                                                                                    This command reboot card (no need to specify bus number if there is only one card): 
                                                                                    # fecmode -r\n
                                                                                     
                                                                                    This command display the current FEC mode: 
                                                                                    # fecmode\n
                                                                                    "},{"location":"sw/fpga_tools/fpgabist/fpgabist/","title":"fpgabist","text":""},{"location":"sw/fpga_tools/fpgabist/fpgabist/#synopsis","title":"SYNOPSIS","text":"
                                                                                    fpgabist [-h] [-i device_id] [-b bus] [-d device] [-f function] [path_to_gbs1 path_to_gbs2 ...]\n
                                                                                    "},{"location":"sw/fpga_tools/fpgabist/fpgabist/#description","title":"DESCRIPTION","text":"
                                                                                    The fpgabist tool performs self-diagnostic tests on supported FPGA platforms.
                                                                                     
                                                                                    The tool accepts one or more Accelerator Function (AF) binaries from a predetermined set of AFs. Depending on the available binaries,  the tool runs appropriate tests and reports hardware issues.
                                                                                     
                                                                                    fpgabist always uses fpgainfo to report system information before running any hardware tests.
                                                                                     
                                                                                    Currently, fpgabist accepts the following AFs:    1. nlb_mode_3: The native loopback (NLB) test implements a loopback from TX to RX. Use it to verify basic functionality    and to measure bandwidth.    2. dma_afu: The direct memory access (DMA) AFU test transfers data from host memory to FPGA-attached local memory. 
                                                                                     
                                                                                    The installation includes the AF files, but you can also compile the AFs from the source. 
                                                                                     
                                                                                    If there are multiple PCIe\u00ae devices, use -b, -d, -f to specify the BDF for the specific PCIe\u00ae device.
                                                                                    "},{"location":"sw/fpga_tools/fpgabist/fpgabist/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                                                                    [path_to_gbs1 path_to_gbs2 ...]
                                                                                     
                                                                                    Paths to Accelerator Function (AF) files.
                                                                                    "},{"location":"sw/fpga_tools/fpgabist/fpgabist/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                                                                    You can use the single letter or the full parameter name for the command line arguments.
                                                                                     
                                                                                    -h, --help
                                                                                     
                                                                                    Prints usage information
                                                                                     
                                                                                    -i device_id, --device-id device_id
                                                                                     
                                                                                    Device ID for Intel FPGA. Default is: 0x09c4
                                                                                     
                                                                                    -B bus, --bus bus
                                                                                     
                                                                                    Bus number for specific FPGA
                                                                                     
                                                                                    -D device, --device device
                                                                                     
                                                                                    Device number for specific FPGA
                                                                                     
                                                                                    -F function, --function function
                                                                                     
                                                                                    Function number for specific FPGA
                                                                                    "},{"location":"sw/fpga_tools/fpgabist/fpgabist/#examples","title":"EXAMPLES","text":"
                                                                                    fpgabist <path_to_gbs_files>/dma_afu.gbs <path_to_gbs_files>/nlb_3.gbs
                                                                                     
                                                                                    Runs fpgabist on any platform in the system that matches the default device ID. This command runs both the DMA and   NLB_MODE_3 tests.
                                                                                     
                                                                                    fpgabist -i 09c4 -b 5 <path to gbs>/dma_afu.gbs
                                                                                     
                                                                                    Runs fpgabist the DMA test on the PCIe\u00ae  Endpoint with device_id 09c4 on bus 5. 
                                                                                    "},{"location":"sw/fpga_tools/fpgaconf/fpgaconf/","title":"fpgaconf","text":""},{"location":"sw/fpga_tools/fpgaconf/fpgaconf/#synopsis","title":"SYNOPSIS","text":"
                                                                                    fpgaconf [-hvVn] [-S <segment>] [-B <bus>] [-D <device>] [-F <function>] [PCI_ADDR] <gbs>
                                                                                    "},{"location":"sw/fpga_tools/fpgaconf/fpgaconf/#description","title":"DESCRIPTION","text":"
                                                                                    fpgaconf configures the FPGA with the accelerator function (AF). It also checks the AF for compatibility with  the targeted FPGA and the FPGA Interface Manager (FIM). fpgaconf takes the following arguments: 
                                                                                     
                                                                                    -h, --help
                                                                                     
                                                                                    Prints usage information.\n
                                                                                     
                                                                                    -v, --version
                                                                                     
                                                                                    Prints version information and exits.\n
                                                                                     
                                                                                    -V, --verbose
                                                                                     
                                                                                    Prints more verbose messages while enumerating and configuring. Can be\nrequested more than once.\n
                                                                                     
                                                                                    -n, --dry-run
                                                                                     
                                                                                    Performs enumeration. Skips any operations with side-effects such as the\nactual AF configuration.\n
                                                                                     
                                                                                    -S, --segment
                                                                                     
                                                                                    PCIe segment number of the target FPGA.\n
                                                                                     
                                                                                    -B, --bus
                                                                                     
                                                                                    PCIe bus number of the target FPGA.\n
                                                                                     
                                                                                    -D, --device
                                                                                     
                                                                                    PCIe device number of the target FPGA.\n
                                                                                     
                                                                                    -F, --function
                                                                                     
                                                                                    PCIe function number of the target FPGA.\n
                                                                                     
                                                                                    --force
                                                                                     
                                                                                    Reconfigure the AFU even if it is in use.\n
                                                                                     
                                                                                    fpgaconf enumerates available FPGA devices in the system and selects compatible FPGAs for configuration. If more than one FPGA is compatible with the AF, fpgaconf exits and asks you to be more specific in selecting the target FPGAs by specifying a a PCIe BDF.
                                                                                    "},{"location":"sw/fpga_tools/fpgaconf/fpgaconf/#examples","title":"EXAMPLES","text":"
                                                                                    fpgaconf my_af.gbs
                                                                                     
                                                                                    Program \"my_af.gbs\" to a compatible FPGA.\n
                                                                                     
                                                                                    fpgaconf -V -B 0x3b my_af.gbs
                                                                                     
                                                                                    Program \"my_af.gbs\" to the FPGA in bus 0x3b, if compatible,\nwhile printing out slightly more verbose information.\n
                                                                                     
                                                                                    fpgaconf 0000:3b:00.0 my_af.gbs
                                                                                     
                                                                                    Program \"my_af.gbs\" to the FPGA at address 0000:3b:00.0.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgad/fpgad/","title":"fpgad","text":""},{"location":"sw/fpga_tools/fpgad/fpgad/#synopsis","title":"SYNOPSIS","text":"
                                                                                    fpgad --daemon [--version] [--directory=<dir>] [--logfile=<file>] [--pidfile=<file>] [--umask=<mode>] [--socket=<sock>] [--null-bitstream=<file>] fpgad [--socket=<sock>] [--null-bitstream=<file>]
                                                                                    "},{"location":"sw/fpga_tools/fpgad/fpgad/#description","title":"DESCRIPTION","text":"
                                                                                    fpgad monitors the device sensors, checking for sensor values that are out of the prescribed range. 
                                                                                     
                                                                                    When any of the sensors is detected to be out of bounds, fpgad will focus on keeping the server from rebooting by masking PCIE AER, and send a message to system administrator. System administrator can take further actions like stop the application and stop the FPGA, but fpgad just focus on monitor the sensors and will not take any cooling actions. 
                                                                                     
                                                                                    Note: fpgad must be running (as root) and actively monitoring devices when a sensor anomaly occurs in order to initiate Graceful Shutdown.  If fpgad is not loaded during such a sensor anomaly, the out-of-bounds scenario will not be detected, and the resulting effect on the hardware is undefined.
                                                                                    "},{"location":"sw/fpga_tools/fpgad/fpgad/#arguments","title":"ARGUMENTS","text":"
                                                                                    -v, --version
                                                                                     
                                                                                    Prints version information and exits.\n
                                                                                     
                                                                                    -d, --daemon
                                                                                     
                                                                                    When specified, fpgad executes as a system daemon process.\n
                                                                                     
                                                                                    -D, --directory <dir>
                                                                                     
                                                                                    When running in daemon mode, run from the specified directory.\nIf omitted when daemonizing, `fpgad` uses /tmp.\n
                                                                                     
                                                                                    -l, --logfile <file>
                                                                                     
                                                                                    When running in daemon mode, send output to file. When not in daemon mode, the output goes to stdout.\nIf omitted when daemonizaing, fpgad uses /tmp/fpgad.log.\n
                                                                                     
                                                                                    -p, --pidfile <file>
                                                                                     
                                                                                    When running in daemon mode, write the daemon's process id to a file.\nIf omitted when daemonizing, fpgad uses /tmp/fpgad.pid.\n
                                                                                     
                                                                                    -m, --umask <mode>
                                                                                     
                                                                                    When running in daemon mode, use the mode value as the file mode creation mask passed to umask.\nIf omitted when daemonizing, fpgad uses 0.\n
                                                                                     
                                                                                    -s, --socket <sock>
                                                                                     
                                                                                    Listen for event API registration requests on the UNIX domain socket on the specified path. \nThe default=/tmp/fpga_event_socket.\n
                                                                                     
                                                                                    -n, --null-bitstream <file>
                                                                                     
                                                                                    Specify the NULL bitstream to program when an AP6 event occurs. This option may be specified multiple\ntimes. The AF, if any, that matches the FPGA's PR interface ID is programmed when an AP6\nevent occurs.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgad/fpgad/#troubleshooting","title":"TROUBLESHOOTING","text":"
                                                                                    If you encounter any issues, you can get debug information in two ways:
                                                                                     
                                                                                       
                                                                                    1. By examining the log file when in daemon mode.
                                                                                      2.  
                                                                                    2. By running in non-daemon mode and viewing stdout.
                                                                                      3.  
                                                                                    "},{"location":"sw/fpga_tools/fpgad/fpgad/#examples","title":"EXAMPLES","text":"
                                                                                    fpgad --daemon --null-bitstream=my_null_bits.gbs
                                                                                     
                                                                                    This command starts fpgad as a system daemon process: 
                                                                                    sudo systemctl start fpgad\n
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/","title":"fpgadiag","text":""},{"location":"sw/fpga_tools/fpgadiag/#synopsis","title":"SYNOPSIS","text":"
                                                                                    fpgadiag [-m | --mode=] <mode> [-t | --target=] <target> [options]\n
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#description","title":"DESCRIPTION","text":"
                                                                                    Includes several tests to diagnose, test, and report on the FPGA hardware.
                                                                                     
                                                                                    <mode> chooses which test to run.  <target> specifies the platform that runs the test. <target> can be either fpga or ase where ase.  <ase> is the abbreviation for Accelerator Simulation Environment.
                                                                                     
                                                                                    The <mode> selects from the  following tests:
                                                                                     
                                                                                    lpbk1
                                                                                     
                                                                                    This test runs a loopback test on the number of cachelines specified with  the BEGIN option. fpgadiag sets up source and  destination buffers in  main memory. The FPGA then performs a memcpy from a source buffer to the  destination buffer, one cacheline at a time. 
                                                                                     
                                                                                    A cacheline is 64 bytes. When BEGIN = END, the test performs one iteration. When  BEGIN = END + x, the test performs x iterations. The first iteration consists  of copying BEGIN cachelines; the second iteration consists of copying  BEGIN+1 cache lines. The third iteration consists of copying BEGIN+2  cache lines, and so on. 
                                                                                     
                                                                                    The latency is shown as the number of clock cycles. 
                                                                                     
                                                                                    When you specify MULTI-CL, you copy MULTI-CL cache lines at a time. The WR-FENCE chooses on which virtual channel the WrFence occurs.
                                                                                     
                                                                                    If you specify continuous mode with --cont, the program iterates until the timeout specified in TIMEOUT completes.
                                                                                     
                                                                                    read
                                                                                     
                                                                                    This test performs reads. Use this test to measure read bandwidth. 
                                                                                     
                                                                                    write 
                                                                                     
                                                                                    This test performs writes. Use it to measure write bandwidth. 
                                                                                     
                                                                                    trput
                                                                                     
                                                                                    This test measures both read and write bandwidth by performing 50% read and  50% write tests.
                                                                                     
                                                                                    sw
                                                                                     
                                                                                    This is a send-and-respond (ping-pong) test. One side sends data and  waits for response.
                                                                                     
                                                                                    Each test requires a particular AF. Before running a test, make sure the required AF is properly configured on the platform. 
                                                                                     
                                                                                       
                                                                                    • The lpbk1 test requires the nlb mode 0 AF.
                                                                                      •  
                                                                                    • The trput test requires the nlb mode 3 AF. 
                                                                                      •  
                                                                                    • The sw test requires the nlb mode 7 AF. This AF is only available for the integrated FPGA platform.      You cannot run it on the PCIe accelerator card (PAC).
                                                                                      •  
                                                                                     
                                                                                    fpgalpbk
                                                                                     
                                                                                    This enable/disable FPGA loopback.
                                                                                     
                                                                                    fpgastats
                                                                                     
                                                                                    This get fpga mac statistics.
                                                                                     
                                                                                    mactest
                                                                                     
                                                                                    This compare mac addresses that read from MAC ROM with mac addresses read from Host side.
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#options","title":"OPTIONS","text":""},{"location":"sw/fpga_tools/fpgadiag/#common-options","title":"Common options","text":"
                                                                                    --help, -h
                                                                                     
                                                                                    Print help information and exit.\n
                                                                                     
                                                                                    --target=, -t
                                                                                     
                                                                                    This switch specifies fpga (hardware) or ase (simulation). The default=fpga.\n
                                                                                     
                                                                                    --mode=, -m
                                                                                     
                                                                                    The test to run. The valid values are `lpbk1`, `read`,\n`write`, `trput`, and `sw`.\n
                                                                                     
                                                                                    --config=, -c
                                                                                     
                                                                                    A configuration file in the JSON format that specifies options for a test.\nIf an option is specified both in the configuration file and on the command \nline, the value in the configuration file takes precedence.\n
                                                                                     
                                                                                    --dsm-timeout-usec
                                                                                     
                                                                                    Timeout in microseconds for test completion. The test fails if not completed by \nspecified timeout. The default=1000000.\n
                                                                                     
                                                                                    --socket-id=, -s
                                                                                     
                                                                                    Socket ID encoded in FPGA Interface Manager (FIM). The default=0.\n
                                                                                     
                                                                                    --bus=, -B
                                                                                     
                                                                                    Bus number of the PCIe device. The default=0.\n
                                                                                     
                                                                                    --device=, -D
                                                                                     
                                                                                    Device number of the PCIe device. The default=0.\n
                                                                                     
                                                                                    --function=, -F
                                                                                     
                                                                                    Function number of the PCIe device. The default=0.\n
                                                                                     
                                                                                    --freq=, -T
                                                                                     
                                                                                    Clock frequency (in Hz) used for bandwidth calculation. The default=400000000 Hz (400 MHz).\n
                                                                                     
                                                                                    eval_rst .. note::     This frequency is used only when the software cannot infer the frequency from the accelerator.
                                                                                     
                                                                                    --suppress-hdr, -S
                                                                                     
                                                                                    Suppress column headers for text output. The default=off.\n
                                                                                     
                                                                                    --csv, -V
                                                                                     
                                                                                    Comma separated value format. The default=off.\n
                                                                                     
                                                                                    --suppress-stats
                                                                                     
                                                                                    Suppress statistics output at the end of test. The default=off.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#lpbk1-test-options","title":"lpbk1 test options","text":"
                                                                                    --guid=, -g
                                                                                     
                                                                                    AFU ID to enumerate. The default=D8424DC4-A4A3-C413-F89E-433683F9040B.\n
                                                                                     
                                                                                    --begin=B, -b
                                                                                     
                                                                                    1 <= B <= 65535. The default=1, B = number of cache lines.\n
                                                                                     
                                                                                    --end=E, -e
                                                                                     
                                                                                    1 <= E <= 65535. The default=B, B and E designate number of cache lines.\n
                                                                                     
                                                                                    --multi-cl=M, -u
                                                                                     
                                                                                    M can equal 1, 2, or 4. The default=1.\n
                                                                                     
                                                                                    --cont, -L
                                                                                     
                                                                                    Continuous mode. The default=off.\n
                                                                                     
                                                                                    --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                                                                     
                                                                                    timeout for --cont mode. The default for all options is 0.\n
                                                                                     
                                                                                    --cache-policy=, -p
                                                                                     
                                                                                    Can be wrline-I, wrline-M, or wrpush-I The default=wrline-M.\n
                                                                                     
                                                                                    --cache-hint=, -i
                                                                                     
                                                                                    Can be rdline-I or rdline-S. The default=rdline-I.\n
                                                                                     
                                                                                    --read-vc=, -r
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. The default=auto.\n
                                                                                     
                                                                                    --write-vc=, -w
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. The default=auto.\n
                                                                                     
                                                                                    --wrfence-vc=, -f
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1. The default=auto.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#read-test-options","title":"read test options","text":"
                                                                                    --guid=, -g
                                                                                     
                                                                                    AFU ID to enumerate. The default=F7DF405C-BD7A-CF72-22F1-44B0B93ACD18.\n
                                                                                     
                                                                                    --begin=B, -b
                                                                                     
                                                                                    1 <= B <= 65535. The default=1, B = number of cache lines.\n
                                                                                     
                                                                                    --end=E, -e
                                                                                     
                                                                                    1 <= E <= 65535. The default=B, B and E designate number of cache lines.\n
                                                                                     
                                                                                    --multi-cl=M, -u
                                                                                     
                                                                                    M can equal 1, 2, or 4. The default=1.\n
                                                                                     
                                                                                    --strided-access=S, -a
                                                                                     
                                                                                    1<= S <= 64. The default=1.\n
                                                                                     
                                                                                    --cont, -L
                                                                                     
                                                                                    Continuous mode. The default=off.\n
                                                                                     
                                                                                    --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                                                                     
                                                                                    timeout for --cont mode. The default for all options is 0.\n
                                                                                     
                                                                                    --cache-hint=, -i
                                                                                     
                                                                                    Can be rdline-I or rdline-S. The default=rdline-I.\n
                                                                                     
                                                                                    --warm-fpga-cache -H; --cool-fpga-cache -M
                                                                                     
                                                                                    Try to prime the cache with hits. The default=off. Try to prime the \ncache with misses. The default=off.\n
                                                                                     
                                                                                    --cool-cpu-cache, -C
                                                                                     
                                                                                    Try to prime the cpu cache with misses. The default=off.\n
                                                                                     
                                                                                    --read-vc=, -r
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. The default=auto\n
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#write-test-options","title":"write test options","text":"
                                                                                    --guid=, -g
                                                                                     
                                                                                    AFU ID to enumerate. The default=F7DF405C-BD7A-CF72-22F1-44B0B93ACD18\n
                                                                                     
                                                                                    --begin=B, -b
                                                                                     
                                                                                    1 <= E <= 65535. The default=B, B and E designate number of cache lines.\n
                                                                                     
                                                                                    --multi-cl=M, -u
                                                                                     
                                                                                    M can equal 1, 2, or 4. The default=1.\n
                                                                                     
                                                                                    --strided-access=S, -a
                                                                                     
                                                                                    1<= S <= 64. The default=1.\n
                                                                                     
                                                                                    --cont, -L
                                                                                     
                                                                                    Continuous mode. The default=off.\n
                                                                                     
                                                                                    --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                                                                     
                                                                                    timeout for --cont mode. The default for all options is 0.\n
                                                                                     
                                                                                    --cache-policy=, -p
                                                                                     
                                                                                    Can be wrline-I, wrline-M, or wrpush-I The default=wrline-M\n
                                                                                     
                                                                                    --warm-fpga-cache -H; --cool-fpga-cache -M
                                                                                     
                                                                                    Try to prime the cache with hits. The default=off. Try to prime the \ncache with misses. The default=off.\n
                                                                                     
                                                                                    --cool-cpu-cache, -C
                                                                                     
                                                                                    Try to prime the cpu cache with misses. The default=off.\n
                                                                                     
                                                                                    --write-vc=, -w
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. The default=auto.\n
                                                                                     
                                                                                    --wrfence-vc=, -f
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. The default=`WRITE-VC`.\n
                                                                                     
                                                                                    --alt-wr-pattern, -l
                                                                                     
                                                                                    Alternate Write Pattern. The default=off.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#trput-test-options","title":"trput test options","text":"
                                                                                    --guid=, -g
                                                                                     
                                                                                    AFU ID to enumerate. The default=F7DF405C-BD7A-CF72-22F1-44B0B93ACD18.\n
                                                                                     
                                                                                    --begin=B, -b
                                                                                     
                                                                                    1 <= B <= 65535. The default=1, B = number of cache lines.\n
                                                                                     
                                                                                    --end=E, -e
                                                                                     
                                                                                    1 <= E <= 65535. The default=B, B and E designate number of cache lines.\n
                                                                                     
                                                                                    --multi-cl=M, -u
                                                                                     
                                                                                    M can equal 1, 2, or 4. The default=1.\n
                                                                                     
                                                                                    --strided-access=S, -a
                                                                                     
                                                                                    1<= S <= 64. The default=1\n
                                                                                     
                                                                                    --cont, -L
                                                                                     
                                                                                    Continuous mode. The default=off.\n
                                                                                     
                                                                                    --timeout-usec=, --timeout-msec=, --timeout-sec=, --timeout-min=, --timeout-hour=
                                                                                     
                                                                                    timeout for --cont mode. The default for all options is 0.\n
                                                                                     
                                                                                    --cache-policy=, -p
                                                                                     
                                                                                    Can be wrline-I, wrline-M, or wrpush-I The default=wrline-M.\n
                                                                                     
                                                                                    --cache-hint=, -i
                                                                                     
                                                                                    Can be rdline-I or rdline-S. The default=rdline-I.\n
                                                                                     
                                                                                    --read-vc=, -r
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. The default=auto.\n
                                                                                     
                                                                                    --write-vc=, -w
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random. The default=auto.\n
                                                                                     
                                                                                    --wrfence-vc=, -f
                                                                                     
                                                                                    Can be  auto, vl0, vh0, vh1. The default=`WRITE-VC`.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#sw-test-options","title":"sw test options","text":"
                                                                                    --guid=, -g
                                                                                     
                                                                                    AFU ID to enumerate. The default=7BAF4DEA-A57C-E91E-168A-455D9BDA88A3.\n
                                                                                     
                                                                                    --begin=B, -b
                                                                                     
                                                                                    1 <= B <= 65535. The default=1, B = number of cache lines.\n
                                                                                     
                                                                                    --end=E, -e
                                                                                     
                                                                                    1 <= E <= 65535. The default=B, B and E designate number of cache lines.\n
                                                                                     
                                                                                    --cache-policy=, -p
                                                                                     
                                                                                    Can be wrline-I, wrline-M, or wrpush-I. The default=wrline-M.\n
                                                                                     
                                                                                    --cache-hint= -i
                                                                                     
                                                                                    Can be rdline-I or rdline-S. The default=rdline-I.\n
                                                                                     
                                                                                    --read-vc=, -r
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random The default=auto.\n
                                                                                     
                                                                                    --write-vc=, -w
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1, random The default=auto.\n
                                                                                     
                                                                                    --wrfence-vc=, -f
                                                                                     
                                                                                    Can be auto, vl0, vh0, vh1. The default=`WRITE-VC`.\n
                                                                                     
                                                                                    --notice=, -N
                                                                                     
                                                                                    Can be poll or csr-write. The default=poll.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#enable-fpga-n3000-ethernet-group-vfio-mdev","title":"Enable FPGA N3000 Ethernet group VFIO mdev","text":"
                                                                                    FPGA DFL driver does not support any ioctls to read/write ethernet group info and registers. Users can read/write eth group registers by enabling VFIO mdev. Unbind the dfl_eth_group driver and bind vfio-mdev-dfl driver for ethernet group dfl-device; then userspace can take full control of ethernet group feature id 10.
                                                                                     
                                                                                    Ethernet group must be enabled before running fpgalpbk, mactest tools.
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#steps-to-enablecreate-vfio-mdev","title":"Steps to enable/create vfio mdev","text":"
                                                                                    unbind eth group feature id 10:\n    echo dfl-fme.0.8 > /sys/bus/dfl/drivers/dfl-eth-group/unbind\n    echo dfl-fme.0.7 > /sys/bus/dfl/drivers/dfl-eth-group/unbind\nbind to vfio-mdev-dfl:\n    echo vfio-mdev-dfl > /sys/bus/dfl/devices/dfl-fme.0.7/driver_override\n    echo vfio-mdev-dfl > /sys/bus/dfl/devices/dfl-fme.0.8/driver_override\nload vfio driver:\n    modprobe vfio_pci\n    modprobe vfio_iommu_type1\n    modprobe vfio_mdev\n    modprobe vfio_mdev_dfl\ntrigger mdev:\n    echo dfl-fme.0.7 >/sys/bus/dfl/drivers_probe\n    echo dfl-fme.0.8 >/sys/bus/dfl/drivers_probe\n    echo 83b8f4f2-509f-382f-3c1e-e6bfe0fa1001 > /sys/bus/dfl/devices/dfl-fme.0.7/mdev_supported_types/vfio-mdev-dfl-1/create\n    echo 83b8f4f2-509f-382f-3c1e-e6bfe0fa1002 > /sys/bus/dfl/devices/dfl-fme.0.8/mdev_supported_types/vfio-mdev-dfl-1/create\n\nlinux kerenl msg after enabling mdev:\n    i40e 0000:b3:00.0 eth1: NIC Link is Down\n    i40e 0000:b1:00.1 eth0: NIC Link is Down\n    vfio-mdev-dfl dfl-fme.2.7: MDEV: Registered\n    vfio-mdev-dfl dfl-fme.2.8: MDEV: Registered\n    vfio_mdev 83b8f4f2-509f-382f-3c1e-e6bfe0fa1005: Adding to iommu group 140\n    vfio_mdev 83b8f4f2-509f-382f-3c1e-e6bfe0fa1005: MDEV: group_id = 140\n    vfio_mdev 83b8f4f2-509f-382f-3c1e-e6bfe0fa1006: Adding to iommu group 141\n    vfio_mdev 83b8f4f2-509f-382f-3c1e-e6bfe0fa1006: MDEV: group_id = 141\n
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#remove-vfio-mdev","title":"Remove vfio mdev","text":"
                                                                                        echo 1 | sudo tee /sys/bus/mdev/devices/83b8f4f2-509f-382f-3c1e-e6bfe0fa1002/remove\n    echo 1 | sudo tee /sys/bus/mdev/devices/83b8f4f2-509f-382f-3c1e-e6bfe0fa1001/remove\n\n    rmmod vfio_mdev_dfl\n    modprobe dfl_eth_group\n\n    echo dfl-fme.0.7 >/sys/bus/dfl/drivers_probe\n    echo dfl-fme.0.8 >/sys/bus/dfl/drivers_probe\n\n    echo dfl-eth-group > /sys/bus/dfl/devices/dfl-fme.0.7/driver_override\n    echo dfl-eth-group > /sys/bus/dfl/devices/dfl-fme.0.8/driver_override\n
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#fpgalpbk-test-options","title":"fpgalpbk test options","text":"
                                                                                    --enable
                                                                                     
                                                                                    Enable fpga phy loopback.\n
                                                                                     
                                                                                    --disable
                                                                                     
                                                                                    Disable fpga phy loopback.\n
                                                                                     
                                                                                    --direction
                                                                                     
                                                                                    Can be local, remote.\n
                                                                                     
                                                                                    --type
                                                                                     
                                                                                    Can be serial, precdr, postcdr.\n
                                                                                     
                                                                                    --side
                                                                                     
                                                                                    Can be line, host.\n
                                                                                     
                                                                                    --port
                                                                                     
                                                                                    0 <= port <= 7, the default is all.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#mactest-test-options","title":"mactest test options","text":"
                                                                                    --offset
                                                                                     
                                                                                    Read mac addresses from an offset, The default=0.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#examples","title":"EXAMPLES","text":"
                                                                                    This command starts a lpbk1 test for the FPGA on bus 0x5e. The test  copies 57535, 57536, 57537 ... up to 65535 cache lines, one line at a time. The test prints output in the comma separated values (CSV) format with the header suppressed. 
                                                                                    ./fpgadiag --mode=lpbk1 --target=fpga -V --bus=0x5e --begin=57535\n--end=65535 --cache-hint=rdline-I --cache-policy=wrpush-I --multi-cl=1\n--write-vc=vl0 --read-vc=vh1 --wrfence-vc=auto\n
                                                                                     
                                                                                    This command starts a read test on the FPGA located on bus 0xbe. The test reads 2045 cache lines in the continuous mode with a 15-second timeout period.  The reads use a strided pattern with a 10-byte stride length. 
                                                                                    ./fpgadiag --mode=read --target=fpga -V --bus=0xbe --begin=2045 --cont\n--timeout-sec=15 --cache-hint=rdline-I --multi-cl=1 -a=10 \n--read-vc=auto --wrfence-vc=auto\n
                                                                                     
                                                                                    This command starts a sw test on the FPGA located on bus 0xbe. The test signals completion using a CSR write. 
                                                                                    ./fpgadiag --mode=sw --target=fpga -V --bus=0xbe --begin=4 --end=8192\n--cache-hint=rdline-I --cache-policy=wrline-I --notice=csr-write --write-vc=vl0\n--wrfence-vc=auto --read-vc=random \n
                                                                                     
                                                                                    This command enable a fpgalpbk on the FPGA located on bus 0xbe. 
                                                                                    ./fpgadiag -m fpgalpbk --bus 0xbe --enable --direction local --type postcdr\n--side host\n
                                                                                     
                                                                                    This command show fpgastats on the FPGA located on bus 0xbe. 
                                                                                    ./fpgadiag -m fpgastats --bus 0xbe\n
                                                                                    "},{"location":"sw/fpga_tools/fpgadiag/#troubleshooting","title":"TROUBLESHOOTING","text":"
                                                                                    When a test fails to run or gives errors, check the following:
                                                                                     
                                                                                       
                                                                                    • Is the Intel FPGA driver properly installed?  See Installation Guide  for driver installation instructions.
                                                                                      •  
                                                                                    • Are FPGA port permissions set properly? Check the permission bits of the port, for example, /dev/intel-fpga-port-0. You need READ and WRITE permissions to run fpgadiag tests.
                                                                                      •  
                                                                                    • Is hugepage properly configured on the system?  See Installation Guide for hugepage configuration steps. In particular, fpgadiag requires a few 1 GB pages. 
                                                                                      •  
                                                                                    • Is the required AFU loaded? See DESCRIPTION for information about what AFU the test requires.
                                                                                      •  
                                                                                    • Are --begin and --end values set properly? --end must be larger than the --begin. Also, --begin must be a multiple of the --multi-cl value.
                                                                                      •  
                                                                                    • The --warm-fpga-cache and --cool-fpga-cache options in the read and write tests are mutually exclusive.
                                                                                      •  
                                                                                    • The timeout options are only meaningful for the continuous mode  (with the --cont option).
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_tools/fpgaflash/fpgaflash/","title":"fpgaflash","text":""},{"location":"sw/fpga_tools/fpgaflash/fpgaflash/#synopsis","title":"SYNOPSIS","text":"
                                                                                    fpgaflash [-h] {user,factory} file [bdf]\n
                                                                                    "},{"location":"sw/fpga_tools/fpgaflash/fpgaflash/#description","title":"DESCRIPTION","text":"
                                                                                    fpgaflash updates the static FIM image loaded from flash at power-on.
                                                                                     
                                                                                    If there are multiple devices in the system, fpgaflash must specify a BDF to select the correct device. If no BDF is specified, fpgaflash prints out the BDFs of any compatible devices.
                                                                                    "},{"location":"sw/fpga_tools/fpgaflash/fpgaflash/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                                                                    {user, factory}
                                                                                     
                                                                                    Specifies the type of flash programming.
                                                                                     
                                                                                    user 
                                                                                     
                                                                                    Only reprograms the user image in flash.  
                                                                                     
                                                                                    factory 
                                                                                     
                                                                                    Reprograms the entire flash.  A catastrophic failure during a factory update such as a power outage  requires a USB cable and quartus_pgm to recover.
                                                                                     
                                                                                    file
                                                                                     
                                                                                    Specifies the Raw Programming Data File (rpd) to program into flash.
                                                                                     
                                                                                    bdf
                                                                                     
                                                                                    Specifies the bus, device and function (BDF) of device to program such as 04:00.0 or 0000:04:00.0. This flag is optional when there is a single device in the system.
                                                                                    "},{"location":"sw/fpga_tools/fpgaflash/fpgaflash/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                                                                    -h, --help
                                                                                     
                                                                                    Print usage information.
                                                                                    "},{"location":"sw/fpga_tools/fpgaflash/fpgaflash/#example","title":"EXAMPLE","text":"
                                                                                    fpgaflash user new_image.rpd 0000:04:00.0
                                                                                     
                                                                                    Programs new_image.rpd to flash of device with BDF 0000:04:00.0.
                                                                                    "},{"location":"sw/fpga_tools/fpgaflash/superrsu/","title":"super-rsu","text":""},{"location":"sw/fpga_tools/fpgaflash/superrsu/#synopsis","title":"SYNOPSIS","text":"
                                                                                    super-rsu [-h] [-n] [--verify] | [ [--log-level {trace,debug,error,warn,info,notset}]\n          [--log-file <filename>] [--rsu-only] [--with-rsu] [--force-flash] ]\n      rsu_config\n
                                                                                    "},{"location":"sw/fpga_tools/fpgaflash/superrsu/#description","title":"DESCRIPTION","text":"
                                                                                    super-rsu is a tool that can be used for flashing image files and commanding an Intel PAC device to perform RSU (remote system update - or a board reboot). Performing an RSU on an Intel PAC device will cause it to reload any firmware or programmable logic and restart its execution, a requirement for any updated firmware or programmable logic to take effect.
                                                                                     
                                                                                    At the core of super-rsu is its configuration file (referred to in this document as 'rsu_config') which is essentially a manifest file for identifying both the target device and the binary images (and their versions) to be flashed.
                                                                                     
                                                                                    At a high level, the flow of super-rsu should be: 1. Read and parse rsu_config file 2. Use product identifiers (like vendor, device and any additional vendor, device    pairs that may be present in the PCIe bus) to locate all compatible    devices on the PCIe bus. 3. For every device found on the system, update the device using the flash    images defined in the \"flash\" section in the rsu_config data (or nvmupdate    section).    Each item in the \"flash\" section is a \"flash spec\" that contains:    * The flash type (\"user\", \"bmc_fw\", \"bmc_img\", ...)    * The filename of the image to flash. super-rsu will look for this file      first in the same directory of the rsu_config file, and then look in the      current working directory.    * The version of the image.    * An optional \"force\" indicator    * An optional \"requires\" indicator    The \"nvmupdate\" section is used to describe an Ethernet firmware file and    its version. 4. Using the data in the \"nvmupdate\" and \"flash\" sections, the update routine    involves:   * If an \"nvmupdate\" section is present:     1. Locate the file on the file system to use to flash the Ethernet device.     2. Call nvmupdate to get an \"inventory\" of devices matching the vendor and        device id in this section.     3. Use this data to dynamically generate an nvmupdate compatible        configuration file.     4. Call nvmupdate with the generated configuration file to flash the        Ethernet interfaces in the Vista Creek card (if version reported by        system does not match the version in this section).   * For each spec in the \"flash\" section:     1. Locate the file on the file system to use to flash.     2. Compare the version listed in the \"flash spec\" to version reported by        the target component.     3. Create a task to call fpgaflash if either of the following conditions is        met (and the revision specified is compatible):        * The \"force\" indicator is present and set to true.        * The version in the spec does not match the version reported by the          system OR the flash type is factory type.   * For each task created from the \"flash\" section:     1. Call fpgaflash with the command line arguments that correspond to the        flash type and the file name in the spec used to create the task.        This opens and controls the execution of fpgaflash in another process.
                                                                                     
                                                                                    NOTE: If the system reports a revision for one of the components being flashed, this revision must be in the set of revisions listed in the manifest. Example: if the system reports 'a' for bmc_img and the manifest includes 'ab', then the image will be flashed.
                                                                                     
                                                                                    NOTE: Each update routine is run in a thread specific to a device located on the PCIe bus. Every task in an update routine involves opening a new process that is controlled and managed by its update routine thread. If a task includes a timeout and the timeout is reached, a termination request will be sent to its process and it will be counted as a failure. If a global timeout is reached in the main thread, a termination request will be sent to each thread performing the update. Consequently, the update routine will give the current task extra time before terminating the process. The RSU operation will only be performed if requested with either --with-rsu command line argument or with the --rsu-only command line argument. The former will perform the RSU command upon successful completion of flash operations. The latter will skip the process of version matching and flashing images and will only perform the RSU command. It is recommended that super-rsu be executed again if any flash operation is interrupted.
                                                                                    "},{"location":"sw/fpga_tools/fpgaflash/superrsu/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                                                                    rsu config
                                                                                     
                                                                                    Specifies the name of the file containing the RSU configuration (in JSON format)
                                                                                    "},{"location":"sw/fpga_tools/fpgaflash/superrsu/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                                                                    -h, --help
                                                                                     
                                                                                    Print usage information.
                                                                                     
                                                                                    --verify
                                                                                     
                                                                                    Compare versions of flashable components on the system against the manifest.   Return non-zero exit if compatible components are not up to date.
                                                                                     
                                                                                    -n, --dry-run
                                                                                     
                                                                                    Don't perform any updates, just a dry run.   This will print out commands that can be executed   in a Linux shell.
                                                                                     
                                                                                    --log-level {trace,debug,error,warn,info,notset}
                                                                                     
                                                                                    Log level to use. Default is 'info'.
                                                                                     
                                                                                    `--log-file  (default: /tmp/super-rsu.log) 
                                                                                    Emit log messages (with DEBUG level) to filename   NOTE: The default log file (/tmp/super-rsu.log) is set to rollover every   time super-rsu is executed. This will create numbered backups before   truncating the log file. The maximum number of backups is 50.
                                                                                     
                                                                                    --rsu-only
                                                                                     
                                                                                    Only perform the RSU command.
                                                                                     
                                                                                    --with-rsu
                                                                                     
                                                                                    Perform RSU after updating flash components(experimental)
                                                                                     
                                                                                    --force-flash
                                                                                     
                                                                                    Flash all images regardless of versions matching or not.
                                                                                    "},{"location":"sw/fpga_tools/fpgaflash/superrsu/#configuration","title":"CONFIGURATION","text":"
                                                                                    The following is the JSON schema expected by super-rsu. Any deviance from this schema may result in errors executing super-rsu.
                                                                                     
                                                                                    {\n  \"definitions\": {},\n  \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n  \"$id\": \"http://example.com/root.json\",\n  \"type\": \"object\",\n  \"title\": \"The Root Schema\",\n  \"required\": [\n    \"product\",\n    \"vendor\",\n    \"device\",\n    \"flash\"\n  ],\n  \"optional\": [\n    \"nvmupdate\",\n  ],\n  \"properties\": {\n    \"product\": {\n      \"$id\": \"#/properties/product\",\n      \"type\": \"string\",\n      \"title\": \"The Product Schema\",\n      \"default\": \"\",\n      \"examples\": [\n        \"n3000\"\n      ],\n      \"pattern\": \"^(.*)$\"\n    },\n    \"vendor\": {\n      \"$id\": \"#/properties/vendor\",\n      \"type\": \"string\",\n      \"title\": \"The Vendor Schema\",\n      \"default\": \"\",\n      \"examples\": [\n        \"0x8086\"\n      ],\n      \"pattern\": \"^((0x)?[A-Fa-f0-9]{4})$\"\n    },\n    \"device\": {\n      \"$id\": \"#/properties/device\",\n      \"type\": \"string\",\n      \"title\": \"The Device Schema\",\n      \"default\": \"\",\n      \"examples\": [\n        \"0x0b30\"\n      ],\n      \"pattern\": \"^((0x)?[A-Fa-f0-9]{4})$\"\n    },\n    \"nvmupdate\": {\n      \"$id\": \"#/properties/nvmupdate\",\n      \"type\": \"object\",\n      \"title\": \"The nvmupdate Schema\",\n      \"required\": [\n        \"vendor\",\n        \"device\",\n        \"filename\",\n        \"version\"\n      ],\n      \"optional\": [\n        \"interfaces\"\n      ],\n      \"properties\": {\n        \"vendor\": {\n          \"$id\": \"#/properties/nvmupdate/vendor\",\n          \"type\": \"string\",\n          \"title\": \"The nvmupdate Vendor Schema\",\n          \"default\": \"\",\n          \"examples\": [\n             \"0x8086\"\n          ],\n          \"pattern\": \"^((0x)?[A-Fa-f0-9]{4})$\"\n        },\n        \"device\": {\n          \"$id\": \"#/properties/nvmupdate/device\",\n          \"type\": \"string\",\n          \"title\": \"The nvmupdate Device Schema\",\n          \"default\": \"\",\n          \"examples\": [\n            \"0x0d58\"\n          ],\n          \"pattern\": \"^((0x)?[A-Fa-f0-9]{4})$\"\n        },\n        \"interfaces\": {\n          \"$id\": \"#/properties/nvmupdate/interfaces\",\n          \"type\": \"number\",\n          \"title\": \"The nvmupdate Interfaces Schema\",\n          \"default\": \"1\",\n          \"examples\": [\n            2, 4\n          ]\n        },\n        \"filename\": {\n          \"$id\": \"#/properties/nvmupdate/filename\",\n          \"type\": \"string\",\n          \"title\": \"The nvmupdate Filename Schema\",\n          \"default\": \"\",\n          \"examples\": [\n            \"PSG_XL710_6p80_XLAUI_NCSI_CFGID2p61_Dual_DID_0D58_800049C6.bin\"\n          ],\n          \"pattern\": \"^(.*)$\"\n        },\n        \"version\": {\n          \"$id\": \"#/properties/nvmupdate/version\",\n          \"type\": \"string\",\n          \"title\": \"The nvmupdate Version Schema\",\n          \"default\": \"\",\n          \"examples\": [\n            \"800049C6\"\n          ],\n          \"pattern\": \"^((0x)?[A-Fa-f0-9]{8})$\"\n        },\n        \"timeout\": {\n          \"$id\": \"#/properties/nvmupdate/timeout\",\n          \"type\": \"string\",\n          \"title\": \"The Timeout Schema\",\n          \"default\": \"\",\n          \"examples\": [\n            \"10m\"\n          ],\n          \"pattern\": \"^([0-9]+(\\\\.[0-9]+)?([dhms]))+$\"\n        }\n      }\n    },\n    \"flash\": {\n      \"$id\": \"#/properties/flash\",\n      \"type\": \"array\",\n      \"title\": \"The Flash Schema\",\n      \"items\": {\n        \"$id\": \"#/properties/flash/items\",\n        \"type\": \"object\",\n        \"title\": \"The Items Schema\",\n        \"required\": [\n          \"filename\",\n          \"type\",\n          \"version\",\n          \"revision\"\n        ],\n    \"optional\": [\n          \"enabled\",\n          \"force\",\n      \"timeout\",\n      \"requires\"\n    ],\n        \"properties\": {\n      \"enabled\": {\n            \"$id\": \"#/properties/flash/items/properties/enabled\",\n        \"type\": \"boolean\",\n        \"title\": \"The Enabled Schema\",\n        \"default\": \"true\"\n      },\n          \"filename\": {\n            \"$id\": \"#/properties/flash/items/properties/filename\",\n            \"type\": \"string\",\n            \"title\": \"The Filename Schema\",\n            \"default\": \"\",\n            \"examples\": [\n              \"vista_creek_qspi_xip_v1.0.6.ihex\"\n            ],\n            \"pattern\": \"^(.*)$\"\n          },\n          \"type\": {\n            \"$id\": \"#/properties/flash/items/properties/type\",\n            \"type\": \"string\",\n            \"title\": \"The Type Schema\",\n            \"default\": \"\",\n            \"examples\": [\n              \"bmc_fw\"\n            ],\n            \"enum\": [\"user\", \"bmc_fw\", \"bmc_img\", \"dtb\", \"factory_only\",\n        \"phy_eeprom\"]\n          },\n          \"version\": {\n            \"$id\": \"#/properties/flash/items/properties/version\",\n            \"type\": \"string\",\n            \"title\": \"The Version Schema\",\n            \"default\": \"\",\n            \"examples\": [\n              \"1.0.6\"\n            ],\n            \"pattern\": \"^\\\\d+\\\\.\\\\d+\\\\.\\\\d+$\"\n          },\n          \"force\": {\n            \"$id\": \"#/properties/flash/items/properties/force\",\n            \"type\": \"boolean\",\n            \"title\": \"The Force Schema\",\n            \"default\": false,\n            \"examples\": [\n              true\n            ]\n          },\n          \"revision\": {\n            \"$id\": \"#/properties/flash/items/properties/revision\",\n            \"type\": \"string\",\n            \"title\": \"The Revision Schema\",\n            \"default\": \"\",\n            \"examples\": [\n              \"C\"\n            ],\n            \"pattern\": \"^([A-Za-z])$\"\n          },\n          \"timeout\": {\n            \"$id\": \"#/properties/nvmupdate/timeout\",\n            \"type\": \"string\",\n            \"title\": \"The Timeout Schema\",\n            \"default\": \"\",\n            \"examples\": [\n              \"10m\"\n            ],\n            \"pattern\": \"^([0-9]+(\\.[0-9]+)?([dhms]))+$\"\n          },\n          \"requires\": {\n            \"$id\": \"#/properties/flash/items/properties/requires\",\n            \"type\": \"string\",\n            \"title\": \"The Requires Schema\",\n            \"default\": \"\",\n            \"examples\": [\n              \"bmc_img >= 1.0.12\"\n            ],\n            \"pattern\": \"^(([a-z_]+) ((<>!=)?=) ([0-9a-z\\\\.]+)$\"\n          }\n        }\n      }\n    }\n  }\n}\n
                                                                                    "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/","title":"fpgainfo","text":""},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#synopsis","title":"SYNOPSIS","text":"
                                                                                       fpgainfo [-h] [-S <segment>] [-B <bus>] [-D <device>] [-F <function>] [PCI_ADDR]\n            {errors,power,temp,fme,port,bmc,mac,phy,security}\n
                                                                                    "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#description","title":"DESCRIPTION","text":"
                                                                                    fpgainfo displays FPGA information derived from sysfs files. The command argument is one of the following: errors, power, temp, port, fme, bmc, phy or mac,security,events. Some commands may also have other arguments or options that control their behavior.
                                                                                     
                                                                                    For systems with multiple FPGA devices, you can specify the BDF to limit the output to the FPGA resource with the corresponding PCIe configuration. If not specified, information displays for all resources for the given command.
                                                                                    "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#fpgainfo-commands","title":"FPGAINFO COMMANDS","text":"
                                                                                    errors
                                                                                     
                                                                                    Show/clear errors of an FPGA resource that the first argument specifies. fpgainfo displays information in human readable form.
                                                                                     Error Description Catfatal Errors Bit 8 indicates an Injected Fatal error, bit 11 indicates an Injected Catastrophic Error. Inject Errors [2:0] are mainly writeable bits. Can read back values. (FME) Next Error [59:0] 60 LSBs are taken from the given error register that was triggered second, [60:61] 0 = FME0 Error, 1 = PCIe0 Error. (FME) First Error [59:0] 60 LSBs are taken from the given error register that was triggered first, [60:61] 0 = FME0 Error, 1 = PCIe0 Error. FME Errors Error from Partial Reconfiguration Block reporting a FIFO Parity Error has occurred. Non-fatal Errors Bit 6 is used to advertise an Injected Warning Error. 
                                                                                    power
                                                                                     
                                                                                    Show total the power in watts that the FPGA hardware consumes.
                                                                                     
                                                                                    temp
                                                                                     
                                                                                    Show FPGA temperature values in degrees Celcius.
                                                                                     
                                                                                    port
                                                                                     
                                                                                    Show information about the port such as the AFU ID of currently loaded AFU.
                                                                                     
                                                                                    fme
                                                                                     
                                                                                    Show information about the FPGA platform including the partial reconfiguration (PR) Interface ID, the OPAE version, and the FPGA Interface Manager (FIM) ID.
                                                                                     
                                                                                    bmc
                                                                                     
                                                                                    Show all Board Management Controller sensor values for the FPGA resource, if available.
                                                                                     
                                                                                    phy
                                                                                     
                                                                                    Show information about the PHY integrated in the FPGA, if available.
                                                                                     
                                                                                    mac
                                                                                     
                                                                                    Show information about the MAC address in ROM attached to the FPGA, if available.
                                                                                     
                                                                                    security
                                                                                     
                                                                                    Show information about the security keys, hashs and flash count, if available.
                                                                                     
                                                                                    events
                                                                                     
                                                                                    Show information about events and sensors, if available.
                                                                                    "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                                                                    --help, -h
                                                                                     
                                                                                    Prints help information and exit.
                                                                                     
                                                                                    --version, -v
                                                                                     
                                                                                    Prints version information and exit.
                                                                                    "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#common-arguments","title":"COMMON ARGUMENTS","text":"
                                                                                    The following arguments are common to all commands and are optional.
                                                                                     
                                                                                    -S, --segment
                                                                                     
                                                                                    PCIe segment number of resource.
                                                                                     
                                                                                    -B, --bus
                                                                                     
                                                                                    PCIe bus number of resource.
                                                                                     
                                                                                    -D, --device
                                                                                     
                                                                                    PCIe device number of resource.
                                                                                     
                                                                                    -F, --function
                                                                                     
                                                                                    PCIe function number of resource.
                                                                                    "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#errors-arguments","title":"ERRORS ARGUMENTS","text":"
                                                                                    The first argument to the errors command specifies the resource type. It must be one of the following:    fme,port,all
                                                                                     
                                                                                    fme
                                                                                     
                                                                                    Show/clear FME errors. 
                                                                                     
                                                                                    port
                                                                                     
                                                                                    Show/clear PORT errors.
                                                                                     
                                                                                    all
                                                                                     
                                                                                    Show/clear errors for all resources.
                                                                                     
                                                                                    The optional <command-args> arguments are:
                                                                                     
                                                                                    --clear, -c
                                                                                     
                                                                                    Clear errors for the given FPGA resource.
                                                                                    "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#phy-arguments","title":"PHY ARGUMENTS","text":"
                                                                                    The optional <command-args> argument is:
                                                                                     
                                                                                    --group, -G
                                                                                     
                                                                                    Select which PHY group(s) information to show.
                                                                                    "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#events-arguments","title":"EVENTS ARGUMENTS","text":"
                                                                                    The optional <command-args> argument is:
                                                                                     
                                                                                    --list,-l
                                                                                     
                                                                                    List boots (implies --all).
                                                                                     
                                                                                    --boot,-b
                                                                                     
                                                                                    Boot index to use, i.e: \u00a0\u00a0\u00a0\u00a00 for current boot (default). \u00a0\u00a0\u00a0\u00a01 for previous boot, etc.
                                                                                     
                                                                                    --count,-c
                                                                                     
                                                                                    Number of events to print.
                                                                                     
                                                                                    --all,-a
                                                                                     
                                                                                    Print all events.
                                                                                     
                                                                                    --sensors,-s
                                                                                     
                                                                                    Print sensor data too.
                                                                                     
                                                                                    --bits,-i
                                                                                     
                                                                                    Print bit values too.
                                                                                     
                                                                                    --help,-h
                                                                                     
                                                                                    Print this help.
                                                                                    "},{"location":"sw/fpga_tools/fpgainfo/fpgainfo/#examples","title":"EXAMPLES","text":"
                                                                                    This command shows the current power telemetry: 
                                                                                    ./fpgainfo power\n
                                                                                     
                                                                                    This command shows the current temperature readings: 
                                                                                    ./fpgainfo temp\n
                                                                                     
                                                                                    This command shows FME resource errors: 
                                                                                    ./fpgainfo errors fme\n
                                                                                     This command clears all errors on all resources: 
                                                                                    ./fpgainfo errors all -c\n
                                                                                     This command shows information of the FME on bus 0x5e 
                                                                                    ./fpgainfo fme -B 0x5e\n
                                                                                     This command shows information of the FPGA security on bus 0x5e 
                                                                                    ./fpgainfo security -B 0x5e\n
                                                                                     This command shows all events and sensors information including sensor bits: 
                                                                                    ./fpgainfo events -asi\n
                                                                                    "},{"location":"sw/fpga_tools/fpgamux/fpgamux/","title":"fpgamux","text":""},{"location":"sw/fpga_tools/fpgamux/fpgamux/#synopsis","title":"SYNOPSIS","text":"
                                                                                    fpgamux [-h] [-S|--socket-id SOCKET_ID] [-B|--bus-number BUS] [-D|--device DEVICE] [-F|--function FUNCTION]\n        [-G|--guid GUID] -m|--muxfile <filepath.json>\n
                                                                                    "},{"location":"sw/fpga_tools/fpgamux/fpgamux/#description","title":"DESCRIPTION","text":"
                                                                                    fpgamux tests multiple AFUs that are synthesized into a single AFU along with the CCIP-MUX basic building block (BBB). The CCIP-MUX uses the upper bits in the MMIO addresses to route MMIO reads and writes to the AFU running on the corresponding CCIP-MUX port. fpgamux uses a configuration file that lists the software components and correct configuration. fpgamux only runs on the Integrated FPGA Platform.  You cannot run it on the PCIe accelerator card (PAC).
                                                                                     
                                                                                    .. note::
                                                                                     
                                                                                      The OPAE driver discovers only the first AFU. The first software component in the configuration \n  determines the GUID to use for enumeration. Use the -G|--guid option to override the GUID\n  for the first software component.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgamux/fpgamux/#options","title":"OPTIONS","text":"
                                                                                    -S SOCKET_ID, --socket-id SOCKET_ID
                                                                                     
                                                                                    socket id of FPGA resource.
                                                                                     
                                                                                    -B BUS, --bus BUS
                                                                                     
                                                                                    bus id of FPGA resource.
                                                                                     
                                                                                    -D DEVICE, --device DEVICE
                                                                                     
                                                                                    The device id of FPGA resource.
                                                                                     
                                                                                    -F FUNCTION, --function FUNCTION
                                                                                     
                                                                                    The function id of FPGA resource.
                                                                                     
                                                                                    -G, --guid
                                                                                     
                                                                                    Specifies the GUID to use for the resource enumeration.
                                                                                     
                                                                                    -m, --muxfile <filepath.json>
                                                                                     
                                                                                    The path to the fpgamux configuration file. This file must be in JSON format following the schema described below.
                                                                                    "},{"location":"sw/fpga_tools/fpgamux/fpgamux/#configuration","title":"CONFIGURATION","text":"
                                                                                    fpgamux uses a configuration file (in JSON format) to determine what software components to instantiate and how to configure them to work with the AFUs. The schema includes the following elements:
                                                                                     
                                                                                        [\n        {\n            \"app\" : \"fpga_app\",\n            \"name\" : \"String\",\n            \"config\" : \"Object\"\n        }\n    ]\n
                                                                                    "},{"location":"sw/fpga_tools/fpgamux/fpgamux/#examples","title":"EXAMPLES","text":"
                                                                                    The following example shows a configuration with two components: 
                                                                                        [\n        {\n            \"app\" : \"nlb0\",\n            \"name\" : \"nlb0\",\n            \"config\" :\n            {\n                \"begin\" : 1,\n                \"end\" : 1,\n                \"multi-cl\" : 1,\n                \"cont\" : false,\n                \"cache-policy\" : \"wrline-M\",\n                \"cache-hint\" : \"rdline-I\",\n                \"read-vc\" : \"vh0\",\n                \"write-vc\" : \"vh1\",\n                \"wrfence-vc\" : \"write-vc\",\n                \"timeout-usec\" : 0,\n                \"timeout-msec\" : 0,\n                \"timeout-sec\" : 1,\n                \"timeout-min\" : 0,\n                \"timeout-hour\" : 0,\n                \"freq\" : 400000000\n            }\n        },\n        {\n            \"app\" : \"nlb3\",\n            \"name\" : \"nlb3\",\n            \"config\" :\n            {\n                \"mode\" : \"read\",\n                \"begin\" : 1,\n                \"end\" : 1,\n                \"multi-cl\" : 1,\n                \"strided-access\" : 1,\n                \"cont\" : false,\n                \"warm-fpga-cache\" : false,\n                \"cool-fpga-cache\" : false,\n                \"cool-cpu-cache\" : false,\n                \"cache-policy\" : \"wrline-M\",\n                \"cache-hint\" : \"rdline-I\",\n                \"read-vc\" : \"vh0\",\n                \"write-vc\" : \"vh1\",\n                \"wrfence-vc\" : \"write-vc\",\n                \"alt-wr-pattern\" : false,\n                \"timeout-usec\" : 0,\n                \"timeout-msec\" : 0,\n                \"timeout-sec\" : 1,\n                \"timeout-min\" : 0,\n                \"timeout-hour\" : 0,\n                \"freq\" : 400000000\n            }\n        }\n    ]\n
                                                                                    "},{"location":"sw/fpga_tools/fpgaport/fpgaport/","title":"fpgaport","text":""},{"location":"sw/fpga_tools/fpgaport/fpgaport/#synopsis","title":"SYNOPSIS","text":"
                                                                                    fpgaport [-h] [-N NUMVFS] [-X] [--debug] {assign,release} device [port]\n
                                                                                    "},{"location":"sw/fpga_tools/fpgaport/fpgaport/#description","title":"DESCRIPTION","text":"
                                                                                    The fpgaport enables and disables virtualization. It assigns and releases control of the port to the virtual function (VF). By default, the driver assigns the port to the physical function (PF) in the non-virtualization use case.
                                                                                    "},{"location":"sw/fpga_tools/fpgaport/fpgaport/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                                                                    {assign, release}
                                                                                     
                                                                                    Action to perform.\n
                                                                                     
                                                                                    device
                                                                                     
                                                                                    The FPGA device being targeted with this action.\n
                                                                                     
                                                                                    port
                                                                                     
                                                                                    The number of the port.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgaport/fpgaport/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                                                                    -N NUMVFS, --numvfs NUMVFS
                                                                                     
                                                                                    Create NUMVFS virtual functions. The typical value is 1.\n
                                                                                     
                                                                                    -X, --destroy-vfs
                                                                                     
                                                                                    Destroy all virtual functions prior to assigning.\n
                                                                                     
                                                                                    --debug
                                                                                     
                                                                                    Display additional log information.\n
                                                                                     
                                                                                    -h, --help
                                                                                     
                                                                                    Print usage information.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgaport/fpgaport/#example","title":"EXAMPLE","text":"
                                                                                    fpgaport release /dev/dfl-fme.0 0
                                                                                     
                                                                                    Release port 0 from physical function control.\n
                                                                                     
                                                                                    fpgaport assign /dev/dfl-fme.0 0
                                                                                     
                                                                                    Assign port 0 to physical function control.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgasupdate/fpgasupdate/","title":"fpgasupdate","text":""},{"location":"sw/fpga_tools/fpgasupdate/fpgasupdate/#synopsis","title":"SYNOPSIS","text":"
                                                                                    fpgasupdate [--log-level=<level>] file [bdf]
                                                                                    "},{"location":"sw/fpga_tools/fpgasupdate/fpgasupdate/#description","title":"DESCRIPTION","text":"
                                                                                    The fpgasupdate command implements a secure firmware update for the following programmable accelerator cards (PACs): * Intel\u00ae PAC with Intel Arria\u00ae 10 GX FPGA * Intel\u00ae FPGA PAC D5005 * Intel\u00ae PAC N3000 * Intel\u00ae FPGA SmartNIC N6001-PL with Intel&reg Agilex&reg FPGA * Intel\u00ae FPGA IPU F2000X-PL
                                                                                     
                                                                                    --log-level <level>
                                                                                     
                                                                                    Specifies the `log-level` which is the level of information output to your command tool.\nThe following seven levels  are available: `state`, `ioctl`, `debug`, `info`, `warning`,\n`error`, `critical`. Setting `--log-level=state` provides the most verbose output.\nSetting `--log-level=ioctl` provides the second most information, and so on. The default\nlevel is `info`.\n
                                                                                     
                                                                                    file
                                                                                     
                                                                                    Specifies the secure update firmware file to be programmed. This file may be to program a\nstatic region (SR), programmable region (PR), root entry hash, key cancellation, or other\ndevice-specific firmware.\n
                                                                                     
                                                                                    bdf
                                                                                     
                                                                                    The PCIe&reg; address of the PAC to program. `bdf` is of the form `[ssss:]bb:dd:f`,\ncorresponding to PCIe segment, bus, device, function. The segment is optional. If\nyou do not specify a segment, the segment defaults to `0000`. If the system has only\none PAC you can omit the `bdf` and let `fpgasupdate`  determine the address\nautomatically.\n
                                                                                    "},{"location":"sw/fpga_tools/fpgasupdate/fpgasupdate/#troubleshooting","title":"TROUBLESHOOTING","text":"
                                                                                    To gather more debug output, decrease the --log-level parameter. 
                                                                                    "},{"location":"sw/fpga_tools/fpgasupdate/fpgasupdate/#examples","title":"EXAMPLES","text":"
                                                                                    fpgasupdate firmware.bin fpgasupdate firmware.bin 05:00.0 fpgasupdate firmware.bin 0001:04:02.0 --log-level=ioctl
                                                                                    "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/","title":"host_exerciser","text":""},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#synopsis","title":"SYNOPSIS","text":"
                                                                                    Usage: host_exerciser [OPTIONS] SUBCOMMAND\n\n\nOptions:\n  -h,--help                   Print this help message and exit\n  -p,--pci-address TEXT       [<domain>:]<bus>:<device>.<function>\n  -l,--log-level TEXT:{trace,debug,info,warning,error,critical,off}=warning\n                              stdout logging level\n  -s,--shared                 open in shared mode, default is off\n  -t,--timeout UINT=60000     test timeout (msec)\n  -m,--mode UINT:value in {lpbk->0,read->1,trput->3,write->2} OR {0,1,3,2}=lpbk\n                              host exerciser mode {lpbk,read, write, trput}\n  --cls UINT:value in {cl_1->0,cl_2->1,cl_4->2,cl_8->3} OR {0,1,2,3}=cl_1\n                              number of CLs per request{cl_1, cl_2, cl_4, cl_8}\n  --continuousmode BOOLEAN=false\n                              test rollover or test termination\n  --atomic UINT:value in {cas_4->9,cas_8->11,fadd_4->1,fadd_8->3,off->0,swap_4->5,swap_8->7} OR {9,11,1,3,0,5,7}=off\n                              atomic requests (only permitted in combination with lpbk/cl_1)\n  --encoding UINT:value in {default->0,dm->1,pu->2,random->3} OR {0,1,2,3}=default\n                              data mover or power user encoding -- random interleaves both in the same stream\n  -d,--delay BOOLEAN=false    Enables random delay insertion between requests\n  --interleave UINT=0         Interleave requests pattern to use in throughput mode {0, 1, 2}\n                              indicating one of the following series of read/write requests:\n                              0: rd-wr-rd-wr\n                              1: rd-rd-wr-wr\n                              2: rd-rd-rd-rd-wr-wr-wr-wr\n  --interrupt UINT:INT in [0 - 3]\n                              The Interrupt Vector Number for the device\n  --contmodetime UINT=1       Continuous mode time in seconds\n  --testall BOOLEAN=false     Run all tests\n  --clock-mhz UINT=0          Clock frequency (MHz) -- when zero, read the frequency from the AFU\n\nSubcommands:\n  lpbk                        run simple loopback test\n  mem                         run simple mem test\n
                                                                                    "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#description","title":"DESCRIPTION","text":"
                                                                                    The host exerciser used to exercise and characterize the various host-FPGA interactions eg. MMIO, Data transfer from host to FPGA , PR, host to FPGA memory etc.
                                                                                    "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#host-exerciser-loopback-he-lbk","title":"Host Exerciser Loopback (HE-LBK)","text":"
                                                                                    HE-LB is responsible for generating traffic with the intention of exercising the path from the AFU to the Host at full bandwidth.  Host Exerciser Loopback (HE-LBK) AFU can move data between host memory and FPGA.
                                                                                     
                                                                                    HE-LBK supports: 1. Latency (AFU to Host memory read) 2. MMIO latency (Write+Read) 3. MMIO BW (64B MMIO writes) 4. BW (Read/Write, Read only, Wr only)
                                                                                    "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#host-exerciser-memory-he-mem","title":"Host Exerciser Memory (HE-MEM)","text":"
                                                                                    HE-MEM is used to exercise use of FPGA connected DDR; data read from the host is  written to DDR, and the same data is read from DDR before sending it back to the  host. HE-MEM uses external DDR memory (i.e. EMIF) to store data. It has a customized version of the AVMM interface to communicate with the EMIF memory controller.
                                                                                     
                                                                                    Execution of these exercisors requires the user to bind specific VF endpoint to vfio-pci Bind the correct endpoint for a device with B/D/F 0000:b1:00.0
                                                                                     
                                                                                    [user@localhost]: sudo opae.io init -d 0000:b1:00.2 user:user
                                                                                    "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#host-exerciser-sub-commands","title":"HOST EXERCISER SUB COMMANDS","text":"
                                                                                    lpbk
                                                                                     
                                                                                    run host exerciser loopback test
                                                                                     
                                                                                    mem
                                                                                     
                                                                                    run host exerciser memory test
                                                                                    "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                                                                    --help, -h
                                                                                     
                                                                                    Prints help information and exit.
                                                                                    "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#common-arguments-options","title":"COMMON ARGUMENTS / OPTIONS","text":"
                                                                                    The following arguments are common to all commands and are optional.
                                                                                     
                                                                                    -p,--pci-address
                                                                                     
                                                                                    PCIe domain, bus, device, function number of fpga resource.
                                                                                     
                                                                                    -l,--log-level
                                                                                     
                                                                                    set host exerciser tool log level, trace, debug, info, warning, error, critical, off
                                                                                     
                                                                                    -s,--shared
                                                                                     
                                                                                    open FPGA PCIe resource in shared mode
                                                                                     
                                                                                    -t,--timeout
                                                                                     
                                                                                    host exerciser tool time out, by default time out 60000
                                                                                     
                                                                                    -m,--mode
                                                                                     
                                                                                    host exerciser test modes are lpbk, read, write, trput
                                                                                     
                                                                                    --cls
                                                                                     
                                                                                    Number of cachelines per request 1, 2, 3, 4.
                                                                                     
                                                                                    --continuousmode
                                                                                     
                                                                                    Configures test rollover or test termination mode.
                                                                                     
                                                                                    --atomic
                                                                                     
                                                                                    atomic requests.
                                                                                     
                                                                                    --encoding
                                                                                     
                                                                                    select data mover mode or power user mode or random.
                                                                                     
                                                                                    -d,--delay
                                                                                     
                                                                                    Enables random delay insertion between requests.
                                                                                     
                                                                                    --interleave
                                                                                     
                                                                                    Enables interleave requests in throughput mode. Value:3'b000-Rd,Wr,Rd,Wr Value:3'b001-Rd,Rd,Wr,Wr Value:3'b010-Rd,Rd,Rd,Rd,Wr,Wr,Wr,Wr Value:3'b011-Not supported
                                                                                     
                                                                                    --interrupt
                                                                                     
                                                                                    Accelerator interrupt vector Number.
                                                                                     
                                                                                    --contmodetime
                                                                                     
                                                                                    Continuous mode time in seconds.
                                                                                     
                                                                                    --testall
                                                                                     
                                                                                    Run all host exerciser tests.
                                                                                     
                                                                                    --clock-mhz
                                                                                     
                                                                                    pcie clock frequency, default value 350Mhz. When specified by the user, will not check value against AFU clock before calculating performance metrics.
                                                                                    "},{"location":"sw/fpga_tools/host_exerciser/host_exerciser/#examples","title":"EXAMPLES","text":"
                                                                                    This command exerciser Loopback afu: 
                                                                                    host_exerciser lpbk\n
                                                                                     
                                                                                    This command exerciser memory afu: 
                                                                                    host_exerciser mem\n
                                                                                     
                                                                                    This command exerciser Loopback afu on pcie 000:3b:00.0: 
                                                                                    host_exerciser --pci-address 000:3b:00.0    lpbk\n
                                                                                     
                                                                                    This command exerciser Loopback afu on pcie 000:3b:00.0 and run in write mode: 
                                                                                    host_exerciser --pci-address 000:3b:00.0   --mode write lpbl\n
                                                                                     
                                                                                    This command exerciser Loopback afu on pcie 000:3b:00.0 and run 2 cache lines per request: 
                                                                                    host_exerciser --pci-address 000:3b:00.0   --cls cl_2  lpbk\n
                                                                                     
                                                                                    This command exerciser Loopback afu on pcie 000:3b:00.0 and run continuous mode for 10 seconds: 
                                                                                    host_exerciser --pci-address 000:3b:00.0   -cls cl_1   -m 0 --continuousmode true --contmodetime 10 lpbk\n
                                                                                    "},{"location":"sw/fpga_tools/hssi/hssi/","title":"hssi","text":""},{"location":"sw/fpga_tools/hssi/hssi/#synopsis","title":"SYNOPSIS","text":"
                                                                                    hssi COMMON_OPTIONS MODE MODE_OPTIONS
                                                                                    "},{"location":"sw/fpga_tools/hssi/hssi/#description","title":"DESCRIPTION","text":"
                                                                                    The hssi application provides a means of interacting with the 10G, 100G, and 200G/400F HE-HSSI AFUs. In all operating modes, the application initializes the AFU and completes the desired transfer as described by the mode- specific options.
                                                                                     
                                                                                    COMMON_OPTIONS - application options common to the 10G, 100g, and 200G/400G modes.
                                                                                     
                                                                                    -h, --help
                                                                                     
                                                                                    Display common command-line help and exit.\n
                                                                                     
                                                                                    -p, --pci-address ADDR
                                                                                     
                                                                                    The PCIe address of the desired accelerator in ssss:bb:dd.f format.\n
                                                                                     
                                                                                    -s, --shared on|off
                                                                                     
                                                                                    Whether to open the accelerator in shared mode. The default is off.\n
                                                                                     
                                                                                    -t, --timeout VALUE
                                                                                     
                                                                                    The application timeout value in milliseconds. The default timeout is 60000 msec.\n
                                                                                     
                                                                                    MODE - select AFU. Valid values are hssi_10g, hssi_100g, hssi_200g_400g.
                                                                                     
                                                                                    MODE_OPTIONS [hssi_10g] - application options specific to the 10G AFU.
                                                                                     
                                                                                    -h, --help
                                                                                     
                                                                                    Display 10G AFU specific command-line help and exit.\n
                                                                                     
                                                                                    --port PORT
                                                                                     
                                                                                    Select the QSFP port in the range 0-7. The default is port 0.\n
                                                                                     
                                                                                    --num-packets PACKETS
                                                                                     
                                                                                    The number of packets to transfer. The default is 1 packet.\n
                                                                                     
                                                                                    --random-length fixed|random
                                                                                     
                                                                                    Specify packet length randomization. Valid values are fixed and\nrandom. The default is fixed (no randomization).\n
                                                                                     
                                                                                    --random-payload incremental|random
                                                                                     
                                                                                    Specify payload randomization. Valid values are incremental and\nrandom. The default is incremental.\n
                                                                                     
                                                                                    --packet-length LENGTH
                                                                                     
                                                                                    Specify packet length. The default is 64 bytes.\n
                                                                                     
                                                                                    --src-addr ADDR
                                                                                     
                                                                                    Specify the source MAC address. The default value is 11:22:33:44:55:66.\n
                                                                                     
                                                                                    --dest-addr ADDR
                                                                                     
                                                                                    Specify the destination MAC address. The default value is 77:88:99:aa:bb:cc.\n
                                                                                     
                                                                                    --rnd-seed0 SEED0
                                                                                     
                                                                                    Specify the prbs generator bits [31:0]. The default is 1592590336.\n
                                                                                     
                                                                                    --rnd-seed1 SEED1
                                                                                     
                                                                                    Specify the prbs generator bits [47:32]. The default is 1592590337.\n
                                                                                     
                                                                                    --rnd-seed2 SEED2
                                                                                     
                                                                                    Specify the prbs generator bits [91:64]. The default is 155373.\n
                                                                                     
                                                                                    MODE_OPTIONS [hssi_100g] - application options specific to the 100G AFU.
                                                                                     
                                                                                    --port PORT
                                                                                     
                                                                                    Select the QSFP port in the range 0-7. The default is port 0.\n
                                                                                     
                                                                                    --eth-loopback on|off
                                                                                     
                                                                                    Whether to enable loopback on the ethernet interface. Valid values are\non and off. The default is on.\n
                                                                                     
                                                                                    --num-packets PACKETS
                                                                                     
                                                                                    The number of packets to transfer. The default is 1 packet.\n
                                                                                     
                                                                                    --gap random|none
                                                                                     
                                                                                    Inter-packet gap. Valid values are random and none. The default is none.\n
                                                                                     
                                                                                    --pattern random|fixed|increment
                                                                                     
                                                                                    Pattern mode. Valid values are random, fixed, or increment. The default\nis random.\n
                                                                                     
                                                                                    --src-addr ADDR
                                                                                     
                                                                                    Specify the source MAC address. The default value is 11:22:33:44:55:66.\n
                                                                                     
                                                                                    --dest-addr ADDR
                                                                                     
                                                                                    Specify the destination MAC address. The default value is 77:88:99:aa:bb:cc.\n
                                                                                     
                                                                                    --start-size SIZE
                                                                                     
                                                                                    Specify the packet size in bytes, or the first packet size for --pattern increment.\n
                                                                                     
                                                                                    --end-size SIZE
                                                                                     
                                                                                    Specify the end packet size in bytes.\n
                                                                                     
                                                                                    --end-select pkt_num|gen_idle
                                                                                     
                                                                                    Specify packet generation end mode.\n
                                                                                     
                                                                                    MODE_OPTIONS [pkt_filt_10g] - application options specific to the Packet Filter 10G AFU.
                                                                                     
                                                                                    --dfl-dev DFL_DEV
                                                                                     
                                                                                    Packet Filter DFL device, eg --dfl-dev dfl_dev.0\n
                                                                                     
                                                                                    MODE_OPTIONS [pkt_filt_100g] - application options specific to the Packet Filter 100G AFU.
                                                                                     
                                                                                    --dfl-dev DFL_DEV
                                                                                     
                                                                                    Packet Filter DFL device, eg --dfl-dev dfl_dev.1\n
                                                                                     
                                                                                    MODE_OPTIONS [hssi_200g_400g] - application options specific to the 200G/400G AFU.
                                                                                     
                                                                                    --num-packets PACKETS
                                                                                     
                                                                                    The number of packets to transfer. Must be a multiple of 32. Default value is 32. Increasing the timeout (--timeout) may be necessary if specifying a large number of packets.\n
                                                                                    "},{"location":"sw/fpga_tools/hssi/hssi/#examples","title":"EXAMPLES","text":"
                                                                                    hssi -h hssi hssi_10g -h sudo hssi --pci-address=0000:3b:00.0 hssi_10g --eth-loopback=on --num-packets=500 sudo hssi --pci-address=0000:3b:00.0 hssi_100g --pattern=increment sudo hssi --pci-address=0000:0d:00.6 hssi_200g_400g --num-packets=640000
                                                                                    "},{"location":"sw/fpga_tools/hssi_config/readme/","title":"hssi_config","text":""},{"location":"sw/fpga_tools/hssi_config/readme/#synopsis","title":"Synopsis","text":"
                                                                                    hssi_config reads or writes HSSI registers on either on an Intel\u00ae FPGA using the  FPGA Interface Manager (FIM) or on an HSSI retimer card attached to the board. hssi_config is only available for the Integrated FPGA Platform. You cannot run it  on the PCIe accelerator card (PAC).
                                                                                    "},{"location":"sw/fpga_tools/hssi_config/readme/#usage","title":"Usage","text":"
                                                                                    hssi_config [--resource|-r <sysfs resource>] [--socket-id|s 0|1] command [command options]
                                                                                     
                                                                                    Where command is one of the following:
                                                                                     
                                                                                        dump [outfile.csv] [--input-file inputfile.csv]\n    iread instance (0,1) device-addr byte-address byte-count\n    iwrite instance (0,1) device-addr byte-address byte1 [byte2 [byte3...]]\n    load [inputfile.csv] [--c-header]\n    read lane(0-15) reg-address\n    rread device(0x30, 0x32, 0x34, 0x36) channel(0-3) address\n    rwrite device(0x30, 0x32, 0x34, 0x36) channel(0-3) address value\n    test (rd|rw) inputfile.csv [--acktimes] [--repeat N]\n    write lane(0-15) reg-address value\n
                                                                                     
                                                                                    The first argument is the command and any additional arguments are command arguments. The following options and commands are available:
                                                                                    "},{"location":"sw/fpga_tools/hssi_config/readme/#options","title":"Options","text":"
                                                                                    [--resource|-r <sysfs resource path>
                                                                                     
                                                                                    The resource path in the sysfs pseudo-filesystem. Example:     /sys/devices/pci0000\\:5e/0000\\:5e\\:00.0/resource0
                                                                                     
                                                                                    [--socket-id 0|1]
                                                                                     
                                                                                    The socket id of the target FPGA. Required on two-socket systems to differentiate between the two possible target FPGAs.
                                                                                    "},{"location":"sw/fpga_tools/hssi_config/readme/#commands","title":"Commands","text":"
                                                                                    dump [outfile.csv] [--input-file inputfile.csv]
                                                                                     
                                                                                    Dump registers to stdout or to a file, if provided. hssi_config has a built-in set of registers to dump. The first argument is the path to a file to write. The command dumps to stdout if you do not  specify a file name. Use the --input-file option to specify a different set of registers. 
                                                                                     
                                                                                    load [inputfile.csv] [--c-header]
                                                                                     
                                                                                    Load a set of register values from either stdin or an input file, if provided. The first argument  is the path to a file containing the registers to load. Loads from stdin if omitted.  
                                                                                     
                                                                                    Use --c-header to generate a C header file with an array of 64-bit numbers to write to the  HSSI_CTRL register. This header file can substitute for the input file.  NOTE: You must perform the acknowledge routine after each write.
                                                                                     
                                                                                    read lane(0-15) reg-address
                                                                                     
                                                                                    Read from a single XCVR (transceiver) register. The first command argument is the XCVR lane. Use -1 to specify a read from all lanes. The second argument is the XCVR address (offset).
                                                                                     
                                                                                    write lane(0-15) reg-address value
                                                                                     
                                                                                    Write to a single XCVR register. The first argument is the XCVR lane. The second argument is the XCVR address(offset). The third argument is the value to write to the register.
                                                                                     
                                                                                    rread device(0x30, 0x32, 0x34, 0x36) channel(0-3) address
                                                                                     
                                                                                    Read from a single retimer register. The first argument is the I2C device address. The second argument is the channel. The third argument is the register address (or I2C byte address).
                                                                                     
                                                                                    rwrite device(0x30, 0x32, 0x34, 0x36) channel(0-3) address value
                                                                                     
                                                                                    Write to a single retimer register. The first argument is the I2C device address. The second argument is the channel. The third argument is the register address (or I2C byte address). The fourth argument is the value to write.
                                                                                     
                                                                                    iread instance (0,1) device-addr byte-address byte-count
                                                                                     
                                                                                    Read from a device on the I2C bus. The first argument is the I2C controller instance (0 or 1). The second argument is the device address to read from. The third argument is the byte address of the register to read from the device. The fourth argument is the number of bytes to read.
                                                                                     
                                                                                    iwrite instance (0,1) device-addr byte-address byte1 [byte2 [byte3...]]
                                                                                     
                                                                                    Write to a device on the I2C bus. The first argument is the I2C controller instance (0 or 1). The second argument is the device address to read from. The third argument is the byte address of the register to read from the device. All subsequent arguments are the bytes to write to the device.
                                                                                     
                                                                                    test (rd|rw) inputfile.csv [--acktimes]
                                                                                     
                                                                                    Perform built-in test for reading or writing XCVR registers. The first argument is the path to a file containing the registers to test.
                                                                                    "},{"location":"sw/fpga_tools/hssi_config/readme/#overview","title":"Overview","text":"
                                                                                    The hssi_config utility reads or writes hssi equalization parameters stored in either the transceiver (XCVR) registers or the registers of the retimer on the I2C bus. To access registers,  the hssi controller writes to the HSSI_CTRL register and reads from the HSSI_STAT register in the FPGA Management Engine (FME). These two registers implement the HSSI AUX bus mailbox protocol to access devices on other buses. Because hssi_config maps the FME MMIO space directly, the FPGA driver is not required.
                                                                                    "},{"location":"sw/fpga_tools/hssi_config/readme/#locating-the-fme-device","title":"Locating the FME Device","text":"
                                                                                    The FME reads and writes the HSSI_CTRL and HSSI_STAT registers on the NIOS device. The FME maps the MMIO address space of the FME identified by its resource in the sysfs psuedo-filesystem in Linux operating systems. To identify resource paths, use the lspci utility to query for Intel devices with device id of bcc0.
                                                                                     
                                                                                    Example:
                                                                                     
                                                                                    lspci -d 8086:bcc0
                                                                                     
                                                                                    This command should print out at least one line like the following example:
                                                                                     
                                                                                    5e:00.0 Processing accelerators: Intel Corporation Device bcc0
                                                                                     
                                                                                    Use the first three numbers (bus:device.function) to locate the device resource in the sysfs filesystem:
                                                                                     
                                                                                    /sys/devices/pci0000\\:<bus>/0000\\:<bus>\\:<device>.<function>/resource0
                                                                                     
                                                                                    For example, the example above with bus of 5e, device of 00 and function of 0 would use a resource path as follows:
                                                                                     
                                                                                    /sys/devices/pci0000\\:5e/0000\\:5e\\:00.0/resource0
                                                                                    "},{"location":"sw/fpga_tools/hssi_config/readme/#csv-file-format","title":"CSV File Format","text":"
                                                                                    Any CSV file parsed by hssi_config must meet have at least four columns. The following table provides the column specifications:
                                                                                     Column Name Description 1 Register type. Can be either FPGA_RX, FPGA_TX, RTMR_RX, RTMR_TX (or their corresponding numeric values, 1-4). 2 Lane or Channel 0-15 for XCVR lanes on FPGA, 0-3 for retimer channels. -1 to designate all lanes or channels. 3 Device address (on I2C bus) Only applies to retimer registers. 0 - 3, -1 to designate all devices. 4 Register address (or offset) Examples: 0x213, 0x100. 5 Register value to write Examples: 0x1, 1, 0. Applies only when loading or writing registers."},{"location":"sw/fpga_tools/hssi_config/readme/#examples-of-commands","title":"Examples of Commands","text":""},{"location":"sw/fpga_tools/hssi_config/readme/#dumping-registers","title":"Dumping Registers","text":"
                                                                                    Dump default register set to stdout:
                                                                                     
                                                                                    >hssi_config dump
                                                                                     
                                                                                    Dump default registers to a file, data.csv:
                                                                                     
                                                                                    >hssi_config dump data.csv
                                                                                     
                                                                                    Dump to an output file, data.csv, Registers specified in an input file, regspec.csv:
                                                                                     
                                                                                    >hssi_config dump data.csv --input-file regspec.csv
                                                                                    "},{"location":"sw/fpga_tools/hssi_config/readme/#reading-single-registers","title":"Reading Single Registers","text":"
                                                                                    Read register from XCVR at 0x2e1 on lane 0:
                                                                                     
                                                                                    >hssi_config read 0 0x2e1
                                                                                     
                                                                                    Read register 0x109 from retimer on channel 0, device 0x30, channel 1:
                                                                                     
                                                                                    >hssi_config rread 0 0x30 0x109
                                                                                    "},{"location":"sw/fpga_tools/hssi_config/readme/#loading-registers","title":"Loading Registers","text":"
                                                                                    Load registers specified in an input file called data.csv:
                                                                                     
                                                                                    >hssi_config load data.csv
                                                                                     
                                                                                    Load registers specified from stdin:
                                                                                     
                                                                                    >hssi_config load
                                                                                     
                                                                                    FPGA_RX,1,-1,0x213,0\nFPGA_RX,2,-1,0x213,0\nFPGA_RX,3,-1,0x213,0\n<CTRL-D>\n
                                                                                    "},{"location":"sw/fpga_tools/hssi_config/readme/#writing-single-registers","title":"Writing Single Registers","text":"
                                                                                    Write 1 to XCVR register at 0x2e1 on lane 0:
                                                                                     
                                                                                    >hssi_config write 0 0x2e1 1
                                                                                     
                                                                                    Read register 0x109 from retimer on channel 0, device 0x30, channel 1:
                                                                                     
                                                                                    >hssi_config rread 0 0x30 0x109
                                                                                    "},{"location":"sw/fpga_tools/hssi_config/readme/#testing-hssi-read-and-write","title":"Testing HSSI Read and Write","text":"
                                                                                    >  test (rd|rw) register-file.csv [--acktimes]
                                                                                     
                                                                                    rd|wr
                                                                                     
                                                                                    Specifies either a rd or wr of transceiver registers. For writes, every register in the file is read from and written to in the following sequence:
                                                                                     
                                                                                    1. Read the register, save the value   2. Write 1 to the register   3. Read the register, verify that the register value is 1   4. Write 0 to the register   5. Read the register, verify that the register value is 0   6. Write the original value to the register   7. Read the register, assert it is the original value
                                                                                     
                                                                                    register=file.csv
                                                                                     
                                                                                    Specifies the path to a file containing the set of registers to test. 
                                                                                     
                                                                                    --acktimes 
                                                                                     
                                                                                    Specifies the time spent in the ack routine. When measured, a summary of ack times prints  to stdout. This argument is optional.
                                                                                    "},{"location":"sw/fpga_tools/hssi_ethernet/hssiloopback/","title":"HSSI ethernet loopback","text":""},{"location":"sw/fpga_tools/hssi_ethernet/hssiloopback/#synopsis","title":"SYNOPSIS","text":"
                                                                                    hssiloopback [-h] [--pcie-address PCIE_ADDRESS, -P PCIE_ADDRESS] --loopback [{enable,disable}]\n
                                                                                    "},{"location":"sw/fpga_tools/hssi_ethernet/hssiloopback/#description","title":"DESCRIPTION","text":"
                                                                                    The hssiloopback  tool enables and disable ethernet loopback.
                                                                                    "},{"location":"sw/fpga_tools/hssi_ethernet/hssiloopback/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                                                                    -h, --help
                                                                                     
                                                                                    Prints usage information
                                                                                     
                                                                                    --pcie-address PCIE_ADDRESS, -P PCIE_ADDRESS
                                                                                     
                                                                                    The PCIe address of the desired fpga  in ssss:bb:dd.f format. sbdf of device to program (e.g. 04:00.0 or 0000:04:00.0). Optional when one device in system.
                                                                                     
                                                                                    --loopback [{enable,disable}]
                                                                                     
                                                                                    Ethernet enable or disable loopback.
                                                                                    "},{"location":"sw/fpga_tools/hssi_ethernet/hssiloopback/#examples","title":"EXAMPLES","text":"
                                                                                    hssiloopback --pcie-address  0000:04:00.0 --loopback enable
                                                                                     
                                                                                    Enables ethernet loopback
                                                                                     
                                                                                    hssiloopback --pcie-address  0000:04:00.0 --loopback disable
                                                                                     
                                                                                    Disable ethernet loopback
                                                                                    "},{"location":"sw/fpga_tools/hssi_ethernet/hssimac/","title":"HSSI ethernet mac","text":""},{"location":"sw/fpga_tools/hssi_ethernet/hssimac/#synopsis","title":"SYNOPSIS","text":"
                                                                                    hssimac [-h] --pcie-address PCIE_ADDRESS [--port PORT]\n
                                                                                    "},{"location":"sw/fpga_tools/hssi_ethernet/hssimac/#description","title":"DESCRIPTION","text":"
                                                                                    The hssimac  tool provides Maximum TX and RX frame size.
                                                                                    "},{"location":"sw/fpga_tools/hssi_ethernet/hssimac/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                                                                    -h, --help
                                                                                     
                                                                                    Prints usage information
                                                                                     
                                                                                    --pcie-address PCIE_ADDRESS, -P PCIE_ADDRESS
                                                                                     
                                                                                    The PCIe address of the desired fpga  in ssss:bb:dd.f format. sbdf of device to program (e.g. 04:00.0 or 0000:04:00.0).
                                                                                     
                                                                                    --port PORT
                                                                                     
                                                                                    hssi port number.
                                                                                    "},{"location":"sw/fpga_tools/hssi_ethernet/hssimac/#examples","title":"EXAMPLES","text":"
                                                                                    hssimac --pcie-address  0000:04:00.0 --port 1
                                                                                     
                                                                                    prints Maximum TX and RX frame size for port 1.
                                                                                    "},{"location":"sw/fpga_tools/hssi_ethernet/hssistats/","title":"HSSI ethernet statistics","text":""},{"location":"sw/fpga_tools/hssi_ethernet/hssistats/#synopsis","title":"SYNOPSIS","text":"
                                                                                    hssistats [-h] [--pcie-address PCIE_ADDRESS, -P PCIE_ADDRESS]\n
                                                                                    "},{"location":"sw/fpga_tools/hssi_ethernet/hssistats/#description","title":"DESCRIPTION","text":"
                                                                                    The hssistats  tool provides the MAC statistics.
                                                                                    "},{"location":"sw/fpga_tools/hssi_ethernet/hssistats/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                                                                    -h, --help
                                                                                     
                                                                                    Prints usage information
                                                                                     
                                                                                    --pcie-address PCIE_ADDRESS, -P PCIE_ADDRESS
                                                                                     
                                                                                    The PCIe address of the desired fpga in ssss:bb:dd.f format. sbdf of device to program (e.g. 04:00.0 or 0000:04:00.0). Optional when one device in system.
                                                                                    "},{"location":"sw/fpga_tools/hssi_ethernet/hssistats/#examples","title":"EXAMPLES","text":"
                                                                                    hssistats --pcie-address  0000:04:00.0
                                                                                     
                                                                                    prints the MAC statistics 
                                                                                    "},{"location":"sw/fpga_tools/hssi_loopback/readme/","title":"hssi_loopback","text":""},{"location":"sw/fpga_tools/hssi_loopback/readme/#name","title":"NAME","text":"
                                                                                    hssi_loopback - Software utility to run HSSI loopback tests on FPGA
                                                                                    "},{"location":"sw/fpga_tools/hssi_loopback/readme/#synopsis","title":"SYNOPSIS","text":"
                                                                                    hssi_loopback [[--bus|-b <bus number>] [--device | -d <device number>] [--function | -f <function number>]]|[--socket-id <socket-id>]       [--mode|-m auto|e40|e10]        [send [<source port> [<destination port>] [--packet-count|-c <count>] [--packet-delay|-d <delay>] [--packet-length|-l <length>]] |status [clear] | stop | readmacs
                                                                                    "},{"location":"sw/fpga_tools/hssi_loopback/readme/#description","title":"DESCRIPTION","text":"
                                                                                    The hssi_loopback utility works in conjunction with a packet generator accelerator function unit (AFU) to test high-speed serial interface (HSSI) cards. The hssi_loopback utility tests both external and internal loopbacks. hssi_loopback runs an external loopback test when the command line arguments include both source and destination ports. hssi_loopback runs an internal loopback test when command line arguments include a single port. hssi_loopback only runs on the Intel Xeon with Arria 10 FPGA. You cannot run it on the Intel PAC (programmable accelerator card).
                                                                                     
                                                                                    NOTE: The following limitations apply to the current version of hssi_loopback:
                                                                                     
                                                                                       
                                                                                    • For the external loopback the two port arguments can be the same. For the e10 design, the ports should be the same.
                                                                                      •  
                                                                                    • The hssi_loopback test supports only the e40 and e10 E2E AFUs.  The e10 E2E AFU tests HSSI with a retimer card.
                                                                                      •  
                                                                                    • The hssi_loopback test uses the control and status registers (CSRs) defined in the AFU.
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_tools/hssi_loopback/readme/#options","title":"OPTIONS","text":"
                                                                                    -S SOCKET_ID, --socket-id SOCKET_ID
                                                                                     
                                                                                    Socket ID FPGA resource.
                                                                                     
                                                                                    -B BUS, --bus BUS
                                                                                     
                                                                                    Bus ID of FPGA resource.
                                                                                     
                                                                                    -D DEVICE, --device DEVICE
                                                                                     
                                                                                    Device ID of FPGA resource.
                                                                                     
                                                                                    -F FUNCTION, --function FUNCTION
                                                                                     
                                                                                    Function ID of FPGA resource.
                                                                                     
                                                                                    -G, --guid
                                                                                     
                                                                                    Specifies guid for the resource enumeration.
                                                                                     
                                                                                    -m, --mode
                                                                                     
                                                                                    One of the following: [auto, e40, e10] auto is the default and indicates that the software runs the mode based on the first accelerator functional unit it identifies.
                                                                                     
                                                                                    -t, --timeout
                                                                                     
                                                                                    Timeout (in seconds) before the application terminates in continuous mode. Continuous mode is the default when you do not specify the number of packets.
                                                                                     
                                                                                    -y, --delay
                                                                                     
                                                                                    Delay (in seconds) between printing out a simple status line. Default is 0.100 seconds (100 milliseconds).
                                                                                     
                                                                                    -c, --packet-count
                                                                                     
                                                                                    The number of packets to send.
                                                                                     
                                                                                    -d, --packet-delay
                                                                                     
                                                                                    The delay in between packets. This delay is the number of 100 MHz clock cycles, roughly 10 nanoseconds.
                                                                                     
                                                                                    -s, --packet-size
                                                                                     
                                                                                    The packet size to send. The minimum is 46 bytes and the maximum is 1500 bytes. The default is 46 bytes.
                                                                                    "},{"location":"sw/fpga_tools/hssi_loopback/readme/#commands","title":"COMMANDS","text":"
                                                                                    send <source port> [<destination port>] [--packet-count|-c <count>] [--packet-delay|-d <delay>] [--packet-length|-l <length>]
                                                                                     
                                                                                    Send packets from one port to the other. If the command line does not specify a destination port, the test runs an internal  loopback. Otherwise, the test runs an external loopback from the source port to the destination port.
                                                                                     
                                                                                    status [clear]
                                                                                     
                                                                                    Read and interpret the status registers and print to the screen. clear clears the status registers.
                                                                                     
                                                                                    stop
                                                                                     
                                                                                    Issue a stop command to all Ethernet controllers in the AFU.
                                                                                     
                                                                                    readmacs
                                                                                     
                                                                                    Read and display the port MAC addresses. An EEPROM stores the MAC addresses.
                                                                                    "},{"location":"sw/fpga_tools/hssi_loopback/readme/#exit-codes","title":"EXIT CODES","text":"
                                                                                    0    Success - Number of packets received are equal to the number of packets sent and no errors           are reported.
                                                                                     
                                                                                    -1    Loopback failure - Either number of packets does not match or the test detected errors.
                                                                                     
                                                                                    -2    Errors parsing arguments.
                                                                                    "},{"location":"sw/fpga_tools/hssi_loopback/readme/#examples","title":"EXAMPLES","text":"
                                                                                    Read the MAC addresses of the AFU loaded on bus 0x5e:
                                                                                     
                                                                                    >sudo hssi_loopback readmacs -B 0x5e\n
                                                                                     
                                                                                    Run an external loopback, sending 100 packets from port 0 to port 1. The AFU is on bus 0x5e:
                                                                                     
                                                                                    >sudo hssi_loopback -B 0x5e send 0 1 -c 100\n
                                                                                     
                                                                                    Run an internal loopback until a timeout of 5 seconds is reached. The AFU is on bus 0x5e:
                                                                                     
                                                                                    >sudo hssi_loopback -B 0x5e send 0 -t 5\n
                                                                                    "},{"location":"sw/fpga_tools/mem_tg/mem_tg/","title":"mem_tg","text":""},{"location":"sw/fpga_tools/mem_tg/mem_tg/#synopsis","title":"SYNOPSIS","text":"
                                                                                    Usage: mem_tg [OPTIONS] SUBCOMMAND\n\nOptions:\n  -h,--help                   Print this help message and exit\n  -g,--guid TEXT=4DADEA34-2C78-48CB-A3DC-5B831F5CECBB\n                              GUID\n  -p,--pci-address TEXT       [<domain>:]<bus>:<device>.<function>\n  -l,--log-level TEXT:{trace,debug,info,warning,error,critical,off}=info\n                              stdout logging level\n  -s,--shared                 open in shared mode, default is off\n  -t,--timeout UINT=60000     test timeout (msec)\n  -m,--mem-channel UINT=0     Target memory bank for test to run on (0 indexed)\n  --loops UINT=1              Number of read/write loops to be run\n  -w,--writes UINT=1          Number of unique write transactions per loop\n  -r,--reads UINT=1           Number of unique read transactions per loop\n  -b,--bls UINT=1             Burst length of each request\n  --stride UINT=1             Address stride for each sequential transaction\n  --data UINT:value in {fixed->0,prbs15->2,prbs31->3,prbs7->1,rot1->3} OR {0,2,3,1,3}=fixed\n                              Memory traffic data pattern: fixed, prbs7, prbs15, prbs31, rot1\n  -f,--mem-frequency UINT=0   Memory traffic clock frequency in MHz\n\nSubcommands:\n  tg_test                     configure & run mem traffic generator test\n
                                                                                    "},{"location":"sw/fpga_tools/mem_tg/mem_tg/#description","title":"DESCRIPTION","text":"
                                                                                    The memory traffic generator (TG) used to exercise and test available memory channels with a configurable traffic pattern.
                                                                                     
                                                                                    Execution of this application requires the user to bind the specific VF endpoint containing the mem_tg AFU id to vfio-pci
                                                                                     
                                                                                    In the TG, read responses are checked against a specified pattern. If the application is configured to perform a read only test on a region of memory that has not previously been initialized to contain that pattern it will flag a test failure.
                                                                                    "},{"location":"sw/fpga_tools/mem_tg/mem_tg/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                                                                    --help, -h
                                                                                     
                                                                                    Prints help information and exit.
                                                                                    "},{"location":"sw/fpga_tools/mem_tg/mem_tg/#common-arguments-options","title":"COMMON ARGUMENTS / OPTIONS","text":"
                                                                                    The following arguments are common to all commands and are optional.
                                                                                     
                                                                                    -p,--pci-address
                                                                                     
                                                                                    PCIe domain, bus, device, function number of fpga resource.
                                                                                     
                                                                                    -l,--log-level
                                                                                     
                                                                                    set application log level, trace, debug, info, warning, error, critical, off
                                                                                     
                                                                                    -s,--shared
                                                                                     
                                                                                    open FPGA PCIe resource in shared mode
                                                                                     
                                                                                    -t,--timeout
                                                                                     
                                                                                    mem_tg application time out, by default time out 60000
                                                                                     
                                                                                    -m,--mem-channel
                                                                                     
                                                                                    Target memory bank for test to run on (0 indexed)  default: 0
                                                                                     
                                                                                    --loops
                                                                                     
                                                                                    Number of read/write loops to be run  default: 1
                                                                                     
                                                                                    -w,--writes
                                                                                     
                                                                                    Number of unique write transactions per loop.  default: 1
                                                                                     
                                                                                    -r,--reads
                                                                                     
                                                                                    Number of unique read transactions per loop  default: 1
                                                                                     
                                                                                    -b,--bls
                                                                                     
                                                                                    AXI4 burst length of each request.  Supports 1-256 transfers beginning from 0. default: 0
                                                                                     
                                                                                    --stride
                                                                                     
                                                                                    Address stride for each sequential transaction (>= burst length)  default: 1
                                                                                     
                                                                                    --data
                                                                                     
                                                                                    Memory traffic data pattern. 0 = fixed {0xFF, 0x00} 1 = prbs7 2 = prbs15 3 = prbs31 4 = rot1
                                                                                     
                                                                                    default: fixed
                                                                                     
                                                                                    -f, --mem-frequency
                                                                                     
                                                                                    Memory traffic clock frequency in MHz  default: 300 MHz
                                                                                    "},{"location":"sw/fpga_tools/mem_tg/mem_tg/#examples","title":"EXAMPLES","text":"
                                                                                    This command will run a basic read/write test on the channel 0 traffic generator: 
                                                                                    mem_tg tg_test\n
                                                                                     
                                                                                    This command will run the application for an afu on pcie 000:b1:00.7: 
                                                                                    mem_tg --pci-address 000:b1:00.7 tg_test\n
                                                                                     
                                                                                    This command will test channel 2 write bandwidth: 
                                                                                    mem_tg -loops 1000 -w 1000 -r 0 -m 2 tg_test\n
                                                                                     
                                                                                    This command will perform a read bandwidth test with a burst of 16 on channel 1 and perform a data comparison with the prbs7 pattern: 
                                                                                    mem_tg -loops 1000 -w 0 -r 1000 -b 0xF --data prbs7 -m 1 tg_test\n
                                                                                     
                                                                                    This command will perform a read/write test with 1 MB strided access to channel 0 memory: 
                                                                                    mem_tg -loops 10000 --stride 0x100000 tg_test\n
                                                                                    "},{"location":"sw/fpga_tools/mmlink/mmlink/","title":"mmlink","text":""},{"location":"sw/fpga_tools/mmlink/mmlink/#synopsis","title":"Synopsis","text":"
                                                                                    mmlink [-v] [-B <bus>] [-D <device>] [-F <function>] [-S <socket>] [-P <TCP port>] [-I <IP Address>]
                                                                                    "},{"location":"sw/fpga_tools/mmlink/mmlink/#description","title":"Description","text":"
                                                                                    The Remote Signal Tap logic analyzer provides real-time hardware debugging for the Accelerator Function Unit (AFU).  It provides a signal trace capability that the Quartus Prime software adds to the AFU. The Remote Signal Tap logic analyzer provides access to the Remote Signal Tap part of the Port MMIO space and then runs the remote protocol.
                                                                                    "},{"location":"sw/fpga_tools/mmlink/mmlink/#examples","title":"Examples","text":"
                                                                                    ./mmlink  -B 0x5e -P 3333
                                                                                     
                                                                                    MMLink app starts and listens for connection.
                                                                                    "},{"location":"sw/fpga_tools/mmlink/mmlink/#options","title":"Options","text":"
                                                                                    -v,--version
                                                                                     
                                                                                    Prints version information and exits.
                                                                                     
                                                                                    -B,--bus 
                                                                                     
                                                                                    FPGA Bus number.
                                                                                     
                                                                                    -D,--device 
                                                                                     
                                                                                    FPGA Device number.
                                                                                     
                                                                                    -F,--function 
                                                                                     
                                                                                    FPGA function number.
                                                                                     
                                                                                    -S,--socket 
                                                                                     
                                                                                    FPGA socket number.
                                                                                     
                                                                                    -P,--port 
                                                                                     
                                                                                    TCP port number.
                                                                                     
                                                                                    -I,--ip 
                                                                                     
                                                                                    IP address of FPGA system. 
                                                                                    "},{"location":"sw/fpga_tools/mmlink/mmlink/#notes","title":"Notes","text":"
                                                                                    Driver privilege:
                                                                                     
                                                                                    Change AFU driver privilege to user:
                                                                                     
                                                                                    $ chmod 777 /dev/intel-fpga-port.0\n
                                                                                     
                                                                                    Change locked memory size:
                                                                                     
                                                                                    edit the file /etc/security/limits.conf
                                                                                     
                                                                                    $ sudo vi /etc/security/limits.conf\n\nuser    hard   memlock           10000\n\nuser    soft   memlock           10000\n
                                                                                     
                                                                                    Exit terminal and log into a new terminal.
                                                                                     
                                                                                    Verify that the locked memory is now set:  ``` $ ulimit -l 10000
                                                                                    "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/","title":"ofs.uio","text":""},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#synopsis","title":"SYNOPSIS","text":"
                                                                                    ofs.uio [-h] [--pcie-address PCIE_ADDRESS] [--uio uiox] [--feature-id FEATURE_ID] [--region-index REGION_INDEX]                   [--mailbox-cmdcsr offset] [--bit-size {8,16,32,64}] [--peek offset] [--poke offset value]                   [--mailbox-read offset] [--mailbox-dump address size] [--mailbox-write address value]
                                                                                     
                                                                                    ofs.uio [--uio uiox] [--peek offset] ofs.uio [--uio uiox] [--poke offset value] ofs.uio [--uio uiox] [--mailbox-read address] ofs.uio [--uio uiox] [--mailbox-write address value] ofs.uio [--uio uiox] [--mailbox-dump address size] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--peek offset] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--poke offset value] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--mailbox-read address] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--mailbox-write address value] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--mailbox-dump address size]
                                                                                    "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#description","title":"DESCRIPTION","text":"
                                                                                    ofs.uio is a tool that provides user space access to DFL UIO devices, command line options like peek, poke, mailbox-read, mailbox-write, mailbox-dump to  access Configuration and Status Registers (CSRs).
                                                                                    "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#options","title":"OPTIONS","text":""},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#peek","title":"Peek","text":"
                                                                                    Peek/Read UIO CSR offset ofs.uio [--uio uio] [--peek offset] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--peek offset]
                                                                                    "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#poke","title":"Poke","text":"
                                                                                    Poke/Write value to UIO CSR offset ofs.uio [--uio uio] [--poke offset value] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--poke offset value]
                                                                                    "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#mailbox-read","title":"Mailbox Read","text":"
                                                                                    Read CSR address using mailbox ofs.uio [--uio uio] [--mailbox-read address] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--mailbox-read address]
                                                                                    "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#mailbox-write","title":"Mailbox Write","text":"
                                                                                    Write value to CSR address using mailbox  ofs.uio [--uio uio] [--mailbox-write address value] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--mailbox-write address value]
                                                                                    "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#mailbox-dump","title":"Mailbox Dump","text":"
                                                                                    Reads/Dumps block size of CSR address using mailbox ofs.uio [--uio uio] [--mailbox-dump address size] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--mailbox-dump address size]
                                                                                    "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#bit-size","title":"Bit size","text":"
                                                                                    Read/Write bit-field 8,16,32,64 sizes ofs.uio [--uio uio] --bit-size 8 [--peek offset] ofs.uio [--uio uio] --bit-size 32 [--peek offset]
                                                                                    "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#pcie-address","title":"PCIe Address","text":"
                                                                                    PCIE_ADDR PCIe address of FPGA device ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] [--peek offset]
                                                                                    "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#uio-region-index","title":"UIO region index","text":"
                                                                                    UIO region index, default region index is 0  ofs.uio [--uio uio] --region-index 0 [--peek offset] ofs.uio [--uio uio] --region-index 1 [--peek offset]
                                                                                    "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#mailbox-command-status-csr-offset","title":"Mailbox command status csr offset","text":"
                                                                                    Mailbox command status csr offset,  default value set to dfl pcie subsystem system feature mailbox command status register offset 0x28  ofs.uio [--uio uio] --mailbox-cmdcsr 0xa8 [--mailbox-read address] ofs.uio [--pcie-address PCIE_ADDRESS] [--feature-id FEATURE_ID] --mailbox-cmdcsr 0xa8  [--mailbox-read address]
                                                                                    "},{"location":"sw/fpga_tools/ofs.uio/ofs.uio/#examples","title":"EXAMPLES","text":"
                                                                                    Peek/Read 
                                                                                    ofs.uio --uio uio0 --peek 0x0\npeek(0x0): 0x3000000010002015\n\nofs.uio --uio uio6 --peek 0x0\npeek(0x0): 0x3000000100000020\n\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x15 --peek 0x0\npeek(0x0): 0x3000000010002015\n\nofs.uio --uio uio0 --peek 0x0 --bit-size 32\npeek(0x0): 0x10002015\n
                                                                                     
                                                                                    Poke/Write 
                                                                                    ofs.uio --uio uio6 --peek 0x8\npeek(0x8): 0x0\nofs.uio --uio uio6 --poke  0x8 0xabcdd12345\npoke(0x8):0xabcdd12345\n\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x15 --peek 0x0\npeek(0x8): 0x0\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x15 --poke  0x8 0x1234\npoke(0x8):0x1234\n
                                                                                     
                                                                                    Mailbox Read 
                                                                                    ofs.uio --uio uio6 --mailbox-read 0x0\nMailboxRead(0x0): 0x1000000\nofs.uio --uio uio6 --mailbox-read 0x8\nMailboxRead(0x8): 0x110c000\n\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x20 --mailbox-read 0x0\nMailboxRead(0x0): 0x1000000\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x20 --mailbox-read 0x8 \nMailboxRead(0x8): 0x110c000\n
                                                                                     
                                                                                    Mailbox Write 
                                                                                    ofs.uio --uio uio6 --mailbox-write 0x0 0x1234\nMailboxWrite(0x0):0x1234\nofs.uio --uio uio6 --mailbox-read 0x0\nMailboxRead(0x0):0x1234\n\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x20 --mailbox-write 0x0 0x1234\nMailboxWrite(0x0):0x1234\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x20 --mailbox-read 0x0 \nMailboxRead(0x0):0x1234\n
                                                                                     
                                                                                    Mailbox Dump 
                                                                                    ofs.uio --uio uio6 --mailbox-dump 0x0 0x10\nMailboxDump(0x0): 0x1000000\nMailboxDump(0x4): 0x1000000\nMailboxDump(0x8): 0x110c000\nMailboxDump(0xc): 0x110c000\nMailboxDump(0x10): 0x0\nMailboxDump(0x14): 0x0\nMailboxDump(0x18): 0x0\nMailboxDump(0x1c): 0x0\nMailboxDump(0x20): 0x0\nMailboxDump(0x24): 0x0\nMailboxDump(0x28): 0x0\nMailboxDump(0x2c): 0x0\nMailboxDump(0x30): 0x0\nMailboxDump(0x34): 0x0\nMailboxDump(0x38): 0x0\nMailboxDump(0x3c): 0x0\n\n\n\nofs.uio --pcie-address 0000:b1:00.0 --feature-id 0x20 --mailbox-dump 0x0 0x10\nMailboxDump(0x0): 0x1000000\nMailboxDump(0x4): 0x1000000\nMailboxDump(0x8): 0x110c000\nMailboxDump(0xc): 0x110c000\nMailboxDump(0x10): 0x0\nMailboxDump(0x14): 0x0\nMailboxDump(0x18): 0x0\nMailboxDump(0x1c): 0x0\nMailboxDump(0x20): 0x0\nMailboxDump(0x24): 0x0\nMailboxDump(0x28): 0x0\nMailboxDump(0x2c): 0x0\nMailboxDump(0x30): 0x0\nMailboxDump(0x34): 0x0\nMailboxDump(0x38): 0x0\nMailboxDump(0x3c): 0x0\n
                                                                                    "},{"location":"sw/fpga_tools/opae.io/opae.io/","title":"opae.io","text":""},{"location":"sw/fpga_tools/opae.io/opae.io/#synopsis","title":"SYNOPSIS","text":"
                                                                                    opae.io ls [-v,--viddid VIDDID] [-s,--sub-viddid SUB_VIDDID] [--all] [--system-class] opae.io init [-d,--device PCI_ADDR] [USER[:GROUP]] opae.io release [-d,--device PCI_ADDR] opae.io [-d,--device PCI_ADDR] [-r,--region REGION] walk [--offset [OFFSET]] [-u,--show-uuid] [-D,--dump] [-c,--count COUNT] [-y,--delay DELAY] [-s,--safe] opae.io [-d,--device PCI_ADDR] [-r,--region REGION] dump [--offset [OFFSET]] [-o,--output OUTPUT] [-f,--format {bin,hex}] [-c,--count COUNT] opae.io [-d,--device PCI_ADDR] [-r,--region REGION] peek OFFSET opae.io [-d,--device PCI_ADDR] [-r,--region REGION] poke OFFSET VALUE opae.io [-d,--device PCI_ADDR] [-r,--region REGION] script SCRIPT ARG1 ARG2 ... ARGN opae.io [-d,--device PCI_ADDR] [-r,--region REGION]
                                                                                    "},{"location":"sw/fpga_tools/opae.io/opae.io/#description","title":"DESCRIPTION","text":"
                                                                                    opae.io is an interactive Python environment packaged on top of libopaevfio.so, which provides user space access to PCIe devices via the vfio-pci driver. The main feature of opae.io is its built-in Python command interpreter, along with some Python bindings that provide a means to access Configuration and Status Registers (CSRs) that reside on the PCIe device.
                                                                                     
                                                                                    opae.io has two operating modes: command line mode and interactive mode.
                                                                                    "},{"location":"sw/fpga_tools/opae.io/opae.io/#command-line-mode","title":"COMMAND LINE MODE","text":"
                                                                                    To view the accelerator devices that are present on the system, opae.io provides the ls command option.
                                                                                     
                                                                                    opae.io ls [-v,--viddid VIDDID] [-s,--sub-viddid SUB_VIDDID] [--all] [--system-class]
                                                                                     
                                                                                    Each accelerator device is listed along with the PCIe address, the PCIe vendor/device ID, a brief description of the device, and the driver to which the device is currently bound.
                                                                                     
                                                                                    Device filtering is available by providing a Vendor ID:Device ID pair, eg -v 8086:bcce. Further filtering can be done by providing a sub- Vendor ID:sub-Device ID pair, eg -s 8086:1771. The --all option provides a list of all of the PCIe devices in the system, which an be quite verbose. The --system-class option prints the PCIe database class of the accelerator device, rather than the product name.
                                                                                     
                                                                                    opae.io provides an option to initialize a PCIe device for use with the vfio-pci driver. In order for the device CSRs to be accessed from user space, the device must first be bound to the vfio-pci driver. This is the job of the init command option.
                                                                                     
                                                                                    opae.io init [-d,--device PCI_ADDR] [USER[:GROUP]]
                                                                                     
                                                                                    The init command unbinds the specified device from its current driver and binds it to vfio-pci. This creates a new vfio group under /dev/vfio. This group path is then used by the libopaevfio.so library to interact with the device.
                                                                                     
                                                                                    To release the PCIe device from vfio-pci and return it to use with its previous driver, the release command option is used.
                                                                                     
                                                                                    opae.io release [-d,--device PCI_ADDR]
                                                                                     
                                                                                    The release command option reverses the actions of the last init command, releasing the device from vfio-pci and binding it to the driver which was bound at the time the init command was issued.
                                                                                     
                                                                                    The walk command option traverses and displays the Device Feature List of the given region.
                                                                                     
                                                                                    opae.io walk [--offset [OFFSET]] [-u,--show-uuid] [-D,--dump] [-c,--count COUNT] [-y,--delay DELAY] [-s,--safe]
                                                                                     
                                                                                    The various fields of each Device Feature Header are displayed. The --show-uuid option additionally displays the GUID for each feature. OFFSET can be used to specify the beginning of the DFL in the MMIO region. --dump displays the raw DFH contents in hex format. COUNT limits the number of DFH entries traversed. DELAY causes a pause between each printout. --safe examines each DFH offset for proper alignment.
                                                                                     
                                                                                    The dump command provides a means to dump the MMIO space in ASCII hex or binary format.
                                                                                     
                                                                                    opae.io dump [--offset [OFFSET]] [-o,--output OUTPUT] [-f,--format {bin,hex}] [-c,--count COUNT]
                                                                                     
                                                                                    OFFSET specifies the starting MMIO offset. OUTPUT gives the name of a file to capture the dump output, where sys.stdout is used by default. --format allows changing the output format. COUNT specifies the number of qwords to dump.
                                                                                     
                                                                                    The peek command option reads and displays a CSR value.
                                                                                     
                                                                                    opae.io peek OFFSET
                                                                                     
                                                                                    The poke command option writes a given value to a CSR.
                                                                                     
                                                                                    opae.io poke OFFSET VALUE
                                                                                     
                                                                                    opae.io can also execute Python scripts from the command line. These Python scripts may contain calls to the device built-in functions that are available during an interactive session. Refer to the description of interactive mode for details.
                                                                                     
                                                                                    opae.io script myscript.py a b c
                                                                                     
                                                                                    In order to enter the interactive mode of opae.io, simply invoke it and optionally pass the desired device address and MMIO region options.
                                                                                     
                                                                                    opae.io [-d,--device PCI_ADDR] [-r,--region REGION]
                                                                                    "},{"location":"sw/fpga_tools/opae.io/opae.io/#interactive-mode","title":"INTERACTIVE MODE","text":"
                                                                                    Upon entering interactive mode, opae.io begins a Python interpreter session and displays the command prompt shown below:
                                                                                     
                                                                                    0000:3f:00.0[0]>>
                                                                                     
                                                                                    The first portion of the prompt shows the address of the active PCIe device, here 0000:3f:00.0. The part in square brackets shows the active MMIO region, here [0].
                                                                                     
                                                                                    The interpreter waits for a valid Python command, then attempts to execute the given command in the usual way. The only differences between the traditional Python command intepreter and opae.io are that opae.io provides 1) the notion of an active PCIe device and MMIO region and 2) several built-in functions and objects that allow manipulating the active device and MMIO region.
                                                                                    "},{"location":"sw/fpga_tools/opae.io/opae.io/#built-in-functions","title":"BUILT-IN FUNCTIONS","text":"
                                                                                    The opae.io built-in functions assume an active device and MMIO region. Attempting to use the built-in functions without first opening a device and region will result in errors.
                                                                                     
                                                                                    peek(OFFSET)
                                                                                     
                                                                                    The peek built-in function reads and displays a CSR value from the active device and region, at the offset supplied by its argument.
                                                                                     
                                                                                    0000:3f:00.0[0]>> peek(0x28) 0xdeadbeef
                                                                                     
                                                                                    poke(OFFSET, VALUE)
                                                                                     
                                                                                    The poke built-in function writes the given VALUE to the current MMIO region at the given OFFSET.
                                                                                     
                                                                                    0000:3f:00.0[0]>> poke(0x28, 0xdeadbeef)
                                                                                     
                                                                                    read_csr(OFFSET)
                                                                                     
                                                                                    The read_csr built-in function returns the value of the CSR at the active MMIO region and the given OFFSET.
                                                                                     
                                                                                    0000:3f:00.0[0]>> print('0x{:0x}'.format(read_csr(0x28))) 0xdeadbeef
                                                                                     
                                                                                    write_csr(OFFSET, VALUE)
                                                                                     
                                                                                    The write_csr built-in function writes the given VALUE to the current MMIO region at the given OFFSET.
                                                                                     
                                                                                    0000:3f:00.0[0]>> write_csr(0x28, 0xdeadbeef)
                                                                                     
                                                                                    device(PCI_ADDR)
                                                                                     
                                                                                    The device built-in function allows changing the active PCIe device.
                                                                                     
                                                                                    0000:3f:00.0[0]>> device('0000:2b:00.0') 0000:2b:00.0>>
                                                                                     
                                                                                    region(REGION)
                                                                                     
                                                                                    The region built-in function allows changing the active MMIO region.
                                                                                     
                                                                                    0000:2b:00.0>> region(0) 0000:2b:00.0[0]>>
                                                                                     
                                                                                    allocate_buffer(SIZE)
                                                                                     
                                                                                    The allocate_buffer built-in function creates and returns a DMA buffer object. The underlying buffer will be SIZE bytes in length.
                                                                                     
                                                                                    0000:2b:00.0[0]>> b1 = allocate_buffer(4096) 0000:2b:00.0[0]>> print(b1.size, '0x{:0x}'.format(b1.address), b1.io_address) 4096 0x7f9361c66000 0
                                                                                     
                                                                                    version()
                                                                                     
                                                                                    The version built-in function returns a tuple containing the four components used to identify the opae.io version:
                                                                                     
                                                                                    0000:2b:00.0[0]>> print(version()) ('opae.io', 0, 2, 0)
                                                                                    "},{"location":"sw/fpga_tools/opae.io/opae.io/#built-in-objects","title":"BUILT-IN OBJECTS","text":"
                                                                                    opae.io interactive mode provides two global objects corresponding to the current device and that device's current MMIO region. These objects are referred to by global variables the_device and the_region, respectively.
                                                                                     
                                                                                    The device class:
                                                                                     
                                                                                    the_device.descriptor() : method that returns the integer file descriptor of the VFIO container.
                                                                                     
                                                                                    0000:2b:00.0[0]>> print(the_device.descriptor()) 5
                                                                                     
                                                                                    the_device.repr() : method that is invoked when a device object is printed.
                                                                                     
                                                                                    0000:2b:00.0[0]>> print(the_device) 0000:2b:00.0
                                                                                     
                                                                                    the_device.allocate(SIZE) : method that allocates and returns a system_buffer object. The buffer will be mapped into the DMA space of the_device.
                                                                                     
                                                                                    0000:2b:00.0[0]>> b1 = the_device.allocate(4096)
                                                                                     
                                                                                    the_device.pci_address() : read-only property that returns the PCIe address of the_device.
                                                                                     
                                                                                    0000:2b:00.0[0]>> print(the_device.pci_address) 0000:2b:00.0
                                                                                     
                                                                                    the_device.num_regions : read-only property that returns the number of MMIO regions in the_device.
                                                                                     
                                                                                    0000:2b:00.0[0]>> print(the_device.num_regions) 2
                                                                                     
                                                                                    the_device.regions : read-only property that returns a list of the active MMIO regions of the_device:
                                                                                     
                                                                                    0000:2b:00.0[0]>> print(the_device.regions) [0, 2]
                                                                                     
                                                                                    The region class:
                                                                                     
                                                                                    the_region.write32(OFFSET, VALUE) : method that writes a 32-bit VALUE to the CSR at OFFSET.
                                                                                     
                                                                                    the_region.read32(OFFSET) : method that returns a 32-bit CSR at the given OFFSET.
                                                                                     
                                                                                    0000:2b:00.0[0]>> the_region.write32(0x28, 0xdeadbeef) 0000:2b:00.0[0]>> print('0x{:0x}'.format(the_region.read32(0x28))) 0xdeadbeef
                                                                                     
                                                                                    the_region.write64(OFFSET, VALUE): method that writes a 64-bit VALUE to the CSR at OFFSET.
                                                                                     
                                                                                    the_region.read64(OFFSET): method that returns a 64-bit CSR at the given OFFSET.
                                                                                     
                                                                                    0000:2b:00.0[0]>> the_region.write64(0x28, 0xbaddecaf) 0000:2b:00.0[0]>> print('0x{:0x}'.format(the_region.read64(0x28))) 0xbaddecaf
                                                                                     
                                                                                    the_region.index(): method that returns the MMIO index of the_region.
                                                                                     
                                                                                    0000:2b:00.0[0]>> print(the_region.index()) 0
                                                                                     
                                                                                    the_region.repr(): method that is invoked when a region object is printed.
                                                                                     
                                                                                    0000:2b:00.0[0]>> print(the_region) 0
                                                                                     
                                                                                    the_region.len(): method that is invoked to determine the MMIO region size.
                                                                                     
                                                                                    0000:2b:00.0[0]>> print(len(the_region)) 524288
                                                                                     
                                                                                    The allocate_buffer() built-in function and the device.allocate() method return objects of type system_buffer.
                                                                                     
                                                                                    The system_buffer class is as follows:
                                                                                     
                                                                                    buf.size: read-only property that gives the buffer size.
                                                                                     
                                                                                    0000:2b:00.0[0]>> print(b1.size) 4096
                                                                                     
                                                                                    buf.address: read-only property that gives the buffer's user mode virtual address.
                                                                                     
                                                                                    0000:2b:00.0[0]>> print('0x{:0x}'.format(b1.address)) 0x7f2c15d8200
                                                                                     
                                                                                    buf.io_address: read-only property that gives the buffer's IO address.
                                                                                     
                                                                                    0000:2b:00.0[0]>> print('0x{:0x}'.format(b1.io_address)) 0x0
                                                                                     
                                                                                    buf.__getitem__ and buf.__setitem__: indexing get/set of 64-bit data item.
                                                                                     
                                                                                    0000:2b:00.0[0]>> b1[0] = 0xdecafbad 0000:2b:00.0[0]>> print('0x{:0x}'.format(b1[0])) 0xdecafbad
                                                                                     
                                                                                    buf.read8(OFFSET) buf.read16(OFFSET) buf.read32(OFFSET) buf.read64(OFFSET) : methods that read the given size data item from the given buffer OFFSET.
                                                                                     
                                                                                    buf.fill8(VALUE) buf.fill16(VALUE) buf.fill32(VALUE) buf.fill64(VALUE) : methods that fill the buffer with the given VALUE, using the given size.
                                                                                     
                                                                                    b1.compare(b2): method that compares buffers. The method returns the index of the first byte that miscompares, or the length of b1.
                                                                                    "},{"location":"sw/fpga_tools/opaeuio/opaeuio/","title":"opaeuio","text":""},{"location":"sw/fpga_tools/opaeuio/opaeuio/#synopsis","title":"SYNOPSIS","text":"
                                                                                    opaeuio [-h] [-i] [-r] [-d DRIVER] [-u USER] [-g GROUP] [-v] [device]
                                                                                    "},{"location":"sw/fpga_tools/opaeuio/opaeuio/#description","title":"DESCRIPTION","text":"
                                                                                    The opaeuio command enables the binding/unbinding of a DFL device to/from the dfl-uio-pdev device driver. See https://kernel.org/doc/html/v4.14/driver-api/uio-howto.html for a description of uio.
                                                                                    "},{"location":"sw/fpga_tools/opaeuio/opaeuio/#options","title":"OPTIONS","text":"
                                                                                    device     The DFL device name, eg dfl_dev.10
                                                                                     
                                                                                    -h, --help
                                                                                     
                                                                                    Display command-line help and exit.\n
                                                                                     
                                                                                    -i, --init
                                                                                     
                                                                                    Specifies binding mode operation - initialize the given device for uio.\nUsed in conjunction with -u, -g, and -d.\n
                                                                                     
                                                                                    -r, --release
                                                                                     
                                                                                    Specifies unbinding mode operation - release the given device from uio.\n
                                                                                     
                                                                                    -d DRIVER, --driver DRIVER
                                                                                     
                                                                                    Specifies the device driver to bind to when binding to uio.\nThe default value is dfl-uio-pdev.\n
                                                                                     
                                                                                    -u USER, --user USER
                                                                                     
                                                                                    The user ID to assign when binding to uio. A new device node is created in\n/dev when the device is bound to uio. Use this option to specify\nthe new device owner.\n
                                                                                     
                                                                                    -g GROUP, --group GROUP
                                                                                     
                                                                                    The group ID to assign when binding to uio. Use this option to specify the\nnew device group for the device created in /dev.\n
                                                                                     
                                                                                    -v, --version
                                                                                     
                                                                                    Display script version information and exit.\n
                                                                                    "},{"location":"sw/fpga_tools/opaeuio/opaeuio/#examples","title":"EXAMPLES","text":"
                                                                                    opaeuio -h opaeuio -v sudo opaeuio -i -u lab -g labusers dfl_dev.10 sudo opaeuio -r dfl_dev.10
                                                                                    "},{"location":"sw/fpga_tools/opaevfio/opaevfio/","title":"opaevfio","text":""},{"location":"sw/fpga_tools/opaevfio/opaevfio/#synopsis","title":"SYNOPSIS","text":"
                                                                                    opaevfio [-h] [-i] [-r] [-d DRIVER] [-u USER] [-g GROUP] [-n] [-v] [addr]
                                                                                    "},{"location":"sw/fpga_tools/opaevfio/opaevfio/#description","title":"DESCRIPTION","text":"
                                                                                    The opaevfio command enables the binding/unbinding of a PCIe device to/from the vfio-pci device driver. See https://kernel.org/doc/Documentation/vfio.txt for a description of vfio-pci.
                                                                                    "},{"location":"sw/fpga_tools/opaevfio/opaevfio/#options","title":"OPTIONS","text":"
                                                                                    addr     The PCIe address of the device in ssss:bb:dd.f format, eg 0000:7f:00.0
                                                                                     
                                                                                    -h, --help
                                                                                     
                                                                                    Display command-line help and exit.\n
                                                                                     
                                                                                    -i, --init
                                                                                     
                                                                                    Specifies binding mode operation - initialize the given addr for vfio.\nUsed in conjunction with -u, -g, and -n.\n
                                                                                     
                                                                                    -r, --release
                                                                                     
                                                                                    Specifies unbinding mode operation - release the given addr from vfio.\nUsed in conjunction with -d.\n
                                                                                     
                                                                                    -d DRIVER, --driver DRIVER
                                                                                     
                                                                                    Specifies the device driver to bind to when releasing from vfio.\nWhen omitted, the device is not rebound to a driver (default).\n
                                                                                     
                                                                                    -u USER, --user USER
                                                                                     
                                                                                    The user ID to assign when binding to vfio. A new device node is created in\n/dev/vfio when the device is bound to vfio-pci. Use this option to specify\nthe new device owner.\n
                                                                                     
                                                                                    -g GROUP, --group GROUP
                                                                                     
                                                                                    The group ID to assign when binding to vfio. Use this option to specify the\nnew device group for the device created in /dev/vfio.\n
                                                                                     
                                                                                    -n, --no-sriov
                                                                                     
                                                                                    Do not enable SR-IOV when binding to vfio. The default value for this option\nis FALSE, ie the script should specify SR-IOV functionality when binding to\nthe vfio-pci driver. When omitted, the modprobe command which loads the vfio-pci\ndriver will contain the `enable_sriov=1` option. When given, it will not.\n
                                                                                     
                                                                                    -v, --version
                                                                                     
                                                                                    Display script version information and exit.\n
                                                                                    "},{"location":"sw/fpga_tools/opaevfio/opaevfio/#examples","title":"EXAMPLES","text":"
                                                                                    opaevfio -h opaevfio -v sudo opaevfio -i -u lab -g labusers 0000:7f:00.0 sudo opaevfio -r 0000:7f:00.0
                                                                                    "},{"location":"sw/fpga_tools/pac_hssi_config/pac_hssi_config/","title":"Pac hssi config","text":"
                                                                                    # pac_hssi_config #\n\n## SYNOPSIS ##\n```console\npac_hssi_config.py [-h] subcommand [subarg] [bdf]\n
                                                                                    "},{"location":"sw/fpga_tools/pac_hssi_config/pac_hssi_config/#description","title":"DESCRIPTION","text":"
                                                                                    The pac_hssi_config.py tool exercises the Ethernet 10 Gbps (10GbE) and 40GbE transceivers for designs using the Intel\u00ae Programmable Acceleration Card (PAC) with Intel Arria\u00ae 10 GX FPGA. This tool does not support the  Intel Xeon\u00ae Processor with Integrated FPGA. 
                                                                                     
                                                                                     
                                                                                    The two required arguments to the pac_hssi_config.py tool specify the subcommand and bus, device, and function (BDF) for the PCIe device under test. You must provide the BDF parameter for systems with more than one PCIe card. 
                                                                                     
                                                                                    .. note::\n    If you do not provide the BDF when required, the command prints a list of valid BDFs for the system. You can also\n    determine the BDF using the ``lspci`` command.\n
                                                                                     
                                                                                    For usage help, type the following at a command prompt:
                                                                                     
                                                                                    pac_hssi_config.py [-h|--help]
                                                                                     
                                                                                    To configure the network ports, send data, and read statistics, use the following form of the pac_hssi_config.py script:
                                                                                     
                                                                                    pac_hssi_config.py subcommand [subarg] [bdf]
                                                                                     
                                                                                    Only a subset of subcommand arguments support subarg. 
                                                                                    "},{"location":"sw/fpga_tools/pac_hssi_config/pac_hssi_config/#table-1-general-subcommands","title":"Table 1. General Subcommands","text":"Subcommand Subarg Description stat N/A Prints high speed serial interface (HSSI) controller statistics. eeprom N/A Reads the 128-bit unique board ID, MAC address, and board-specific IDs from EEPROM."},{"location":"sw/fpga_tools/pac_hssi_config/pac_hssi_config/#table-2-1040-gbe-traffic-generation-subcommands","title":"Table 2. 10/40 GbE Traffic Generation Subcommands","text":"Subcommand Subarg Description e10init and e40init N/A Initializes HSSI PHY to 10GbE or 40GbE mode.  Clears statistics and enable internal HSSI transceiver loopback. e10loop and e40loop On/Off Turns on or off internal HSSI transceiver loopback. e10reset and e40reset On/Off Asserts or deasserts AFU reset.  Clears packet statistics and disables internal HSSI transceiver loopback. e10send and e40send N/A Sends 1,000,000 1500-byte packets. For 10GbE sends packets on all four ports. 40GbE has a single port. e10stat and e40stat N/A Prints packet statistics. e10statclr and e40statclr N/A Clears packet statistics.  Use this command after switching loopback modes to clear any transient statistics accumulated during the mode switch. 
                                                                                    The transceiver equalization eqwrite and eqread subcommands write and read transceiver equalization settings.  These subcommands require you to specify the transceiver channel, the equalization setting, and the value (for writes).  Use the following form for the eqwrite command:
                                                                                     
                                                                                    pac_hssi_config.py eqwrite [transceiver channel number] [equalization setting] [equalization value] [bdf]
                                                                                     
                                                                                    Use the following form for the eqreadcommand:
                                                                                     
                                                                                    pac_hssi_config.py eqread [transceiver channel number] [equalization setting] [bdf] 
                                                                                    "},{"location":"sw/fpga_tools/pac_hssi_config/pac_hssi_config/#table-3-transceiver-equalization-subcommands","title":"Table 3. Transceiver Equalization Subcommands","text":"Subcommand Channel Number Equalization Setting Value eqwrite 0-3 0 = Continuous time-linear equalization (CTLE)  1 = Variable gain amplifier (VGA)  2 = DCGAIN  3 = Pre-emphasis first post-tap  4 = Pre-emphasis second post-tap  5 = Pre-emphasis first pre-tap  6 = Pre-emphasis second pre-tap  7 = Differential output voltage (VOD) Specifies the value for the specified equalization setting. eqread 0-3 0 = Continuous time-linear equalization (CTLE)  1 = Variable gain amplifier (VGA)  2 = DCGAIN  3 = Pre-emphasis first post-tap  4 = Pre-emphasis second post-tap  5 = Pre-emphasis first pre-tap  6 = Pre-emphasis second pre-tap  7 = Differential output voltage (VOD) N/A 
                                                                                    For more information about reconfiguring transceiver analog parameter settings In Arria\u00ae 10 devices, refer to \"Changing PMA Analog Parameters\" in the  Intel\u00ae Arria\u00ae 10 Transceiver PHY User Guide. 
                                                                                    "},{"location":"sw/fpga_tools/packager/packager/","title":"packager","text":""},{"location":"sw/fpga_tools/packager/packager/#synopsis","title":"SYNOPSIS","text":"
                                                                                    packager <cmd> [arguments]
                                                                                    "},{"location":"sw/fpga_tools/packager/packager/#description","title":"Description","text":"
                                                                                    The packager provides tools that Accelerator Functional Unit (AFU) developers use to create Accelerator Function (AF)  files. The AF file is the programming file for an AFU on Intel\u00ae FPGA platforms. The packager tool concatenates the metadata from the JSON file to a raw binary file (.rbf) that the Intel Quartus\u00ae Prime software generates. 
                                                                                     
                                                                                    The packager's only function is to create an AF file. Refer to Packager Command Syntax for more information about invoking the packager. The packager depends on a JSON file to describe AFU metadata. Refer to  Accelerator Description File for more information about the JSON file.
                                                                                     
                                                                                    The packager requires Python 2.7.1 and Python 2.7.3. The tool indicates if it is being called with a compatible  of Python.
                                                                                    "},{"location":"sw/fpga_tools/packager/packager/#packager-command-syntax","title":"Packager Command Syntax","text":"
                                                                                    The packager is a command line tool with the following syntax:
                                                                                     
                                                                                    $ packager <cmd> [arguments]
                                                                                     
                                                                                    The following table describes the <CMD> arguments:
                                                                                     Command Arguments Description create-gbs --rbf=<RBF_PATH> --afu=<AFU_JSON_PATH> --gbs=<GBS_PATH> --set-value=<key>.<value> Creates the AF file. The engineering name for this file is the green bit stream, abbreviated gbs. The --rbf and --afu   arguments are required.  <RBF_PATH> is the path to the RBF file for the AFU. The Quartus\u00ae Prime software generates this RBF by compiling the AFU design. <AFU_JSON_PATH> is the path to the Accelerator Description file. This is a JSON file that describes the metadata that create-gbs appends to the RBF. <GBS_PATH> is the path to the RBF file for the FPGA Interface Manager (FIM) that contains the FPGA interface unit and other interfaces. If you do not specify the --gbs, the command defaults to <rbf_name>.gbs. You can use the optional --set-value=<key>.<value> argument to set values for JSON metadata. To set more than one JSON value, list a series of <key>.<value>  pairs. modify-gbs --gbs=<gbs_PATH> Modifies the AF file. The --input-gbsargument is required. If you do not provide the --output-gbs argument, modify-gbs overwrites the --input-gbs file. Use the --set-value=<key>.<value> argument to set values for JSON metadata. To set more than one JSON value, list a series of <key>.<value>  pairs. gbs-info --input-gbs=<gbs_PATH> Prints information about the AF file. The --input-gbs argument is required. get-rbf --gbs=<GBS_PATH> --rbf=<RBF_PATH> Creates the RBF by extracting it from the AF file. The --gbsargument is required. If you do not specify the --rbf argument, the command defaults to <gbs_name.rbf . None, or any <CMD> --help Summarizes the <CMD> options. Typing packager --help gives a list of <CMD> values. Typing packager <CMD> --help provides detailed help for <CMD>"},{"location":"sw/fpga_tools/packager/packager/#examples","title":"Examples","text":"
                                                                                    To generate an AF file, run:
                                                                                     
                                                                                    $ packager create-gbs --rbf=<RBF_PATH> --afu=<AFU_JSON_PATH> --gbs=<GBS_PATH>
                                                                                     
                                                                                    TIP: JSON files are very particular about syntax such as trailing commas. If you are getting errors, use jsonlint.com to validate that your JSON is formatted correctly. 
                                                                                     
                                                                                    To modify metadata in an existing AF, run the following command:
                                                                                     
                                                                                    $ packager modify-gbs --input-gbs=<PATH_TO_GBS_TO_BE_MODIFIED> --outputgbs=<NAME_FOR_NEW_GBS> --set-value <key>:<value>
                                                                                     
                                                                                    You can pass in a number of : pairs with --set-value to update values in an AF.  
                                                                                    To print the metadata of an existing AF: 
                                                                                     
                                                                                    $ packager get-info --gbs=<GBS_PATH> 
                                                                                     
                                                                                    To extract the RBF from the AF:
                                                                                     
                                                                                    $ packager get-rbf --gbs=<GBS_PATH> --rbf=<NAME_FOR_RBF>
                                                                                    "},{"location":"sw/fpga_tools/packager/packager/#accelerator-description-file","title":"Accelerator Description File","text":"
                                                                                    The Accelerator Description File is a JSON file that describes the metadata associated with an AFU. The Open Progammable Accelerator Engine (OPAE) uses this metadata during reconfiguration. Here is an example file:
                                                                                     
                                                                                    {\n   \"version\": 1,\n   \"platform-name\": \"DCP\",\n   \"afu-image\": {\n      \"magic-no\": 488605312,\n      \"interface-uuid\": \"01234567-89AB-CDEF-0123-456789ABCDEF\",\n      \"power\": 0,\n      \"accelerator-clusters\": [{\n         \"name\": \"dma_test_afu\",\n         \"total-contexts\": 1,   \n         \"accelerator-type-uuid\": \"331DB30C-9885-41EA-9081-F88B8F655CAA\"\n      }\n      ]  \n   }\n}\n
                                                                                     The packager stores these parameter values in the resultant AF. After reprogramming the AFU using partial reconfiguration (PR), the  software driver reconfigures the PLLs by writing the clock-frequency-high and clock-frequency-low values (if present) over the  PCIe\u00ae and CCI interfaces. 
                                                                                     
                                                                                    .. note:: 
                                                                                    The JSON file format may change as the architecture evolves. Any changes to the current format trigger an update\nto the version number.  \n
                                                                                     
                                                                                    CATEGORY | NAME | TYPE | DESCRIPTION | MANDATORY ---------|------|------|-------------|:----------:| Per-AFU  | version | Integer | Version of the metadata format. | Yes Per-AFU  | magic-no (to be deprecated)| Integer | Magic no. Associated with the FPGA Interface Manager. | No Per-AFU  | platform-name | String | Name of the platform for which the metadata is intended. The field value is \u201cDCP\u201d for Intel  Acceleration Stack for FPGAs. | No Per-AFU  | interface-uuid | UUID | Interface id associated with the FPGA Interface Manager. | Yes Per-AFU  | power | Integer | Accelerator Function power consumption, in watts. Set to 0 for Intel Acceleration Stack for FPGAs platforms. | Yes Per-AFU  | clock-frequency-low | Float | Clock frequency for 1st PLL (Clock network)1 in MHz. | No Per-AFU  | clock-frequency-high | Float | Clock frequency for 2nd PLL (0 if absent) in MHz. | No Per-AFC Cluster | total-contexts | Integer | Number of AFCs in this cluster. Always be 1 in current architectures. | Yes Per-AFC Cluster | afc-type-uuid |  UUID | AFC type = AFU ID in current architectures. | Yes Per-AFC Cluster | name | string | AFC name = AFU name in current architectures. | Yes
                                                                                     Date Intel Acceleration Stack Version Changes Made 2018.05.21 DCP 1.1 Beta (works with Quartus Prime Pro 17.1.1) Fixed typos."},{"location":"sw/fpga_tools/pci_device/pci_device/","title":"pci_device","text":""},{"location":"sw/fpga_tools/pci_device/pci_device/#synopsis","title":"SYNOPSIS","text":"
                                                                                    pci_device [-h] [-E] device-filter [{aer,bind,plug,remove,rescan,topology,unbind,unplug,vf}]
                                                                                    "},{"location":"sw/fpga_tools/pci_device/pci_device/#description","title":"DESCRIPTION","text":"
                                                                                    pci_device is a tool to aid in common operations for managing PCIe devices and drivers.
                                                                                    "},{"location":"sw/fpga_tools/pci_device/pci_device/#options","title":"OPTIONS","text":""},{"location":"sw/fpga_tools/pci_device/pci_device/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                                                                    `device filter`\n\nPCIe address of a device or vendor/device ID pair.\nThe PCIe address follows the format of [segment:]bus:device.function\nwhile the vendor/device ID pair follows the format [vendor ID]:[device ID]\nwhere at least one of these must be present.\n\n`{aer,bind,plug,remove,rescan,topology,unbind,unplug,vf}`\n\naction to perform on device\n\n`aer`\nPerform AER (Advanced Error Reporting) operations.\nThe aer action has its own sub-commands which are listed below:\n\n* `dump` sub-command will print out the AER error counters as reported\n   by the sysfs files for the device.\n* `mask` can either print out the current AER mask bits or set them\n  * If `show` or `print` (or nothing) is given after the `mask`\n    command, it will show the current mask bits for AER.\nBy default output will be written in stdout but can be written to an\noutput file if `-o|--output FILENAME` argument is given.\n  * If `all` is given after the `mask` command, it will mask all bits\n    (by setting the values to 0xffffffff and 0xffffffff).\n  * If `off` is given after the `mask` command, it will unmask all\n    bits (by setting the values to 0x0 and 0x0).\n  * If two numbers are present after the `mask` command, those two\n    numbers will be used to set the mask bits.\nValues for setting the mask can also be read in from an input file if\n`-i|--input FILENAME` argument is given.\n\n_NOTE_: mask related operations require root privileges\n\n`bind`\n\nAssociate a device with its driver.\n\n`plug`\n\nRestore a device that was previously given to `pci_device <device> unplug`\n\n`remove`\n\nRemove the pci device from the pci bus\n\n`rescan`\n\nRescan the bus as identified by the bus component of the PCIe device address\n\n'topology`\n\nPrint the PCIe topology from the root port to the PCIe device.\nThis shows the PCIe tree rooted at the PCIe root port.\nEach line shows the the PCIe address, vendor ID, and device ID along with\nthe driver bound to the device. The indentation is used to show\nparent/child relationship of devices.\n\nThe line listing the target PCIe device as identified by the given PCIe\naddress will be highlighted in green while the endpoints will be\nhighlighted in cyan.\n\nThe example below shows the topology of an N3000 device with eight virtual\nfunctions created from one of the Ethernet controllers:\n\n```console\n[pci_address(0000:3a:00.0), pci_id(0x8086, 0x2030)] (pcieport)\n    [pci_address(0000:3b:00.0), pci_id(0x10b5, 0x8747)] (pcieport)\n        [pci_address(0000:3c:09.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:3f:00.0), pci_id(0x8086, 0x0b30)] (dfl-pci)\n        [pci_address(0000:3c:11.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:43:00.0), pci_id(0x8086, 0x0b32)] (no driver)\n    [pci_address(0000:3c:08.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:3d:02.0), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:00.1), pci_id(0x8086, 0x0d58)] (i40e)\n            [pci_address(0000:3d:02.7), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:02.5), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:02.3), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:02.1), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:00.0), pci_id(0x8086, 0x0d58)] (i40e)\n            [pci_address(0000:3d:02.6), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:02.4), pci_id(0x8086, 0x154c)] (iavf)\n            [pci_address(0000:3d:02.2), pci_id(0x8086, 0x154c)] (iavf)\n        [pci_address(0000:3c:10.0), pci_id(0x10b5, 0x8747)] (pcieport)\n            [pci_address(0000:41:00.0), pci_id(0x8086, 0x0d58)] (i40e)\n            [pci_address(0000:41:00.1), pci_id(0x8086, 0x0d58)] (i40e)\n\n```\n\n`unbind`\n\nUnbind the driver bound to the device.\n\n`unplug`\n\nRemove device from PCI bus in anticipation of a RSU event by configuring its root port and associated endpoints.\n\n`vf`\n\nCreate/destroy VFs (virtual functions) by setting the number here.\nThe number given here will be written to sriov_numvfs sysfs file triggering\nthe PCIe subsystem to create/destroy VFs so that the current number of VFs\nwill be equal to the given number. If the number given is outside of the total VFs supported, an error message will be displayed to indicate this.\n
                                                                                    "},{"location":"sw/fpga_tools/pci_device/pci_device/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                                                                    `-h, --help`\n\nshow this help message and exit\n\n`-E, --other-endpoints`\n\nperform action on peer PCIe devices\n
                                                                                    "},{"location":"sw/fpga_tools/pci_device/pci_device/#examples","title":"EXAMPLES","text":"
                                                                                    pci_device 0000:3d:00.0 remove\npci_device 0000:3d:00.0 rescan\npci_device 3d:00.0 topology\npci_device :0b30 topology\npci_device :0b30 aer\npci_device :0b30 aer mask\npci_device :0b30 aer mask all\npci_device :0b30 aer mask -o mask.dat\npci_device :0b30 aer mask -i mask.dat\n
                                                                                    "},{"location":"sw/fpga_tools/rsu/rsu/","title":"rsu","text":""},{"location":"sw/fpga_tools/rsu/rsu/#synopsis","title":"SYNOPSIS","text":"
                                                                                    rsu [-h] [-d] {bmc,bmcimg,retimer,fpga,sdm,fpgadefault} [PCIE_ADDR]\n
                                                                                    "},{"location":"sw/fpga_tools/rsu/rsu/#description","title":"DESCRIPTION","text":""},{"location":"sw/fpga_tools/rsu/rsu/#mode-1-rsu","title":"Mode 1: RSU","text":"
                                                                                    rsu bmc --page=(user|factory) [PCIE_ADDR]\nrsu retimer [PCIE_ADDR]\nrsu fpga --page=(user1|user2|factory) [PCIE_ADDR]\nrsu sdm --type=(sr|pr|sr_cancel|pr_cancel) [PCIE_ADDR]\n
                                                                                     
                                                                                    Perform RSU (remote system update) operation on PAC device given its PCIe address. An RSU operation sends an instruction to the device to trigger a power cycle of the card only. This will force reconfiguration from flash for either BMC, Retimer, SDM, (on devices that support these) or the FPGA.
                                                                                    "},{"location":"sw/fpga_tools/rsu/rsu/#mode-2-default-fpga-image","title":"Mode 2: Default FPGA Image","text":"
                                                                                    rsu fpgadefault --page=(user1|user2|factory) --fallback=<csv> [PCIE_ADDR]\n
                                                                                     
                                                                                    Set the default FPGA boot sequence. The --page option determines the primary FPGA boot image. The --fallback option allows a comma-separated list of values to specify fallback images.
                                                                                    "},{"location":"sw/fpga_tools/rsu/rsu/#positional-arguments","title":"POSITIONAL ARGUMENTS","text":"
                                                                                    {bmc,bmcimg,retimer,fpga,sdm,fpgadefault}
                                                                                     
                                                                                    type of RSU operation or set Default FPGA Image operation.
                                                                                     
                                                                                    PCIE_ADDR  PCIe address of device to do rsu (e.g. 04:00.0 or 0000:04:00.0) 
                                                                                    "},{"location":"sw/fpga_tools/rsu/rsu/#optional-arguments","title":"OPTIONAL ARGUMENTS","text":"
                                                                                    -h, --help show this help message and exit
                                                                                     
                                                                                    -d, --debug log debug statements
                                                                                     
                                                                                    --force force rsu operation
                                                                                    "},{"location":"sw/fpga_tools/rsu/rsu/#example","title":"EXAMPLE","text":"
                                                                                    # rsu bmc --page=user 25:00.0\n
                                                                                     
                                                                                    Triggers a boot of the BMC image (user page) for the device with PCIe  address 25:00.0.
                                                                                     
                                                                                    # rsu bmc --page=factory 25:00.0\n
                                                                                     
                                                                                    Triggers a factory boot of the BMC image for the device with  PCIe address 25:00.0.
                                                                                     
                                                                                    # rsu fpga --page=user2 25:00.0\n
                                                                                     
                                                                                    Triggers a reconfiguration of the FPGA (user2 page) for the  device with PCIe address 25:00.0.
                                                                                     
                                                                                    # rsu --force fpga --page=user2 25:00.0\n
                                                                                     
                                                                                    Forces a reconfiguration of the FPGA (user2 page) for the  device with PCIe address 25:00.0. Default behavior is to not perform  the rsu operation if DPC (downstream port containment) is not supported  and AER (advanced error reporting) is also not supported. Using --force  changes this behavior to perform rsu operation regardless but may result  in a surprise removal of pci devices which may cause the Linux kernel  to panic.
                                                                                     
                                                                                    # rsu fpga --page=factory 25:00.0\n
                                                                                     
                                                                                    Triggers a factory reconfiguration of the FPGA for the device  with PCIe address 25:00.0.
                                                                                     
                                                                                    # rsu sdm --type=sr 25:00.0\n
                                                                                     
                                                                                    Triggers Static Region key programming for the device with  PCIE address 25:00.0.
                                                                                     
                                                                                    # rsu fpgadefault --page=factory --fallback=user1,user2 25:00.0\n
                                                                                     
                                                                                    Sets the FPGA boot sequence to factory with fallbacks user1, user2.
                                                                                    "},{"location":"sw/fpga_tools/super-rsu/super-rsu/","title":"Super Remote System Update User Guide","text":"
                                                                                    .. toctree::\n.. highlight:: c\n.. highlight:: console\n
                                                                                    "},{"location":"sw/fpga_tools/super-rsu/super-rsu/#overview","title":"Overview","text":"
                                                                                    Intel Programmable Acceleration Card (PAC) devices are comprised of multiple processors and controllers that execute firmware. Maintaining and updating these firmware images manually is error-prone and does not scale well within the Data Center. The solution described here is derived with the following goals in mind:
                                                                                     
                                                                                       
                                                                                    • The ability to update one or more (possibly all) firwmare images with a single package.
                                                                                      •  
                                                                                    • The ability to complete all firmware updates within a stipulated time window.
                                                                                      •  
                                                                                    • The ability to update each PAC in the server, all servers in a Data Center, and multiple Data Centers remotely.
                                                                                      •  
                                                                                    • The ability to remotely initiate download of the package and its installation with a single command per server instance.
                                                                                      •  
                                                                                    • The ability to roll back firmware to a previous revision.
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_tools/super-rsu/super-rsu/#implementation","title":"Implementation","text":"
                                                                                    A single package containing firmware images for all programmable parts on a PAC is delivered as an RPM, eg opae-super-rsu-n3000.M.m.p-r.noarch.rpm. The RPM revision will sequentially increase with every update.
                                                                                     
                                                                                    Installing or upgrading the RPM invokes the complete update of all programmable parts on all PAC boards in the system.
                                                                                     
                                                                                    The standard RPM dependency framework ensures that correct versions of dependecy packages opae-intel-fpga-driver and fpga-tools-extra are installed on the system.
                                                                                     
                                                                                    Rolling back is achieved by uninstalling the current version and re-installing a previous version of the RPM.
                                                                                     
                                                                                    .. note:: 
                                                                                    Note: once Secure Update is deployed, roll back restrictions shall be implemented to prevent\nrollback attacks.\n
                                                                                     
                                                                                    RPM management on remote systems is standard practice, requiring no new infrastructure/training.
                                                                                    "},{"location":"sw/fpga_tools/super-rsu/super-rsu/#details","title":"Details","text":"
                                                                                    The post-install hook of the opae-super-rsu-n3000 RPM is leveraged to call out to the super-rsu Python script to update all PAC boards. super-rsu uses the manifest file packaged within opae-super-rsu-n3000 to associate a firmware image with its version. Each of the firmware images contained in opae-super-rsu-n3000 is placed on the target system in /usr/share/opae/n3000.
                                                                                    "},{"location":"sw/fpga_tools/super-rsu/super-rsu/#algorithm","title":"Algorithm","text":"
                                                                                       
                                                                                    • Acquire the current firmware versions of all programmable parts.
                                                                                      •  
                                                                                    • For each programmable image, if the installed version of firmware does not equal the version provided in the RPM manifest file, then update the firmware image, and set image_updated to True.
                                                                                      •  
                                                                                    • After all updates, if image_updated, then initiate a safe reboot of all boards in the system.
                                                                                      •  
                                                                                    • After safe reboot, verify that the reported firmware versions match those of the RPM manifest. If they do not match, then RPM installation exits with a failing status.
                                                                                      •  
                                                                                    • Run board self test. If the self test fails, then the RPM installation exits with a failing status.
                                                                                      •  
                                                                                    • If all of the above checks is successful, then RPM installation exits with a success status.
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_tools/super-rsu/super-rsu/#dependencies","title":"Dependencies","text":"
                                                                                       
                                                                                    • The standard Python package for the distro (version 2.7).
                                                                                      •  
                                                                                    • The opae-intel-fpga-driver RPM. (version determined by opae-super-rsu-n3000)
                                                                                      •  
                                                                                    • The opae-tools-extra RPM. (version determined by opae-super-rsu-n3000)
                                                                                      •  
                                                                                    "},{"location":"sw/fpga_tools/userclk/userclk/","title":"userclk","text":""},{"location":"sw/fpga_tools/userclk/userclk/#synopsis","title":"SYNOPSIS","text":"
                                                                                    userclk [-hv] [-S <segment>] [-B <bus>] [-D <device>] [-F <function>] [PCI_ADDR] [-H <User clock high frequency>] -L <User clock low frequency>]
                                                                                    "},{"location":"sw/fpga_tools/userclk/userclk/#description","title":"DESCRIPTION","text":"
                                                                                    userclk sets the frequency range for an AFU. 
                                                                                    "},{"location":"sw/fpga_tools/userclk/userclk/#examples","title":"EXAMPLES","text":"
                                                                                    ./userclk -B 0x5e -H 400 -L 200
                                                                                     
                                                                                    Sets AFU frequency.
                                                                                    "},{"location":"sw/fpga_tools/userclk/userclk/#options","title":"OPTIONS","text":"
                                                                                    -v,--version
                                                                                     
                                                                                    Prints version information and exits.
                                                                                     
                                                                                    -S,--segment 
                                                                                     
                                                                                    FPGA segment number.
                                                                                     
                                                                                    -B,--bus 
                                                                                     
                                                                                    FPGA Bus number.
                                                                                     
                                                                                    -D,--device 
                                                                                     
                                                                                    FPGA Device number.
                                                                                     
                                                                                    -F,--function 
                                                                                     
                                                                                    FPGA function number.
                                                                                     
                                                                                    -H,--freq-high 
                                                                                     
                                                                                    User clock high frequency. 
                                                                                     
                                                                                    -L,--freq-low 
                                                                                     
                                                                                    User clock low frequency. 
                                                                                     Date Intel Acceleration Stack Version Changes Made 2018.05.21 DCP 1.1 Beta (works with Quartus Prime Pro 17.1.1) Fixed typos."},{"location":"sw/fpga_tools/vabtool/vabtool/","title":"vabtool","text":""},{"location":"sw/fpga_tools/vabtool/vabtool/#synopsis","title":"SYNOPSIS","text":"
                                                                                    vabtool [-r RETRIES] [-d] [-y] [-v] <ACTION>
                                                                                     
                                                                                    Where ACTION is defined as one of the following:
                                                                                     
                                                                                    vabtool sr_key_provision PCIE_ADDRESS SR_RKH_FILE FPGA_IMG_FILE vabtool sr_status PCIE_ADDRESS vabtool pr_key_provision PCIE_ADDRESS PR_AUTH_CERT_FILE PR_RKH_FILE vabtool pr_status PCIE_ADDRESS vabtool sr_key_cancel PCIE_ADDRESS SR_RKH_CANCEL_FILE vabtool sr_cancel_status PCIE_ADDRESS vabtool pr_key_cancel PCIE_ADDRESS PR_RKH_CANCEL_FILE vabtool pr_cancel_status PCIE_ADDRESS
                                                                                    "},{"location":"sw/fpga_tools/vabtool/vabtool/#description","title":"DESCRIPTION","text":"
                                                                                    The vabtool command helps perform Vendor Authenticated Boot provisioning of Static Region and Partial Reconfiguration Region key hashes and helps perform SR and PR hash cancellation and status reporting.
                                                                                    "},{"location":"sw/fpga_tools/vabtool/vabtool/#options","title":"OPTIONS","text":"
                                                                                    -r RETRIES, --retries RETRIES
                                                                                     
                                                                                    Specifies the number of times a failed SR or PR key provision is to be\nretried. The default value for RETRIES is 3.\n
                                                                                     
                                                                                    -d, --dry-run
                                                                                     
                                                                                    Don't execute the actual fpgasupdate and rsu commands, but only print\nthe commands that would be executed during a normal run of the script.\n
                                                                                     
                                                                                    -y, --yes
                                                                                     
                                                                                    The tool will respond with an automatic Yes answer to all confirmation\nprompts posed by the sub-tools.\n
                                                                                     
                                                                                    -v, --version
                                                                                     
                                                                                    Display script version information and exit.\n
                                                                                    "},{"location":"sw/fpga_tools/vabtool/vabtool/#examples","title":"EXAMPLES","text":"
                                                                                    sudo vabtool -y sr_key_provision 0000:bc:00.0 my_sr_rkh.bin my_fpga.bin sudo vabtool sr_status 0000:bc:00.0 sudo vabtool -y pr_key_provision 0000:bc:00.0 pr_auth_cert.bin my_pr_rkh.bin sudo vabtool pr_status 0000:bc:00.0 sudo vabtool sr_key_cancel 0000:bc:00.0 my_sr_rhk_cancel.bin sudo vabtool sr_cancel_status 0000:bc:00.0 sudo vabtool pr_key_cancel 0000:bc:00.0 my_pr_rhk_cancel.bin sudo vabtool pr_cancel_status 0000:bc:00.0
                                                                                    "},{"location":"sw/install_guide/installation_guide/","title":"OPAE Installation Guide","text":""},{"location":"sw/install_guide/installation_guide/#how-to-download-the-opae-sdk","title":"How to download the OPAE SDK","text":"
                                                                                    OPAE SDK releases are available on GitHub. Source code for the OPAE DFL device driver for Linux is also available on GitHub.
                                                                                    "},{"location":"sw/install_guide/installation_guide/#install-the-fedora","title":"Install the Fedora","text":"
                                                                                    Download the Fedora  (x86_64 version) installation file in fedora, and install the Fedora  in yourserver. You can choose Fedora Workstation or Fedora server.
                                                                                    "},{"location":"sw/install_guide/installation_guide/#build-the-kernel-and-dfl-drivers","title":"Build the kernel and DFL drivers","text":"
                                                                                    For building the OPAE kernel and kernel driver, the kernel development environment is required. So before you build the kernel, you must install the required packages. Run the following commands:
                                                                                     
                                                                                    $ sudo dnf install gcc gcc-c++ make kernel-headers kernel-devel elfutils-libelf-devel ncurses-devel openssl-devel bison flex\n
                                                                                     
                                                                                    Download the OPAE upstream kernel tree from github, for example download from fpga-ofs-dev-5.15-lts branch. 
                                                                                    $ git clone https://github.com/OPAE/linux-dfl.git -b fpga-ofs-dev-5.15-lts\n
                                                                                     
                                                                                    Configure the kernel. 
                                                                                    $ cd linux-dfl\n$ cp /boot/config-`uname -r` .config\n$ cat configs/dfl-config >> .config\n$ echo 'CONFIG_LOCALVERSION=\"-dfl\"' >> .config\n$ echo 'CONFIG_LOCALVERSION_AUTO=y' >> .config\n$ sed -i -r 's/CONFIG_SYSTEM_TRUSTED_KEYS=.*/CONFIG_SYSTEM_TRUSTED_KEYS=\"\"/' .config\n$ sed -i '/^CONFIG_DEBUG_INFO_BTF/ s/./#&/' .config\n$ echo 'CONFIG_DEBUG_ATOMIC_SLEEP=y' >> .config\n$ make olddefconfig\n
                                                                                     
                                                                                    Compile and install the new kernel. 
                                                                                    $ make -j $(nproc)\n$ sudo make modules_install -j $(nproc)\n$ sudo make install\n
                                                                                     
                                                                                    Build linux DFL Kernel instructions please also refer to: https://github.com/OPAE/linux-dfl/wiki/Build-the-linux-dfl-kernel
                                                                                     
                                                                                    When install finished, reboot your system. When the system login again, verify the kernel version is correct. For example: 
                                                                                    [figo@localhost linux-dfl]$ uname -a\nLinux localhost.localdomain 5.15.lts-dfl-g73e16386cda0 #6 SMP Mon Jun 13 21:21:31 -04 2022 x86_64 x86_64 x86_64\n
                                                                                     
                                                                                    And also you can check the OPAE dfl drivers have auto-loaded. 
                                                                                    [figo@localhost linux-dfl]$ lsmod | grep fpga\nifpga_sec_mgr          20480  1 intel_m10_bmc_secure\nfpga_region            20480  3 dfl_fme_region,dfl_fme,dfl\nfpga_bridge            24576  4 dfl_fme_region,fpga_region,dfl_fme,dfl_fme_br\nfpga_mgr               16384  4 dfl_fme_region,fpga_region,dfl_fme_mgr,dfl_fme\n[figo@localhost linux-dfl]$ lsmod | grep dfl\ndfl_eth_group          36864  0\ndfl_fme_region         20480  0\ndfl_emif               16384  0\ndfl_n3000_nios         20480  0\ndfl_fme_br             16384  0\ndfl_fme_mgr            20480  1\ndfl_fme                49152  0\ndfl_afu                36864  0\ndfl_pci                20480  0\ndfl                    40960  7 dfl_pci,dfl_fme,dfl_fme_br,dfl_eth_group,dfl_n3000_nios,dfl_afu,dfl_emif\nfpga_region            20480  3 dfl_fme_region,dfl_fme,dfl\nfpga_bridge            24576  4 dfl_fme_region,fpga_region,dfl_fme,dfl_fme_br\nfpga_mgr               16384  4 dfl_fme_region,fpga_region,dfl_fme_mgr,dfl_fme\n
                                                                                    "},{"location":"sw/install_guide/installation_guide/#build-the-opae-sdk","title":"Build the OPAE-SDK","text":"
                                                                                    Before you build the OPAE SDK, you must install the required packages. Run the following commands:
                                                                                    "},{"location":"sw/install_guide/installation_guide/#rocky-linux-85","title":"Rocky Linux 8.5","text":"
                                                                                    # dnf install -y 'dnf-command(config-manager)'\n# dnf config-manager --set-enabled powertools\n# dnf install -y epel-release\n# dnf check-update\n# dnf upgrade -y\n# dnf install -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml python3-pybind11 git gcc gcc-c++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel cli11-devel spdlog-devel libedit-devel systemd-devel rpm-build rpmdevtools pybind11-devel yaml-cpp-devel libudev-devel linuxptp numactl-devel\n# python3 -m pip install jsonschema virtualenv pyyaml\n
                                                                                    "},{"location":"sw/install_guide/installation_guide/#fedora","title":"Fedora","text":"
                                                                                    # dnf check-update\n# dnf upgrade -y\n# dnf install -y python3 python3-pip python3-devel python3-jsonschema python3-pyyaml python3-pybind11 git gcc g++ make cmake libuuid-devel json-c-devel hwloc-devel tbb-devel libedit-devel rpm-build rpmdevtools pybind11-devel yaml-cpp-devel libudev-devel cli11-devel spdlog-devel linuxptp numactl-devel\n# pip3 install jsonschema virtualenv pyyaml\n
                                                                                    "},{"location":"sw/install_guide/installation_guide/#ubuntu-2204","title":"Ubuntu 22.04","text":"
                                                                                    # apt-get update\n# apt-get upgrade -y\n# apt-get install -y python3 python3-pip python3-dev git gcc g++ make cmake uuid-dev libjson-c-dev libhwloc-dev libtbb-dev libedit-dev libudev-dev linuxptp pandoc devscripts debhelper doxygen libnuma-dev\n# pip3 install jsonschema virtualenv pyyaml pybind11\n
                                                                                    "},{"location":"sw/install_guide/installation_guide/#rhel-82","title":"RHEL 8.2","text":"
                                                                                    Register and enable Red Hat subscription to install any packages on the system.
                                                                                     
                                                                                    # subscription-manager register --proxy=PROXY --username=USER --password=PASSWORD --auto-attach\n
                                                                                     
                                                                                    Set the RHEL version and install packages. Set proxy name and port number.
                                                                                     
                                                                                    # subscription-manager release --set=8.2 --proxy proxy-name.com:port number\n# subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms\n# dnf upgrade -y\n# dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm\n# dnf install -y python3 python3-pip python3-devel gdb vim git gcc gcc-c++ make cmake libuuid-devel rpm-build systemd-devel  nmap\n# dnf install -y python3-jsonschema json-c-devel tbb-devel rpmdevtools libcap-devel numactl-devel\n# dnf check-update || true\n# dnf install -y spdlog-devel cli11-devel python3-pyyaml python3-pybind11 hwloc-devel libedit-devel\n# python3 -m pip install --user jsonschema virtualenv pudb pyyaml\n
                                                                                     
                                                                                    Install the latest version of cmake on top of the outdated cmake package from the package manager.
                                                                                     
                                                                                    # cd cmake-3.25.1/\n# ./bootstrap --prefix=/usr\n# make\n# make install\n# which cmake\n/usr/bin/cmake\n
                                                                                    "},{"location":"sw/install_guide/installation_guide/#create-opae-sdk-packages","title":"Create opae-sdk packages","text":"
                                                                                    Download the OPAE-SDK source code from github. For example, download from Master branch.
                                                                                     
                                                                                    $ git clone https://github.com/OPAE/opae-sdk.git\n
                                                                                     
                                                                                    Compile and build the OPAE-SDK RPMs (Fedora, Rocky, RHEL 8.2). 
                                                                                    $ cd opae-sdk/packaging/opae/rpm\n$ ./create fedora\n
                                                                                     
                                                                                    Note that if you find that your distribution has changed package names such that there is a conflict when building RPMs, you can install all of the build dependencies so that the SDK compiles and then build the RPMs in unrestricted mode: 
                                                                                    $ cd opae-sdk/packaging/opae/rpm\n$ ./create unrestricted\n
                                                                                     
                                                                                    After a successful compile, there are 3 rpm packages generated (Fedora, Rocky, RHEL8.2). For example: 
                                                                                    opae-2.1.0-1.fc34.x86_64.rpm\nopae-devel-2.1.0-1.fc34.x86_64.rpm\nopae-extra-tools-2.1.0-1.fc34.x86_64.rpm\n
                                                                                     
                                                                                    Compile and build the OPAE-SDK deb packages (Ubuntu 22.04). 
                                                                                    $ cd opae-sdk/packaging/opae/deb\n$ ./create\n
                                                                                     
                                                                                    After a successful compile, there are 3 deb packages generated (Ubuntu 22.04). For example: 
                                                                                    opae_2.1.1-1_amd64.deb  \nopae-devel_2.1.1-1_amd64.deb  \nopae-extra-tools_2.1.1-1_amd64.deb\n
                                                                                    "},{"location":"sw/install_guide/installation_guide/#opae-sdk-installation-with-rpmdeb-packages","title":"OPAE SDK installation with rpm/deb packages","text":"
                                                                                    The rpm packages generated in the previous step can be installed using these commands:
                                                                                     
                                                                                    $ sudo dnf install ./*.rpm\n
                                                                                     
                                                                                    The deb packages generated in the previous step can be installed using these commands:
                                                                                     
                                                                                    $ sudo dpkg -i  ./*.deb\n
                                                                                     
                                                                                    When you installed the rpms, you can run fpgainfo command to check the FPGA FME infomation. For example: 
                                                                                    [figo@localhost install_guide]$ fpgainfo fme\nBoard Management Controller, MAX10 NIOS FW version: D.2.1.24\nBoard Management Controller, MAX10 Build version: D.2.0.7\n//****** FME ******//\nObject Id                        : 0xEF00000\nPCIe s:b:d.f                     : 0000:08:00.0\nDevice Id                        : 0x0B30\nSocket Id                        : 0x00\nPorts Num                        : 01\nBitstream Id                     : 0x2300011001030F\nBitstream Version                : 0.2.3\nPr Interface Id                  : f3c99413-5081-4aad-bced-07eb84a6d0bb\nBoot Page                        : user\n
                                                                                     
                                                                                    To uninstall the OPAE rpms, you can use this commands 
                                                                                    $ dnf list installed | grep opae\n$ sudo dnf remove opae*.x86_64\n
                                                                                     
                                                                                    To uninstall the OPAE deb, you can use this commands 
                                                                                    $ dpkg -l  | grep opae\n$ dpkg -r opae-extra-tools:amd64\n$ dpkg -r opae-devel:amd64\n$ dpkg -r opae\n
                                                                                    "},{"location":"sw/install_guide/installation_guide/#fpga-device-access-permissions","title":"FPGA Device Access Permissions","text":"
                                                                                    Access to FPGA accelerators and devices is controlled using file access permissions on the Intel\u00ae FPGA device files, /dev/dfl-fme.* and /dev/dfl-port.*, as well as to the files reachable through /sys/class/fpga_region/.
                                                                                     
                                                                                    In order to allow regular (non-root) users to access accelerators, you need to grant them read and write permissions on /dev/dfl-port.* (with * denoting the respective socket, i.e. 0 or 1). E.g.:
                                                                                     
                                                                                    $ sudo chmod a+rw /dev/dfl-port.0\n
                                                                                    "},{"location":"sw/install_guide/installation_guide/#memlock-limit","title":"Memlock limit","text":"
                                                                                    Depending on the requirements of your application, you may also want to increase the maximum amount of memory a user process is allowed to lock. The exact way to do this depends on your Linux distribution.
                                                                                     
                                                                                    You can check the current memlock limit using
                                                                                     
                                                                                    $ ulimit -l\n
                                                                                     
                                                                                    A way to permanently remove the limit for locked memory for a regular user is to add the following lines to your /etc/security/limits.conf:
                                                                                     
                                                                                    user1    hard   memlock           unlimited\nuser1    soft   memlock           unlimited\n
                                                                                     
                                                                                    This removes the limit on locked memory for user user1. To remove it for all users, you can replace user1 with *:
                                                                                     
                                                                                    *    hard   memlock           unlimited\n*    soft   memlock           unlimited\n
                                                                                     
                                                                                    Note that settings in the /etc/security/limits.conf file don't apply to services.  To increase the locked memory limit for a service you need to modify the application's systemd service file and add the line:
                                                                                     
                                                                                    [Service]\nLimitMEMLOCK=infinity\n
                                                                                    "},{"location":"sw/install_guide/installation_guide/#hugepage-settings","title":"Hugepage Settings","text":"
                                                                                    Users need to configure system hugepages to reserve 2MB-hugepages or 1GB-hugepages. For example, the 'hello_fpga' sample requires several 2MB-hugepages. And the fpgadiag tool requires several 1GB-hugepages.
                                                                                     
                                                                                    The command below reserves 20 2M-hugepages:
                                                                                     
                                                                                    $ sudo sh -c 'echo 20 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages'\n
                                                                                     
                                                                                    The command below reserves 4 1GB-hugepages:
                                                                                     
                                                                                    $ sudo sh -c 'echo 4 > /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages'\n
                                                                                     
                                                                                    For x86_64 architecture processors, user can use following command to find out avaiable hugepage sizes:
                                                                                     
                                                                                    $ grep pse /proc/cpuinfo | uniq\nflags : ... pse ...\n
                                                                                     
                                                                                    If this commands returns a non-empty string, 2MB pages are supported.
                                                                                     
                                                                                    $ grep pse /proc/cpuinfo | uniq\nflags : ... pdpe1gb ...\n
                                                                                     
                                                                                    If this commands returns a non-empty string, 1GB pages are supported.
                                                                                    "},{"location":"sw/pyopae/python_bindings/","title":"OPAE Python Bindings","text":"
                                                                                    OPAE (Open Programmable Acceleration Engine) now includes Python bindings for interacting with FPGA resources. The OPAE Python API is built on top of the OPAE C++ Core API and its object model. Because of this, developing OPAE applications in Python is very similar to developing OPAE applications in C++ which significantly reduces the learning curve required to adapt to the Python API. While the object model remains the same, some static factory functions in the OPAE C++ Core API have been moved to module level methods in the OPAE Python API with the exception of the properties class. The goal of the OPAE Python API is to enable fast prototyping, test automation, infrastructure managment, and an easy to use framework for FPGA resource interactions that don\u2019t rely on software algorithms with a high runtime complexity.
                                                                                     
                                                                                    Currently, the only Python package that is part of OPAE is opae.fpga
                                                                                    "},{"location":"sw/pyopae/python_bindings/#implementation","title":"Implementation","text":"
                                                                                    The OPAE Python API is implemented by creating a Python extension using pybind11 <http://pybind11.readthedocs.io/en/stable>. This extension is created by using the pybind11 API which relies mostly on macros and compile time introspection to define the module initialization point as well as type converters between OPAE C++ Core types and OPAE Python types.
                                                                                    "},{"location":"sw/pyopae/python_bindings/#benefits","title":"Benefits","text":"
                                                                                    The major benefits of using pybind11 for developing the OPAE Python API include, but are not limited to, the following features of pybind11:
                                                                                     
                                                                                       
                                                                                    • Uses C++ 11 standard library although it can use C++ 14 or C++17.
                                                                                      •  
                                                                                    • Automatic conversions of shared_ptr types
                                                                                      •  
                                                                                    • Built-in support for numpy and Eigen numerical libraries
                                                                                      •  
                                                                                    • Interoperable with the Python C API
                                                                                      •  
                                                                                    "},{"location":"sw/pyopae/python_bindings/#runtime-requirements","title":"Runtime Requirements","text":"
                                                                                    Because opae.fpga is built on top of the opae-cxx-core API, it does require that the runtime libraries for both opae-cxx-core and opae-c be installed on the system (as well as any other libraries they depend on). Those libraries can be installed using the opae-libs package (from either RPM or DEB format - depending on your Linux distribution).
                                                                                    "},{"location":"sw/pyopae/python_bindings/#installation","title":"Installation","text":""},{"location":"sw/pyopae/python_bindings/#python-wheels","title":"Python Wheels","text":"
                                                                                    The preferred method of installation is to use a binary wheel package for your version of Python.
                                                                                     
                                                                                    The following table lists example names for different Python versions and platforms.
                                                                                     
                                                                                    | Python Version | Python ABI | Linux Platform | Package Name | |\u2014\u2014\u2014\u2014\u2014-|\u2014\u2014\u2014\u2014\u2014\u2013|\u2014\u2014\u2014\u2014\u2014-|\u2014\u2014\u2014\u2014\u2013| | 2.7 | CPython w/ UCS4 | x86_64 | opae.fpga.-cp27-cp27mu-linux_x86_64.whl | | 3.4 | CPython w/ UCS4 | x86_64 | opae.fpga.-cp34-cp34mu-linux_x86_64.whl | | 3.6 | CPython w/ UCS4 | x86_64 | opae.fpga.-cp36-cp36mu-linux_x86_64.whl |
                                                                                     
                                                                                    opae.fpga is currently not available in the Python Package Index but once it does become available, one should be able to install using pip by simply typing the following:
                                                                                     
                                                                                    > pip install --user opae.fpga\n
                                                                                    "},{"location":"sw/pyopae/python_bindings/#installing-from-source","title":"Installing From Source","text":"
                                                                                    In addition to the runtime libraries mentioned above, installing from source does require that the OPAE header files be installed as well as those header files for pybind11. The former can be installed with the opae-devel package and the latter can be installed by installing pybind11 Python module.
                                                                                    "},{"location":"sw/pyopae/python_bindings/#example-installation","title":"Example Installation","text":"
                                                                                    The following example shows how to build from source by installing the prerequisites before running the setup.py file.
                                                                                     
                                                                                    >sudo yum install opae-libs-<release>.x86_64.rpm\n>sudo yum install opae-devel-<release>.x86_64.rpm\n>pip install --user pybind11\n>pip install --user opae.fpga-<release>.tar.gz\n
                                                                                     
                                                                                    NOTE: The pip examples above use the --user flag to avoid requiring root permissions. Those packages will be installed in the user\u2019s site-packages directory found in the user\u2019s .local directory.
                                                                                    "},{"location":"sw/pyopae/python_bindings/#example-scripts","title":"Example Scripts","text":"
                                                                                    The following example is an implementation of the sample, hello_fpga.c, which is designed to configure the NLB0 diagnostic accelerator for a simple loopback.
                                                                                     
                                                                                    import time\nfrom opae import fpga\n\nNLB0 = \"d8424dc4-a4a3-c413-f89e-433683f9040b\"\nCTL = 0x138\nCFG = 0x140\nNUM_LINES = 0x130\nSRC_ADDR = 0x0120\nDST_ADDR = 0x0128\nDSM_ADDR = 0x0110\nDSM_STATUS = 0x40\n\ndef cl_align(addr):\n    return addr >> 6\n\ntokens = fpga.enumerate(type=fpga.ACCELERATOR, guid=NLB0)\nassert tokens, \"Could not enumerate accelerator: {}\".format(NlB0)\n\nwith fpga.open(tokens[0], fpga.OPEN_SHARED) as handle:\n    src = fpga.allocate_shared_buffer(handle, 4096)\n    dst = fpga.allocate_shared_buffer(handle, 4096)\n    dsm = fpga.allocate_shared_buffer(handle, 4096)\n    handle.write_csr32(CTL, 0)\n    handle.write_csr32(CTL, 1)\n    handle.write_csr64(DSM_ADDR, dsm.io_address())\n    handle.write_csr64(SRC_ADDR, cl_align(src.io_address())) # cacheline-aligned\n    handle.write_csr64(DST_ADDR, cl_align(dst.io_address())) # cacheline-aligned\n    handle.write_csr32(CFG, 0x42000)\n    handle.write_csr32(NUM_LINES, 4096/64)\n    handle.write_csr32(CTL, 3)\n    while dsm[DSM_STATUS] & 0x1 == 0:\n        time.sleep(0.001)\n    handle.write_csr32(CTL, 7)\n
                                                                                     
                                                                                    This example shows how one might reprogram (Partial Reconfiguration) an accelerator on a given bus, 0x5e, using a bitstream file, m0.gbs.
                                                                                     
                                                                                    from opae import fpga\n\nBUS = 0x5e\nGBS = 'm0.gbs'\ntokens = fpga.enumerate(type=fpga.DEVICE, bus=BUS)\nassert tokens, \"Could not enumerate device on bus: {}\".format(BUS)\nwith open(GBS, 'rb') as fd, fpga.open(tokens[0]) as device:\n    device.reconfigure(0, fd)\n
                                                                                    "},{"location":"sw/tod/tod/","title":"Time of Day (ToD)","text":""},{"location":"sw/tod/tod/#synopsis","title":"SYNOPSIS","text":"
                                                                                    The Intel FPGA ToD driver in the kernel space exposes ToD IP as PHC (PTP Hardware Clock) device to the Linux PTP (Precision Time Protocol) stack to synchronize the system clock to its ToD information. The phc2sys utility of Linux PTP stack is used to access ToD information and synchronize the system clock.
                                                                                     
                                                                                    Install the Linux PTP utilities: 
                                                                                    # sudo yum install linuxptp\n
                                                                                     
                                                                                    phc_ctl and phc2sys utilities (linuxptp package) are used to control the PHC device and synchronize the system clock to its ToD information.
                                                                                     
                                                                                    phc_ctl: directly controls PHC device clock. 
                                                                                    Usage: phc_ctl [options] <device> -- [command]\n\n    device         ethernet or ptp clock device\n\nOptions:\n    -l [num]       set the logging level to 'num'\n    -q             do not print messages to the syslog\n    -Q             do not print messages to stdout\n    -v             prints the software version and exits\n    -h             prints this message and exits\n\nCommands:\n specify commands with arguments. Can specify multiple commands to be executed in order. \n Seconds are read as double precision floating point values.\n\n    set  [seconds]  set PHC time (defaults to time on CLOCK_REALTIME)\n    get             get PHC time\n    adj  <seconds>  adjust PHC time by offset\n    freq [ppb]      adjust PHC frequency (default returns current offset)\n    cmp             compare PHC offset to CLOCK_REALTIME\n    caps            display device capabilities (default if no command given)\n    wait <seconds>  pause between commands\n                    This command may be useful for sanity checking whether the PHC clock is\n                    running as expected.\n                    The arguments specified in seconds are read as double precision floating point\n                    values, and will scale to nanoseconds. This means providing a value of 5.5 means 5\n                    and one half seconds. This allows specifying fairly precise values for time.\n
                                                                                     
                                                                                    phc2sys: synchronize two clocks. 
                                                                                    Usage: phc2sys [options]\n\nAutomatic configuration:\n    -a             turn on autoconfiguration\n    -r             synchronize system (realtime) clock\n                   repeat -r to consider it also as a time source\n\nManual configuration:\n    -c [dev|name]  slave clock (CLOCK_REALTIME)\n    -d [dev]       master PPS device\n    -s [dev|name]  master clock\n    -O [offset]    slave-master time offset (0)\n    -w             wait for ptp4l\n\nOptions:\n    -f [file]      configuration file\n    -E [pi|linreg] clock servo (pi)\n    -P [kp]        proportional constant (0.7)\n    -I [ki]        integration constant (0.3)\n    -S [step]      step threshold (disabled)\n    -F [step]      step threshold only on start (0.00002)\n    -R [rate]      slave clock update rate in HZ (1.0)\n    -N [num]       number of master clock readings per update (5)\n    -L [limit]     sanity frequency limit in ppb (200000000)\n    -M [num]       NTP SHM segment number (0)\n    -u [num]       number of clock updates in summary stats (0)\n    -n [num]       domain number (0)\n    -x             apply leap seconds by servo instead of kernel\n    -z [path]      server address for UDS (/var/run/ptp4l)\n    -l [num]       set the logging level to 'num' (6)\n    -t [tag]       add tag to log messages\n    -m             print messages to stdout\n    -q             do not print messages to the syslog\n    -v             prints the software version and exits\n    -h             prints this message and exits\n
                                                                                    "},{"location":"sw/tod/tod/#description","title":"DESCRIPTION","text":"
                                                                                    The phc2sys utility is used to synchronize the system clock to the PTP Hardware Clock (PHC) or ToD clock. The phc_ctl utility is used to directly control PHC clock device.
                                                                                    "},{"location":"sw/tod/tod/#configuring-the-ptp-service","title":"Configuring the PTP service","text":"
                                                                                       
                                                                                    1. Install the linuxptp package: 
                                                                                      # sudo yum install linuxptp\n
                                                                                      2.  
                                                                                    2. Check PTP device is created successfully by the ToD driver. 
                                                                                      3.  
                                                                                     
                                                                                    ToD driver registering as PHC device (clock_name: dfl_tod) to the Linux PTP stack and exposing to the Linux kernel to synchronize the system clock to its ToD information. 
                                                                                    # cat /sys/class/ptp/ptp0/clock_name\ndfl_tod\n
                                                                                     
                                                                                       
                                                                                    1. Configure phc2sys service on a system:
                                                                                      2.  
                                                                                     
                                                                                    The phc2sys service is configured in the /etc/sysconfig/phc2sys configuration file. Define start-up option for phc2sys daemon in /etc/sysconfig/phc2sys. The master clock is /dev/ptp0 device and the slave clock is system clock/CLOCK_REALTIME: 
                                                                                     OPTIONS=\"-s /dev/ptp0 -c CLOCK_REALTIME -r -O 0 -R 16\"\n
                                                                                     
                                                                                       
                                                                                    1.  
                                                                                      Start phc2sys service: 
                                                                                      # service phc2sys start\n
                                                                                       
                                                                                      2.  
                                                                                    2.  
                                                                                      Stop phc2sys service: 
                                                                                      # service phc2sys stop\n
                                                                                       
                                                                                      3.  
                                                                                    "},{"location":"sw/tod/tod/#examples","title":"Examples","text":""},{"location":"sw/tod/tod/#using-phc_ctl-utility","title":"using phc_ctl utility","text":"
                                                                                    Read the current clock time from the PHC clock device: 
                                                                                    # sudo phc_ctl /dev/ptp0 get\n
                                                                                     
                                                                                    Set the PHC clock time to CLOCK_REALTIME: 
                                                                                    # sudo phc_ctl /dev/ptp0 set\n
                                                                                     
                                                                                    Set PHC clock time to 0: 
                                                                                    # sudo phc_ctl /dev/ptp0 set 0.0\n
                                                                                     
                                                                                    Set PHC clock time to 0 and wait for 10 sec and read the clock time: 
                                                                                    # sudo phc_ctl /dev/ptp0 set 0.0 wait 10.0 get\n
                                                                                     
                                                                                    Set and compare PHC clock time to CLOCK_REALTIME: 
                                                                                    # sudo phc_ctl /dev/ptp0 set cmp\n
                                                                                     
                                                                                    Read the PHC device capabilities: 
                                                                                    # sudo phc_ctl /dev/ptp0 caps\n
                                                                                    "},{"location":"sw/tod/tod/#using-phc2sys-utility","title":"using phc2sys utility","text":"
                                                                                    To synchronize the system clock to the PHC clock: 
                                                                                    # sudo phc2sys -s /dev/ptp0 -c CLOCK_REALTIME -r -O 0 -R 16 -m\nphc2sys[7896.789]: CLOCK_REALTIME phc offset  -1259509 s0 freq  -31462 delay   1338\nphc2sys[7896.852]: CLOCK_REALTIME phc offset  -1261498 s1 freq  -63144 delay   1328\nphc2sys[7896.914]: CLOCK_REALTIME phc offset       -15 s2 freq  -63159 delay   1328\nphc2sys[7896.977]: CLOCK_REALTIME phc offset       -19 s2 freq  -63167 delay   1327\nphc2sys[7897.039]: CLOCK_REALTIME phc offset       -35 s2 freq  -63189 delay   1328\nphc2sys[7897.102]: CLOCK_REALTIME phc offset       -37 s2 freq  -63201 delay   1331\nphc2sys[7897.165]: CLOCK_REALTIME phc offset       -30 s2 freq  -63205 delay   1328\nphc2sys[7897.227]: CLOCK_REALTIME phc offset       -50 s2 freq  -63234 delay   1331\nphc2sys[7897.290]: CLOCK_REALTIME phc offset       -50 s2 freq  -63249 delay   1329\nphc2sys[7897.353]: CLOCK_REALTIME phc offset       -62 s2 freq  -63276 delay   1334\nphc2sys[7897.415]: CLOCK_REALTIME phc offset       -53 s2 freq  -63286 delay   1335\nphc2sys[7897.478]: CLOCK_REALTIME phc offset       -46 s2 freq  -63295 delay   1325\nphc2sys[7897.541]: CLOCK_REALTIME phc offset       -57 s2 freq  -63320 delay   1334\nphc2sys[7897.603]: CLOCK_REALTIME phc offset       -39 s2 freq  -63319 delay   1327\nphc2sys[7897.666]: CLOCK_REALTIME phc offset       -31 s2 freq  -63323 delay   1328\nphc2sys[7897.728]: CLOCK_REALTIME phc offset       -48 s2 freq  -63349 delay   1327\nphc2sys[7897.791]: CLOCK_REALTIME phc offset       -42 s2 freq  -63357 delay   1323\nphc2sys[7897.854]: CLOCK_REALTIME phc offset       -41 s2 freq  -63369 delay   1324\nphc2sys[7897.916]: CLOCK_REALTIME phc offset       -44 s2 freq  -63384 delay   1328\nphc2sys[7897.979]: CLOCK_REALTIME phc offset       -13 s2 freq  -63366 delay   1327\nphc2sys[7898.042]: CLOCK_REALTIME phc offset       -16 s2 freq  -63373 delay   1327\nphc2sys[7898.104]: CLOCK_REALTIME phc offset       -19 s2 freq  -63381 delay   1328\nphc2sys[7898.167]: CLOCK_REALTIME phc offset       -16 s2 freq  -63384 delay   1327\nphc2sys[7898.229]: CLOCK_REALTIME phc offset         3 s2 freq  -63370 delay   1327\nphc2sys[7898.292]: CLOCK_REALTIME phc offset        16 s2 freq  -63356 delay   1325\nphc2sys[7898.355]: CLOCK_REALTIME phc offset        10 s2 freq  -63357 delay   1319\nphc2sys[7898.417]: CLOCK_REALTIME phc offset        23 s2 freq  -63341 delay   1327\nphc2sys[7898.480]: CLOCK_REALTIME phc offset        13 s2 freq  -63344 delay   1335\nphc2sys[7898.543]: CLOCK_REALTIME phc offset        23 s2 freq  -63330 delay   1323\nphc2sys[7898.605]: CLOCK_REALTIME phc offset        29 s2 freq  -63317 delay   1312\nphc2sys[7898.668]: CLOCK_REALTIME phc offset        22 s2 freq  -63315 delay   1324\nphc2sys[7898.730]: CLOCK_REALTIME phc offset        42 s2 freq  -63289 delay   1325\nphc2sys[7898.793]: CLOCK_REALTIME phc offset        29 s2 freq  -63289 delay   1333\nphc2sys[7898.856]: CLOCK_REALTIME phc offset        34 s2 freq  -63276 delay   1327\nphc2sys[7898.918]: CLOCK_REALTIME phc offset        21 s2 freq  -63278 delay   1331\nphc2sys[7898.981]: CLOCK_REALTIME phc offset        22 s2 freq  -63271 delay   1335\nphc2sys[7899.044]: CLOCK_REALTIME phc offset        30 s2 freq  -63256 delay   1327\nphc2sys[7899.106]: CLOCK_REALTIME phc offset        30 s2 freq  -63247 delay   1328\nphc2sys[7899.169]: CLOCK_REALTIME phc offset        37 s2 freq  -63231 delay   1333\nphc2sys[7899.232]: CLOCK_REALTIME phc offset        29 s2 freq  -63228 delay   1331\nphc2sys[7899.294]: CLOCK_REALTIME phc offset        24 s2 freq  -63225 delay   1330\n
                                                                                    "},{"location":"opae-code/annotated/","title":"Class List","text":"
                                                                                    Here are the classes, structs, unions and interfaces with brief descriptions:
                                                                                     
                                                                                       
                                                                                    • struct _fpga_token_header Internal token type header. 
                                                                                      •  
                                                                                    • struct _opae_hash_map Hash map object. 
                                                                                      •  
                                                                                    • struct _opae_hash_map_item List link item. 
                                                                                      •  
                                                                                    • struct cache_line 
                                                                                      •  
                                                                                    • struct config 
                                                                                      •  
                                                                                    • struct fpga_error_info 
                                                                                      •  
                                                                                    • struct fpga_metric Metric struct. 
                                                                                      •  
                                                                                    • struct fpga_metric_info Metric info struct. 
                                                                                      •  
                                                                                    • struct fpga_version Semantic version. 
                                                                                      •  
                                                                                    • struct mem_alloc 
                                                                                      •  
                                                                                    • struct mem_link Provides an API for allocating/freeing a logical address space. 
                                                                                      •  
                                                                                    • struct metric_threshold 
                                                                                      •  
                                                                                    • union metric_value Metric value union. 
                                                                                      •  
                                                                                    • namespace opae 
                                                                                         
                                                                                      • namespace fpga 
                                                                                           
                                                                                        • namespace types 
                                                                                             
                                                                                          • class busy busy exception 
                                                                                            •  
                                                                                          • namespace detail 
                                                                                            •  
                                                                                          • class error An error object represents an error register for a resource. 
                                                                                            •  
                                                                                          • class event Wraps fpga event routines in OPAE C. 
                                                                                               
                                                                                            • struct type_t C++ struct that is interchangeable with fpga_event_type enum. 
                                                                                              •  
                                                                                             
                                                                                            •  
                                                                                          • class except Generic OPAE exception. 
                                                                                            •  
                                                                                          • class exception exception exception 
                                                                                            •  
                                                                                          • struct guid_t Representation of the guid member of a properties object. 
                                                                                            •  
                                                                                          • class handle An allocated accelerator resource. 
                                                                                            •  
                                                                                          • class invalid_param invalid_param exception 
                                                                                            •  
                                                                                          • class no_access no_access exception 
                                                                                            •  
                                                                                          • class no_daemon no_daemon exception 
                                                                                            •  
                                                                                          • class no_driver no_driver exception 
                                                                                            •  
                                                                                          • class no_memory no_memory exception 
                                                                                            •  
                                                                                          • class not_found not_found exception 
                                                                                            •  
                                                                                          • class not_supported not_supported exception 
                                                                                            •  
                                                                                          • class properties Wraps an OPAE fpga_properties object. 
                                                                                            •  
                                                                                          • struct pvalue Wraps OPAE properties defined in the OPAE C API by associating an fpga_properties reference with the getters and setters defined for a property. 
                                                                                            •  
                                                                                          • class reconf_error reconf_error exception 
                                                                                            •  
                                                                                          • class shared_buffer Host/AFU shared memory blocks. 
                                                                                            •  
                                                                                          • class src_location Identify a particular line in a source file. 
                                                                                            •  
                                                                                          • class sysobject Wraps the OPAE fpga_object primitive. 
                                                                                            •  
                                                                                          • class token Wraps the OPAE fpga_token primitive. 
                                                                                            •  
                                                                                          • class version 
                                                                                            •  
                                                                                           
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    • struct opae_uio OPAE UIO device abstraction. 
                                                                                      •  
                                                                                    • struct opae_uio_device_region MMIO region info. 
                                                                                      •  
                                                                                    • struct opae_vfio OPAE VFIO device abstraction. 
                                                                                      •  
                                                                                    • struct opae_vfio_buffer System DMA buffer. 
                                                                                      •  
                                                                                    • struct opae_vfio_device VFIO device. 
                                                                                      •  
                                                                                    • struct opae_vfio_device_irq Interrupt info. 
                                                                                      •  
                                                                                    • struct opae_vfio_device_region MMIO region info. 
                                                                                      •  
                                                                                    • struct opae_vfio_group VFIO group. 
                                                                                      •  
                                                                                    • struct opae_vfio_iova_range IO Virtual Address Range. 
                                                                                      •  
                                                                                    • struct opae_vfio_sparse_info MMIO sparse-mappable region info. 
                                                                                      •  
                                                                                    • struct ras_inject_error 
                                                                                      •  
                                                                                    • namespace std 
                                                                                      •  
                                                                                    • struct threshold Threshold struct. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/files/","title":"File List","text":"
                                                                                    Here is a list of all files with brief descriptions:
                                                                                     
                                                                                       
                                                                                    • dir docs 
                                                                                         
                                                                                      • dir sw 
                                                                                           
                                                                                        • dir include 
                                                                                             
                                                                                          • dir opae 
                                                                                               
                                                                                            • file access.h Functions to acquire, release, and reset OPAE FPGA resources. 
                                                                                              •  
                                                                                            • file buffer.h Functions for allocating and sharing system memory with an FPGA accelerator. 
                                                                                              •  
                                                                                            • dir cxx 
                                                                                                 
                                                                                              • file core.h 
                                                                                                •  
                                                                                              • dir core 
                                                                                                   
                                                                                                • file errors.h 
                                                                                                  •  
                                                                                                • file events.h 
                                                                                                  •  
                                                                                                • file except.h 
                                                                                                  •  
                                                                                                • file handle.h 
                                                                                                  •  
                                                                                                • file properties.h 
                                                                                                  •  
                                                                                                • file pvalue.h 
                                                                                                  •  
                                                                                                • file shared_buffer.h 
                                                                                                  •  
                                                                                                • file sysobject.h 
                                                                                                  •  
                                                                                                • file token.h 
                                                                                                  •  
                                                                                                • file version.h 
                                                                                                  •  
                                                                                                 
                                                                                                •  
                                                                                               
                                                                                              •  
                                                                                            • file enum.h APIs for resource enumeration and managing tokens. 
                                                                                              •  
                                                                                            • file error.h Functions for reading and clearing errors in resources. 
                                                                                              •  
                                                                                            • file event.h Functions for registering events and managing the lifecycle for fpga_event_handle s. 
                                                                                              •  
                                                                                            • file fpga.h FPGA API. 
                                                                                              •  
                                                                                            • file hash_map.h A general-purpose hybrid array/list hash map implementation. 
                                                                                              •  
                                                                                            • file init.h Initialization routine. 
                                                                                              •  
                                                                                            • file log.h 
                                                                                              •  
                                                                                            • file manage.h Functions for managing FPGA configurations. 
                                                                                              •  
                                                                                            • file mem_alloc.h 
                                                                                              •  
                                                                                            • file metrics.h Functions for Discover/ Enumerates metrics and retrieves values. 
                                                                                              •  
                                                                                            • file mmio.h Functions for mapping and accessing MMIO space. 
                                                                                              •  
                                                                                            • file properties.h Functions for examining and manipulating fpga_properties objects. 
                                                                                              •  
                                                                                            • file sysobject.h Functions to read/write from system objects. 
                                                                                              •  
                                                                                            • file types.h Type definitions for FPGA API. 
                                                                                              •  
                                                                                            • file types_enum.h Definitions of enumerated types for the OPAE C API. 
                                                                                              •  
                                                                                            • file uio.h APIs to manage a PCIe device via UIO. 
                                                                                              •  
                                                                                            • file umsg.h FPGA UMsg API. 
                                                                                              •  
                                                                                            • file userclk.h Functions for setting and get afu user clock. 
                                                                                              •  
                                                                                            • file utils.h Utility functions and macros for the FPGA API. 
                                                                                              •  
                                                                                            • file version.h 
                                                                                              •  
                                                                                            • file vfio.h APIs to manage a PCIe device via vfio-pci. 
                                                                                              •  
                                                                                             
                                                                                            •  
                                                                                           
                                                                                          •  
                                                                                        • dir samples 
                                                                                             
                                                                                          • dir hello_events 
                                                                                               
                                                                                            • file hello_events.c A code sample of using OPAE event API. 
                                                                                              •  
                                                                                             
                                                                                            •  
                                                                                          • dir hello_fpga 
                                                                                               
                                                                                            • file hello_fpga.c A code sample illustrates the basic usage of the OPAE C API. 
                                                                                              •  
                                                                                             
                                                                                            •  
                                                                                           
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/","title":"Struct _fpga_token_header","text":"
                                                                                    ClassList > _fpga_token_header
                                                                                     
                                                                                    Internal token type header. More...
                                                                                     
                                                                                       
                                                                                    • #include <types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#public-attributes","title":"Public Attributes","text":"Type Name uint8_t bus uint8_t device uint16_t device_id uint8_t function fpga_guid guid fpga_interface interface uint64_t magic uint64_t object_id fpga_objtype objtype uint16_t segment uint16_t subsystem_device_id uint16_t subsystem_vendor_id uint16_t vendor_id"},{"location":"opae-code/struct__fpga__token__header/#detailed-description","title":"Detailed Description","text":"
                                                                                    Each plugin (dfl: libxfpga.so, vfio: libopae-v.so) implements its own proprietary token type. This header must appear at offset zero within that structure.
                                                                                     
                                                                                    eg, see lib/plugins/xfpga/types_int.h:struct _fpga_token and lib/plugins/vfio/opae_vfio.h:struct _vfio_token. 
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/struct__fpga__token__header/#variable-bus","title":"variable bus","text":"
                                                                                    uint8_t _fpga_token_header::bus;\n
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#variable-device","title":"variable device","text":"
                                                                                    uint8_t _fpga_token_header::device;\n
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#variable-device_id","title":"variable device_id","text":"
                                                                                    uint16_t _fpga_token_header::device_id;\n
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#variable-function","title":"variable function","text":"
                                                                                    uint8_t _fpga_token_header::function;\n
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#variable-guid","title":"variable guid","text":"
                                                                                    fpga_guid _fpga_token_header::guid;\n
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#variable-interface","title":"variable interface","text":"
                                                                                    fpga_interface _fpga_token_header::interface;\n
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#variable-magic","title":"variable magic","text":"
                                                                                    uint64_t _fpga_token_header::magic;\n
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#variable-object_id","title":"variable object_id","text":"
                                                                                    uint64_t _fpga_token_header::object_id;\n
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#variable-objtype","title":"variable objtype","text":"
                                                                                    fpga_objtype _fpga_token_header::objtype;\n
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#variable-segment","title":"variable segment","text":"
                                                                                    uint16_t _fpga_token_header::segment;\n
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#variable-subsystem_device_id","title":"variable subsystem_device_id","text":"
                                                                                    uint16_t _fpga_token_header::subsystem_device_id;\n
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#variable-subsystem_vendor_id","title":"variable subsystem_vendor_id","text":"
                                                                                    uint16_t _fpga_token_header::subsystem_vendor_id;\n
                                                                                    "},{"location":"opae-code/struct__fpga__token__header/#variable-vendor_id","title":"variable vendor_id","text":"
                                                                                    uint16_t _fpga_token_header::vendor_id;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                                                                    "},{"location":"opae-code/struct__opae__hash__map/","title":"Struct _opae_hash_map","text":"
                                                                                    ClassList > _opae_hash_map
                                                                                     
                                                                                    Hash map object. More...
                                                                                     
                                                                                       
                                                                                    • #include <hash_map.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/struct__opae__hash__map/#public-attributes","title":"Public Attributes","text":"Type Name opae_hash_map_item ** buckets void * cleanup_context Optional second parameter to key_cleanup and value_cleanup. int flags uint32_t hash_seed void(* key_cleanup (optional) int(* key_compare (required) uint32_t(* key_hash uint32_t num_buckets void(* value_cleanup (optional)"},{"location":"opae-code/struct__opae__hash__map/#detailed-description","title":"Detailed Description","text":"
                                                                                    This structure defines the internals of the hash map. Each of the parameters supplied to opae_hash_map_init() is stored in the structure. All parameters are required, except key_cleanup and value_cleanup, which may optionally be NULL. 
                                                                                    "},{"location":"opae-code/struct__opae__hash__map/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/struct__opae__hash__map/#variable-buckets","title":"variable buckets","text":"
                                                                                    opae_hash_map_item** _opae_hash_map::buckets;\n
                                                                                    "},{"location":"opae-code/struct__opae__hash__map/#variable-cleanup_context","title":"variable cleanup_context","text":"
                                                                                    void* _opae_hash_map::cleanup_context;\n
                                                                                    "},{"location":"opae-code/struct__opae__hash__map/#variable-flags","title":"variable flags","text":"
                                                                                    int _opae_hash_map::flags;\n
                                                                                    "},{"location":"opae-code/struct__opae__hash__map/#variable-hash_seed","title":"variable hash_seed","text":"
                                                                                    uint32_t _opae_hash_map::hash_seed;\n
                                                                                    "},{"location":"opae-code/struct__opae__hash__map/#variable-key_cleanup","title":"variable key_cleanup","text":"
                                                                                    void(* _opae_hash_map::key_cleanup) (void *key, void *context);\n
                                                                                    "},{"location":"opae-code/struct__opae__hash__map/#variable-key_compare","title":"variable key_compare","text":"
                                                                                    int(* _opae_hash_map::key_compare) (void *keya, void *keyb);\n
                                                                                    "},{"location":"opae-code/struct__opae__hash__map/#variable-key_hash","title":"variable key_hash","text":"
                                                                                    uint32_t(* _opae_hash_map::key_hash) (uint32_t num_buckets, uint32_t hash_seed, void *key);\n
                                                                                    "},{"location":"opae-code/struct__opae__hash__map/#variable-num_buckets","title":"variable num_buckets","text":"
                                                                                    uint32_t _opae_hash_map::num_buckets;\n
                                                                                    "},{"location":"opae-code/struct__opae__hash__map/#variable-value_cleanup","title":"variable value_cleanup","text":"
                                                                                    void(* _opae_hash_map::value_cleanup) (void *value, void *context);\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/hash_map.h
                                                                                    "},{"location":"opae-code/struct__opae__hash__map__item/","title":"Struct _opae_hash_map_item","text":"
                                                                                    ClassList > _opae_hash_map_item
                                                                                     
                                                                                    List link item. More...
                                                                                     
                                                                                       
                                                                                    • #include <hash_map.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/struct__opae__hash__map__item/#public-attributes","title":"Public Attributes","text":"Type Name void * key struct _opae_hash_map_item * next void * value"},{"location":"opae-code/struct__opae__hash__map__item/#detailed-description","title":"Detailed Description","text":"
                                                                                    This structure provides the association between key and value. When the supplied hash function for keys A and B returns the same bucket index, both A and B can co-exist on the same list rooted at the bucket index. 
                                                                                    "},{"location":"opae-code/struct__opae__hash__map__item/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/struct__opae__hash__map__item/#variable-key","title":"variable key","text":"
                                                                                    void* _opae_hash_map_item::key;\n
                                                                                    "},{"location":"opae-code/struct__opae__hash__map__item/#variable-next","title":"variable next","text":"
                                                                                    struct _opae_hash_map_item* _opae_hash_map_item::next;\n
                                                                                    "},{"location":"opae-code/struct__opae__hash__map__item/#variable-value","title":"variable value","text":"
                                                                                    void* _opae_hash_map_item::value;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/hash_map.h
                                                                                    "},{"location":"opae-code/structcache__line/","title":"Struct cache_line","text":"
                                                                                    ClassList > cache_line
                                                                                    "},{"location":"opae-code/structcache__line/#public-attributes","title":"Public Attributes","text":"Type Name uint32_t uint"},{"location":"opae-code/structcache__line/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structcache__line/#variable-uint","title":"variable uint","text":"
                                                                                    uint32_t cache_line::uint[16];\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/samples/hello_fpga/hello_fpga.c
                                                                                    "},{"location":"opae-code/structconfig/","title":"Struct config","text":"
                                                                                    ClassList > config
                                                                                    "},{"location":"opae-code/structconfig/#public-attributes","title":"Public Attributes","text":"Type Name int open_flags int run_n3000"},{"location":"opae-code/structconfig/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structconfig/#variable-open_flags","title":"variable open_flags","text":"
                                                                                    int config::open_flags;\n
                                                                                    "},{"location":"opae-code/structconfig/#variable-run_n3000","title":"variable run_n3000","text":"
                                                                                    int config::run_n3000;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/samples/hello_fpga/hello_fpga.c
                                                                                    "},{"location":"opae-code/structfpga__error__info/","title":"Struct fpga_error_info","text":"
                                                                                    ClassList > fpga_error_info
                                                                                     
                                                                                       
                                                                                    • #include <types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structfpga__error__info/#public-attributes","title":"Public Attributes","text":"Type Name bool can_clear name of the error char name"},{"location":"opae-code/structfpga__error__info/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structfpga__error__info/#variable-can_clear","title":"variable can_clear","text":"
                                                                                    bool fpga_error_info::can_clear;\n
                                                                                    "},{"location":"opae-code/structfpga__error__info/#variable-name","title":"variable name","text":"
                                                                                    char fpga_error_info::name[FPGA_ERROR_NAME_MAX];\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                                                                    "},{"location":"opae-code/structfpga__metric/","title":"Struct fpga_metric","text":"
                                                                                    ClassList > fpga_metric
                                                                                     
                                                                                    Metric struct. 
                                                                                     
                                                                                       
                                                                                    • #include <types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structfpga__metric/#public-attributes","title":"Public Attributes","text":"Type Name bool isvalid uint64_t metric_num metric_value value"},{"location":"opae-code/structfpga__metric/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structfpga__metric/#variable-isvalid","title":"variable isvalid","text":"
                                                                                    bool fpga_metric::isvalid;\n
                                                                                    "},{"location":"opae-code/structfpga__metric/#variable-metric_num","title":"variable metric_num","text":"
                                                                                    uint64_t fpga_metric::metric_num;\n
                                                                                    "},{"location":"opae-code/structfpga__metric/#variable-value","title":"variable value","text":"
                                                                                    metric_value fpga_metric::value;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                                                                    "},{"location":"opae-code/structfpga__metric__info/","title":"Struct fpga_metric_info","text":"
                                                                                    ClassList > fpga_metric_info
                                                                                     
                                                                                    Metric info struct. 
                                                                                     
                                                                                       
                                                                                    • #include <types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structfpga__metric__info/#public-attributes","title":"Public Attributes","text":"Type Name char group_name enum fpga_metric_datatype metric_datatype fpga_guid metric_guid char metric_name uint64_t metric_num enum fpga_metric_type metric_type char metric_units char qualifier_name"},{"location":"opae-code/structfpga__metric__info/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structfpga__metric__info/#variable-group_name","title":"variable group_name","text":"
                                                                                    char fpga_metric_info::group_name[FPGA_METRIC_STR_SIZE];\n
                                                                                    "},{"location":"opae-code/structfpga__metric__info/#variable-metric_datatype","title":"variable metric_datatype","text":"
                                                                                    enum fpga_metric_datatype fpga_metric_info::metric_datatype;\n
                                                                                    "},{"location":"opae-code/structfpga__metric__info/#variable-metric_guid","title":"variable metric_guid","text":"
                                                                                    fpga_guid fpga_metric_info::metric_guid;\n
                                                                                    "},{"location":"opae-code/structfpga__metric__info/#variable-metric_name","title":"variable metric_name","text":"
                                                                                    char fpga_metric_info::metric_name[FPGA_METRIC_STR_SIZE];\n
                                                                                    "},{"location":"opae-code/structfpga__metric__info/#variable-metric_num","title":"variable metric_num","text":"
                                                                                    uint64_t fpga_metric_info::metric_num;\n
                                                                                    "},{"location":"opae-code/structfpga__metric__info/#variable-metric_type","title":"variable metric_type","text":"
                                                                                    enum fpga_metric_type fpga_metric_info::metric_type;\n
                                                                                    "},{"location":"opae-code/structfpga__metric__info/#variable-metric_units","title":"variable metric_units","text":"
                                                                                    char fpga_metric_info::metric_units[FPGA_METRIC_STR_SIZE];\n
                                                                                    "},{"location":"opae-code/structfpga__metric__info/#variable-qualifier_name","title":"variable qualifier_name","text":"
                                                                                    char fpga_metric_info::qualifier_name[FPGA_METRIC_STR_SIZE];\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                                                                    "},{"location":"opae-code/structfpga__version/","title":"Struct fpga_version","text":"
                                                                                    ClassList > fpga_version
                                                                                     
                                                                                    Semantic version. More...
                                                                                     
                                                                                       
                                                                                    • #include <types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structfpga__version/#public-attributes","title":"Public Attributes","text":"Type Name uint8_t major Major version. uint8_t minor Minor version. uint16_t patch Revision or patchlevel."},{"location":"opae-code/structfpga__version/#detailed-description","title":"Detailed Description","text":"
                                                                                    Data structure for expressing version identifiers following the semantic versioning scheme. Used in various properties for tracking component versions. 
                                                                                    "},{"location":"opae-code/structfpga__version/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structfpga__version/#variable-major","title":"variable major","text":"
                                                                                    uint8_t fpga_version::major;\n
                                                                                    "},{"location":"opae-code/structfpga__version/#variable-minor","title":"variable minor","text":"
                                                                                    uint8_t fpga_version::minor;\n
                                                                                    "},{"location":"opae-code/structfpga__version/#variable-patch","title":"variable patch","text":"
                                                                                    uint16_t fpga_version::patch;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                                                                    "},{"location":"opae-code/structmem__alloc/","title":"Struct mem_alloc","text":"
                                                                                    ClassList > mem_alloc
                                                                                     
                                                                                       
                                                                                    • #include <mem_alloc.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structmem__alloc/#public-attributes","title":"Public Attributes","text":"Type Name struct mem_link allocated struct mem_link free"},{"location":"opae-code/structmem__alloc/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structmem__alloc/#variable-allocated","title":"variable allocated","text":"
                                                                                    struct mem_link mem_alloc::allocated;\n
                                                                                    "},{"location":"opae-code/structmem__alloc/#variable-free","title":"variable free","text":"
                                                                                    struct mem_link mem_alloc::free;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/mem_alloc.h
                                                                                    "},{"location":"opae-code/structmem__link/","title":"Struct mem_link","text":"
                                                                                    ClassList > mem_link
                                                                                     
                                                                                    Provides an API for allocating/freeing a logical address space. More...
                                                                                     
                                                                                       
                                                                                    • #include <mem_alloc.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structmem__link/#public-attributes","title":"Public Attributes","text":"Type Name uint64_t address struct mem_link * next struct mem_link * prev uint64_t size"},{"location":"opae-code/structmem__link/#detailed-description","title":"Detailed Description","text":"
                                                                                    There is no interaction with any OS memory allocation infrastructure, whether malloc, mmap, etc. The \"address ranges\" tracked by this allocator are arbitrary 64-bit integers. The allocator simply provides the bookeeping logic that ensures that a unique address with the appropriate size is returned for each allocation request, and that an allocation can be freed, ie released back to the available pool of logical address space for future allocations. The memory backing the allocator's internal data structures is managed by malloc()/free(). 
                                                                                    "},{"location":"opae-code/structmem__link/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structmem__link/#variable-address","title":"variable address","text":"
                                                                                    uint64_t mem_link::address;\n
                                                                                    "},{"location":"opae-code/structmem__link/#variable-next","title":"variable next","text":"
                                                                                    struct mem_link* mem_link::next;\n
                                                                                    "},{"location":"opae-code/structmem__link/#variable-prev","title":"variable prev","text":"
                                                                                    struct mem_link* mem_link::prev;\n
                                                                                    "},{"location":"opae-code/structmem__link/#variable-size","title":"variable size","text":"
                                                                                    uint64_t mem_link::size;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/mem_alloc.h
                                                                                    "},{"location":"opae-code/structmetric__threshold/","title":"Struct metric_threshold","text":"
                                                                                    ClassList > metric_threshold
                                                                                     
                                                                                       
                                                                                    • #include <types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structmetric__threshold/#public-attributes","title":"Public Attributes","text":"Type Name threshold hysteresis threshold lower_c_threshold threshold lower_nc_threshold threshold lower_nr_threshold char metric_name threshold upper_c_threshold threshold upper_nc_threshold threshold upper_nr_threshold"},{"location":"opae-code/structmetric__threshold/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structmetric__threshold/#variable-hysteresis","title":"variable hysteresis","text":"
                                                                                    threshold metric_threshold::hysteresis;\n
                                                                                    "},{"location":"opae-code/structmetric__threshold/#variable-lower_c_threshold","title":"variable lower_c_threshold","text":"
                                                                                    threshold metric_threshold::lower_c_threshold;\n
                                                                                    "},{"location":"opae-code/structmetric__threshold/#variable-lower_nc_threshold","title":"variable lower_nc_threshold","text":"
                                                                                    threshold metric_threshold::lower_nc_threshold;\n
                                                                                    "},{"location":"opae-code/structmetric__threshold/#variable-lower_nr_threshold","title":"variable lower_nr_threshold","text":"
                                                                                    threshold metric_threshold::lower_nr_threshold;\n
                                                                                    "},{"location":"opae-code/structmetric__threshold/#variable-metric_name","title":"variable metric_name","text":"
                                                                                    char metric_threshold::metric_name[FPGA_METRIC_STR_SIZE];\n
                                                                                    "},{"location":"opae-code/structmetric__threshold/#variable-upper_c_threshold","title":"variable upper_c_threshold","text":"
                                                                                    threshold metric_threshold::upper_c_threshold;\n
                                                                                    "},{"location":"opae-code/structmetric__threshold/#variable-upper_nc_threshold","title":"variable upper_nc_threshold","text":"
                                                                                    threshold metric_threshold::upper_nc_threshold;\n
                                                                                    "},{"location":"opae-code/structmetric__threshold/#variable-upper_nr_threshold","title":"variable upper_nr_threshold","text":"
                                                                                    threshold metric_threshold::upper_nr_threshold;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                                                                    "},{"location":"opae-code/unionmetric__value/","title":"Union metric_value","text":"
                                                                                    ClassList > metric_value
                                                                                     
                                                                                    Metric value union. 
                                                                                     
                                                                                       
                                                                                    • #include <types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/unionmetric__value/#public-attributes","title":"Public Attributes","text":"Type Name bool bvalue double dvalue float fvalue uint64_t ivalue"},{"location":"opae-code/unionmetric__value/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/unionmetric__value/#variable-bvalue","title":"variable bvalue","text":"
                                                                                    bool metric_value::bvalue;\n
                                                                                    "},{"location":"opae-code/unionmetric__value/#variable-dvalue","title":"variable dvalue","text":"
                                                                                    double metric_value::dvalue;\n
                                                                                    "},{"location":"opae-code/unionmetric__value/#variable-fvalue","title":"variable fvalue","text":"
                                                                                    float metric_value::fvalue;\n
                                                                                    "},{"location":"opae-code/unionmetric__value/#variable-ivalue","title":"variable ivalue","text":"
                                                                                    uint64_t metric_value::ivalue;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                                                                    "},{"location":"opae-code/namespaceopae/","title":"Namespace opae","text":"
                                                                                    Namespace List > opae
                                                                                    "},{"location":"opae-code/namespaceopae/#namespaces","title":"Namespaces","text":"Type Name namespace fpga 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/errors.h
                                                                                    "},{"location":"opae-code/namespaceopae_1_1fpga/","title":"Namespace opae::fpga","text":"
                                                                                    Namespace List > opae > fpga
                                                                                    "},{"location":"opae-code/namespaceopae_1_1fpga/#namespaces","title":"Namespaces","text":"Type Name namespace types 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/errors.h
                                                                                    "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types/","title":"Namespace opae::fpga::types","text":"
                                                                                    Namespace List > opae > fpga > types
                                                                                    "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types/#namespaces","title":"Namespaces","text":"Type Name namespace detail"},{"location":"opae-code/namespaceopae_1_1fpga_1_1types/#classes","title":"Classes","text":"Type Name class busy busy exception class error An error object represents an error register for a resource. class event Wraps fpga event routines in OPAE C. class except Generic OPAE exception. class exception exception exception struct guid_t Representation of the guid member of a properties object. class handle An allocated accelerator resource. class invalid_param invalid_param exception class no_access no_access exception class no_daemon no_daemon exception class no_driver no_driver exception class no_memory no_memory exception class not_found not_found exception class not_supported not_supported exception class properties Wraps an OPAE fpga_properties object. struct pvalue <typename T>Wraps OPAE properties defined in the OPAE C API by associating an fpga_properties reference with the getters and setters defined for a property. class reconf_error reconf_error exception class shared_buffer Host/AFU shared memory blocks. class src_location Identify a particular line in a source file. class sysobject Wraps the OPAE fpga_object primitive. class token Wraps the OPAE fpga_token primitive. class version 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/errors.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/","title":"Class opae::fpga::types::busy","text":"
                                                                                    ClassList > opae > fpga > types > busy
                                                                                     
                                                                                    busy exception More...
                                                                                     
                                                                                       
                                                                                    • #include <except.h>
                                                                                      •  
                                                                                     
                                                                                    Inherits the following classes: opae::fpga::types::except
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#public-functions","title":"Public Functions","text":"Type Name busy (src_location loc) noexceptbusy constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#detailed-description","title":"Detailed Description","text":"
                                                                                    busy tracks the source line of origin for exceptions thrown when the error code FPGA_BUSY is returned from a call to an OPAE C API function 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1busy/#function-busy","title":"function busy","text":"
                                                                                    busy constructor 
                                                                                    inline opae::fpga::types::busy::busy (\n    src_location loc\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • loc Location where the exception was constructed. 
                                                                                      •  
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/","title":"Namespace opae::fpga::types::detail","text":"
                                                                                    Namespace List > opae > fpga > types > detail
                                                                                    "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-types","title":"Public Types","text":"Type Name typedef bool(* exception_fn typedef function pointer that returns bool if result is FPGA_OK"},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-static-attributes","title":"Public Static Attributes","text":"Type Name exception_fn opae_exceptions   = = { is_ok<opae::fpga::types::invalid_param>, is_ok<opae::fpga::types::busy>, is_ok<opae::fpga::types::exception>, is_ok<opae::fpga::types::not_found>, is_ok<opae::fpga::types::no_memory>, is_ok<opae::fpga::types::not_supported>, is_ok<opae::fpga::types::no_driver>, is_ok<opae::fpga::types::no_daemon>, is_ok<opae::fpga::types::no_access>, is_ok<opae::fpga::types::reconf_error>}"},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-functions","title":"Public Functions","text":"Type Name constexpr bool is_ok (fpga_result result, const opae::fpga::types::src_location & loc) is_ok is a template function that throws an excpetion of its template argument type if the result code is not FPGA_OK."},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-static-functions","title":"Public Static Functions","text":"Type Name void assert_fpga_ok (fpga_result result, const opae::fpga::types::src_location & loc)"},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#typedef-exception_fn","title":"typedef exception_fn","text":"
                                                                                    typedef bool(* opae::fpga::types::detail::exception_fn) (fpga_result, const opae::fpga::types::src_location &loc);\n
                                                                                    "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-static-attributes-documentation","title":"Public Static Attributes Documentation","text":""},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#variable-opae_exceptions","title":"variable opae_exceptions","text":"
                                                                                    exception_fn opae::fpga::types::detail::opae_exceptions[12];\n
                                                                                    "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#function-is_ok","title":"function is_ok","text":"
                                                                                    is_ok is a template function that throws an excpetion of its template argument type if the result code is not FPGA_OK. 
                                                                                    template<typename T typename T>\nconstexpr bool opae::fpga::types::detail::is_ok (\n    fpga_result result,\n    const opae::fpga::types::src_location & loc\n) \n
                                                                                     
                                                                                    Otherwise it returns true. 
                                                                                    "},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/namespaceopae_1_1fpga_1_1types_1_1detail/#function-assert_fpga_ok","title":"function assert_fpga_ok","text":"
                                                                                    static inline void opae::fpga::types::detail::assert_fpga_ok (\n    fpga_result result,\n    const opae::fpga::types::src_location & loc\n) \n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/","title":"Class opae::fpga::types::error","text":"
                                                                                    ClassList > opae > fpga > types > error
                                                                                     
                                                                                    An error object represents an error register for a resource. More...
                                                                                     
                                                                                       
                                                                                    • #include <errors.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< error > ptr_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#public-functions","title":"Public Functions","text":"Type Name fpga_error_info c_type () constGet the C data structure. bool can_clear () Indicates whether an error register can be cleared. error (const error & e) = delete std::string name () Get the error register name. error & operator= (const error & e) = delete uint64_t read_value () Read the raw value contained in the associated error register. ~error ()"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#public-static-functions","title":"Public Static Functions","text":"Type Name error::ptr_t get (token::ptr_t tok, uint32_t num) Factory function for creating an error object."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#detailed-description","title":"Detailed Description","text":"
                                                                                    This is used to read out the raw value in the register. No parsing is done by this class. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                                                                    typedef std::shared_ptr<error> opae::fpga::types::error::ptr_t;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-c_type","title":"function c_type","text":"
                                                                                    Get the C data structure. 
                                                                                    inline fpga_error_info opae::fpga::types::error::c_type () const\n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    The fpga_error_info that contains the name and the can_clear boolean. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-can_clear","title":"function can_clear","text":"
                                                                                    Indicates whether an error register can be cleared. 
                                                                                    inline bool opae::fpga::types::error::can_clear () \n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A boolean value indicating if the error register can be cleared. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-error-12","title":"function error [\u00bd]","text":"
                                                                                    opae::fpga::types::error::error (\n    const error & e\n) = delete\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-name","title":"function name","text":"
                                                                                    Get the error register name. 
                                                                                    inline std::string opae::fpga::types::error::name () \n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A std::string object set to the error name. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-operator","title":"function operator=","text":"
                                                                                    error & opae::fpga::types::error::operator= (\n    const error & e\n) = delete\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-read_value","title":"function read_value","text":"
                                                                                    Read the raw value contained in the associated error register. 
                                                                                    uint64_t opae::fpga::types::error::read_value () \n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A 64-bit value (unparsed) read from the error register 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-error","title":"function ~error","text":"
                                                                                    inline opae::fpga::types::error::~error () \n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1error/#function-get","title":"function get","text":"
                                                                                    Factory function for creating an error object. 
                                                                                    static error::ptr_t opae::fpga::types::error::get (\n    token::ptr_t tok,\n    uint32_t num\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • tok The token object representing a resource. 
                                                                                      •  
                                                                                    • num The index of the error register. This must be lower than the num_errors property of the resource.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A shared_ptr containing the error object 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/errors.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/","title":"Class opae::fpga::types::event","text":"
                                                                                    ClassList > opae > fpga > types > event
                                                                                     
                                                                                    Wraps fpga event routines in OPAE C. 
                                                                                     
                                                                                       
                                                                                    • #include <events.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#classes","title":"Classes","text":"Type Name struct type_t C++ struct that is interchangeable with fpga_event_type enum."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< event > ptr_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#public-functions","title":"Public Functions","text":"Type Name fpga_event_handle get () Get the fpga_event_handle contained in this object. operator fpga_event_handle () Coversion operator for converting to fpga_event_handle objects. int os_object () constGet OS Object from the event object. virtual ~event () Destroy event and associated resources."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#public-static-functions","title":"Public Static Functions","text":"Type Name event::ptr_t register_event (handle::ptr_t h, event::type_t t, int flags=0) Factory function to create event objects."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                                                                    typedef std::shared_ptr<event> opae::fpga::types::event::ptr_t;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#function-get","title":"function get","text":"
                                                                                    Get the fpga_event_handle contained in this object. 
                                                                                    inline fpga_event_handle opae::fpga::types::event::get () \n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    The fpga_event_handle contained in this object 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#function-operator-fpga_event_handle","title":"function operator fpga_event_handle","text":"
                                                                                    Coversion operator for converting to fpga_event_handle objects. 
                                                                                    opae::fpga::types::event::operator fpga_event_handle () \n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    The fpga_event_handle contained in this object 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#function-os_object","title":"function os_object","text":"
                                                                                    Get OS Object from the event object. 
                                                                                    int opae::fpga::types::event::os_object () const\n
                                                                                     
                                                                                    Get an OS specific object from the event which can be used to subscribe for events. On Linux, the object corresponds to a file descriptor that can be used with select/poll/epoll calls.
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    An integer object representing the OS object 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#function-event","title":"function ~event","text":"
                                                                                    virtual opae::fpga::types::event::~event () \n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1event/#function-register_event","title":"function register_event","text":"
                                                                                    Factory function to create event objects. 
                                                                                    static event::ptr_t opae::fpga::types::event::register_event (\n    handle::ptr_t h,\n    event::type_t t,\n    int flags=0\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • h A shared ptr of a resource handle 
                                                                                      •  
                                                                                    • t The resource type 
                                                                                      •  
                                                                                    • flags Event registration flags passed on to fpgaRegisterEvent
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A shared ptr to an event object 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/events.h
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/","title":"Struct opae::fpga::types::event::type_t","text":"
                                                                                    ClassList > opae > fpga > types > event > type_t
                                                                                     
                                                                                    C++ struct that is interchangeable with fpga_event_type enum. 
                                                                                     
                                                                                       
                                                                                    • #include <events.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#public-static-attributes","title":"Public Static Attributes","text":"Type Name constexpr fpga_event_type error   = = FPGA_EVENT_ERROR constexpr fpga_event_type interrupt   = = FPGA_EVENT_INTERRUPT constexpr fpga_event_type power_thermal   = = FPGA_EVENT_POWER_THERMAL"},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#public-functions","title":"Public Functions","text":"Type Name operator fpga_event_type ()  type_t (fpga_event_type c_type)"},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#public-static-attributes-documentation","title":"Public Static Attributes Documentation","text":""},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#variable-error","title":"variable error","text":"
                                                                                    constexpr fpga_event_type opae::fpga::types::event::type_t::error;\n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#variable-interrupt","title":"variable interrupt","text":"
                                                                                    constexpr fpga_event_type opae::fpga::types::event::type_t::interrupt;\n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#variable-power_thermal","title":"variable power_thermal","text":"
                                                                                    constexpr fpga_event_type opae::fpga::types::event::type_t::power_thermal;\n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#function-operator-fpga_event_type","title":"function operator fpga_event_type","text":"
                                                                                    inline opae::fpga::types::event::type_t::operator fpga_event_type () \n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1event_1_1type__t/#function-type_t","title":"function type_t","text":"
                                                                                    inline opae::fpga::types::event::type_t::type_t (\n    fpga_event_type c_type\n) \n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/events.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/","title":"Class opae::fpga::types::except","text":"
                                                                                    ClassList > opae > fpga > types > except
                                                                                     
                                                                                    Generic OPAE exception. More...
                                                                                     
                                                                                       
                                                                                    • #include <except.h>
                                                                                      •  
                                                                                     
                                                                                    Inherits the following classes: std::exception
                                                                                     
                                                                                    Inherited by the following classes: opae::fpga::types::busy,  opae::fpga::types::exception,  opae::fpga::types::invalid_param,  opae::fpga::types::no_access,  opae::fpga::types::no_daemon,  opae::fpga::types::no_driver,  opae::fpga::types::no_memory,  opae::fpga::types::not_found,  opae::fpga::types::not_supported,  opae::fpga::types::reconf_error
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#public-static-attributes","title":"Public Static Attributes","text":"Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#public-functions","title":"Public Functions","text":"Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#protected-attributes","title":"Protected Attributes","text":"Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#detailed-description","title":"Detailed Description","text":"
                                                                                    An except tracks the source line of origin and an optional fpga_result. If no fpga_result is given, then FPGA_EXCEPTION is used. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#public-static-attributes-documentation","title":"Public Static Attributes Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#variable-max_except","title":"variable MAX_EXCEPT","text":"
                                                                                    const std::size_t opae::fpga::types::except::MAX_EXCEPT;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#function-except-13","title":"function except [\u2153]","text":"
                                                                                    except constructor The fpga_result value is FPGA_EXCEPTION. 
                                                                                    opae::fpga::types::except::except (\n    src_location loc\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • loc Location where the exception was constructed. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#function-except-23","title":"function except [\u2154]","text":"
                                                                                    except constructor 
                                                                                    opae::fpga::types::except::except (\n    fpga_result res,\n    src_location loc\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • res The fpga_result value associated with this exception. 
                                                                                      •  
                                                                                    • loc Location where the exception was constructed. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#function-except-33","title":"function except [3/3]","text":"
                                                                                    except constructor 
                                                                                    opae::fpga::types::except::except (\n    fpga_result res,\n    const char * msg,\n    src_location loc\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • res The fpga_result value associated with this exception. 
                                                                                      •  
                                                                                    • msg The error message as a string 
                                                                                      •  
                                                                                    • loc Location where the exception was constructed. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#function-operator-fpga_result","title":"function operator fpga_result","text":"
                                                                                    inline opae::fpga::types::except::operator fpga_result () noexcept const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#function-what","title":"function what","text":"
                                                                                    virtual const char * opae::fpga::types::except::what () noexcept override const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#protected-attributes-documentation","title":"Protected Attributes Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#variable-buf_","title":"variable buf_","text":"
                                                                                    char opae::fpga::types::except::buf_[MAX_EXCEPT];\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#variable-loc_","title":"variable loc_","text":"
                                                                                    src_location opae::fpga::types::except::loc_;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#variable-msg_","title":"variable msg_","text":"
                                                                                    const char* opae::fpga::types::except::msg_;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1except/#variable-res_","title":"variable res_","text":"
                                                                                    fpga_result opae::fpga::types::except::res_;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/","title":"Class opae::fpga::types::exception","text":"
                                                                                    ClassList > opae > fpga > types > exception
                                                                                     
                                                                                    exception exception More...
                                                                                     
                                                                                       
                                                                                    • #include <except.h>
                                                                                      •  
                                                                                     
                                                                                    Inherits the following classes: opae::fpga::types::except
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#public-functions","title":"Public Functions","text":"Type Name exception (src_location loc) noexceptexception constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#detailed-description","title":"Detailed Description","text":"
                                                                                    exception tracks the source line of origin for exceptions thrown when the error code FPGA_EXCEPTION is returned from a call to an OPAE C API function 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1exception/#function-exception","title":"function exception","text":"
                                                                                    exception constructor 
                                                                                    inline opae::fpga::types::exception::exception (\n    src_location loc\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • loc Location where the exception was constructed. 
                                                                                      •  
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/","title":"Struct opae::fpga::types::guid_t","text":"
                                                                                    ClassList > opae > fpga > types > guid_t
                                                                                     
                                                                                    Representation of the guid member of a properties object. 
                                                                                     
                                                                                       
                                                                                    • #include <pvalue.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#public-functions","title":"Public Functions","text":"Type Name const uint8_t * c_type () constReturn a raw pointer to the guid. guid_t (fpga_properties * p) Construct the guid_t given its containing fpga_properties. void invalidate () Invalidate the cached local copy of the guid. bool is_set () constTracks whether the cached local copy of the guid is valid. operator uint8_t * () Return a raw pointer to the guid. guid_t & operator= (fpga_guid g) Assign from fpga_guid Sets the guid field of the associated properties object using the OPAE properties API. bool operator== (const fpga_guid & g) Compare contents with an fpga_guid. void parse (const char * str) Convert a string representation of a guid to binary. void update () Update the local cached copy of the guid."},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-c_type","title":"function c_type","text":"
                                                                                    inline const uint8_t * opae::fpga::types::guid_t::c_type () const\n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-guid_t","title":"function guid_t","text":"
                                                                                    inline opae::fpga::types::guid_t::guid_t (\n    fpga_properties * p\n) \n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-invalidate","title":"function invalidate","text":"
                                                                                    inline void opae::fpga::types::guid_t::invalidate () \n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-is_set","title":"function is_set","text":"
                                                                                    inline bool opae::fpga::types::guid_t::is_set () const\n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-operator-uint8_t","title":"function operator uint8_t *","text":"
                                                                                    Return a raw pointer to the guid. 
                                                                                    inline opae::fpga::types::guid_t::operator uint8_t * () \n
                                                                                     
                                                                                    Return value:
                                                                                     
                                                                                       
                                                                                    • nullptr if the guid could not be queried. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-operator","title":"function operator=","text":"
                                                                                    Assign from fpga_guid Sets the guid field of the associated properties object using the OPAE properties API. 
                                                                                    inline guid_t & opae::fpga::types::guid_t::operator= (\n    fpga_guid g\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • g The given fpga_guid. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    a reference to this guid_t. 
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-operator_1","title":"function operator==","text":"
                                                                                    Compare contents with an fpga_guid. 
                                                                                    inline bool opae::fpga::types::guid_t::operator== (\n    const fpga_guid & g\n) \n
                                                                                     
                                                                                    Return value:
                                                                                     
                                                                                       
                                                                                    • The result of memcmp of the two objects. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-parse","title":"function parse","text":"
                                                                                    Convert a string representation of a guid to binary. 
                                                                                    inline void opae::fpga::types::guid_t::parse (\n    const char * str\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • str The guid string. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#function-update","title":"function update","text":"
                                                                                    inline void opae::fpga::types::guid_t::update () \n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#friends-documentation","title":"Friends Documentation","text":""},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1guid__t/#friend-operator","title":"friend operator<<","text":"
                                                                                    inline std::ostream & opae::fpga::types::guid_t::operator<< (\n    std::ostream & ostr,\n    const guid_t & g\n) \n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/pvalue.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/","title":"Class opae::fpga::types::handle","text":"
                                                                                    ClassList > opae > fpga > types > handle
                                                                                     
                                                                                    An allocated accelerator resource. More...
                                                                                     
                                                                                       
                                                                                    • #include <handle.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< handle > ptr_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#public-functions","title":"Public Functions","text":"Type Name fpga_handle c_type () constRetrieve the underlying OPAE handle. fpga_result close () Close an accelerator resource (if opened) token::ptr_t get_token () constRetrieve the token corresponding to this handle object. handle (const handle &) = delete uint8_t * mmio_ptr (uint64_t offset, uint32_t csr_space=0) constRetrieve a pointer to the MMIO region. operator fpga_handle () constRetrieve the underlying OPAE handle. handle & operator= (const handle &) = delete uint32_t read_csr32 (uint64_t offset, uint32_t csr_space=0) constRead 32 bits from a CSR belonging to a resource associated with a handle. uint64_t read_csr64 (uint64_t offset, uint32_t csr_space=0) constRead 64 bits from a CSR belonging to a resource associated with a handle. void reconfigure (uint32_t slot, const uint8_t * bitstream, size_t size, int flags) Load a bitstream into an FPGA slot. virtual void reset () Reset the accelerator identified by this handle. void write_csr32 (uint64_t offset, uint32_t value, uint32_t csr_space=0) Write 32 bit to a CSR belonging to a resource associated with a handle. void write_csr512 (uint64_t offset, const void * value, uint32_t csr_space=0) Write 512 bits to a CSR belonging to a resource associated with a handle. void write_csr64 (uint64_t offset, uint64_t value, uint32_t csr_space=0) Write 64 bits to a CSR belonging to a resource associated with a handle. virtual ~handle ()"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#public-static-functions","title":"Public Static Functions","text":"Type Name handle::ptr_t open (fpga_token token, int flags) Open an accelerator resource, given a raw fpga_token. handle::ptr_t open (token::ptr_t token, int flags) Open an accelerator resource, given a token object."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#detailed-description","title":"Detailed Description","text":"
                                                                                    Represents an accelerator resource that has been allocated by OPAE. Depending on the type of resource, its register space may be read/written using a handle object. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                                                                    typedef std::shared_ptr<handle> opae::fpga::types::handle::ptr_t;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-c_type","title":"function c_type","text":"
                                                                                    inline fpga_handle opae::fpga::types::handle::c_type () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-close","title":"function close","text":"
                                                                                    Close an accelerator resource (if opened) 
                                                                                    fpga_result opae::fpga::types::handle::close () \n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    fpga_result indication the result of closing the handle or FPGA_EXCEPTION if handle is not opened
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This is available for explicitly closing a handle. The destructor for handle will call close. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-get_token","title":"function get_token","text":"
                                                                                    token::ptr_t opae::fpga::types::handle::get_token () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-handle-12","title":"function handle [\u00bd]","text":"
                                                                                    opae::fpga::types::handle::handle (\n    const handle &\n) = delete\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-mmio_ptr","title":"function mmio_ptr","text":"
                                                                                    Retrieve a pointer to the MMIO region. 
                                                                                    uint8_t * opae::fpga::types::handle::mmio_ptr (\n    uint64_t offset,\n    uint32_t csr_space=0\n) const\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • offset The byte offset to add to MMIO base. 
                                                                                      •  
                                                                                    • csr_space The desired CSR space. Default is 0. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    MMIO base + offset 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-operator-fpga_handle","title":"function operator fpga_handle","text":"
                                                                                    inline opae::fpga::types::handle::operator fpga_handle () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-operator","title":"function operator=","text":"
                                                                                    handle & opae::fpga::types::handle::operator= (\n    const handle &\n) = delete\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-read_csr32","title":"function read_csr32","text":"
                                                                                    Read 32 bits from a CSR belonging to a resource associated with a handle. 
                                                                                    uint32_t opae::fpga::types::handle::read_csr32 (\n    uint64_t offset,\n    uint32_t csr_space=0\n) const\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • offset The register offset 
                                                                                      •  
                                                                                    • csr_space The CSR space to read from. Default is 0.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    The 32-bit value read from the CSR 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-read_csr64","title":"function read_csr64","text":"
                                                                                    Read 64 bits from a CSR belonging to a resource associated with a handle. 
                                                                                    uint64_t opae::fpga::types::handle::read_csr64 (\n    uint64_t offset,\n    uint32_t csr_space=0\n) const\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • offset The register offset 
                                                                                      •  
                                                                                    • csr_space The CSR space to read from. Default is 0.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    The 64-bit value read from the CSR 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-reconfigure","title":"function reconfigure","text":"
                                                                                    Load a bitstream into an FPGA slot. 
                                                                                    void opae::fpga::types::handle::reconfigure (\n    uint32_t slot,\n    const uint8_t * bitstream,\n    size_t size,\n    int flags\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • slot The slot number to program 
                                                                                      •  
                                                                                    • bitstream The bitstream binary data 
                                                                                      •  
                                                                                    • size The size of the bitstream 
                                                                                      •  
                                                                                    • flags Flags that control behavior of reconfiguration. Value of 0 indicates no flags. FPGA_RECONF_FORCE indicates that the bitstream is programmed into the slot without checking if the resource is currently in use.
                                                                                      •  
                                                                                     
                                                                                    Exception:
                                                                                     
                                                                                       
                                                                                    • invalid_param if the handle is not an FPGA device handle or if the other parameters are not valid. 
                                                                                      •  
                                                                                    • exception if an internal error is encountered. 
                                                                                      •  
                                                                                    • busy if the accelerator for the given slot is in use. 
                                                                                      •  
                                                                                    • reconf_error if errors are reported by the driver (CRC or protocol errors). 
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-reset","title":"function reset","text":"
                                                                                    virtual void opae::fpga::types::handle::reset () \n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-write_csr32","title":"function write_csr32","text":"
                                                                                    Write 32 bit to a CSR belonging to a resource associated with a handle. 
                                                                                    void opae::fpga::types::handle::write_csr32 (\n    uint64_t offset,\n    uint32_t value,\n    uint32_t csr_space=0\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • offset The register offset. 
                                                                                      •  
                                                                                    • value The 32-bit value to write to the register. 
                                                                                      •  
                                                                                    • csr_space The CSR space to read from. Default is 0. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-write_csr512","title":"function write_csr512","text":"
                                                                                    Write 512 bits to a CSR belonging to a resource associated with a handle. 
                                                                                    void opae::fpga::types::handle::write_csr512 (\n    uint64_t offset,\n    const void * value,\n    uint32_t csr_space=0\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • offset The register offset. 
                                                                                      •  
                                                                                    • value Pointer to the 512-bit value to write to the register. 
                                                                                      •  
                                                                                    • csr_space The CSR space to read from. Default is 0. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-write_csr64","title":"function write_csr64","text":"
                                                                                    Write 64 bits to a CSR belonging to a resource associated with a handle. 
                                                                                    void opae::fpga::types::handle::write_csr64 (\n    uint64_t offset,\n    uint64_t value,\n    uint32_t csr_space=0\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • offset The register offset. 
                                                                                      •  
                                                                                    • value The 64-bit value to write to the register. 
                                                                                      •  
                                                                                    • csr_space The CSR space to read from. Default is 0. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-handle","title":"function ~handle","text":"
                                                                                    virtual opae::fpga::types::handle::~handle () \n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-open-12","title":"function open [\u00bd]","text":"
                                                                                    Open an accelerator resource, given a raw fpga_token. 
                                                                                    static handle::ptr_t opae::fpga::types::handle::open (\n    fpga_token token,\n    int flags\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • token A token describing the accelerator resource to be allocated.
                                                                                      •  
                                                                                    • flags The flags parameter to fpgaOpen().
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    pointer to the mmio base + offset for the given csr space 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1handle/#function-open-22","title":"function open [2/2]","text":"
                                                                                    Open an accelerator resource, given a token object. 
                                                                                    static handle::ptr_t opae::fpga::types::handle::open (\n    token::ptr_t token,\n    int flags\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • token A token object describing the accelerator resource to be allocated.
                                                                                      •  
                                                                                    • flags The flags parameter to fpgaOpen().
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    shared ptr to a handle object 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/handle.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/","title":"Class opae::fpga::types::invalid_param","text":"
                                                                                    ClassList > opae > fpga > types > invalid_param
                                                                                     
                                                                                    invalid_param exceptionMore...
                                                                                     
                                                                                       
                                                                                    • #include <except.h>
                                                                                      •  
                                                                                     
                                                                                    Inherits the following classes: opae::fpga::types::except
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#public-functions","title":"Public Functions","text":"Type Name invalid_param (src_location loc) noexceptinvalid_param constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#detailed-description","title":"Detailed Description","text":"
                                                                                    invalid_param tracks the source line of origin for exceptions thrown when the error code FPGA_INVALID_PARAM is returned from a call to an OPAE C API function 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1invalid__param/#function-invalid_param","title":"function invalid_param","text":"
                                                                                    invalid_param constructor 
                                                                                    inline opae::fpga::types::invalid_param::invalid_param (\n    src_location loc\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • loc Location where the exception was constructed. 
                                                                                      •  
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/","title":"Class opae::fpga::types::no_access","text":"
                                                                                    ClassList > opae > fpga > types > no_access
                                                                                     
                                                                                    no_access exceptionMore...
                                                                                     
                                                                                       
                                                                                    • #include <except.h>
                                                                                      •  
                                                                                     
                                                                                    Inherits the following classes: opae::fpga::types::except
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#public-functions","title":"Public Functions","text":"Type Name no_access (src_location loc) noexceptno_access constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#detailed-description","title":"Detailed Description","text":"
                                                                                    no_access tracks the source line of origin for exceptions thrown when the error code FPGA_NO_ACCESS is returned from a call to an OPAE C API function 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__access/#function-no_access","title":"function no_access","text":"
                                                                                    no_access constructor 
                                                                                    inline opae::fpga::types::no_access::no_access (\n    src_location loc\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • loc Location where the exception was constructed. 
                                                                                      •  
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/","title":"Class opae::fpga::types::no_daemon","text":"
                                                                                    ClassList > opae > fpga > types > no_daemon
                                                                                     
                                                                                    no_daemon exceptionMore...
                                                                                     
                                                                                       
                                                                                    • #include <except.h>
                                                                                      •  
                                                                                     
                                                                                    Inherits the following classes: opae::fpga::types::except
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#public-functions","title":"Public Functions","text":"Type Name no_daemon (src_location loc) noexceptno_daemon constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#detailed-description","title":"Detailed Description","text":"
                                                                                    no_daemon tracks the source line of origin for exceptions thrown when the error code FPGA_NO_DAEMON is returned from a call to an OPAE C API function 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__daemon/#function-no_daemon","title":"function no_daemon","text":"
                                                                                    no_daemon constructor 
                                                                                    inline opae::fpga::types::no_daemon::no_daemon (\n    src_location loc\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • loc Location where the exception was constructed. 
                                                                                      •  
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/","title":"Class opae::fpga::types::no_driver","text":"
                                                                                    ClassList > opae > fpga > types > no_driver
                                                                                     
                                                                                    no_driver exceptionMore...
                                                                                     
                                                                                       
                                                                                    • #include <except.h>
                                                                                      •  
                                                                                     
                                                                                    Inherits the following classes: opae::fpga::types::except
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#public-functions","title":"Public Functions","text":"Type Name no_driver (src_location loc) noexceptno_driver constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#detailed-description","title":"Detailed Description","text":"
                                                                                    no_driver tracks the source line of origin for exceptions thrown when the error code FPGA_NO_DRIVER is returned from a call to an OPAE C API function 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__driver/#function-no_driver","title":"function no_driver","text":"
                                                                                    no_driver constructor 
                                                                                    inline opae::fpga::types::no_driver::no_driver (\n    src_location loc\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • loc Location where the exception was constructed. 
                                                                                      •  
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/","title":"Class opae::fpga::types::no_memory","text":"
                                                                                    ClassList > opae > fpga > types > no_memory
                                                                                     
                                                                                    no_memory exceptionMore...
                                                                                     
                                                                                       
                                                                                    • #include <except.h>
                                                                                      •  
                                                                                     
                                                                                    Inherits the following classes: opae::fpga::types::except
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#public-functions","title":"Public Functions","text":"Type Name no_memory (src_location loc) noexceptno_memory constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#detailed-description","title":"Detailed Description","text":"
                                                                                    no_memory tracks the source line of origin for exceptions thrown when the error code FPGA_NO_MEMORY is returned from a call to an OPAE C API function 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1no__memory/#function-no_memory","title":"function no_memory","text":"
                                                                                    no_memory constructor 
                                                                                    inline opae::fpga::types::no_memory::no_memory (\n    src_location loc\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • loc Location where the exception was constructed. 
                                                                                      •  
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/","title":"Class opae::fpga::types::not_found","text":"
                                                                                    ClassList > opae > fpga > types > not_found
                                                                                     
                                                                                    not_found exceptionMore...
                                                                                     
                                                                                       
                                                                                    • #include <except.h>
                                                                                      •  
                                                                                     
                                                                                    Inherits the following classes: opae::fpga::types::except
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#public-functions","title":"Public Functions","text":"Type Name not_found (src_location loc) noexceptnot_found constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#detailed-description","title":"Detailed Description","text":"
                                                                                    not_found tracks the source line of origin for exceptions thrown when the error code FPGA_NOT_FOUND is returned from a call to an OPAE C API function 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__found/#function-not_found","title":"function not_found","text":"
                                                                                    not_found constructor 
                                                                                    inline opae::fpga::types::not_found::not_found (\n    src_location loc\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • loc Location where the exception was constructed. 
                                                                                      •  
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/","title":"Class opae::fpga::types::not_supported","text":"
                                                                                    ClassList > opae > fpga > types > not_supported
                                                                                     
                                                                                    not_supported exceptionMore...
                                                                                     
                                                                                       
                                                                                    • #include <except.h>
                                                                                      •  
                                                                                     
                                                                                    Inherits the following classes: opae::fpga::types::except
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#public-functions","title":"Public Functions","text":"Type Name not_supported (src_location loc) noexceptnot_supported constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#detailed-description","title":"Detailed Description","text":"
                                                                                    not_supported tracks the source line of origin for exceptions thrown when the error code FPGA_NOT_SUPPORTED is returned from a call to an OPAE C API function 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1not__supported/#function-not_supported","title":"function not_supported","text":"
                                                                                    not_supported constructor 
                                                                                    inline opae::fpga::types::not_supported::not_supported (\n    src_location loc\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • loc Location where the exception was constructed. 
                                                                                      •  
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/","title":"Class opae::fpga::types::properties","text":"
                                                                                    ClassList > opae > fpga > types > properties
                                                                                     
                                                                                    Wraps an OPAE fpga_properties object. More...
                                                                                     
                                                                                       
                                                                                    • #include <properties.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< properties > ptr_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-attributes","title":"Public Attributes","text":"Type Name pvalue< fpga_accelerator_state > accelerator_state pvalue< uint64_t > bbs_id pvalue< fpga_version > bbs_version pvalue< uint8_t > bus pvalue< uint64_t > capabilities pvalue< uint8_t > device pvalue< uint16_t > device_id pvalue< uint8_t > function guid_t guid pvalue< fpga_interface > interface pvalue< uint64_t > local_memory_size pvalue< char * > model pvalue< uint32_t > num_errors pvalue< uint32_t > num_interrupts pvalue< uint32_t > num_mmio pvalue< uint32_t > num_slots pvalue< uint64_t > object_id pvalue< fpga_token > parent pvalue< uint16_t > segment pvalue< uint8_t > socket_id pvalue< uint16_t > subsystem_device_id pvalue< uint16_t > subsystem_vendor_id pvalue< fpga_objtype > type pvalue< uint16_t > vendor_id"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-static-attributes","title":"Public Static Attributes","text":"Type Name const std::vector< properties::ptr_t > none An empty vector of properties."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-functions","title":"Public Functions","text":"Type Name fpga_properties c_type () constGet the underlying fpga_properties object. operator fpga_properties () constGet the underlying fpga_properties object. properties & operator= (const properties & p) = delete properties (const properties & p) = delete ~properties ()"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-static-functions","title":"Public Static Functions","text":"Type Name properties::ptr_t get () Create a new properties object. properties::ptr_t get (fpga_guid guid_in) Create a new properties object from a guid. properties::ptr_t get (fpga_objtype objtype) Create a new properties object from an fpga_objtype. properties::ptr_t get (std::shared_ptr< token > t) Retrieve the properties for a given token object. properties::ptr_t get (fpga_token t) Retrieve the properties for a given fpga_token. properties::ptr_t get (std::shared_ptr< handle > h) Retrieve the properties for a given handle object."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#detailed-description","title":"Detailed Description","text":"
                                                                                    properties are information describing an accelerator resource that is identified by its token. The properties are used during enumeration to narrow the search for an accelerator resource, and after enumeration to provide the configuration of that resource. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                                                                    typedef std::shared_ptr<properties> opae::fpga::types::properties::ptr_t;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-accelerator_state","title":"variable accelerator_state","text":"
                                                                                    pvalue<fpga_accelerator_state> opae::fpga::types::properties::accelerator_state;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-bbs_id","title":"variable bbs_id","text":"
                                                                                    pvalue<uint64_t> opae::fpga::types::properties::bbs_id;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-bbs_version","title":"variable bbs_version","text":"
                                                                                    pvalue<fpga_version> opae::fpga::types::properties::bbs_version;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-bus","title":"variable bus","text":"
                                                                                    pvalue<uint8_t> opae::fpga::types::properties::bus;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-capabilities","title":"variable capabilities","text":"
                                                                                    pvalue<uint64_t> opae::fpga::types::properties::capabilities;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-device","title":"variable device","text":"
                                                                                    pvalue<uint8_t> opae::fpga::types::properties::device;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-device_id","title":"variable device_id","text":"
                                                                                    pvalue<uint16_t> opae::fpga::types::properties::device_id;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-function","title":"variable function","text":"
                                                                                    pvalue<uint8_t> opae::fpga::types::properties::function;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-guid","title":"variable guid","text":"
                                                                                    guid_t opae::fpga::types::properties::guid;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-interface","title":"variable interface","text":"
                                                                                    pvalue<fpga_interface> opae::fpga::types::properties::interface;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-local_memory_size","title":"variable local_memory_size","text":"
                                                                                    pvalue<uint64_t> opae::fpga::types::properties::local_memory_size;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-model","title":"variable model","text":"
                                                                                    pvalue<char *> opae::fpga::types::properties::model;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-num_errors","title":"variable num_errors","text":"
                                                                                    pvalue<uint32_t> opae::fpga::types::properties::num_errors;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-num_interrupts","title":"variable num_interrupts","text":"
                                                                                    pvalue<uint32_t> opae::fpga::types::properties::num_interrupts;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-num_mmio","title":"variable num_mmio","text":"
                                                                                    pvalue<uint32_t> opae::fpga::types::properties::num_mmio;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-num_slots","title":"variable num_slots","text":"
                                                                                    pvalue<uint32_t> opae::fpga::types::properties::num_slots;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-object_id","title":"variable object_id","text":"
                                                                                    pvalue<uint64_t> opae::fpga::types::properties::object_id;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-parent","title":"variable parent","text":"
                                                                                    pvalue<fpga_token> opae::fpga::types::properties::parent;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-segment","title":"variable segment","text":"
                                                                                    pvalue<uint16_t> opae::fpga::types::properties::segment;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-socket_id","title":"variable socket_id","text":"
                                                                                    pvalue<uint8_t> opae::fpga::types::properties::socket_id;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-subsystem_device_id","title":"variable subsystem_device_id","text":"
                                                                                    pvalue<uint16_t> opae::fpga::types::properties::subsystem_device_id;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-subsystem_vendor_id","title":"variable subsystem_vendor_id","text":"
                                                                                    pvalue<uint16_t> opae::fpga::types::properties::subsystem_vendor_id;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-type","title":"variable type","text":"
                                                                                    pvalue<fpga_objtype> opae::fpga::types::properties::type;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-vendor_id","title":"variable vendor_id","text":"
                                                                                    pvalue<uint16_t> opae::fpga::types::properties::vendor_id;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-static-attributes-documentation","title":"Public Static Attributes Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#variable-none","title":"variable none","text":"
                                                                                    An empty vector of properties. 
                                                                                    const std::vector<properties::ptr_t> opae::fpga::types::properties::none;\n
                                                                                     
                                                                                    Useful for enumerating based on a \"match all\" criteria. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-c_type","title":"function c_type","text":"
                                                                                    inline fpga_properties opae::fpga::types::properties::c_type () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-operator-fpga_properties","title":"function operator fpga_properties","text":"
                                                                                    inline opae::fpga::types::properties::operator fpga_properties () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-operator","title":"function operator=","text":"
                                                                                    properties & opae::fpga::types::properties::operator= (\n    const properties & p\n) = delete\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-properties-12","title":"function properties [\u00bd]","text":"
                                                                                    opae::fpga::types::properties::properties (\n    const properties & p\n) = delete\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-properties","title":"function ~properties","text":"
                                                                                    opae::fpga::types::properties::~properties () \n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-get-16","title":"function get [\u2159]","text":"
                                                                                    Create a new properties object. 
                                                                                    static properties::ptr_t opae::fpga::types::properties::get () \n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A properties smart pointer. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-get-26","title":"function get [2/6]","text":"
                                                                                    Create a new properties object from a guid. 
                                                                                    static properties::ptr_t opae::fpga::types::properties::get (\n    fpga_guid guid_in\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • guid_in A guid to set in the properties 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A properties smart pointer with its guid initialized to guid_in 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-get-36","title":"function get [3/6]","text":"
                                                                                    Create a new properties object from an fpga_objtype. 
                                                                                    static properties::ptr_t opae::fpga::types::properties::get (\n    fpga_objtype objtype\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • objtype An object type to set in the properties 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A properties smart pointer with its object type set to objtype. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-get-46","title":"function get [4/6]","text":"
                                                                                    Retrieve the properties for a given token object. 
                                                                                    static properties::ptr_t opae::fpga::types::properties::get (\n    std::shared_ptr< token > t\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • t A token identifying the accelerator resource. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A properties smart pointer for the given token. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-get-56","title":"function get [\u215a]","text":"
                                                                                    Retrieve the properties for a given fpga_token. 
                                                                                    static properties::ptr_t opae::fpga::types::properties::get (\n    fpga_token t\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • t An fpga_token identifying the accelerator resource. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A properties smart pointer for the given fpga_token. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1properties/#function-get-66","title":"function get [6/6]","text":"
                                                                                    Retrieve the properties for a given handle object. 
                                                                                    static properties::ptr_t opae::fpga::types::properties::get (\n    std::shared_ptr< handle > h\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • h A handle identifying the accelerator resource. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A properties smart pointer for the given handle. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/properties.h
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/","title":"Struct opae::fpga::types::pvalue","text":"
                                                                                    template <typename T typename T>
                                                                                     
                                                                                    ClassList > opae > fpga > types > pvalue
                                                                                     
                                                                                    Wraps OPAE properties defined in the OPAE C API by associating an fpga_properties reference with the getters and setters defined for a property.More...
                                                                                     
                                                                                       
                                                                                    • #include <pvalue.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#public-types","title":"Public Types","text":"Type Name typedef std::conditional< std::is_same< T, char * >::value, typename std::string, T >::type copy_t Define the type of our copy variable For char* types use std::string as the copy. typedef std::conditional< std::is_same< T, char * >::value, fpga_result(*)(fpga_properties, T), fpga_result(*)(fpga_properties, T *)>::type getter_t Define getter function as getter_t For char* types, do not use T* as the second argument but instead use T. typedef fpga_result(* setter_t Define the setter function as setter_t."},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#public-functions","title":"Public Functions","text":"Type Name fpga_result get_value (T & value) const void invalidate () Invalidate the cached local copy of the pvalue. bool is_set () constTracks whether the cached local copy of the pvalue is valid. operator copy_t () Implicit converter operator - calls the wrapped getter. pvalue< T > & operator= (const T & v) Overload of = operator that calls the wrapped setter. bool operator== (const T & other) Compare a property for equality with a value. pvalue ()  pvalue (fpga_properties * p, getter_t g, setter_t s) pvalue contructor that takes in a reference to fpga_properties and corresponding accessor methods for a property void update ()  void update () Template specialization of char* type property updater."},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#detailed-description","title":"Detailed Description","text":"
                                                                                    Template parameters:
                                                                                     
                                                                                       
                                                                                    • T The type of the property value being wrapped 
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#typedef-copy_t","title":"typedef copy_t","text":"
                                                                                    typedef std::conditional<std::is_same<T, char *>::value, typename std::string, T>::type opae::fpga::types::pvalue< T >::copy_t;\n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#typedef-getter_t","title":"typedef getter_t","text":"
                                                                                    typedef std::conditional< std::is_same<T, char *>::value, fpga_result (*)(fpga_properties, T), fpga_result (*)(fpga_properties, T *)>::type opae::fpga::types::pvalue< T >::getter_t;\n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#typedef-setter_t","title":"typedef setter_t","text":"
                                                                                    typedef fpga_result(* opae::fpga::types::pvalue< T >::setter_t) (fpga_properties, T);\n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-get_value","title":"function get_value","text":"
                                                                                    inline fpga_result opae::fpga::types::pvalue::get_value (\n    T & value\n) const\n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-invalidate","title":"function invalidate","text":"
                                                                                    inline void opae::fpga::types::pvalue::invalidate () \n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-is_set","title":"function is_set","text":"
                                                                                    inline bool opae::fpga::types::pvalue::is_set () const\n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-operator-copy_t","title":"function operator copy_t","text":"
                                                                                    Implicit converter operator - calls the wrapped getter. 
                                                                                    inline opae::fpga::types::pvalue::operator copy_t () \n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    The property value after calling the getter or a default value of the value type 
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-operator","title":"function operator=","text":"
                                                                                    Overload of = operator that calls the wrapped setter. 
                                                                                    inline pvalue < T > & opae::fpga::types::pvalue::operator= (\n    const T & v\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • v The value to set
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A reference to itself 
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-operator_1","title":"function operator==","text":"
                                                                                    Compare a property for equality with a value. 
                                                                                    inline bool opae::fpga::types::pvalue::operator== (\n    const T & other\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • other The value being compared to
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Whether or not the property is equal to the value 
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-pvalue-12","title":"function pvalue [\u00bd]","text":"
                                                                                    inline opae::fpga::types::pvalue::pvalue () \n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-pvalue-22","title":"function pvalue [2/2]","text":"
                                                                                    pvalue contructor that takes in a reference to fpga_properties and corresponding accessor methods for a property 
                                                                                    inline opae::fpga::types::pvalue::pvalue (\n    fpga_properties * p,\n    getter_t g,\n    setter_t s\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • p A reference to an fpga_properties 
                                                                                      •  
                                                                                    • g The getter function 
                                                                                      •  
                                                                                    • s The setter function 
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-update-12","title":"function update [\u00bd]","text":"
                                                                                    inline void opae::fpga::types::pvalue::update () \n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#function-update-22","title":"function update [2/2]","text":"
                                                                                    Template specialization of char* type property updater. 
                                                                                    inline void opae::fpga::types::pvalue::update () \n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    The result of the property getter function. 
                                                                                     
                                                                                        ## Friends Documentation\n
                                                                                    "},{"location":"opae-code/structopae_1_1fpga_1_1types_1_1pvalue/#friend-operator","title":"friend operator<<","text":"
                                                                                    Stream overalod operator. 
                                                                                    inline std::ostream & opae::fpga::types::pvalue::operator<< (\n    std::ostream & ostr,\n    const pvalue < T > & p\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • ostr The output stream 
                                                                                      •  
                                                                                    • p A reference to a pvalue<T> object
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    The stream operator after streaming the property value 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/pvalue.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/","title":"Class opae::fpga::types::reconf_error","text":"
                                                                                    ClassList > opae > fpga > types > reconf_error
                                                                                     
                                                                                    reconf_error exceptionMore...
                                                                                     
                                                                                       
                                                                                    • #include <except.h>
                                                                                      •  
                                                                                     
                                                                                    Inherits the following classes: opae::fpga::types::except
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#public-static-attributes-inherited-from-opaefpgatypesexcept","title":"Public Static Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name const std::size_t MAX_EXCEPT   = = 256"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#public-functions","title":"Public Functions","text":"Type Name reconf_error (src_location loc) noexceptreconf_error constructor"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#public-functions-inherited-from-opaefpgatypesexcept","title":"Public Functions inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name except (src_location loc) noexceptexcept constructor The fpga_result value is FPGA_EXCEPTION. except (fpga_result res, src_location loc) noexceptexcept constructor except (fpga_result res, const char * msg, src_location loc) noexceptexcept constructor operator fpga_result () noexcept constConvert this except to its fpga_result. virtual const char * what () noexcept override constConvert this except to an informative string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#protected-attributes-inherited-from-opaefpgatypesexcept","title":"Protected Attributes inherited from opae::fpga::types::except","text":"
                                                                                    See opae::fpga::types::except
                                                                                     Type Name char buf_ src_location loc_ const char * msg_ fpga_result res_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#detailed-description","title":"Detailed Description","text":"
                                                                                    reconf_error tracks the source line of origin for exceptions thrown when the error code FPGA_RECONF_ERROR is returned from a call to an OPAE C API function 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1reconf__error/#function-reconf_error","title":"function reconf_error","text":"
                                                                                    reconf_error constructor 
                                                                                    inline opae::fpga::types::reconf_error::reconf_error (\n    src_location loc\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • loc Location where the exception was constructed. 
                                                                                      •  
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/","title":"Class opae::fpga::types::shared_buffer","text":"
                                                                                    ClassList > opae > fpga > types > shared_buffer
                                                                                     
                                                                                    Host/AFU shared memory blocks. More...
                                                                                     
                                                                                       
                                                                                    • #include <shared_buffer.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< shared_buffer > ptr_t typedef std::size_t size_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#public-functions","title":"Public Functions","text":"Type Name uint8_t * c_type () constRetrieve the virtual address of the buffer base. int compare (ptr_t other, size_t len) constCompare this shared_buffer (the first len bytes) to that held in other, using memcmp(). void fill (int c) Write c to each byte location in the buffer. uint64_t io_address () constRetrieve the address of the buffer suitable for programming into the accelerator device. shared_buffer & operator= (const shared_buffer &) = delete handle::ptr_t owner () constRetrieve the handle smart pointer associated with this buffer. T read (size_t offset) constRead a T-sized block of memory at the given location. void release () Disassociate the shared_buffer object from the resource used to create it. shared_buffer (const shared_buffer &) = delete size_t size () constRetrieve the length of the buffer in bytes. void write (const T & value, size_t offset) Write a T-sized block of memory to the given location. uint64_t wsid () constRetrieve the underlying buffer's workspace id. virtual ~shared_buffer () shared_buffer destructor."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#public-static-functions","title":"Public Static Functions","text":"Type Name shared_buffer::ptr_t allocate (handle::ptr_t handle, size_t len, bool read_only=false) shared_buffer factory method - allocate ashared_buffer . shared_buffer::ptr_t attach (handle::ptr_t handle, uint8_t * base, size_t len, bool read_only=false) Attach a pre-allocated buffer to a shared_buffer object."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#protected-attributes","title":"Protected Attributes","text":"Type Name handle::ptr_t handle_ uint64_t io_address_ size_t len_ uint8_t * virt_ uint64_t wsid_"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#protected-functions","title":"Protected Functions","text":"Type Name shared_buffer (handle::ptr_t handle, size_t len, uint8_t * virt, uint64_t wsid, uint64_t io_address)"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#detailed-description","title":"Detailed Description","text":"
                                                                                    shared_buffer abstracts a memory block that may be shared between the host cpu and an accelerator. The block may be allocated by the shared_buffer class itself (see allocate), or it may be allocated elsewhere and then attached to a shared_buffer object via attach. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                                                                    typedef std::shared_ptr<shared_buffer> opae::fpga::types::shared_buffer::ptr_t;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#typedef-size_t","title":"typedef size_t","text":"
                                                                                    typedef std::size_t opae::fpga::types::shared_buffer::size_t;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-c_type","title":"function c_type","text":"
                                                                                    Retrieve the virtual address of the buffer base. 
                                                                                    inline uint8_t * opae::fpga::types::shared_buffer::c_type () const\n
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    Instances of a shared buffer can only be created using either 'allocate' or 'attach' static factory function. Because these functions return a shared pointer (std::shared_ptr) to the instance, references to an instance are counted automatically by design of the shared_ptr class. Calling 'c_type()' function is provided to get access to the raw data but isn't used in tracking its reference count. Assigning this to a variable should be done in limited scopes as this variable can be defined in an outer scope and may outlive the shared_buffer object. Once the reference count in the shared_ptr reaches zero, the shared_buffer object will be released and deallocated, turning any variables assigned from a call to 'c_type()' into dangling pointers. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-compare","title":"function compare","text":"
                                                                                    int opae::fpga::types::shared_buffer::compare (\n    ptr_t other,\n    size_t len\n) const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-fill","title":"function fill","text":"
                                                                                    void opae::fpga::types::shared_buffer::fill (\n    int c\n) \n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-io_address","title":"function io_address","text":"
                                                                                    inline uint64_t opae::fpga::types::shared_buffer::io_address () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-operator","title":"function operator=","text":"
                                                                                    shared_buffer & opae::fpga::types::shared_buffer::operator= (\n    const shared_buffer &\n) = delete\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-owner","title":"function owner","text":"
                                                                                    inline handle::ptr_t opae::fpga::types::shared_buffer::owner () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-read","title":"function read","text":"
                                                                                    Read a T-sized block of memory at the given location. 
                                                                                    template<typename T typename T>\ninline T opae::fpga::types::shared_buffer::read (\n    size_t offset\n) const\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • offset The byte offset from the start of the buffer. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A T from buffer base + offset. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-release","title":"function release","text":"
                                                                                    Disassociate the shared_buffer object from the resource used to create it. 
                                                                                    void opae::fpga::types::shared_buffer::release () \n
                                                                                     
                                                                                    If the buffer was allocated using the allocate function then the buffer is freed. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-shared_buffer-12","title":"function shared_buffer [\u00bd]","text":"
                                                                                    opae::fpga::types::shared_buffer::shared_buffer (\n    const shared_buffer &\n) = delete\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-size","title":"function size","text":"
                                                                                    inline size_t opae::fpga::types::shared_buffer::size () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-write","title":"function write","text":"
                                                                                    Write a T-sized block of memory to the given location. 
                                                                                    template<typename T typename T>\ninline void opae::fpga::types::shared_buffer::write (\n    const T & value,\n    size_t offset\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • value The value to write. 
                                                                                      •  
                                                                                    • offset The byte offset from the start of the buffer. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-wsid","title":"function wsid","text":"
                                                                                    inline uint64_t opae::fpga::types::shared_buffer::wsid () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-shared_buffer","title":"function ~shared_buffer","text":"
                                                                                    virtual opae::fpga::types::shared_buffer::~shared_buffer () \n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-allocate","title":"function allocate","text":"
                                                                                    shared_buffer factory method - allocate ashared_buffer . 
                                                                                    static shared_buffer::ptr_t opae::fpga::types::shared_buffer::allocate (\n    handle::ptr_t handle,\n    size_t len,\n    bool read_only=false\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle The handle used to allocate the buffer. 
                                                                                      •  
                                                                                    • len The length in bytes of the requested buffer. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A valid shared_buffer smart pointer on success, or an empty smart pointer on failure. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-attach","title":"function attach","text":"
                                                                                    Attach a pre-allocated buffer to a shared_buffer object. 
                                                                                    static shared_buffer::ptr_t opae::fpga::types::shared_buffer::attach (\n    handle::ptr_t handle,\n    uint8_t * base,\n    size_t len,\n    bool read_only=false\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle The handle used to attach the buffer. 
                                                                                      •  
                                                                                    • base The base of the pre-allocated memory. 
                                                                                      •  
                                                                                    • len The size of the pre-allocated memory, which must be a multiple of the page size. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A valid shared_buffer smart pointer on success, or an empty smart pointer on failure. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#protected-attributes-documentation","title":"Protected Attributes Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#variable-handle_","title":"variable handle_","text":"
                                                                                    handle::ptr_t opae::fpga::types::shared_buffer::handle_;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#variable-io_address_","title":"variable io_address_","text":"
                                                                                    uint64_t opae::fpga::types::shared_buffer::io_address_;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#variable-len_","title":"variable len_","text":"
                                                                                    size_t opae::fpga::types::shared_buffer::len_;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#variable-virt_","title":"variable virt_","text":"
                                                                                    uint8_t* opae::fpga::types::shared_buffer::virt_;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#variable-wsid_","title":"variable wsid_","text":"
                                                                                    uint64_t opae::fpga::types::shared_buffer::wsid_;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#protected-functions-documentation","title":"Protected Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1shared__buffer/#function-shared_buffer-22","title":"function shared_buffer [2/2]","text":"
                                                                                    opae::fpga::types::shared_buffer::shared_buffer (\n    handle::ptr_t handle,\n    size_t len,\n    uint8_t * virt,\n    uint64_t wsid,\n    uint64_t io_address\n) \n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/shared_buffer.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/","title":"Class opae::fpga::types::src_location","text":"
                                                                                    ClassList > opae > fpga > types > src_location
                                                                                     
                                                                                    Identify a particular line in a source file. 
                                                                                     
                                                                                       
                                                                                    • #include <except.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#public-functions","title":"Public Functions","text":"Type Name const char * file () noexcept constRetrieve the file name component of the location. const char * fn () noexcept constRetrieve the function name component of the location. int line () noexcept constRetrieve the line number component of the location. src_location & operator= (const src_location & other) noexcept src_location (const char * file, const char * fn, int line) noexceptsrc_location constructor src_location (const src_location & other) noexcept"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#function-file","title":"function file","text":"
                                                                                    const char * opae::fpga::types::src_location::file () noexcept const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#function-fn","title":"function fn","text":"
                                                                                    inline const char * opae::fpga::types::src_location::fn () noexcept const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#function-line","title":"function line","text":"
                                                                                    inline int opae::fpga::types::src_location::line () noexcept const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#function-operator","title":"function operator=","text":"
                                                                                    src_location & opae::fpga::types::src_location::operator= (\n    const src_location & other\n) noexcept\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#function-src_location-12","title":"function src_location [\u00bd]","text":"
                                                                                    src_location constructor 
                                                                                    opae::fpga::types::src_location::src_location (\n    const char * file,\n    const char * fn,\n    int line\n) noexcept\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • file The source file name, typically FILE. 
                                                                                      •  
                                                                                    • fn The current function, typically func. 
                                                                                      •  
                                                                                    • line The current line number, typically LINE. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1src__location/#function-src_location-22","title":"function src_location [2/2]","text":"
                                                                                    opae::fpga::types::src_location::src_location (\n    const src_location & other\n) noexcept\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/","title":"Class opae::fpga::types::sysobject","text":"
                                                                                    ClassList > opae > fpga > types > sysobject
                                                                                     
                                                                                    Wraps the OPAE fpga_object primitive. More...
                                                                                     
                                                                                       
                                                                                    • #include <sysobject.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< sysobject > ptr_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#public-functions","title":"Public Functions","text":"Type Name std::vector< uint8_t > bytes (int flags=0) constGet all raw bytes from the object. std::vector< uint8_t > bytes (uint32_t offset, uint32_t size, int flags=0) constGet a subset of raw bytes from the object. fpga_object c_type () constRetrieve the underlying fpga_object primitive. sysobject::ptr_t get (const std::string & name, int flags=0) Get a sysobject from an object. sysobject::ptr_t get (int index) Get a sysobject from a container object. operator fpga_object () constRetrieve the underlying fpga_object primitive. sysobject & operator= (const sysobject & o) = delete uint64_t read64 (int flags=0) constRead a 64-bit value from an FPGA object. uint32_t size () constGet the size (in bytes) of the object. sysobject () = delete sysobject (const sysobject & o) = delete enum fpga_sysobject_type type () constGet the object type (attribute or container) void write64 (uint64_t value, int flags=0) constWrite 64-bit value to an FPGA object. virtual ~sysobject ()"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#public-static-functions","title":"Public Static Functions","text":"Type Name sysobject::ptr_t get (token::ptr_t t, const std::string & name, int flags=0) Get a sysobject from a token. sysobject::ptr_t get (handle::ptr_t h, const std::string & name, int flags=0) Get a sysobject from a handle."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#detailed-description","title":"Detailed Description","text":"
                                                                                    sysobject's are created from a call to fpgaTokenGetObject, fpgaHandleGetObject, or fpgaObjectGetObject 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                                                                    typedef std::shared_ptr<sysobject> opae::fpga::types::sysobject::ptr_t;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-bytes-12","title":"function bytes [\u00bd]","text":"
                                                                                    Get all raw bytes from the object. 
                                                                                    std::vector< uint8_t > opae::fpga::types::sysobject::bytes (\n    int flags=0\n) const\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • flags Flags that control how object is read If FPGA_OBJECT_SYNC is used then object will update its buffered copy before retrieving the data.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A vector of all bytes in the object. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-bytes-22","title":"function bytes [2/2]","text":"
                                                                                    Get a subset of raw bytes from the object. 
                                                                                    std::vector< uint8_t > opae::fpga::types::sysobject::bytes (\n    uint32_t offset,\n    uint32_t size,\n    int flags=0\n) const\n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • offset The bytes offset for the start of the returned vector. 
                                                                                      •  
                                                                                    • size The number of bytes for the returned vector. 
                                                                                      •  
                                                                                    • flags Flags that control how object is read If FPGA_OBJECT_SYNC is used then object will update its buffered copy before retrieving the data.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A vector of size bytes in the object starting at offset. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-c_type","title":"function c_type","text":"
                                                                                    inline fpga_object opae::fpga::types::sysobject::c_type () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-get-14","title":"function get [\u00bc]","text":"
                                                                                    Get a sysobject from an object. 
                                                                                    sysobject::ptr_t opae::fpga::types::sysobject::get (\n    const std::string & name,\n    int flags=0\n) \n
                                                                                     
                                                                                    This will be read-write if its parent was created from a handle..
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • name An identifier representing an object belonging to this object. 
                                                                                      •  
                                                                                    • flags Control behavior of object identification and creation. FPGA_OBJECT_GLOB is used to indicate that the name should be treated as a globbing expression. FPGA_OBJECT_RECURSE_ONE indicates that subobjects be created for objects one level down from the object identified by name. FPGA_OBJECT_RECURSE_ALL indicates that subobjects be created for all objects. Flags are defaulted to 0 meaning no flags.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A shared_ptr to a sysobject instance. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-get-24","title":"function get [2/4]","text":"
                                                                                    Get a sysobject from a container object. 
                                                                                    sysobject::ptr_t opae::fpga::types::sysobject::get (\n    int index\n) \n
                                                                                     
                                                                                    This will be read-write if its parent was created from a handle..
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • index An index number to get.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A shared_ptr to a sysobject instance. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-operator-fpga_object","title":"function operator fpga_object","text":"
                                                                                    inline opae::fpga::types::sysobject::operator fpga_object () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-operator","title":"function operator=","text":"
                                                                                    sysobject & opae::fpga::types::sysobject::operator= (\n    const sysobject & o\n) = delete\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-read64","title":"function read64","text":"
                                                                                    Read a 64-bit value from an FPGA object. 
                                                                                    uint64_t opae::fpga::types::sysobject::read64 (\n    int flags=0\n) const\n
                                                                                     
                                                                                    The value is assumed to be in string format and will be parsed. See flags below for changing that behavior.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • flags Flags that control how the object is read If FPGA_OBJECT_SYNC is used then object will update its buffered copy before retrieving the data. If FPGA_OBJECT_RAW is used, then the data will be read as raw bytes into the uint64_t pointer variable. Flags are defaulted to 0 meaning no flags.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A 64-bit value from the object. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-size","title":"function size","text":"
                                                                                    Get the size (in bytes) of the object. 
                                                                                    uint32_t opae::fpga::types::sysobject::size () const\n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    The number of bytes that the object occupies in memory. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-sysobject-13","title":"function sysobject [\u2153]","text":"
                                                                                    opae::fpga::types::sysobject::sysobject () = delete\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-sysobject-23","title":"function sysobject [\u2154]","text":"
                                                                                    opae::fpga::types::sysobject::sysobject (\n    const sysobject & o\n) = delete\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-type","title":"function type","text":"
                                                                                    enum fpga_sysobject_type opae::fpga::types::sysobject::type () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-write64","title":"function write64","text":"
                                                                                    Write 64-bit value to an FPGA object. 
                                                                                    void opae::fpga::types::sysobject::write64 (\n    uint64_t value,\n    int flags=0\n) const\n
                                                                                     
                                                                                    The value will be converted to string before writing. See flags below for changing that behavior.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • value The value to write to the object. 
                                                                                      •  
                                                                                    • flags Flags that control how the object is written If FPGA_OBJECT_RAW is used, then the value will be written as raw bytes. Flags are defaulted to 0 meaning no flags.
                                                                                      •  
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This operation will force a sync operation to update its cached buffer 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-sysobject","title":"function ~sysobject","text":"
                                                                                    virtual opae::fpga::types::sysobject::~sysobject () \n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-get-34","title":"function get [\u00be]","text":"
                                                                                    Get a sysobject from a token. 
                                                                                    static sysobject::ptr_t opae::fpga::types::sysobject::get (\n    token::ptr_t t,\n    const std::string & name,\n    int flags=0\n) \n
                                                                                     
                                                                                    This will be read-only.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • t Token object representing a resource. 
                                                                                      •  
                                                                                    • name An identifier representing an object belonging to a resource represented by the token. 
                                                                                      •  
                                                                                    • flags Control behavior of object identification and creation. FPGA_OBJECT_GLOB is used to indicate that the name should be treated as a globbing expression. FPGA_OBJECT_RECURSE_ONE indicates that subobjects be created for objects one level down from the object identified by name. FPGA_OBJECT_RECURSE_ALL indicates that subobjects be created for all objects below the current object identified by name.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A shared_ptr to a sysobject instance. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1sysobject/#function-get-44","title":"function get [4/4]","text":"
                                                                                    Get a sysobject from a handle. 
                                                                                    static sysobject::ptr_t opae::fpga::types::sysobject::get (\n    handle::ptr_t h,\n    const std::string & name,\n    int flags=0\n) \n
                                                                                     
                                                                                    This will be read-write.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • h Handle object representing an open resource. 
                                                                                      •  
                                                                                    • name An identifier representing an object belonging to a resource represented by the handle. 
                                                                                      •  
                                                                                    • flags Control behavior of object identification and creation. FPGA_OBJECT_GLOB is used to indicate that the name should be treated as a globbing expression. FPGA_OBJECT_RECURSE_ONE indicates that subobjects be created for objects one level down from the object identified by name. FPGA_OBJECT_RECURSE_ALL indicates that subobjects be created for all objects below the current object identified by name.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A shared_ptr to a sysobject instance. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/sysobject.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/","title":"Class opae::fpga::types::token","text":"
                                                                                    ClassList > opae > fpga > types > token
                                                                                     
                                                                                    Wraps the OPAE fpga_token primitive. More...
                                                                                     
                                                                                       
                                                                                    • #include <token.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#public-types","title":"Public Types","text":"Type Name typedef std::shared_ptr< token > ptr_t"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#public-functions","title":"Public Functions","text":"Type Name fpga_token c_type () constRetrieve the underlying fpga_token primitive. ptr_t get_parent () constRetrieve the parent token, or an empty pointer if there is none. operator fpga_token () constRetrieve the underlying fpga_token primitive. ~token ()"},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#public-static-functions","title":"Public Static Functions","text":"Type Name std::vector< token::ptr_t > enumerate (const std::vector< properties::ptr_t > & props) Obtain a vector of token smart pointers for given search criteria."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#detailed-description","title":"Detailed Description","text":"
                                                                                    token's are created from an enumeration operation that uses properties describing an accelerator resource as search criteria. 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#typedef-ptr_t","title":"typedef ptr_t","text":"
                                                                                    typedef std::shared_ptr<token> opae::fpga::types::token::ptr_t;\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#function-c_type","title":"function c_type","text":"
                                                                                    inline fpga_token opae::fpga::types::token::c_type () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#function-get_parent","title":"function get_parent","text":"
                                                                                    ptr_t opae::fpga::types::token::get_parent () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#function-operator-fpga_token","title":"function operator fpga_token","text":"
                                                                                    inline opae::fpga::types::token::operator fpga_token () const\n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#function-token","title":"function ~token","text":"
                                                                                    opae::fpga::types::token::~token () \n
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1token/#function-enumerate","title":"function enumerate","text":"
                                                                                    Obtain a vector of token smart pointers for given search criteria. 
                                                                                    static std::vector< token::ptr_t > opae::fpga::types::token::enumerate (\n    const std::vector< properties::ptr_t > & props\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • props The search criteria. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    A set of known tokens that match the search. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/token.h
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1version/","title":"Class opae::fpga::types::version","text":"
                                                                                    ClassList > opae > fpga > types > version
                                                                                     
                                                                                       
                                                                                    • #include <version.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1version/#public-static-functions","title":"Public Static Functions","text":"Type Name std::string as_string () Get the package version information as a string. fpga_version as_struct () Get the package version information as a struct. std::string build () Get the package build information as a string."},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1version/#public-static-functions-documentation","title":"Public Static Functions Documentation","text":""},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1version/#function-as_string","title":"function as_string","text":"
                                                                                    Get the package version information as a string. 
                                                                                    static std::string opae::fpga::types::version::as_string () \n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    The package version as an std::string object 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1version/#function-as_struct","title":"function as_struct","text":"
                                                                                    Get the package version information as a struct. 
                                                                                    static fpga_version opae::fpga::types::version::as_struct () \n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    The package version as an fpga_version struct 
                                                                                    "},{"location":"opae-code/classopae_1_1fpga_1_1types_1_1version/#function-build","title":"function build","text":"
                                                                                    Get the package build information as a string. 
                                                                                    static std::string opae::fpga::types::version::build () \n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    The package build as an std::string object 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/version.h
                                                                                    "},{"location":"opae-code/structopae__uio/","title":"Struct opae_uio","text":"
                                                                                    ClassList > opae_uio
                                                                                     
                                                                                    OPAE UIO device abstraction. More...
                                                                                     
                                                                                       
                                                                                    • #include <uio.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae__uio/#public-attributes","title":"Public Attributes","text":"Type Name int device_fd char device_path struct opae_uio_device_region * regions"},{"location":"opae-code/structopae__uio/#detailed-description","title":"Detailed Description","text":"
                                                                                    This structure is used to interact with the OPAE UIO API. Each UIO device has a file descriptor that is used to mmap its regions into user address space. Each device also has one or more MMIO regions. 
                                                                                    "},{"location":"opae-code/structopae__uio/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__uio/#variable-device_fd","title":"variable device_fd","text":"
                                                                                    int opae_uio::device_fd;\n
                                                                                    "},{"location":"opae-code/structopae__uio/#variable-device_path","title":"variable device_path","text":"
                                                                                    char opae_uio::device_path[OPAE_UIO_PATH_MAX];\n
                                                                                    "},{"location":"opae-code/structopae__uio/#variable-regions","title":"variable regions","text":"
                                                                                    struct opae_uio_device_region* opae_uio::regions;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/uio.h
                                                                                    "},{"location":"opae-code/structopae__uio__device__region/","title":"Struct opae_uio_device_region","text":"
                                                                                    ClassList > opae_uio_device_region
                                                                                     
                                                                                    MMIO region info. More...
                                                                                     
                                                                                       
                                                                                    • #include <uio.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae__uio__device__region/#public-attributes","title":"Public Attributes","text":"Type Name struct opae_uio_device_region * next uint32_t region_index size_t region_page_offset uint8_t * region_ptr size_t region_size"},{"location":"opae-code/structopae__uio__device__region/#detailed-description","title":"Detailed Description","text":"
                                                                                    Describes a range of mappable MMIO. 
                                                                                    "},{"location":"opae-code/structopae__uio__device__region/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__uio__device__region/#variable-next","title":"variable next","text":"
                                                                                    struct opae_uio_device_region* opae_uio_device_region::next;\n
                                                                                    "},{"location":"opae-code/structopae__uio__device__region/#variable-region_index","title":"variable region_index","text":"
                                                                                    uint32_t opae_uio_device_region::region_index;\n
                                                                                    "},{"location":"opae-code/structopae__uio__device__region/#variable-region_page_offset","title":"variable region_page_offset","text":"
                                                                                    size_t opae_uio_device_region::region_page_offset;\n
                                                                                    "},{"location":"opae-code/structopae__uio__device__region/#variable-region_ptr","title":"variable region_ptr","text":"
                                                                                    uint8_t* opae_uio_device_region::region_ptr;\n
                                                                                    "},{"location":"opae-code/structopae__uio__device__region/#variable-region_size","title":"variable region_size","text":"
                                                                                    size_t opae_uio_device_region::region_size;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/uio.h
                                                                                    "},{"location":"opae-code/structopae__vfio/","title":"Struct opae_vfio","text":"
                                                                                    ClassList > opae_vfio
                                                                                     
                                                                                    OPAE VFIO device abstraction. More...
                                                                                     
                                                                                       
                                                                                    • #include <vfio.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae__vfio/#public-attributes","title":"Public Attributes","text":"Type Name opae_hash_map cont_buffers Map of allocated DMA buffers. char * cont_device \"/dev/vfio/vfio\" int cont_fd Container file descriptor. char * cont_pciaddr PCIe address, eg 0000:00:00.0. struct opae_vfio_iova_range * cont_ranges List of IOVA ranges. struct opae_vfio_device device The VFIO device. struct opae_vfio_group group The VFIO device group. struct mem_alloc iova_alloc Allocator for IOVA space. pthread_mutex_t lock For thread safety."},{"location":"opae-code/structopae__vfio/#detailed-description","title":"Detailed Description","text":"
                                                                                    This structure is used to interact with the OPAE VFIO API. It tracks data related to the VFIO container, group, and device. A mutex is provided for thread safety. 
                                                                                    "},{"location":"opae-code/structopae__vfio/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio/#variable-cont_buffers","title":"variable cont_buffers","text":"
                                                                                    opae_hash_map opae_vfio::cont_buffers;\n
                                                                                    "},{"location":"opae-code/structopae__vfio/#variable-cont_device","title":"variable cont_device","text":"
                                                                                    char* opae_vfio::cont_device;\n
                                                                                    "},{"location":"opae-code/structopae__vfio/#variable-cont_fd","title":"variable cont_fd","text":"
                                                                                    int opae_vfio::cont_fd;\n
                                                                                    "},{"location":"opae-code/structopae__vfio/#variable-cont_pciaddr","title":"variable cont_pciaddr","text":"
                                                                                    char* opae_vfio::cont_pciaddr;\n
                                                                                    "},{"location":"opae-code/structopae__vfio/#variable-cont_ranges","title":"variable cont_ranges","text":"
                                                                                    struct opae_vfio_iova_range* opae_vfio::cont_ranges;\n
                                                                                    "},{"location":"opae-code/structopae__vfio/#variable-device","title":"variable device","text":"
                                                                                    struct opae_vfio_device opae_vfio::device;\n
                                                                                    "},{"location":"opae-code/structopae__vfio/#variable-group","title":"variable group","text":"
                                                                                    struct opae_vfio_group opae_vfio::group;\n
                                                                                    "},{"location":"opae-code/structopae__vfio/#variable-iova_alloc","title":"variable iova_alloc","text":"
                                                                                    struct mem_alloc opae_vfio::iova_alloc;\n
                                                                                    "},{"location":"opae-code/structopae__vfio/#variable-lock","title":"variable lock","text":"
                                                                                    pthread_mutex_t opae_vfio::lock;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                                                                    "},{"location":"opae-code/structopae__vfio__buffer/","title":"Struct opae_vfio_buffer","text":"
                                                                                    ClassList > opae_vfio_buffer
                                                                                     
                                                                                    System DMA buffer. More...
                                                                                     
                                                                                       
                                                                                    • #include <vfio.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae__vfio__buffer/#public-attributes","title":"Public Attributes","text":"Type Name uint64_t buffer_iova Buffer IOVA address. uint8_t * buffer_ptr Buffer virtual address. size_t buffer_size Buffer size. int flags See opae_vfio_buffer_flags."},{"location":"opae-code/structopae__vfio__buffer/#detailed-description","title":"Detailed Description","text":"
                                                                                    Describes a system memory address space that is capable of DMA. 
                                                                                    "},{"location":"opae-code/structopae__vfio__buffer/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__buffer/#variable-buffer_iova","title":"variable buffer_iova","text":"
                                                                                    uint64_t opae_vfio_buffer::buffer_iova;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__buffer/#variable-buffer_ptr","title":"variable buffer_ptr","text":"
                                                                                    uint8_t* opae_vfio_buffer::buffer_ptr;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__buffer/#variable-buffer_size","title":"variable buffer_size","text":"
                                                                                    size_t opae_vfio_buffer::buffer_size;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__buffer/#variable-flags","title":"variable flags","text":"
                                                                                    int opae_vfio_buffer::flags;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                                                                    "},{"location":"opae-code/structopae__vfio__device/","title":"Struct opae_vfio_device","text":"
                                                                                    ClassList > opae_vfio_device
                                                                                     
                                                                                    VFIO device. More...
                                                                                     
                                                                                       
                                                                                    • #include <vfio.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae__vfio__device/#public-attributes","title":"Public Attributes","text":"Type Name uint64_t device_config_offset Offset of PCIe config space. int device_fd Device file descriptor. uint32_t device_num_irqs IRQ index count. uint32_t device_num_regions Total MMIO region count. struct opae_vfio_device_irq * irqs IRQ list pointer. struct opae_vfio_device_region * regions Region list pointer."},{"location":"opae-code/structopae__vfio__device/#detailed-description","title":"Detailed Description","text":"
                                                                                    Each VFIO device has a file descriptor that is used to query information about the device MMIO regions and config space. 
                                                                                    "},{"location":"opae-code/structopae__vfio__device/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__device/#variable-device_config_offset","title":"variable device_config_offset","text":"
                                                                                    uint64_t opae_vfio_device::device_config_offset;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__device/#variable-device_fd","title":"variable device_fd","text":"
                                                                                    int opae_vfio_device::device_fd;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__device/#variable-device_num_irqs","title":"variable device_num_irqs","text":"
                                                                                    uint32_t opae_vfio_device::device_num_irqs;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__device/#variable-device_num_regions","title":"variable device_num_regions","text":"
                                                                                    uint32_t opae_vfio_device::device_num_regions;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__device/#variable-irqs","title":"variable irqs","text":"
                                                                                    struct opae_vfio_device_irq* opae_vfio_device::irqs;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__device/#variable-regions","title":"variable regions","text":"
                                                                                    struct opae_vfio_device_region* opae_vfio_device::regions;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                                                                    "},{"location":"opae-code/structopae__vfio__device__irq/","title":"Struct opae_vfio_device_irq","text":"
                                                                                    ClassList > opae_vfio_device_irq
                                                                                     
                                                                                    Interrupt info. More...
                                                                                     
                                                                                       
                                                                                    • #include <vfio.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae__vfio__device__irq/#public-attributes","title":"Public Attributes","text":"Type Name uint32_t count Number of IRQs at this index. int32_t * event_fds Event file descriptors. uint32_t flags Flags. uint32_t index The IRQ index. int32_t * masks IRQ masks. struct opae_vfio_device_irq * next Pointer to next in list."},{"location":"opae-code/structopae__vfio__device__irq/#detailed-description","title":"Detailed Description","text":"
                                                                                    Describes an interrupt capability. 
                                                                                    "},{"location":"opae-code/structopae__vfio__device__irq/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__device__irq/#variable-count","title":"variable count","text":"
                                                                                    uint32_t opae_vfio_device_irq::count;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__device__irq/#variable-event_fds","title":"variable event_fds","text":"
                                                                                    int32_t* opae_vfio_device_irq::event_fds;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__device__irq/#variable-flags","title":"variable flags","text":"
                                                                                    Flags. 
                                                                                    uint32_t opae_vfio_device_irq::flags;\n
                                                                                     
                                                                                    See struct vfio_irq_info. 
                                                                                    "},{"location":"opae-code/structopae__vfio__device__irq/#variable-index","title":"variable index","text":"
                                                                                    uint32_t opae_vfio_device_irq::index;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__device__irq/#variable-masks","title":"variable masks","text":"
                                                                                    int32_t* opae_vfio_device_irq::masks;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__device__irq/#variable-next","title":"variable next","text":"
                                                                                    struct opae_vfio_device_irq* opae_vfio_device_irq::next;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                                                                    "},{"location":"opae-code/structopae__vfio__device__region/","title":"Struct opae_vfio_device_region","text":"
                                                                                    ClassList > opae_vfio_device_region
                                                                                     
                                                                                    MMIO region info. More...
                                                                                     
                                                                                       
                                                                                    • #include <vfio.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae__vfio__device__region/#public-attributes","title":"Public Attributes","text":"Type Name struct opae_vfio_device_region * next Pointer to next in list. uint32_t region_index Region index, from 0. uint8_t * region_ptr Virtual address of region. size_t region_size Size of region. struct opae_vfio_sparse_info * region_sparse For sparse regions."},{"location":"opae-code/structopae__vfio__device__region/#detailed-description","title":"Detailed Description","text":"
                                                                                    Describes a range of mappable MMIO. 
                                                                                    "},{"location":"opae-code/structopae__vfio__device__region/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__device__region/#variable-next","title":"variable next","text":"
                                                                                    struct opae_vfio_device_region* opae_vfio_device_region::next;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__device__region/#variable-region_index","title":"variable region_index","text":"
                                                                                    uint32_t opae_vfio_device_region::region_index;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__device__region/#variable-region_ptr","title":"variable region_ptr","text":"
                                                                                    uint8_t* opae_vfio_device_region::region_ptr;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__device__region/#variable-region_size","title":"variable region_size","text":"
                                                                                    size_t opae_vfio_device_region::region_size;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__device__region/#variable-region_sparse","title":"variable region_sparse","text":"
                                                                                    struct opae_vfio_sparse_info* opae_vfio_device_region::region_sparse;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                                                                    "},{"location":"opae-code/structopae__vfio__group/","title":"Struct opae_vfio_group","text":"
                                                                                    ClassList > opae_vfio_group
                                                                                     
                                                                                    VFIO group. More...
                                                                                     
                                                                                       
                                                                                    • #include <vfio.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae__vfio__group/#public-attributes","title":"Public Attributes","text":"Type Name char * group_device Full path to the group device. int group_fd File descriptor for the group device."},{"location":"opae-code/structopae__vfio__group/#detailed-description","title":"Detailed Description","text":"
                                                                                    Each device managed by vfio-pci belongs to a VFIO group rooted at /dev/vfio/N, where N is the group number. 
                                                                                    "},{"location":"opae-code/structopae__vfio__group/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__group/#variable-group_device","title":"variable group_device","text":"
                                                                                    char* opae_vfio_group::group_device;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__group/#variable-group_fd","title":"variable group_fd","text":"
                                                                                    int opae_vfio_group::group_fd;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                                                                    "},{"location":"opae-code/structopae__vfio__iova__range/","title":"Struct opae_vfio_iova_range","text":"
                                                                                    ClassList > opae_vfio_iova_range
                                                                                     
                                                                                    IO Virtual Address Range. More...
                                                                                     
                                                                                       
                                                                                    • #include <vfio.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae__vfio__iova__range/#public-attributes","title":"Public Attributes","text":"Type Name uint64_t end End of this range of offsets. struct opae_vfio_iova_range * next Pointer to next in list. uint64_t start Start of this range of offsets."},{"location":"opae-code/structopae__vfio__iova__range/#detailed-description","title":"Detailed Description","text":"
                                                                                    A range of allocatable IOVA offsets. Used for mapping DMA buffers. 
                                                                                    "},{"location":"opae-code/structopae__vfio__iova__range/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__iova__range/#variable-end","title":"variable end","text":"
                                                                                    uint64_t opae_vfio_iova_range::end;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__iova__range/#variable-next","title":"variable next","text":"
                                                                                    struct opae_vfio_iova_range* opae_vfio_iova_range::next;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__iova__range/#variable-start","title":"variable start","text":"
                                                                                    uint64_t opae_vfio_iova_range::start;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                                                                    "},{"location":"opae-code/structopae__vfio__sparse__info/","title":"Struct opae_vfio_sparse_info","text":"
                                                                                    ClassList > opae_vfio_sparse_info
                                                                                     
                                                                                    MMIO sparse-mappable region info. More...
                                                                                     
                                                                                       
                                                                                    • #include <vfio.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structopae__vfio__sparse__info/#public-attributes","title":"Public Attributes","text":"Type Name uint32_t index Region index, from 0. struct opae_vfio_sparse_info * next Pointer to next in list. uint32_t offset Offset of sparse region. uint8_t * ptr Virtual address of sparse region. uint32_t size Size of sparse region."},{"location":"opae-code/structopae__vfio__sparse__info/#detailed-description","title":"Detailed Description","text":"
                                                                                    Describes a range of sparse-mappable MMIO, for MMIO ranges that have non-contiguous addresses. 
                                                                                    "},{"location":"opae-code/structopae__vfio__sparse__info/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structopae__vfio__sparse__info/#variable-index","title":"variable index","text":"
                                                                                    uint32_t opae_vfio_sparse_info::index;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__sparse__info/#variable-next","title":"variable next","text":"
                                                                                    struct opae_vfio_sparse_info* opae_vfio_sparse_info::next;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__sparse__info/#variable-offset","title":"variable offset","text":"
                                                                                    uint32_t opae_vfio_sparse_info::offset;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__sparse__info/#variable-ptr","title":"variable ptr","text":"
                                                                                    uint8_t* opae_vfio_sparse_info::ptr;\n
                                                                                    "},{"location":"opae-code/structopae__vfio__sparse__info/#variable-size","title":"variable size","text":"
                                                                                    uint32_t opae_vfio_sparse_info::size;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                                                                    "},{"location":"opae-code/structras__inject__error/","title":"Struct ras_inject_error","text":"
                                                                                    ClassList > ras_inject_error
                                                                                    "},{"location":"opae-code/structras__inject__error/#public-attributes","title":"Public Attributes","text":"Type Name union ras_inject_error::@0 @1 uint64_t catastrophicr_error uint64_t csr uint64_t fatal_error uint64_t nonfatal_error uint64_t rsvd"},{"location":"opae-code/structras__inject__error/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structras__inject__error/#variable-1","title":"variable @1","text":"
                                                                                    union ras_inject_error::@0 ras_inject_error::@1;\n
                                                                                    "},{"location":"opae-code/structras__inject__error/#variable-catastrophicr_error","title":"variable catastrophicr_error","text":"
                                                                                    uint64_t ras_inject_error::catastrophicr_error;\n
                                                                                    "},{"location":"opae-code/structras__inject__error/#variable-csr","title":"variable csr","text":"
                                                                                    uint64_t ras_inject_error::csr;\n
                                                                                    "},{"location":"opae-code/structras__inject__error/#variable-fatal_error","title":"variable fatal_error","text":"
                                                                                    uint64_t ras_inject_error::fatal_error;\n
                                                                                    "},{"location":"opae-code/structras__inject__error/#variable-nonfatal_error","title":"variable nonfatal_error","text":"
                                                                                    uint64_t ras_inject_error::nonfatal_error;\n
                                                                                    "},{"location":"opae-code/structras__inject__error/#variable-rsvd","title":"variable rsvd","text":"
                                                                                    uint64_t ras_inject_error::rsvd;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/samples/hello_events/hello_events.c
                                                                                    "},{"location":"opae-code/namespacestd/","title":"Namespace std","text":"
                                                                                    Namespace List > std
                                                                                     
                                                                                    The documentation for this class was generated from the following file [generated]
                                                                                    "},{"location":"opae-code/structthreshold/","title":"Struct threshold","text":"
                                                                                    ClassList > threshold
                                                                                     
                                                                                    Threshold struct. 
                                                                                     
                                                                                       
                                                                                    • #include <types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/structthreshold/#public-attributes","title":"Public Attributes","text":"Type Name uint32_t is_valid char threshold_name double value"},{"location":"opae-code/structthreshold/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/structthreshold/#variable-is_valid","title":"variable is_valid","text":"
                                                                                    uint32_t threshold::is_valid;\n
                                                                                    "},{"location":"opae-code/structthreshold/#variable-threshold_name","title":"variable threshold_name","text":"
                                                                                    char threshold::threshold_name[FPGA_METRIC_STR_SIZE];\n
                                                                                    "},{"location":"opae-code/structthreshold/#variable-value","title":"variable value","text":"
                                                                                    double threshold::value;\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                                                                    "},{"location":"opae-code/dir_49e56c817e5e54854c35e136979f97ca/","title":"Dir docs","text":"
                                                                                    FileList > docs
                                                                                    "},{"location":"opae-code/dir_49e56c817e5e54854c35e136979f97ca/#directories","title":"Directories","text":"Type Name dir sw 
                                                                                    The documentation for this class was generated from the following file docs/
                                                                                    "},{"location":"opae-code/dir_55721a669a8e0900d975c02921addb49/","title":"Dir docs/sw","text":"
                                                                                    FileList > docs > sw
                                                                                    "},{"location":"opae-code/dir_55721a669a8e0900d975c02921addb49/#directories","title":"Directories","text":"Type Name dir include dir samples 
                                                                                    The documentation for this class was generated from the following file docs/sw/
                                                                                    "},{"location":"opae-code/dir_97b4588afba69bf89bbe554642ac6431/","title":"Dir docs/sw/include","text":"
                                                                                    FileList > docs > sw > include
                                                                                    "},{"location":"opae-code/dir_97b4588afba69bf89bbe554642ac6431/#directories","title":"Directories","text":"Type Name dir opae 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/
                                                                                    "},{"location":"opae-code/dir_ade97cd9199f278c0723672dd8647ba4/","title":"Dir docs/sw/include/opae","text":"
                                                                                    FileList > docs > sw > include > opae
                                                                                    "},{"location":"opae-code/dir_ade97cd9199f278c0723672dd8647ba4/#files","title":"Files","text":"Type Name file access.h Functions to acquire, release, and reset OPAE FPGA resources. file buffer.h Functions for allocating and sharing system memory with an FPGA accelerator. file enum.h APIs for resource enumeration and managing tokens. file error.h Functions for reading and clearing errors in resources. file event.h Functions for registering events and managing the lifecycle for fpga_event_handle s. file fpga.h FPGA API. file hash_map.h A general-purpose hybrid array/list hash map implementation. file init.h Initialization routine. file log.h file manage.h Functions for managing FPGA configurations. file mem_alloc.h file metrics.h Functions for Discover/ Enumerates metrics and retrieves values. file mmio.h Functions for mapping and accessing MMIO space. file properties.h Functions for examining and manipulating fpga_properties objects. file sysobject.h Functions to read/write from system objects. file types.h Type definitions for FPGA API. file types_enum.h Definitions of enumerated types for the OPAE C API. file uio.h APIs to manage a PCIe device via UIO. file umsg.h FPGA UMsg API. file userclk.h Functions for setting and get afu user clock. file utils.h Utility functions and macros for the FPGA API. file version.h file vfio.h APIs to manage a PCIe device via vfio-pci."},{"location":"opae-code/dir_ade97cd9199f278c0723672dd8647ba4/#directories","title":"Directories","text":"Type Name dir cxx 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/
                                                                                    "},{"location":"opae-code/access_8h/","title":"File access.h","text":"
                                                                                    FileList > docs > sw > include > opae > access.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Functions to acquire, release, and reset OPAE FPGA resources. 
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/access_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaClose (fpga_handle handle) Close a previously opened FPGA object. fpga_result fpgaOpen (fpga_token token, fpga_handle * handle, int flags) Open an FPGA object. fpga_result fpgaReset (fpga_handle handle) Reset an FPGA object."},{"location":"opae-code/access_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/access_8h/#function-fpgaclose","title":"function fpgaClose","text":"
                                                                                    Close a previously opened FPGA object. 
                                                                                    fpga_result fpgaClose (\n    fpga_handle handle\n) \n
                                                                                     
                                                                                    Relinquishes ownership of a previously fpgaOpen()ed resource. This enables others to acquire ownership if the resource was opened exclusively. Also deallocates / unmaps MMIO and UMsg memory areas.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened FPGA object 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if handle does not refer to an acquired resource, or if handle is NULL. FPGA_EXCEPTION if an internal error occurred while accessing the handle. 
                                                                                    "},{"location":"opae-code/access_8h/#function-fpgaopen","title":"function fpgaOpen","text":"
                                                                                    Open an FPGA object. 
                                                                                    fpga_result fpgaOpen (\n    fpga_token token,\n    fpga_handle * handle,\n    int flags\n) \n
                                                                                     
                                                                                    Acquires ownership of the FPGA resource referred to by 'token'.
                                                                                     
                                                                                    Most often this will be used to open an accelerator object to directly interact with an accelerator function, or to open an FPGA object to perform management functions.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • token Pointer to token identifying resource to acquire ownership of 
                                                                                      •  
                                                                                    • handle Pointer to preallocated memory to place a handle in. This handle will be used in subsequent API calls. 
                                                                                      •  
                                                                                    • flags One of the following flags:
                                                                                         
                                                                                      • FPGA_OPEN_SHARED allows the resource to be opened multiple times (not supported in ASE) Shared resources (including buffers) are released when all associated handles have been closed (either explicitly with fpgaClose() or by process termination). 
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_NOT_FOUND if the resource for 'token' could not be found. FPGA_INVALID_PARAM if 'token' does not refer to a resource that can be opened, or if either argument is NULL or invalid. FPGA_EXCEPTION if an internal exception occurred while creating the handle. FPGA_NO_DRIVER if the driver is not loaded. FPGA_BUSY if trying to open a resource that has already been opened in exclusive mode. FPGA_NO_ACCESS if the current process' privileges are not sufficient to open the resource. 
                                                                                    "},{"location":"opae-code/access_8h/#function-fpgareset","title":"function fpgaReset","text":"
                                                                                    Reset an FPGA object. 
                                                                                    fpga_result fpgaReset (\n    fpga_handle handle\n) \n
                                                                                     
                                                                                    Performs an accelerator reset.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened FPGA object 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if handle does not refer to an acquired resource or to a resource that cannot be reset. FPGA_EXCEPTION if an internal error occurred while trying to access the handle or resetting the resource. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/access.h
                                                                                    "},{"location":"opae-code/access_8h_source/","title":"File access.h","text":"
                                                                                    File List > docs > sw > include > opae > access.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_ACCESS_H__\n#define __FPGA_ACCESS_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaOpen(fpga_token token, fpga_handle *handle,\n             int flags);\n\nfpga_result fpgaClose(fpga_handle handle);\n\nfpga_result fpgaReset(fpga_handle handle);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_ACCESS_H__\n
                                                                                    "},{"location":"opae-code/buffer_8h/","title":"File buffer.h","text":"
                                                                                    FileList > docs > sw > include > opae > buffer.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Functions for allocating and sharing system memory with an FPGA accelerator. More...
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/buffer_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaGetIOAddress (fpga_handle handle, uint64_t wsid, uint64_t * ioaddr) Retrieve base IO address for buffer. fpga_result fpgaPrepareBuffer (fpga_handle handle, uint64_t len, void ** buf_addr, uint64_t * wsid, int flags) Prepare a shared memory buffer. fpga_result fpgaReleaseBuffer (fpga_handle handle, uint64_t wsid) Release a shared memory buffer."},{"location":"opae-code/buffer_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    To share memory between a software application and an FPGA accelerator, these functions set up system components (e.g. an IOMMU) to allow accelerator access to a provided memory region.
                                                                                     
                                                                                    There are a number of restrictions on what memory can be shared, depending on platform capabilities. Usually, FPGA accelerators to not have access to virtual address mappings of the CPU, so they can only access physical addresses. To support this, the OPAE C library on Linux uses hugepages to allocate large, contiguous pages of physical memory that can be shared with an accelerator. It also supports sharing memory that has already been allocated by an application, as long as that memory satisfies the requirements of being physically contigous and page-aligned. 
                                                                                    "},{"location":"opae-code/buffer_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/buffer_8h/#function-fpgagetioaddress","title":"function fpgaGetIOAddress","text":"
                                                                                    Retrieve base IO address for buffer. 
                                                                                    fpga_result fpgaGetIOAddress (\n    fpga_handle handle,\n    uint64_t wsid,\n    uint64_t * ioaddr\n) \n
                                                                                     
                                                                                    This function is used to acquire the physical base address (on some platforms called IO Virtual Address or IOVA) for a shared buffer identified by wsid.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This function will disappear once the APIs for secure sharing of buffer addresses is implemented.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource 
                                                                                      •  
                                                                                    • wsid Buffer handle / workspace ID referring to the buffer for which the IO address is requested 
                                                                                      •  
                                                                                    • ioaddr Pointer to memory where the IO address will be returned 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if invalid parameters were provided, or if the parameter combination is not valid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. FPGA_NOT_FOUND if wsid does not refer to a previously shared buffer. 
                                                                                    "},{"location":"opae-code/buffer_8h/#function-fpgapreparebuffer","title":"function fpgaPrepareBuffer","text":"
                                                                                    Prepare a shared memory buffer. 
                                                                                    fpga_result fpgaPrepareBuffer (\n    fpga_handle handle,\n    uint64_t len,\n    void ** buf_addr,\n    uint64_t * wsid,\n    int flags\n) \n
                                                                                     
                                                                                    Prepares a memory buffer for shared access between an accelerator and the calling process. This may either include allocation of physical memory, or preparation of already allocated memory for sharing. The latter case is indicated by supplying the FPGA_BUF_PREALLOCATED flag.
                                                                                     
                                                                                    This function will ask the driver to pin the indicated memory (make it non-swappable), and program the IOMMU to allow access from the accelerator. If the buffer was not pre-allocated (flag FPGA_BUF_PREALLOCATED), the function will also allocate physical memory of the requested size and map the memory into the caller's process' virtual address space. It returns in 'wsid' an fpga_buffer object that can be used to program address registers in the accelerator for shared access to the memory.
                                                                                     
                                                                                    When using FPGA_BUF_PREALLOCATED, the input len must be a non-zero multiple of the page size, else the function returns FPGA_INVALID_PARAM. When not using FPGA_BUF_PREALLOCATED, the input len is rounded up to the nearest multiple of page size.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource 
                                                                                      •  
                                                                                    • len Length of the buffer to allocate/prepare in bytes 
                                                                                      •  
                                                                                    • buf_addr Virtual address of buffer. Contents may be NULL (OS will choose mapping) or non-NULL (OS will take contents as a hint for the virtual address). 
                                                                                      •  
                                                                                    • wsid Handle to the allocated/prepared buffer to be used with other functions 
                                                                                      •  
                                                                                    • flags Flags. FPGA_BUF_PREALLOCATED indicates that memory pointed at in '*buf_addr' is already allocated an mapped into virtual memory. FPGA_BUF_READ_ONLY pins pages with only read access from the FPGA. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_NO_MEMORY if the requested memory could not be allocated. FPGA_INVALID_PARAM if invalid parameters were provided, or if the parameter combination is not valid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    As a special case, when FPGA_BUF_PREALLOCATED is present in flags, if len == 0 and buf_addr == NULL, then the function returns FPGA_OK if pre-allocated buffers are supported. In this case, a return value other than FPGA_OK indicates that pre-allocated buffers are not supported. 
                                                                                    "},{"location":"opae-code/buffer_8h/#function-fpgareleasebuffer","title":"function fpgaReleaseBuffer","text":"
                                                                                    Release a shared memory buffer. 
                                                                                    fpga_result fpgaReleaseBuffer (\n    fpga_handle handle,\n    uint64_t wsid\n) \n
                                                                                     
                                                                                    Releases a previously prepared shared buffer. If the buffer was allocated using fpgaPrepareBuffer (FPGA_BUF_PREALLOCATED was not specified), this call will deallocate/free that memory. Otherwise, it will only be returned to it's previous state (pinned/unpinned, cached/non-cached).
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource 
                                                                                      •  
                                                                                    • wsid Handle to the allocated/prepared buffer 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if invalid parameters were provided, or if the parameter combination is not valid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/buffer.h
                                                                                    "},{"location":"opae-code/buffer_8h_source/","title":"File buffer.h","text":"
                                                                                    File List > docs > sw > include > opae > buffer.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_BUFFER_H__\n#define __FPGA_BUFFER_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaPrepareBuffer(fpga_handle handle,\n                  uint64_t len,\n                  void **buf_addr, uint64_t *wsid, int flags);\n\nfpga_result fpgaReleaseBuffer(fpga_handle handle, uint64_t wsid);\n\nfpga_result fpgaGetIOAddress(fpga_handle handle, uint64_t wsid,\n                 uint64_t *ioaddr);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_BUFFER_H__\n
                                                                                    "},{"location":"opae-code/dir_3731a7e5669218b938d292e51b4e531c/","title":"Dir docs/sw/include/opae/cxx","text":"
                                                                                    FileList > cxx
                                                                                    "},{"location":"opae-code/dir_3731a7e5669218b938d292e51b4e531c/#files","title":"Files","text":"Type Name file core.h"},{"location":"opae-code/dir_3731a7e5669218b938d292e51b4e531c/#directories","title":"Directories","text":"Type Name dir core 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/
                                                                                    "},{"location":"opae-code/core_8h/","title":"File core.h","text":"
                                                                                    FileList > cxx > core.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <opae/cxx/core/errors.h>
                                                                                      •  
                                                                                    • #include <opae/cxx/core/events.h>
                                                                                      •  
                                                                                    • #include <opae/cxx/core/except.h>
                                                                                      •  
                                                                                    • #include <opae/cxx/core/handle.h>
                                                                                      •  
                                                                                    • #include <opae/cxx/core/properties.h>
                                                                                      •  
                                                                                    • #include <opae/cxx/core/pvalue.h>
                                                                                      •  
                                                                                    • #include <opae/cxx/core/shared_buffer.h>
                                                                                      •  
                                                                                    • #include <opae/cxx/core/token.h>
                                                                                      •  
                                                                                    • #include <opae/cxx/core/version.h>
                                                                                      •  
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core.h
                                                                                    "},{"location":"opae-code/core_8h_source/","title":"File core.h","text":"
                                                                                    File List > cxx > core.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2020, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/cxx/core/errors.h>\n#include <opae/cxx/core/events.h>\n#include <opae/cxx/core/except.h>\n#include <opae/cxx/core/handle.h>\n#include <opae/cxx/core/properties.h>\n#include <opae/cxx/core/pvalue.h>\n#include <opae/cxx/core/shared_buffer.h>\n#include <opae/cxx/core/token.h>\n#include <opae/cxx/core/version.h>\n
                                                                                    "},{"location":"opae-code/dir_23b1b9d7ef54caa3fa7bb54d9bc2d47a/","title":"Dir docs/sw/include/opae/cxx/core","text":"
                                                                                    FileList > core
                                                                                    "},{"location":"opae-code/dir_23b1b9d7ef54caa3fa7bb54d9bc2d47a/#files","title":"Files","text":"Type Name file errors.h file events.h file except.h file handle.h file properties.h file pvalue.h file shared_buffer.h file sysobject.h file token.h file version.h 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/
                                                                                    "},{"location":"opae-code/errors_8h/","title":"File errors.h","text":"
                                                                                    FileList > core > errors.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <opae/cxx/core/token.h>
                                                                                      •  
                                                                                    • #include <opae/types_enum.h>
                                                                                      •  
                                                                                    • #include <memory>
                                                                                      •  
                                                                                    "},{"location":"opae-code/errors_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/errors_8h/#classes","title":"Classes","text":"Type Name class error An error object represents an error register for a resource. 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/errors.h
                                                                                    "},{"location":"opae-code/errors_8h_source/","title":"File errors.h","text":"
                                                                                    File List > core > errors.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n\n#include <opae/cxx/core/token.h>\n#include <opae/types_enum.h>\n\n#include <memory>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass error {\n public:\n  typedef std::shared_ptr<error> ptr_t;\n\n  error(const error &e) = delete;\n\n  error &operator=(const error &e) = delete;\n\n  static error::ptr_t get(token::ptr_t tok, uint32_t num);\n\n  std::string name() { return error_info_.name; }\n\n  bool can_clear() { return error_info_.can_clear; }\n\n  uint64_t read_value();\n\n  ~error() {}\n\n  fpga_error_info c_type() const { return error_info_; }\n\n private:\n  error(token::ptr_t token, uint32_t num);\n  token::ptr_t token_;\n  fpga_error_info error_info_;\n  uint32_t error_num_;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                                                                    "},{"location":"opae-code/events_8h/","title":"File events.h","text":"
                                                                                    FileList > core > events.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <opae/cxx/core/handle.h>
                                                                                      •  
                                                                                    • #include <opae/types_enum.h>
                                                                                      •  
                                                                                    • #include <memory>
                                                                                      •  
                                                                                    "},{"location":"opae-code/events_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/events_8h/#classes","title":"Classes","text":"Type Name class event Wraps fpga event routines in OPAE C. struct type_t C++ struct that is interchangeable with fpga_event_type enum. 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/events.h
                                                                                    "},{"location":"opae-code/events_8h_source/","title":"File events.h","text":"
                                                                                    File List > core > events.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n\n#include <opae/cxx/core/handle.h>\n#include <opae/types_enum.h>\n\n#include <memory>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass event {\n public:\n  typedef std::shared_ptr<event> ptr_t;\n\n  virtual ~event();\n\n  struct type_t {\n    type_t(fpga_event_type c_type) : type_(c_type) {}\n\n    operator fpga_event_type() { return type_; }\n\n    static constexpr fpga_event_type interrupt = FPGA_EVENT_INTERRUPT;\n    static constexpr fpga_event_type error = FPGA_EVENT_ERROR;\n    static constexpr fpga_event_type power_thermal = FPGA_EVENT_POWER_THERMAL;\n\n   private:\n    fpga_event_type type_;\n  };\n\n  fpga_event_handle get() { return event_handle_; }\n\n  operator fpga_event_handle();\n\n  static event::ptr_t register_event(handle::ptr_t h, event::type_t t,\n                                     int flags = 0);\n\n  int os_object() const;\n\n private:\n  event(handle::ptr_t h, event::type_t t, fpga_event_handle event_h);\n  handle::ptr_t handle_;\n  event::type_t type_;\n  fpga_event_handle event_handle_;\n  int os_object_;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                                                                    "},{"location":"opae-code/except_8h/","title":"File except.h","text":"
                                                                                    FileList > core > except.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <opae/types_enum.h>
                                                                                      •  
                                                                                    • #include <cstdint>
                                                                                      •  
                                                                                    • #include <exception>
                                                                                      •  
                                                                                    "},{"location":"opae-code/except_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types namespace detail"},{"location":"opae-code/except_8h/#classes","title":"Classes","text":"Type Name class busy busy exception class except Generic OPAE exception. class exception exception exception class invalid_param invalid_param exception class no_access no_access exception class no_daemon no_daemon exception class no_driver no_driver exception class no_memory no_memory exception class not_found not_found exception class not_supported not_supported exception class reconf_error reconf_error exception class src_location Identify a particular line in a source file."},{"location":"opae-code/except_8h/#macros","title":"Macros","text":"Type Name define ASSERT_FPGA_OK \u00ae Macro to check of result is FPGA_OK If not, throw exception that corresponds to the result code. define OPAECXX_HERE opae::fpga::types::src_location(__FILE__, __func__, __LINE__)Construct a src_location object for the current source line."},{"location":"opae-code/except_8h/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/except_8h/#define-assert_fpga_ok","title":"define ASSERT_FPGA_OK","text":"
                                                                                    #define ASSERT_FPGA_OK (\n    r\n) opae::fpga::types::detail::assert_fpga_ok( \\\n      r, opae::fpga::types::src_location (__FILE__, __func__, __LINE__));\n
                                                                                    "},{"location":"opae-code/except_8h/#define-opaecxx_here","title":"define OPAECXX_HERE","text":"
                                                                                    #define OPAECXX_HERE opae::fpga::types::src_location (__FILE__, __func__, __LINE__)\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/except.h
                                                                                    "},{"location":"opae-code/except_8h_source/","title":"File except.h","text":"
                                                                                    File List > core > except.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/types_enum.h>\n\n#include <cstdint>\n#include <exception>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass src_location {\n public:\n  src_location(const char *file, const char *fn, int line) noexcept;\n\n  src_location(const src_location &other) noexcept;\n\n  src_location &operator=(const src_location &other) noexcept;\n\n  const char *file() const noexcept;\n\n  const char *fn() const noexcept { return fn_; }\n\n  int line() const noexcept { return line_; }\n\n private:\n  const char *file_;\n  const char *fn_;\n  int line_;\n};\n\n#define OPAECXX_HERE \\\n  opae::fpga::types::src_location(__FILE__, __func__, __LINE__)\n\nclass except : public std::exception {\n public:\n  static const std::size_t MAX_EXCEPT = 256;\n\n  except(src_location loc) noexcept;\n\n  except(fpga_result res, src_location loc) noexcept;\n\n  except(fpga_result res, const char *msg, src_location loc) noexcept;\n\n  virtual const char *what() const noexcept override;\n\n  operator fpga_result() const noexcept { return res_; }\n\n protected:\n  fpga_result res_;\n  const char *msg_;\n  src_location loc_;\n  mutable char buf_[MAX_EXCEPT];\n};\n\nclass invalid_param : public except {\n public:\n  invalid_param(src_location loc) noexcept\n      : except(FPGA_INVALID_PARAM, \"failed with return code FPGA_INVALID_PARAM\",\n               loc) {}\n};\n\nclass busy : public except {\n public:\n  busy(src_location loc) noexcept\n      : except(FPGA_BUSY, \"failed with return code FPGA_BUSY\", loc) {}\n};\n\nclass exception : public except {\n public:\n  exception(src_location loc) noexcept\n      : except(FPGA_EXCEPTION, \"failed with return code FPGA_EXCEPTION\", loc) {}\n};\n\nclass not_found : public except {\n public:\n  not_found(src_location loc) noexcept\n      : except(FPGA_NOT_FOUND, \"failed with return code FPGA_NOT_FOUND\", loc) {}\n};\n\nclass no_memory : public except {\n public:\n  no_memory(src_location loc) noexcept\n      : except(FPGA_NO_MEMORY, \"failed with return code FPGA_NO_MEMORY\", loc) {}\n};\n\nclass not_supported : public except {\n public:\n  not_supported(src_location loc) noexcept\n      : except(FPGA_NOT_SUPPORTED, \"failed with return code FPGA_NOT_SUPPORTED\",\n               loc) {}\n};\n\nclass no_driver : public except {\n public:\n  no_driver(src_location loc) noexcept\n      : except(FPGA_NO_DRIVER, \"failed with return code FPGA_NO_DRIVER\", loc) {}\n};\n\nclass no_daemon : public except {\n public:\n  no_daemon(src_location loc) noexcept\n      : except(FPGA_NO_DAEMON, \"failed with return code FPGA_NO_DAEMON\", loc) {}\n};\n\nclass no_access : public except {\n public:\n  no_access(src_location loc) noexcept\n      : except(FPGA_NO_ACCESS, \"failed with return code FPGA_NO_ACCESS\", loc) {}\n};\n\nclass reconf_error : public except {\n public:\n  reconf_error(src_location loc) noexcept\n      : except(FPGA_RECONF_ERROR, \"failed with return code FPGA_RECONF_ERROR\",\n               loc) {}\n};\n\nnamespace detail {\n\ntypedef bool (*exception_fn)(fpga_result,\n                             const opae::fpga::types::src_location &loc);\n\ntemplate <typename T>\nconstexpr bool is_ok(fpga_result result,\n                     const opae::fpga::types::src_location &loc) {\n  return result == FPGA_OK ? true : throw T(loc);\n}\n\nstatic exception_fn opae_exceptions[12] = {\n    is_ok<opae::fpga::types::invalid_param>,\n    is_ok<opae::fpga::types::busy>,\n    is_ok<opae::fpga::types::exception>,\n    is_ok<opae::fpga::types::not_found>,\n    is_ok<opae::fpga::types::no_memory>,\n    is_ok<opae::fpga::types::not_supported>,\n    is_ok<opae::fpga::types::no_driver>,\n    is_ok<opae::fpga::types::no_daemon>,\n    is_ok<opae::fpga::types::no_access>,\n    is_ok<opae::fpga::types::reconf_error>};\n\nstatic inline void assert_fpga_ok(fpga_result result,\n                                  const opae::fpga::types::src_location &loc) {\n  if (result > FPGA_OK && result <= FPGA_RECONF_ERROR)\n    // our exception table above starts at invalid_param with index 0\n    // but FPGA_INVALID_PARAM is actually enum 1 - let's account for that\n    opae_exceptions[result - 1](result, loc);\n}\n\n}  // end of namespace detail\n\n#define ASSERT_FPGA_OK(r)                    \\\n  opae::fpga::types::detail::assert_fpga_ok( \\\n      r, opae::fpga::types::src_location(__FILE__, __func__, __LINE__));\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                                                                    "},{"location":"opae-code/handle_8h/","title":"File handle.h","text":"
                                                                                    FileList > core > handle.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <opae/cxx/core/token.h>
                                                                                      •  
                                                                                    • #include <opae/enum.h>
                                                                                      •  
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    • #include <memory>
                                                                                      •  
                                                                                    • #include <vector>
                                                                                      •  
                                                                                    "},{"location":"opae-code/handle_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/handle_8h/#classes","title":"Classes","text":"Type Name class handle An allocated accelerator resource. 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/handle.h
                                                                                    "},{"location":"opae-code/handle_8h_source/","title":"File handle.h","text":"
                                                                                    File List > core > handle.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018-2021, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/cxx/core/token.h>\n#include <opae/enum.h>\n#include <opae/types.h>\n\n#include <memory>\n#include <vector>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass handle {\n public:\n  typedef std::shared_ptr<handle> ptr_t;\n\n  handle(const handle &) = delete;\n  handle &operator=(const handle &) = delete;\n\n  virtual ~handle();\n\n  fpga_handle c_type() const { return handle_; }\n\n  operator fpga_handle() const { return handle_; }\n\n  void reconfigure(uint32_t slot, const uint8_t *bitstream, size_t size,\n                   int flags);\n\n  uint32_t read_csr32(uint64_t offset, uint32_t csr_space = 0) const;\n\n  void write_csr32(uint64_t offset, uint32_t value, uint32_t csr_space = 0);\n\n  uint64_t read_csr64(uint64_t offset, uint32_t csr_space = 0) const;\n\n  void write_csr64(uint64_t offset, uint64_t value, uint32_t csr_space = 0);\n\n  void write_csr512(uint64_t offset, const void *value, uint32_t csr_space = 0);\n\n  uint8_t *mmio_ptr(uint64_t offset, uint32_t csr_space = 0) const;\n\n  static handle::ptr_t open(fpga_token token, int flags);\n\n  static handle::ptr_t open(token::ptr_t token, int flags);\n\n  virtual void reset();\n\n  fpga_result close();\n\n  token::ptr_t get_token() const;\n\n private:\n  handle(fpga_handle h);\n\n  fpga_handle handle_;\n  fpga_token token_;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                                                                    "},{"location":"opae-code/cxx_2core_2properties_8h/","title":"File properties.h","text":"
                                                                                    FileList > core > properties.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <opae/cxx/core/pvalue.h>
                                                                                      •  
                                                                                    • #include <opae/properties.h>
                                                                                      •  
                                                                                    • #include <iostream>
                                                                                      •  
                                                                                    • #include <map>
                                                                                      •  
                                                                                    • #include <memory>
                                                                                      •  
                                                                                    • #include <vector>
                                                                                      •  
                                                                                    "},{"location":"opae-code/cxx_2core_2properties_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/cxx_2core_2properties_8h/#classes","title":"Classes","text":"Type Name class properties Wraps an OPAE fpga_properties object. 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/properties.h
                                                                                    "},{"location":"opae-code/cxx_2core_2properties_8h_source/","title":"File properties.h","text":"
                                                                                    File List > core > properties.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018-2022, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/cxx/core/pvalue.h>\n#include <opae/properties.h>\n\n#include <iostream>\n#include <map>\n#include <memory>\n#include <vector>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass token;\nclass handle;\nclass properties {\n public:\n  typedef std::shared_ptr<properties> ptr_t;\n\n  const static std::vector<properties::ptr_t> none;\n\n  properties(const properties &p) = delete;\n\n  properties &operator=(const properties &p) = delete;\n\n  ~properties();\n\n  fpga_properties c_type() const { return props_; }\n\n  operator fpga_properties() const { return props_; }\n\n  static properties::ptr_t get();\n\n  static properties::ptr_t get(fpga_guid guid_in);\n\n  static properties::ptr_t get(fpga_objtype objtype);\n\n  static properties::ptr_t get(std::shared_ptr<token> t);\n\n  static properties::ptr_t get(fpga_token t);\n\n  static properties::ptr_t get(std::shared_ptr<handle> h);\n\n private:\n  properties(bool alloc_props = true);\n  fpga_properties props_;\n\n public:\n  pvalue<fpga_objtype> type;\n  pvalue<uint32_t> num_errors;\n  pvalue<uint16_t> segment;\n  pvalue<uint8_t> bus;\n  pvalue<uint8_t> device;\n  pvalue<uint8_t> function;\n  pvalue<uint8_t> socket_id;\n  pvalue<uint32_t> num_slots;\n  pvalue<uint64_t> bbs_id;\n  pvalue<fpga_version> bbs_version;\n  pvalue<uint16_t> vendor_id;\n  pvalue<uint16_t> device_id;\n  pvalue<uint16_t> subsystem_vendor_id;\n  pvalue<uint16_t> subsystem_device_id;\n  pvalue<char *> model;\n  pvalue<uint64_t> local_memory_size;\n  pvalue<uint64_t> capabilities;\n  pvalue<uint32_t> num_mmio;\n  pvalue<uint32_t> num_interrupts;\n  pvalue<fpga_accelerator_state> accelerator_state;\n  pvalue<uint64_t> object_id;\n  pvalue<fpga_token> parent;\n  pvalue<fpga_interface> interface;\n  guid_t guid;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                                                                    "},{"location":"opae-code/pvalue_8h/","title":"File pvalue.h","text":"
                                                                                    FileList > core > pvalue.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <opae/cxx/core/except.h>
                                                                                      •  
                                                                                    • #include <opae/properties.h>
                                                                                      •  
                                                                                    • #include <opae/utils.h>
                                                                                      •  
                                                                                    • #include <uuid/uuid.h>
                                                                                      •  
                                                                                    • #include <algorithm>
                                                                                      •  
                                                                                    • #include <array>
                                                                                      •  
                                                                                    • #include <cstring>
                                                                                      •  
                                                                                    • #include <iostream>
                                                                                      •  
                                                                                    • #include <type_traits>
                                                                                      •  
                                                                                    "},{"location":"opae-code/pvalue_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/pvalue_8h/#classes","title":"Classes","text":"Type Name struct guid_t Representation of the guid member of a properties object. struct pvalue <typename T>Wraps OPAE properties defined in the OPAE C API by associating an fpga_properties reference with the getters and setters defined for a property. 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/pvalue.h
                                                                                    "},{"location":"opae-code/pvalue_8h_source/","title":"File pvalue.h","text":"
                                                                                    File List > core > pvalue.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018-2020, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/cxx/core/except.h>\n#include <opae/properties.h>\n#include <opae/utils.h>\n#include <uuid/uuid.h>\n\n#include <algorithm>\n#include <array>\n#include <cstring>\n#include <iostream>\n#include <type_traits>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nstruct guid_t {\n  guid_t(fpga_properties *p) : props_(p), is_set_(false) {}\n\n  void update() {\n    fpga_result res = fpgaPropertiesGetGUID(\n        *props_, reinterpret_cast<fpga_guid *>(data_.data()));\n    ASSERT_FPGA_OK(res);\n    is_set_ = true;\n  }\n\n  operator uint8_t *() {\n    update();\n    return data_.data();\n  }\n\n  const uint8_t *c_type() const { return data_.data(); }\n\n  guid_t &operator=(fpga_guid g) {\n    is_set_ = false;\n    ASSERT_FPGA_OK(fpgaPropertiesSetGUID(*props_, g));\n    is_set_ = true;\n    uint8_t *begin = &g[0];\n    uint8_t *end = begin + sizeof(fpga_guid);\n    std::copy(begin, end, data_.begin());\n    return *this;\n  }\n\n  bool operator==(const fpga_guid &g) {\n    return is_set() && (0 == std::memcmp(data_.data(), g, sizeof(fpga_guid)));\n  }\n\n  void parse(const char *str) {\n    is_set_ = false;\n    if (0 != uuid_parse(str, data_.data())) {\n      throw except(OPAECXX_HERE);\n    }\n    ASSERT_FPGA_OK(fpgaPropertiesSetGUID(*props_, data_.data()));\n    is_set_ = true;\n  }\n\n  friend std::ostream &operator<<(std::ostream &ostr, const guid_t &g) {\n    fpga_properties props = *g.props_;\n    fpga_guid guid_value;\n    fpga_result res;\n    if ((res = fpgaPropertiesGetGUID(props, &guid_value)) == FPGA_OK) {\n      char guid_str[84];\n      uuid_unparse(guid_value, guid_str);\n      ostr << guid_str;\n    } else if (FPGA_NOT_FOUND == res) {\n      std::cerr << \"[guid_t::<<] GUID property not set\\n\";\n    } else {\n      ASSERT_FPGA_OK(res);\n    }\n    return ostr;\n  }\n\n  bool is_set() const { return is_set_; }\n\n  void invalidate() { is_set_ = false; }\n\n private:\n  fpga_properties *props_;\n  bool is_set_;\n  std::array<uint8_t, 16> data_;\n};\n\ntemplate <typename T>\nstruct pvalue {\n  typedef typename std::conditional<\n      std::is_same<T, char *>::value, fpga_result (*)(fpga_properties, T),\n      fpga_result (*)(fpga_properties, T *)>::type getter_t;\n\n  typedef fpga_result (*setter_t)(fpga_properties, T);\n\n  typedef typename std::conditional<std::is_same<T, char *>::value,\n                                    typename std::string, T>::type copy_t;\n\n  pvalue() : props_(0), is_set_(false), get_(nullptr), set_(nullptr), copy_() {}\n\n  pvalue(fpga_properties *p, getter_t g, setter_t s)\n      : props_(p), is_set_(false), get_(g), set_(s), copy_() {}\n\n  pvalue<T> &operator=(const T &v) {\n    is_set_ = false;\n    ASSERT_FPGA_OK(set_(*props_, v));\n    is_set_ = true;\n    copy_ = v;\n    return *this;\n  }\n\n  bool operator==(const T &other) { return is_set() && (copy_ == other); }\n\n  void update() {\n    ASSERT_FPGA_OK(get_(*props_, &copy_));\n    is_set_ = true;\n  }\n\n  operator copy_t() {\n    update();\n    return copy_;\n  }\n\n  // TODO: Remove this once all properties are tested\n  fpga_result get_value(T &value) const { return get_(*props_, &value); }\n\n  friend std::ostream &operator<<(std::ostream &ostr, const pvalue<T> &p) {\n    T value;\n    fpga_properties props = *p.props_;\n    fpga_result res;\n    if ((res = p.get_(props, &value)) == FPGA_OK) {\n      ostr << +(value);\n    } else if (FPGA_NOT_FOUND == res) {\n      std::cerr << \"property getter returned (\" << res << \") \"\n                << fpgaErrStr(res);\n    } else {\n      ASSERT_FPGA_OK(res);\n    }\n    return ostr;\n  }\n\n  bool is_set() const { return is_set_; }\n\n  void invalidate() { is_set_ = false; }\n\n private:\n  fpga_properties *props_;\n  bool is_set_;\n  getter_t get_;\n  setter_t set_;\n  copy_t copy_;\n};\n\ntemplate <>\ninline void pvalue<char *>::update() {\n  char buf[256];\n  ASSERT_FPGA_OK(get_(*props_, buf));\n  copy_.assign(buf);\n  is_set_ = true;\n}\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                                                                    "},{"location":"opae-code/shared__buffer_8h/","title":"File shared_buffer.h","text":"
                                                                                    FileList > core > shared_buffer.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <opae/buffer.h>
                                                                                      •  
                                                                                    • #include <opae/cxx/core/except.h>
                                                                                      •  
                                                                                    • #include <opae/cxx/core/handle.h>
                                                                                      •  
                                                                                    • #include <chrono>
                                                                                      •  
                                                                                    • #include <cstdint>
                                                                                      •  
                                                                                    • #include <initializer_list>
                                                                                      •  
                                                                                    • #include <memory>
                                                                                      •  
                                                                                    • #include <thread>
                                                                                      •  
                                                                                    • #include <vector>
                                                                                      •  
                                                                                    "},{"location":"opae-code/shared__buffer_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/shared__buffer_8h/#classes","title":"Classes","text":"Type Name class shared_buffer Host/AFU shared memory blocks. 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/shared_buffer.h
                                                                                    "},{"location":"opae-code/shared__buffer_8h_source/","title":"File shared_buffer.h","text":"
                                                                                    File List > core > shared_buffer.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/buffer.h>\n#include <opae/cxx/core/except.h>\n#include <opae/cxx/core/handle.h>\n\n#include <chrono>\n#include <cstdint>\n#include <initializer_list>\n#include <memory>\n#include <thread>\n#include <vector>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass shared_buffer {\n public:\n  typedef std::size_t size_t;\n  typedef std::shared_ptr<shared_buffer> ptr_t;\n\n  shared_buffer(const shared_buffer &) = delete;\n  shared_buffer &operator=(const shared_buffer &) = delete;\n\n  virtual ~shared_buffer();\n\n  static shared_buffer::ptr_t allocate(handle::ptr_t handle, size_t len,\n                                       bool read_only = false);\n\n  static shared_buffer::ptr_t attach(handle::ptr_t handle, uint8_t *base,\n                                     size_t len, bool read_only = false);\n\n  void release();\n\n  volatile uint8_t *c_type() const { return virt_; }\n\n  handle::ptr_t owner() const { return handle_; }\n\n  size_t size() const { return len_; }\n\n  uint64_t wsid() const { return wsid_; }\n\n  uint64_t io_address() const { return io_address_; }\n\n  void fill(int c);\n\n  int compare(ptr_t other, size_t len) const;\n\n  template <typename T>\n  T read(size_t offset) const {\n    if ((offset < len_) && (virt_ != nullptr)) {\n      return *reinterpret_cast<T *>(virt_ + offset);\n    } else if (offset >= len_) {\n      throw except(OPAECXX_HERE);\n    } else {\n      throw except(OPAECXX_HERE);\n    }\n    return T();\n  }\n\n  template <typename T>\n  void write(const T &value, size_t offset) {\n    if ((offset < len_) && (virt_ != nullptr)) {\n      *reinterpret_cast<T *>(virt_ + offset) = value;\n    } else if (offset >= len_) {\n      throw except(OPAECXX_HERE);\n    } else {\n      throw except(OPAECXX_HERE);\n    }\n  }\n\n protected:\n  shared_buffer(handle::ptr_t handle, size_t len, uint8_t *virt, uint64_t wsid,\n                uint64_t io_address);\n\n  handle::ptr_t handle_;\n  size_t len_;\n  uint8_t *virt_;\n  uint64_t wsid_;\n  uint64_t io_address_;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                                                                    "},{"location":"opae-code/cxx_2core_2sysobject_8h/","title":"File sysobject.h","text":"
                                                                                    FileList > core > sysobject.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <opae/cxx/core/handle.h>
                                                                                      •  
                                                                                    • #include <opae/cxx/core/token.h>
                                                                                      •  
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    • #include <memory>
                                                                                      •  
                                                                                    • #include <vector>
                                                                                      •  
                                                                                    "},{"location":"opae-code/cxx_2core_2sysobject_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/cxx_2core_2sysobject_8h/#classes","title":"Classes","text":"Type Name class sysobject Wraps the OPAE fpga_object primitive. 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/sysobject.h
                                                                                    "},{"location":"opae-code/cxx_2core_2sysobject_8h_source/","title":"File sysobject.h","text":"
                                                                                    File List > core > sysobject.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/cxx/core/handle.h>\n#include <opae/cxx/core/token.h>\n#include <opae/types.h>\n\n#include <memory>\n#include <vector>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass sysobject {\n public:\n  typedef std::shared_ptr<sysobject> ptr_t;\n\n  sysobject() = delete;\n\n  sysobject(const sysobject &o) = delete;\n\n  sysobject &operator=(const sysobject &o) = delete;\n\n  static sysobject::ptr_t get(token::ptr_t t, const std::string &name,\n                              int flags = 0);\n\n  static sysobject::ptr_t get(handle::ptr_t h, const std::string &name,\n                              int flags = 0);\n\n  sysobject::ptr_t get(const std::string &name, int flags = 0);\n\n  sysobject::ptr_t get(int index);\n\n  virtual ~sysobject();\n\n  uint32_t size() const;\n\n  uint64_t read64(int flags = 0) const;\n\n  void write64(uint64_t value, int flags = 0) const;\n\n  std::vector<uint8_t> bytes(int flags = 0) const;\n\n  std::vector<uint8_t> bytes(uint32_t offset, uint32_t size,\n                             int flags = 0) const;\n\n  enum fpga_sysobject_type type() const;\n\n  fpga_object c_type() const { return sysobject_; }\n\n  operator fpga_object() const { return sysobject_; }\n\n private:\n  sysobject(fpga_object sysobj, token::ptr_t token, handle::ptr_t hnd);\n  fpga_object sysobject_;\n  token::ptr_t token_;\n  handle::ptr_t handle_;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                                                                    "},{"location":"opae-code/token_8h/","title":"File token.h","text":"
                                                                                    FileList > core > token.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <opae/access.h>
                                                                                      •  
                                                                                    • #include <opae/cxx/core/properties.h>
                                                                                      •  
                                                                                    • #include <opae/enum.h>
                                                                                      •  
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    • #include <memory>
                                                                                      •  
                                                                                    • #include <vector>
                                                                                      •  
                                                                                    "},{"location":"opae-code/token_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/token_8h/#classes","title":"Classes","text":"Type Name class token Wraps the OPAE fpga_token primitive. 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/token.h
                                                                                    "},{"location":"opae-code/token_8h_source/","title":"File token.h","text":"
                                                                                    File List > core > token.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018-2021, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/access.h>\n#include <opae/cxx/core/properties.h>\n#include <opae/enum.h>\n#include <opae/types.h>\n\n#include <memory>\n#include <vector>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass token {\n public:\n  typedef std::shared_ptr<token> ptr_t;\n\n  static std::vector<token::ptr_t> enumerate(\n      const std::vector<properties::ptr_t>& props);\n\n  ~token();\n\n  fpga_token c_type() const { return token_; }\n\n  operator fpga_token() const { return token_; }\n\n  ptr_t get_parent() const;\n\n private:\n  token(fpga_token tok);\n\n  fpga_token token_;\n\n  friend class handle;\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                                                                    "},{"location":"opae-code/cxx_2core_2version_8h/","title":"File version.h","text":"
                                                                                    FileList > core > version.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    • #include <string>
                                                                                      •  
                                                                                    "},{"location":"opae-code/cxx_2core_2version_8h/#namespaces","title":"Namespaces","text":"Type Name namespace opae namespace fpga namespace types"},{"location":"opae-code/cxx_2core_2version_8h/#classes","title":"Classes","text":"Type Name class version 
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/cxx/core/version.h
                                                                                    "},{"location":"opae-code/cxx_2core_2version_8h_source/","title":"File version.h","text":"
                                                                                    File List > core > version.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#pragma once\n#include <opae/types.h>\n\n#include <string>\n\nnamespace opae {\nnamespace fpga {\nnamespace types {\n\nclass version {\n public:\n  static fpga_version as_struct();\n\n  static std::string as_string();\n\n  static std::string build();\n};\n\n}  // end of namespace types\n}  // end of namespace fpga\n}  // end of namespace opae\n
                                                                                    "},{"location":"opae-code/enum_8h/","title":"File enum.h","text":"
                                                                                    FileList > docs > sw > include > opae > enum.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    APIs for resource enumeration and managing tokens. More...
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/enum_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaCloneToken (fpga_token src, fpga_token * dst) Clone a fpga_token object. fpga_result fpgaDestroyToken (fpga_token * token) Destroy a Token. fpga_result fpgaEnumerate (const fpga_properties * filters, uint32_t num_filters, fpga_token * tokens, uint32_t max_tokens, uint32_t * num_matches) Enumerate FPGA resources present in the system."},{"location":"opae-code/enum_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    These APIs are the first step for any application using OPAE to discover resources that are present on the system. They allow selective enumeration (i.e. getting a list of resources that match a given list of criteria) and methods to manage the lifecycle of tokens generated by fpgaEnumerate(). 
                                                                                    "},{"location":"opae-code/enum_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/enum_8h/#function-fpgaclonetoken","title":"function fpgaCloneToken","text":"
                                                                                    Clone a fpga_token object. 
                                                                                    fpga_result fpgaCloneToken (\n    fpga_token src,\n    fpga_token * dst\n) \n
                                                                                     
                                                                                    Creates a copy of an fpga_token object.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This call creates a new token object and allocates memory for it. It is the responsibility of the using application to free this memory after use by calling fpgaDestroyToken() for the cloned token.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • src fpga_token object to copy 
                                                                                      •  
                                                                                    • dst New fpga_token object cloned from 'src' 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success 
                                                                                    "},{"location":"opae-code/enum_8h/#function-fpgadestroytoken","title":"function fpgaDestroyToken","text":"
                                                                                    Destroy a Token. 
                                                                                    fpga_result fpgaDestroyToken (\n    fpga_token * token\n) \n
                                                                                     
                                                                                    This function destroys a token created by fpgaEnumerate() and frees the associated memory.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    fpgaDestroyToken() requires the address of an fpga_token as previously created by fpgaEnumerate() or fpgaCloneToken(). Passing any other value results in undefined behavior.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • token fpga_token to destroy 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success 
                                                                                    "},{"location":"opae-code/enum_8h/#function-fpgaenumerate","title":"function fpgaEnumerate","text":"
                                                                                    Enumerate FPGA resources present in the system. 
                                                                                    fpga_result fpgaEnumerate (\n    const fpga_properties * filters,\n    uint32_t num_filters,\n    fpga_token * tokens,\n    uint32_t max_tokens,\n    uint32_t * num_matches\n) \n
                                                                                     
                                                                                    This call allows the user to query the system for FPGA resources that match a certain set of criteria, e.g. all accelerators that are assigned to a host interface and available, all FPGAs of a specific type, etc.
                                                                                     
                                                                                    fpgaEnumerate() will create a number of fpga_tokens to represent the matching resources and populate the array tokens with these tokens. The max_tokens argument can be used to limit the number of tokens allocated/returned by fpgaEnumerate(); i.e., the number of tokens in the returned tokens array will be either max_tokens or num_matches (the number of resources matching the filter), whichever is smaller. Use fpgaDestroyToken() to destroy tokens that are no longer needed.
                                                                                     
                                                                                    To query the number of matches for a particular set of filters (e.g. to allocate a tokens array of the appropriate size), call fpgaEnumerate() with the parameter tokens set to NULL; this will only return the number of matches in num_matches.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    fpgaEnumerate() will allocate memory for the created tokens returned in tokens. It is the responsibility of the using application to free this memory after use by calling fpgaDestroyToken() for each of the returned tokens.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • filters Array of fpga_properties objects describing the properties of the objects that should be returned. A resource is considered matching if its properties match any one of the supplied filters. To match all FPGA resources, pass an empty filters object (one without any filter criteria set) or pass a NULL filters parameter with num_filters set to 0. 
                                                                                      •  
                                                                                    • num_filters Number of entries in the filters array, or 0 to match all FPGA resources when filters is NULL. 
                                                                                      •  
                                                                                    • tokens Pointer to an array of fpga_token variables to be populated. If NULL is supplied, fpgaEnumerate() will not create any tokens, but it will return the number of possible matches in num_match. 
                                                                                      •  
                                                                                    • max_tokens Maximum number of tokens that fpgaEnumerate() shall return (length of tokens array). There may be more or fewer matches than this number; num_matches is set to the number of actual matches. 
                                                                                      •  
                                                                                    • num_matches Number of resources matching the filter criteria. This number can be higher than the number of tokens returned in the tokens array (depending on the value of max_tokens). 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if invalid pointers or objects are passed into the function. FPGA_NO_DRIVER if OPAE can't find the respective enumeration data structures usually provided by the driver. FPGA_NO_MEMORY if there was not enough memory to create tokens. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/enum.h
                                                                                    "},{"location":"opae-code/enum_8h_source/","title":"File enum.h","text":"
                                                                                    File List > docs > sw > include > opae > enum.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_ENUM_H__\n#define __FPGA_ENUM_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaEnumerate(const fpga_properties *filters,\n              uint32_t num_filters, fpga_token *tokens,\n              uint32_t max_tokens, uint32_t *num_matches);\n\nfpga_result fpgaCloneToken(fpga_token src, fpga_token *dst);\n\nfpga_result fpgaDestroyToken(fpga_token *token);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_ENUM_H__\n
                                                                                    "},{"location":"opae-code/error_8h/","title":"File error.h","text":"
                                                                                    FileList > docs > sw > include > opae > error.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Functions for reading and clearing errors in resources. More...
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/error_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaClearAllErrors (fpga_token token) Clear all error registers of a particular resource. fpga_result fpgaClearError (fpga_token token, uint32_t error_num) Clear error register. fpga_result fpgaGetErrorInfo (fpga_token token, uint32_t error_num, struct fpga_error_info * error_info) Get information about a particular error register. fpga_result fpgaReadError (fpga_token token, uint32_t error_num, uint64_t * value) Read error value."},{"location":"opae-code/error_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    Many FPGA resources have the ability to track the occurrence of errors. This file provides functions to retrieve information about errors within resources. 
                                                                                    "},{"location":"opae-code/error_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/error_8h/#function-fpgaclearallerrors","title":"function fpgaClearAllErrors","text":"
                                                                                    Clear all error registers of a particular resource. 
                                                                                    fpga_result fpgaClearAllErrors (\n    fpga_token token\n) \n
                                                                                     
                                                                                    This function will clear all error registers of the resource referenced by token, observing the necessary order of clearing errors, if any.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • token Token to accelerator resource to query 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the token, and FPGA_BUSY if error could not be cleared. 
                                                                                    "},{"location":"opae-code/error_8h/#function-fpgaclearerror","title":"function fpgaClearError","text":"
                                                                                    Clear error register. 
                                                                                    fpga_result fpgaClearError (\n    fpga_token token,\n    uint32_t error_num\n) \n
                                                                                     
                                                                                    This function will clear the error register error_num of the resource referenced by token.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • token Token to accelerator resource to query 
                                                                                      •  
                                                                                    • error_num Number of error register to clear 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the token, and FPGA_BUSY if error could not be cleared. 
                                                                                    "},{"location":"opae-code/error_8h/#function-fpgageterrorinfo","title":"function fpgaGetErrorInfo","text":"
                                                                                    Get information about a particular error register. 
                                                                                    fpga_result fpgaGetErrorInfo (\n    fpga_token token,\n    uint32_t error_num,\n    struct fpga_error_info * error_info\n) \n
                                                                                     
                                                                                    This function will populate a fpga_error_info struct with information about error number error_num of the resource referenced by token.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • token Token to accelerator resource to query 
                                                                                      •  
                                                                                    • error_num Error register to retrieve information about 
                                                                                      •  
                                                                                    • error_info Pointer to memory to store information into 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the token. 
                                                                                    "},{"location":"opae-code/error_8h/#function-fpgareaderror","title":"function fpgaReadError","text":"
                                                                                    Read error value. 
                                                                                    fpga_result fpgaReadError (\n    fpga_token token,\n    uint32_t error_num,\n    uint64_t * value\n) \n
                                                                                     
                                                                                    This function will read the value of error register error_num of the resource referenced by token into the memory location pointed to by value.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • token Token to accelerator resource to query 
                                                                                      •  
                                                                                    • error_num Number of error register to read 
                                                                                      •  
                                                                                    • value Pointer to memory to store error value into (64 bit) 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the token. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/error.h
                                                                                    "},{"location":"opae-code/error_8h_source/","title":"File error.h","text":"
                                                                                    File List > docs > sw > include > opae > error.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_ERROR_H__\n#define __FPGA_ERROR_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n\nfpga_result fpgaReadError(fpga_token token, uint32_t error_num, uint64_t *value);\n\nfpga_result fpgaClearError(fpga_token token, uint32_t error_num);\n\nfpga_result fpgaClearAllErrors(fpga_token token);\n\nfpga_result fpgaGetErrorInfo(fpga_token token,\n                 uint32_t error_num,\n                 struct fpga_error_info *error_info);\n\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_ERROR_H__\n
                                                                                    "},{"location":"opae-code/event_8h/","title":"File event.h","text":"
                                                                                    FileList > docs > sw > include > opae > event.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Functions for registering events and managing the lifecycle for fpga_event_handle s.More...
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/event_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaCreateEventHandle (fpga_event_handle * event_handle) Initialize an event_handle. fpga_result fpgaDestroyEventHandle (fpga_event_handle * event_handle) Destroy an event_handle. fpga_result fpgaGetOSObjectFromEventHandle (const fpga_event_handle eh, int * fd) Get OS object from event handle. fpga_result fpgaRegisterEvent (fpga_handle handle, fpga_event_type event_type, fpga_event_handle event_handle, uint32_t flags) Register an FPGA event. fpga_result fpgaUnregisterEvent (fpga_handle handle, fpga_event_type event_type, fpga_event_handle event_handle) Unregister an FPGA event."},{"location":"opae-code/event_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    OPAE provides an interface to asynchronous events that can be generated by different FPGA resources. The event API provides functions to register for these events; associated with every event a process has registered for is an fpga_event_handle, which encapsulates the OS-specific data structure for event objects. On Linux, an fpga_event_handle can be used as a file descriptor and passed to select(), poll(), epoll() and similar functions to wait for asynchronous events. 
                                                                                    "},{"location":"opae-code/event_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/event_8h/#function-fpgacreateeventhandle","title":"function fpgaCreateEventHandle","text":"
                                                                                    Initialize an event_handle. 
                                                                                    fpga_result fpgaCreateEventHandle (\n    fpga_event_handle * event_handle\n) \n
                                                                                     
                                                                                    Platform independent way to initialize an event_handle used for notifications from the driver to application. For Linux, this function creates an eventfd and returns the eventfd file descriptor in *event_handle.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • event_handle Pointer to event handle variable.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if event_handle is NULL. FPGA_NOT_SUPPORTED if platform does not support events. 
                                                                                    "},{"location":"opae-code/event_8h/#function-fpgadestroyeventhandle","title":"function fpgaDestroyEventHandle","text":"
                                                                                    Destroy an event_handle. 
                                                                                    fpga_result fpgaDestroyEventHandle (\n    fpga_event_handle * event_handle\n) \n
                                                                                     
                                                                                    Destroy handle and free resources. On Linux this corresponds to closing the file descriptor pointed to by handle
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    fpgaDestroyEventHandle() requires the address of an event_handle as created by fpgaCreateEventHandle(). Passing any other value results in undefined behavior.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • event_handle Pointer to handle to be destroyed
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if event_handle is NULL. 
                                                                                    "},{"location":"opae-code/event_8h/#function-fpgagetosobjectfromeventhandle","title":"function fpgaGetOSObjectFromEventHandle","text":"
                                                                                    Get OS object from event handle. 
                                                                                    fpga_result fpgaGetOSObjectFromEventHandle (\n    const fpga_event_handle eh,\n    int * fd\n) \n
                                                                                     
                                                                                    Check validity of event handle, and get the OS object used to subscribe and unsubscribe to events. On Linux, the object corresponds to a file descriptor.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • eh Event handle to get the descriptor value from 
                                                                                      •  
                                                                                    • fd integer to store the descriptor value
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if event_handle is invalid. 
                                                                                    "},{"location":"opae-code/event_8h/#function-fpgaregisterevent","title":"function fpgaRegisterEvent","text":"
                                                                                    Register an FPGA event. 
                                                                                    fpga_result fpgaRegisterEvent (\n    fpga_handle handle,\n    fpga_event_type event_type,\n    fpga_event_handle event_handle,\n    uint32_t flags\n) \n
                                                                                     
                                                                                    This function tells the driver that the caller is interested in notification for the event specified by the type and flags pair.
                                                                                     
                                                                                    The event_handle points to an OS specific mechanism for event notification. An event_handle is associated with only a single event.
                                                                                     
                                                                                    In case of user interrupts, the flags parameter will be used to specify the vector ID. The value of the flags parameter indicates the vector ID, no bit encoding is used.
                                                                                     
                                                                                    Todo
                                                                                     
                                                                                    define if calling fpgaRegisterEvent multiple times with the same event_handle is an error condition or if it is silently ignored.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened FPGA resource. 
                                                                                      •  
                                                                                    • event_type Type of event 
                                                                                      •  
                                                                                    • event_handle Handle to previously opened resource for event notification. 
                                                                                      •  
                                                                                    • flags Optional argument for specifying additional information about event. For example irq number for interrupt events. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if handle does not refer to a resource supporting the requested event, or if event_handle is not valid. FPGA_EXCEPTION if an internal exception occurred while accessing the handle or the event_handle. On Linux: FPGA_NO_DAEMON if the driver does not support the requested event and there is no FPGA Daemon (fpgad) running to proxy it. 
                                                                                    "},{"location":"opae-code/event_8h/#function-fpgaunregisterevent","title":"function fpgaUnregisterEvent","text":"
                                                                                    Unregister an FPGA event. 
                                                                                    fpga_result fpgaUnregisterEvent (\n    fpga_handle handle,\n    fpga_event_type event_type,\n    fpga_event_handle event_handle\n) \n
                                                                                     
                                                                                    This function tells the driver that the caller is no longer interested in notification for the event associated with the event_handle
                                                                                     
                                                                                    The event_handle points to an OS specific mechanism for event notification. An event_handle is associated with only a single event.
                                                                                     
                                                                                    Todo
                                                                                     
                                                                                    define if calling fpgaUnregisterEvent multiple times with the same event_handle is an error condition or if it is silently ignored.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened FPGA resource. 
                                                                                      •  
                                                                                    • event_type Type of event to unregister. 
                                                                                      •  
                                                                                    • event_handle Handle to previously registered resource for event notification. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if handle does not refer to a resource supporting the requested event, or if event_handle is not valid. FPGA_EXCEPTION if an internal error occurred accessing the handle or the event_handle. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/event.h
                                                                                    "},{"location":"opae-code/event_8h_source/","title":"File event.h","text":"
                                                                                    File List > docs > sw > include > opae > event.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_EVENT_H__\n#define __FPGA_EVENT_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaCreateEventHandle(fpga_event_handle *event_handle);\n\nfpga_result fpgaDestroyEventHandle(fpga_event_handle *event_handle);\n\n\nfpga_result fpgaGetOSObjectFromEventHandle(const fpga_event_handle eh, int *fd);\n\nfpga_result fpgaRegisterEvent(fpga_handle handle,\n                  fpga_event_type event_type,\n                  fpga_event_handle event_handle,\n                  uint32_t flags);\n\nfpga_result fpgaUnregisterEvent(fpga_handle handle,\n                         fpga_event_type event_type,\n                         fpga_event_handle event_handle);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_EVENT_H__\n
                                                                                    "},{"location":"opae-code/fpga_8h/","title":"File fpga.h","text":"
                                                                                    FileList > docs > sw > include > opae > fpga.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    FPGA API. More...
                                                                                     
                                                                                       
                                                                                    • #include <opae/log.h>
                                                                                      •  
                                                                                    • #include <opae/init.h>
                                                                                      •  
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    • #include <opae/access.h>
                                                                                      •  
                                                                                    • #include <opae/buffer.h>
                                                                                      •  
                                                                                    • #include <opae/enum.h>
                                                                                      •  
                                                                                    • #include <opae/event.h>
                                                                                      •  
                                                                                    • #include <opae/manage.h>
                                                                                      •  
                                                                                    • #include <opae/mmio.h>
                                                                                      •  
                                                                                    • #include <opae/properties.h>
                                                                                      •  
                                                                                    • #include <opae/umsg.h>
                                                                                      •  
                                                                                    • #include <opae/utils.h>
                                                                                      •  
                                                                                    • #include <opae/error.h>
                                                                                      •  
                                                                                    • #include <opae/version.h>
                                                                                      •  
                                                                                    • #include <opae/sysobject.h>
                                                                                      •  
                                                                                    • #include <opae/userclk.h>
                                                                                      •  
                                                                                    • #include <opae/metrics.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/fpga_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    This conveniently includes all APIs that a part of the OPAE release (base and extensions). 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/fpga.h
                                                                                    "},{"location":"opae-code/fpga_8h_source/","title":"File fpga.h","text":"
                                                                                    File List > docs > sw > include > opae > fpga.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_FPGA_H__\n#define __FPGA_FPGA_H__\n\n#include <opae/log.h>\n#include <opae/init.h>\n#include <opae/types.h>\n#include <opae/access.h>\n#include <opae/buffer.h>\n#include <opae/enum.h>\n#include <opae/event.h>\n#include <opae/manage.h>\n#include <opae/mmio.h>\n#include <opae/properties.h>\n#include <opae/umsg.h>\n#include <opae/utils.h>\n#include <opae/error.h>\n#include <opae/version.h>\n#include <opae/sysobject.h>\n#include <opae/userclk.h>\n#include <opae/metrics.h>\n\n#endif // __FPGA_FPGA_H__\n
                                                                                    "},{"location":"opae-code/hash__map_8h/","title":"File hash_map.h","text":"
                                                                                    FileList > docs > sw > include > opae > hash_map.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    A general-purpose hybrid array/list hash map implementation. More...
                                                                                     
                                                                                       
                                                                                    • #include <stdint.h>
                                                                                      •  
                                                                                    • #include <stdbool.h>
                                                                                      •  
                                                                                    • #include <opae/types_enum.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/hash__map_8h/#classes","title":"Classes","text":"Type Name struct _opae_hash_map Hash map object. struct _opae_hash_map_item List link item."},{"location":"opae-code/hash__map_8h/#public-types","title":"Public Types","text":"Type Name enum _opae_hash_map_flags Flags used to initialize a hash map. typedef struct _opae_hash_map opae_hash_map Hash map object. typedef enum _opae_hash_map_flags opae_hash_map_flags Flags used to initialize a hash map. typedef struct _opae_hash_map_item opae_hash_map_item List link item."},{"location":"opae-code/hash__map_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result opae_hash_map_add (opae_hash_map * hm, void * key, void * value) Map a key to a value. fpga_result opae_hash_map_destroy (opae_hash_map * hm) Tear down a hash map. fpga_result opae_hash_map_find (opae_hash_map * hm, void * key, void ** value) Retrieve the value for a given key. fpga_result opae_hash_map_init (opae_hash_map * hm, uint32_t num_buckets, uint32_t hash_seed, int flags, uint32_t(*)(uint32_t num_buckets, uint32_t hash_seed, void *key) key_hash, int(*)(void *keya, void *keyb) key_compare, void(*)(void *key, void *context) key_cleanup, void(*)(void *value, void *context) value_cleanup) Initialize a hash map. bool opae_hash_map_is_empty (opae_hash_map * hm) Determine whether a hash map is empty. fpga_result opae_hash_map_remove (opae_hash_map * hm, void * key) Remove a key/value association. int opae_u64_key_compare (void * keya, void * keyb) Convenience key comparison function for 64-bit values. uint32_t opae_u64_key_hash (uint32_t num_buckets, uint32_t hash_seed, void * key) Convenience hash function for arbitrary pointers/64-bit values."},{"location":"opae-code/hash__map_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    Presents a generic interface for mapping key objects to value objects. Both keys and values may be arbitrary data structures. The user supplies the means by which the hash of values is generated and by which the keys are compared to each other. 
                                                                                    "},{"location":"opae-code/hash__map_8h/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/hash__map_8h/#enum-_opae_hash_map_flags","title":"enum _opae_hash_map_flags","text":"
                                                                                    Flags used to initialize a hash map. 
                                                                                    enum _opae_hash_map_flags {\n    OPAE_HASH_MAP_UNIQUE_KEYSPACE = (1u << 0)\n};\n
                                                                                     
                                                                                    OPAE_HASH_MAP_UNIQUE_KEYSPACE says that the user provides a guarantee that the key space is truly unique. In other words, when the provided hash function for keys A and B returns the same bucket index, the key comparison function when comparing A and B will never return a result saying that the keys are equal in value. This is helpful in situations where the key space is guaranteed to produce unique values, for example a memory allocator. When the key space is guaranteed to be unique, opae_hash_map_add() can implement a small performance improvement. 
                                                                                    "},{"location":"opae-code/hash__map_8h/#typedef-opae_hash_map","title":"typedef opae_hash_map","text":"
                                                                                    Hash map object. 
                                                                                    typedef struct _opae_hash_map opae_hash_map;\n
                                                                                     
                                                                                    This structure defines the internals of the hash map. Each of the parameters supplied to opae_hash_map_init() is stored in the structure. All parameters are required, except key_cleanup and value_cleanup, which may optionally be NULL. 
                                                                                    "},{"location":"opae-code/hash__map_8h/#typedef-opae_hash_map_flags","title":"typedef opae_hash_map_flags","text":"
                                                                                    Flags used to initialize a hash map. 
                                                                                    typedef enum _opae_hash_map_flags opae_hash_map_flags;\n
                                                                                     
                                                                                    OPAE_HASH_MAP_UNIQUE_KEYSPACE says that the user provides a guarantee that the key space is truly unique. In other words, when the provided hash function for keys A and B returns the same bucket index, the key comparison function when comparing A and B will never return a result saying that the keys are equal in value. This is helpful in situations where the key space is guaranteed to produce unique values, for example a memory allocator. When the key space is guaranteed to be unique, opae_hash_map_add() can implement a small performance improvement. 
                                                                                    "},{"location":"opae-code/hash__map_8h/#typedef-opae_hash_map_item","title":"typedef opae_hash_map_item","text":"
                                                                                    List link item. 
                                                                                    typedef struct _opae_hash_map_item opae_hash_map_item;\n
                                                                                     
                                                                                    This structure provides the association between key and value. When the supplied hash function for keys A and B returns the same bucket index, both A and B can co-exist on the same list rooted at the bucket index. 
                                                                                    "},{"location":"opae-code/hash__map_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/hash__map_8h/#function-opae_hash_map_add","title":"function opae_hash_map_add","text":"
                                                                                    Map a key to a value. 
                                                                                    fpga_result opae_hash_map_add (\n    opae_hash_map * hm,\n    void * key,\n    void * value\n) \n
                                                                                     
                                                                                    Inserts a mapping from key to value in the given hash map object. Subsequent calls to opae_hash_map_find() that are given the key will retrieve the value.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • hm A pointer to the storage for the hash map object. 
                                                                                      •  
                                                                                    • key The hash map key. 
                                                                                      •  
                                                                                    • value The hash map value. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success, FPGA_INVALID_PARAM if hm is NULL, FPGA_NO_MEMORY if malloc() fails when allocating the list item, or FPGA_INVALID_PARAM if the key hash produced by key_hash is out of bounds. 
                                                                                    "},{"location":"opae-code/hash__map_8h/#function-opae_hash_map_destroy","title":"function opae_hash_map_destroy","text":"
                                                                                    Tear down a hash map. 
                                                                                    fpga_result opae_hash_map_destroy (\n    opae_hash_map * hm\n) \n
                                                                                     
                                                                                    Given a hash map that was previously initialized by opae_hash_map_init(), destroy the hash map, releasing all keys, values, and the bucket array.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • hm A pointer to the storage for the hash map object. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success or FPGA_INVALID_PARAM is hm is NULL. 
                                                                                    "},{"location":"opae-code/hash__map_8h/#function-opae_hash_map_find","title":"function opae_hash_map_find","text":"
                                                                                    Retrieve the value for a given key. 
                                                                                    fpga_result opae_hash_map_find (\n    opae_hash_map * hm,\n    void * key,\n    void ** value\n) \n
                                                                                     
                                                                                    Given a key that was previously passed to opae_hash_map_add(), retrieve its associated value.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • hm A pointer to the storage for the hash map object. 
                                                                                      •  
                                                                                    • key The hash map key. 
                                                                                      •  
                                                                                    • value A pointer to receive the hash map value. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success, FPGA_INVALID_PARAM if hm is NULL or if the key hash produced by key_hash is out of bounds, or FPGA_NOT_FOUND if the given key was not found in the hash map. 
                                                                                    "},{"location":"opae-code/hash__map_8h/#function-opae_hash_map_init","title":"function opae_hash_map_init","text":"
                                                                                    Initialize a hash map. 
                                                                                    fpga_result opae_hash_map_init (\n    opae_hash_map * hm,\n    uint32_t num_buckets,\n    uint32_t hash_seed,\n    int flags,\n    uint32_t(*)(uint32_t num_buckets, uint32_t hash_seed, void *key) key_hash,\n    int(*)(void *keya, void *keyb) key_compare,\n    void(*)(void *key, void *context) key_cleanup,\n    void(*)(void *value, void *context) value_cleanup\n) \n
                                                                                     
                                                                                    Populates the hash map data structure and allocates the buckets array.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • hm A pointer to the storage for the hash map object. 
                                                                                      •  
                                                                                    • num_buckets The desired size of the buckets array. Each array entry may be empty (NULL), or may contain a list of opae_hash_map_item structures for which the given key_hash function returned the same key hash value. 
                                                                                      •  
                                                                                    • hash_seed A seed value used during key hash computation. This value will be the hash_seed parameter to the key hash function. 
                                                                                      •  
                                                                                    • flags Initialization flags. See opae_hash_map_flags. 
                                                                                      •  
                                                                                    • key_hash A pointer to a function that produces the hash value, given the number of buckets, the hash seed, and the key. Valid values are between 0 and num_buckets - 1, inclusively. 
                                                                                      •  
                                                                                    • key_compare A pointer to a function that compares two keys. The return value is similar to that of strcmp(), where a negative value means that keya < keyb, 0 means that keya == keyb, and a positive values means that keya > keyb. 
                                                                                      •  
                                                                                    • key_cleanup A pointer to a function that is called when a key is being removed from the map. This function is optional and may be NULL. When supplied, the function is responsible for freeing any resources allocated when the key was created. 
                                                                                      •  
                                                                                    • value_cleanup A pointer to a function that is called when a value is being removed from the map. This function is optional and may be NULL. When supplied, the function is responsible for freeing any resources allocated when the value was created. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success, FPGA_INVALID_PARAM if any of the required parameters are NULL, or FPGA_NO_MEMORY if the bucket array could not be allocated. 
                                                                                    "},{"location":"opae-code/hash__map_8h/#function-opae_hash_map_is_empty","title":"function opae_hash_map_is_empty","text":"
                                                                                    Determine whether a hash map is empty. 
                                                                                    bool opae_hash_map_is_empty (\n    opae_hash_map * hm\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • hm A pointer to the storage for the hash map object. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    true if there are no key/value mappings present, false otherwise. 
                                                                                    "},{"location":"opae-code/hash__map_8h/#function-opae_hash_map_remove","title":"function opae_hash_map_remove","text":"
                                                                                    Remove a key/value association. 
                                                                                    fpga_result opae_hash_map_remove (\n    opae_hash_map * hm,\n    void * key\n) \n
                                                                                     
                                                                                    Given a key that was previously passed to opae_hash_map_add(), remove the key and its associated value, calling the cleanup functions as needed.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • hm A pointer to the storage for the hash map object. 
                                                                                      •  
                                                                                    • key The hash map key. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success, FPGA_INVALID_PARAM when hm is NULL or when the key hash produced by key_hash is out of bounds, or FPGA_NOT_FOUND if the key is not found in the hash map. 
                                                                                    "},{"location":"opae-code/hash__map_8h/#function-opae_u64_key_compare","title":"function opae_u64_key_compare","text":"
                                                                                    Convenience key comparison function for 64-bit values. 
                                                                                    int opae_u64_key_compare (\n    void * keya,\n    void * keyb\n) \n
                                                                                     
                                                                                    Simply converts the key pointers to uint64_t's and performs unsigned integer comparison. 
                                                                                    "},{"location":"opae-code/hash__map_8h/#function-opae_u64_key_hash","title":"function opae_u64_key_hash","text":"
                                                                                    Convenience hash function for arbitrary pointers/64-bit values. 
                                                                                    uint32_t opae_u64_key_hash (\n    uint32_t num_buckets,\n    uint32_t hash_seed,\n    void * key\n) \n
                                                                                     
                                                                                    Simply converts the key to a uint64_t and then performs the modulus operation with the configured num_buckets. hash_seed is unused. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/hash_map.h
                                                                                    "},{"location":"opae-code/hash__map_8h_source/","title":"File hash_map.h","text":"
                                                                                    File List > docs > sw > include > opae > hash_map.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2022-2023, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __OPAE_HASH_MAP_H__\n#define __OPAE_HASH_MAP_H__\n#include <stdint.h>\n#include <stdbool.h>\n#include <opae/types_enum.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif // __cplusplus\n\ntypedef enum _opae_hash_map_flags {\n    OPAE_HASH_MAP_UNIQUE_KEYSPACE = (1u << 0)\n} opae_hash_map_flags;\n\ntypedef struct _opae_hash_map_item {\n    void *key;\n    void *value;\n    struct _opae_hash_map_item *next;\n} opae_hash_map_item;\n\ntypedef struct _opae_hash_map {\n    uint32_t num_buckets;\n    uint32_t hash_seed;\n    opae_hash_map_item **buckets;\n    int flags;\n    void *cleanup_context; \n    uint32_t (*key_hash)(uint32_t num_buckets,     \n                 uint32_t hash_seed,\n                 void *key);\n    int (*key_compare)(void *keya, void *keyb);    \n    void (*key_cleanup)(void *key, void *context);     \n    void (*value_cleanup)(void *value, void *context); \n} opae_hash_map;\n\nfpga_result opae_hash_map_init(opae_hash_map *hm,\n                   uint32_t num_buckets,\n                   uint32_t hash_seed,\n                   int flags,\n                   uint32_t (*key_hash)(uint32_t num_buckets,\n                            uint32_t hash_seed,\n                            void *key),\n                   int (*key_compare)(void *keya, void *keyb),\n                   void (*key_cleanup)(void *key, void *context),\n                   void (*value_cleanup)(void *value, void *context));\n\nfpga_result opae_hash_map_add(opae_hash_map *hm,\n                  void *key,\n                  void *value);\n\nfpga_result opae_hash_map_find(opae_hash_map *hm,\n                   void *key,\n                   void **value);\n\nfpga_result opae_hash_map_remove(opae_hash_map *hm,\n                 void *key);\n\nfpga_result opae_hash_map_destroy(opae_hash_map *hm);\n\nbool opae_hash_map_is_empty(opae_hash_map *hm);\n\nuint32_t opae_u64_key_hash(uint32_t num_buckets,\n               uint32_t hash_seed,\n               void *key);\n\nint opae_u64_key_compare(void *keya, void *keyb);\n\n#ifdef __cplusplus\n}\n#endif // __cplusplus\n\n#endif // __OPAE_HASH_MAP_H__\n
                                                                                    "},{"location":"opae-code/init_8h/","title":"File init.h","text":"
                                                                                    FileList > docs > sw > include > opae > init.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Initialization routine. 
                                                                                     
                                                                                       
                                                                                    • #include <opae/types_enum.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/init_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaFinalize (void) Finalize the OPAE library. fpga_result fpgaInitialize (const char * config_file) Initialize the OPAE library."},{"location":"opae-code/init_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/init_8h/#function-fpgafinalize","title":"function fpgaFinalize","text":"
                                                                                    Finalize the OPAE library. 
                                                                                    fpga_result fpgaFinalize (\n    void\n) \n
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Whether OPAE finalized successfully. 
                                                                                    "},{"location":"opae-code/init_8h/#function-fpgainitialize","title":"function fpgaInitialize","text":"
                                                                                    Initialize the OPAE library. 
                                                                                    fpga_result fpgaInitialize (\n    const char * config_file\n) \n
                                                                                     
                                                                                    Initialize OPAE using the given configuration file path, or perform default initialization if config_file is NULL.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • config_file Path to OPAE configuration file. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Whether OPAE initialized successfully. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/init.h
                                                                                    "},{"location":"opae-code/init_8h_source/","title":"File init.h","text":"
                                                                                    File List > docs > sw > include > opae > init.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_INIT_H__\n#define __FPGA_INIT_H__\n\n#include <opae/types_enum.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaInitialize(const char *config_file);\n\nfpga_result fpgaFinalize(void);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_INIT_H__\n
                                                                                    "},{"location":"opae-code/log_8h/","title":"File log.h","text":"
                                                                                    FileList > docs > sw > include > opae > log.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <stdint.h>
                                                                                      •  
                                                                                    • #include <stdlib.h>
                                                                                      •  
                                                                                    • #include <string.h>
                                                                                      •  
                                                                                    • #include <errno.h>
                                                                                      •  
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/log_8h/#public-types","title":"Public Types","text":"Type Name enum opae_loglevel"},{"location":"opae-code/log_8h/#public-functions","title":"Public Functions","text":"Type Name void opae_print (int loglevel, const char * fmt, ...)"},{"location":"opae-code/log_8h/#macros","title":"Macros","text":"Type Name define OPAE_DBG (format, ...) {  } define OPAE_DEFAULT_LOGLEVEL  OPAE_LOG_ERROR define OPAE_ERR (format, ...)  define OPAE_MSG (format, ...)  define __SHORT_FILE__"},{"location":"opae-code/log_8h/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/log_8h/#enum-opae_loglevel","title":"enum opae_loglevel","text":"
                                                                                    enum opae_loglevel {\n    OPAE_LOG_ERROR = 0,\n    OPAE_LOG_MESSAGE,\n    OPAE_LOG_DEBUG\n};\n
                                                                                    "},{"location":"opae-code/log_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/log_8h/#function-opae_print","title":"function opae_print","text":"
                                                                                    void opae_print (\n    int loglevel,\n    const char * fmt,\n    ...\n) \n
                                                                                    "},{"location":"opae-code/log_8h/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/log_8h/#define-opae_dbg","title":"define OPAE_DBG","text":"
                                                                                    #define OPAE_DBG (\n    format,\n    ...\n) { }\n
                                                                                    "},{"location":"opae-code/log_8h/#define-opae_default_loglevel","title":"define OPAE_DEFAULT_LOGLEVEL","text":"
                                                                                    #define OPAE_DEFAULT_LOGLEVEL OPAE_LOG_ERROR\n
                                                                                    "},{"location":"opae-code/log_8h/#define-opae_err","title":"define OPAE_ERR","text":"
                                                                                    #define OPAE_ERR (\n    format,\n    ...\n) opae_print ( OPAE_LOG_ERROR ,                                \\\n    \"%s:%u:%s() **ERROR** : \" format \"\\n\",                    \\\n    __SHORT_FILE__, __LINE__, __func__, ##__VA_ARGS__)\n
                                                                                    "},{"location":"opae-code/log_8h/#define-opae_msg","title":"define OPAE_MSG","text":"
                                                                                    #define OPAE_MSG (\n    format,\n    ...\n) opae_print ( OPAE_LOG_MESSAGE , \"%s:%u:%s() : \" format \"\\n\", \\\n    __SHORT_FILE__, __LINE__, __func__, ##__VA_ARGS__)\n
                                                                                    "},{"location":"opae-code/log_8h/#define-__short_file__","title":"define __SHORT_FILE__","text":"
                                                                                    #define __SHORT_FILE__ ({                                                     \\\n    const char *file = __FILE__;                           \\\n    const char *p = file;                                  \\\n    while (*p)                                             \\\n        ++p;                                           \\\n    while ((p > file) && ('/' != *p) && ('\\\\' != *p))      \\\n        --p;                                           \\\n    if (p > file)                                          \\\n        ++p;                                           \\\n    p;                                                     \\\n    })\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/log.h
                                                                                    "},{"location":"opae-code/log_8h_source/","title":"File log.h","text":"
                                                                                    File List > docs > sw > include > opae > log.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018-2021, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __OPAE_LOG_H__\n#define __OPAE_LOG_H__\n\n#include <stdint.h>\n#include <stdlib.h>\n#include <string.h>\n#include <errno.h>\n\n#include <opae/types.h>\n\n/*\n* Convenience macros for printing messages and errors.\n*/\n#ifdef __SHORT_FILE__\n#undef __SHORT_FILE__\n#endif // __SHORT_FILE__\n#define __SHORT_FILE__                                         \\\n    ({                                                     \\\n    const char *file = __FILE__;                           \\\n    const char *p = file;                                  \\\n    while (*p)                                             \\\n        ++p;                                           \\\n    while ((p > file) && ('/' != *p) && ('\\\\' != *p))      \\\n        --p;                                           \\\n    if (p > file)                                          \\\n        ++p;                                           \\\n    p;                                                     \\\n    })\n\n#ifdef OPAE_MSG\n#undef OPAE_MSG\n#endif // OPAE_MSG\n#define OPAE_MSG(format, ...)                                     \\\n    opae_print(OPAE_LOG_MESSAGE, \"%s:%u:%s() : \" format \"\\n\", \\\n    __SHORT_FILE__, __LINE__, __func__, ##__VA_ARGS__)\n\n#ifdef OPAE_ERR\n#undef OPAE_ERR\n#endif // OPAE_ERR\n#define OPAE_ERR(format, ...)                                     \\\n    opae_print(OPAE_LOG_ERROR,                                \\\n    \"%s:%u:%s() **ERROR** : \" format \"\\n\",                    \\\n    __SHORT_FILE__, __LINE__, __func__, ##__VA_ARGS__)\n\n#ifdef OPAE_DBG\n#undef OPAE_DBG\n#endif // OPAE_DBG\n#ifdef LIBOPAE_DEBUG\n#define OPAE_DBG(format, ...)                                    \\\n    opae_print(OPAE_LOG_DEBUG,                               \\\n    \"%s:%u:%s() *DEBUG* : \" format \"\\n\",                     \\\n    __SHORT_FILE__, __LINE__, __func__, ##__VA_ARGS__)\n#else\n#define OPAE_DBG(format, ...)                                    \\\n{   }\n#endif // LIBOPAE_DEBUG\n\n/*\n* Logging functions\n*/\nenum opae_loglevel {\n    OPAE_LOG_ERROR = 0, /* critical errors (always print) */\n    OPAE_LOG_MESSAGE,   /* information (i.e. explain return code */\n    OPAE_LOG_DEBUG      /* debugging (also needs #define DEBUG 1) */\n};\n\n#define OPAE_DEFAULT_LOGLEVEL OPAE_LOG_ERROR\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif // __cplusplus\n\nvoid opae_print(int loglevel, const char *fmt, ...);\n\n#ifdef __cplusplus\n}\n#endif // __cplusplus\n\n#endif // __OPAE_LOG_H__\n
                                                                                    "},{"location":"opae-code/manage_8h/","title":"File manage.h","text":"
                                                                                    FileList > docs > sw > include > opae > manage.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Functions for managing FPGA configurations. More...
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/manage_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaAssignPortToInterface (fpga_handle fpga, uint32_t interface_num, uint32_t slot_num, int flags) Assign Port to a host interface. fpga_result fpgaAssignToInterface (fpga_handle fpga, fpga_token accelerator, uint32_t host_interface, int flags) Assign an accelerator to a host interface. fpga_result fpgaReconfigureSlot (fpga_handle fpga, uint32_t slot, const uint8_t * bitstream, size_t bitstream_len, int flags) Reconfigure a slot. fpga_result fpgaReleaseFromInterface (fpga_handle fpga, fpga_token accelerator) Unassign a previously assigned accelerator."},{"location":"opae-code/manage_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    FPGA accelerators can be reprogrammed at run time by providing new partial bitstreams (\"green bitstreams\"). This file defines API functions for programming green bitstreams as well as for assigning accelerators to host interfaces for more complex deployment setups, such as virtualized systems. 
                                                                                    "},{"location":"opae-code/manage_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/manage_8h/#function-fpgaassignporttointerface","title":"function fpgaAssignPortToInterface","text":"
                                                                                    Assign Port to a host interface. 
                                                                                    fpga_result fpgaAssignPortToInterface (\n    fpga_handle fpga,\n    uint32_t interface_num,\n    uint32_t slot_num,\n    int flags\n) \n
                                                                                     
                                                                                    This function assign Port to a host interface for subsequent use. Only Port that have been assigned to a host interface can be opened by fpgaOpen().
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • fpga Handle to an FPGA object previously opened that both the host interface and the slot belong to 
                                                                                      •  
                                                                                    • interface_num Host interface number 
                                                                                      •  
                                                                                    • slot_num Slot number 
                                                                                      •  
                                                                                    • flags Flags (to be defined) 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success FPGA_INVALID_PARAM if input parameter combination is not valid. FPGA_EXCEPTION if an exception occcurred accessing the fpga handle. FPGA_NOT_SUPPORTED if driver does not support assignment. 
                                                                                    "},{"location":"opae-code/manage_8h/#function-fpgaassigntointerface","title":"function fpgaAssignToInterface","text":"
                                                                                    Assign an accelerator to a host interface. 
                                                                                    fpga_result fpgaAssignToInterface (\n    fpga_handle fpga,\n    fpga_token accelerator,\n    uint32_t host_interface,\n    int flags\n) \n
                                                                                     
                                                                                    This function assigns an accelerator to a host interface for subsequent use. Only accelerators that have been assigned to a host interface can be opened by fpgaOpen().
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This function is currently not supported.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • fpga Handle to an FPGA object previously opened that both the host interface and the accelerator belong to 
                                                                                      •  
                                                                                    • accelerator accelerator to assign 
                                                                                      •  
                                                                                    • host_interface Host interface to assign accelerator to 
                                                                                      •  
                                                                                    • flags Flags (to be defined) 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success 
                                                                                    "},{"location":"opae-code/manage_8h/#function-fpgareconfigureslot","title":"function fpgaReconfigureSlot","text":"
                                                                                    Reconfigure a slot. 
                                                                                    fpga_result fpgaReconfigureSlot (\n    fpga_handle fpga,\n    uint32_t slot,\n    const uint8_t * bitstream,\n    size_t bitstream_len,\n    int flags\n) \n
                                                                                     
                                                                                    Sends a green bitstream file to an FPGA to reconfigure a specific slot. This call, if successful, will overwrite the currently programmed AFU in that slot with the AFU in the provided bitstream.
                                                                                     
                                                                                    As part of the reconfiguration flow, all accelerators associated with this slot will be unassigned and reset.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • fpga Handle to an FPGA object previously opened 
                                                                                      •  
                                                                                    • slot Token identifying the slot to reconfigure 
                                                                                      •  
                                                                                    • bitstream Pointer to memory holding the bitstream 
                                                                                      •  
                                                                                    • bitstream_len Length of the bitstream in bytes 
                                                                                      •  
                                                                                    • flags Flags that control behavior of reconfiguration. Value of 0 indicates no flags. FPGA_RECONF_FORCE indicates that the bitstream is programmed into the slot without checking if the resource is currently in use. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if the provided parameters are not valid. FPGA_EXCEPTION if an internal error occurred accessing the handle or while sending the bitstream data to the driver. FPGA_BUSY if the accelerator for the given slot is in use. FPGA_RECONF_ERROR on errors reported by the driver (such as CRC or protocol errors).
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    By default, fpgaReconfigureSlot will not allow reconfiguring a slot with an accelerator in use. Add the flag FPGA_RECONF_FORCE to force reconfiguration without checking for accelerators in use. 
                                                                                    "},{"location":"opae-code/manage_8h/#function-fpgareleasefrominterface","title":"function fpgaReleaseFromInterface","text":"
                                                                                    Unassign a previously assigned accelerator. 
                                                                                    fpga_result fpgaReleaseFromInterface (\n    fpga_handle fpga,\n    fpga_token accelerator\n) \n
                                                                                     
                                                                                    This function removes the assignment of an accelerator to an host interface (e.g. to be later assigned to a different host interface). As a consequence, the accelerator referred to by token 'accelerator' will be reset during the course of this function.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This function is currently not supported.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • fpga Handle to an FPGA object previously opened that both the host interface and the accelerator belong to 
                                                                                      •  
                                                                                    • accelerator accelerator to unassign/release 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/manage.h
                                                                                    "},{"location":"opae-code/manage_8h_source/","title":"File manage.h","text":"
                                                                                    File List > docs > sw > include > opae > manage.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_MANAGE_H__\n#define __FPGA_MANAGE_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaAssignPortToInterface(fpga_handle fpga,\n                    uint32_t interface_num,\n                    uint32_t slot_num,\n                    int flags);\n\nfpga_result fpgaAssignToInterface(fpga_handle fpga,\n                  fpga_token accelerator,\n                  uint32_t host_interface,\n                  int flags);\n\nfpga_result fpgaReleaseFromInterface(fpga_handle fpga,\n                     fpga_token accelerator);\n\nfpga_result fpgaReconfigureSlot(fpga_handle fpga,\n                uint32_t slot,\n                const uint8_t *bitstream,\n                size_t bitstream_len, int flags);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_MANAGE_H__\n
                                                                                    "},{"location":"opae-code/mem__alloc_8h/","title":"File mem_alloc.h","text":"
                                                                                    FileList > docs > sw > include > opae > mem_alloc.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <stdint.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/mem__alloc_8h/#classes","title":"Classes","text":"Type Name struct mem_alloc struct mem_link Provides an API for allocating/freeing a logical address space."},{"location":"opae-code/mem__alloc_8h/#public-functions","title":"Public Functions","text":"Type Name int mem_alloc_add_free (struct mem_alloc * m, uint64_t address, uint64_t size) Add a memory region to an allocator. void mem_alloc_destroy (struct mem_alloc * m) Destroy a memory allocator object. int mem_alloc_get (struct mem_alloc * m, uint64_t * address, uint64_t size) Allocate memory. void mem_alloc_init (struct mem_alloc * m) Initialize a memory allocator object. int mem_alloc_put (struct mem_alloc * m, uint64_t address) Free memory."},{"location":"opae-code/mem__alloc_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/mem__alloc_8h/#function-mem_alloc_add_free","title":"function mem_alloc_add_free","text":"
                                                                                    Add a memory region to an allocator. 
                                                                                    int mem_alloc_add_free (\n    struct mem_alloc * m,\n    uint64_t address,\n    uint64_t size\n) \n
                                                                                     
                                                                                    The memory region is added to the allocatable space and is immediately ready for allocation.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • m The memory allocator object. 
                                                                                      •  
                                                                                    • address The beginning address of the memory region. 
                                                                                      •  
                                                                                    • size The size of the memory region. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error. Zero on success.
                                                                                     
                                                                                    Example  struct mem_alloc m;
                                                                                     
                                                                                    mem_alloc_init(&m);
                                                                                     
                                                                                    if (mem_alloc_add_free(&m, 0x4000, 4096)) {   // handle error }
                                                                                    "},{"location":"opae-code/mem__alloc_8h/#function-mem_alloc_destroy","title":"function mem_alloc_destroy","text":"
                                                                                    Destroy a memory allocator object. 
                                                                                    void mem_alloc_destroy (\n    struct mem_alloc * m\n) \n
                                                                                     
                                                                                    Frees all of the allocator's internal resources.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • m The address of the memory allocator to destroy. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/mem__alloc_8h/#function-mem_alloc_get","title":"function mem_alloc_get","text":"
                                                                                    Allocate memory. 
                                                                                    int mem_alloc_get (\n    struct mem_alloc * m,\n    uint64_t * address,\n    uint64_t size\n) \n
                                                                                     
                                                                                    Retrieve an available memory address for a free block that is at least size bytes.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • m The memory allocator object. 
                                                                                      •  
                                                                                    • address The retrieved address for the allocation. 
                                                                                      •  
                                                                                    • size The request size in bytes. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error. Zero on success.
                                                                                     
                                                                                    Example  struct mem_alloc m; uint64_t addr = 0;
                                                                                     
                                                                                    mem_alloc_init(&m);
                                                                                     
                                                                                    if (mem_alloc_add_free(&m, 0x4000, 4096)) {   // handle error }
                                                                                     
                                                                                    ...
                                                                                     
                                                                                    if (mem_alloc_get(&m, &addr, 4096)) {   // handle allocation error }
                                                                                    "},{"location":"opae-code/mem__alloc_8h/#function-mem_alloc_init","title":"function mem_alloc_init","text":"
                                                                                    Initialize a memory allocator object. 
                                                                                    void mem_alloc_init (\n    struct mem_alloc * m\n) \n
                                                                                     
                                                                                    After the call, the allocator is initialized but \"empty\". To add allocatable memory regions, further initialize the allocator with mem_alloc_add_free().
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • m The address of the memory allocator to initialize. 
                                                                                      •  
                                                                                    "},{"location":"opae-code/mem__alloc_8h/#function-mem_alloc_put","title":"function mem_alloc_put","text":"
                                                                                    Free memory. 
                                                                                    int mem_alloc_put (\n    struct mem_alloc * m,\n    uint64_t address\n) \n
                                                                                     
                                                                                    Release a previously-allocated memory block.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • m The memory allocator object. 
                                                                                      •  
                                                                                    • address The address to free. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error. Zero on success.
                                                                                     
                                                                                    Example  struct mem_alloc m; uint64_t addr = 0;
                                                                                     
                                                                                    mem_alloc_init(&m);
                                                                                     
                                                                                    if (mem_alloc_add_free(&m, 0x4000, 4096)) {   // handle error }
                                                                                     
                                                                                    ...
                                                                                     
                                                                                    if (mem_alloc_get(&m, &addr, 4096)) {   // handle allocation error }
                                                                                     
                                                                                    ...
                                                                                     
                                                                                    if (mem_alloc_put(&m, addr)) {   // handle free error }
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/mem_alloc.h
                                                                                    "},{"location":"opae-code/mem__alloc_8h_source/","title":"File mem_alloc.h","text":"
                                                                                    File List > docs > sw > include > opae > mem_alloc.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2020, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n#ifndef __OPAE_MEM_ALLOC_H__\n#define __OPAE_MEM_ALLOC_H__\n\n#include <stdint.h>\n\nstruct mem_link {\n    uint64_t address;\n    uint64_t size;\n    struct mem_link *prev;\n    struct mem_link *next;\n};\n\nstruct mem_alloc {\n    struct mem_link free;\n    struct mem_link allocated;\n};\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif // __cplusplus\n\nvoid mem_alloc_init(struct mem_alloc *m);\n\nvoid mem_alloc_destroy(struct mem_alloc *m);\n\nint mem_alloc_add_free(struct mem_alloc *m,\n               uint64_t address,\n               uint64_t size);\n\nint mem_alloc_get(struct mem_alloc *m,\n          uint64_t *address,\n          uint64_t size);\n\nint mem_alloc_put(struct mem_alloc *m,\n          uint64_t address);\n\n#ifdef __cplusplus\n}\n#endif // __cplusplus\n\n#endif // __OPAE_MEM_ALLOC_H__\n
                                                                                    "},{"location":"opae-code/metrics_8h/","title":"File metrics.h","text":"
                                                                                    FileList > docs > sw > include > opae > metrics.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Functions for Discover/ Enumerates metrics and retrieves values. 
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/metrics_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaGetMetricsByIndex (fpga_handle handle, uint64_t * metric_num, uint64_t num_metric_indexes, fpga_metric * metrics) Retrieve metrics values by index. fpga_result fpgaGetMetricsByName (fpga_handle handle, char ** metrics_names, uint64_t num_metric_names, fpga_metric * metrics) Retrieve metric values by names. fpga_result fpgaGetMetricsInfo (fpga_handle handle, fpga_metric_info * metric_info, uint64_t * num_metrics) Retrieve metrics information. fpga_result fpgaGetMetricsThresholdInfo (fpga_handle handle, struct metric_threshold * metric_thresholds, uint32_t * num_thresholds) Retrieve metrics / sendor threshold information and values. fpga_result fpgaGetNumMetrics (fpga_handle handle, uint64_t * num_metrics) Enumerates number of metrics."},{"location":"opae-code/metrics_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/metrics_8h/#function-fpgagetmetricsbyindex","title":"function fpgaGetMetricsByIndex","text":"
                                                                                    Retrieve metrics values by index. 
                                                                                    fpga_result fpgaGetMetricsByIndex (\n    fpga_handle handle,\n    uint64_t * metric_num,\n    uint64_t num_metric_indexes,\n    fpga_metric * metrics\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened fpga resource 
                                                                                      •  
                                                                                    • metric_num Pointer to array of metric index user allocates metric array 
                                                                                      •  
                                                                                    • num_metric_indexes Size of metric array 
                                                                                      •  
                                                                                    • metrics pointer to array of metric struct
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_NOT_FOUND if the Metrics are not found. FPGA_NO_MEMORY if there was not enough memory to enumerates metrics. 
                                                                                    "},{"location":"opae-code/metrics_8h/#function-fpgagetmetricsbyname","title":"function fpgaGetMetricsByName","text":"
                                                                                    Retrieve metric values by names. 
                                                                                    fpga_result fpgaGetMetricsByName (\n    fpga_handle handle,\n    char ** metrics_names,\n    uint64_t num_metric_names,\n    fpga_metric * metrics\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened fpga resource 
                                                                                      •  
                                                                                    • metrics_names Pointer to array of metrics name user allocates metrics name array 
                                                                                      •  
                                                                                    • num_metric_names Size of metric name array 
                                                                                      •  
                                                                                    • metrics Pointer to array of metric struct
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_NOT_FOUND if the Metrics are not found 
                                                                                    "},{"location":"opae-code/metrics_8h/#function-fpgagetmetricsinfo","title":"function fpgaGetMetricsInfo","text":"
                                                                                    Retrieve metrics information. 
                                                                                    fpga_result fpgaGetMetricsInfo (\n    fpga_handle handle,\n    fpga_metric_info * metric_info,\n    uint64_t * num_metrics\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened fpga resource 
                                                                                      •  
                                                                                    • metric_info Pointer to array of metric info struct user allocates metrics info array
                                                                                      •  
                                                                                    • num_metrics Size of metric info array
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_NOT_FOUND if the Metrics are not found. FPGA_NO_MEMORY if there was not enough memory to enumerates metrics. 
                                                                                    "},{"location":"opae-code/metrics_8h/#function-fpgagetmetricsthresholdinfo","title":"function fpgaGetMetricsThresholdInfo","text":"
                                                                                    Retrieve metrics / sendor threshold information and values. 
                                                                                    fpga_result fpgaGetMetricsThresholdInfo (\n    fpga_handle handle,\n    struct metric_threshold * metric_thresholds,\n    uint32_t * num_thresholds\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened fpga resource 
                                                                                      •  
                                                                                    • metrics_threshold pointer to array of metric thresholds user allocates threshold array memory Number of thresholds returns enumerated thresholds if user pass NULL metrics_thresholds 
                                                                                      •  
                                                                                    • num_thresholds number of thresholds
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_NOT_FOUND if the Metrics are not found. FPGA_NO_MEMORY if there was not enough memory to enumerates metrics. 
                                                                                    "},{"location":"opae-code/metrics_8h/#function-fpgagetnummetrics","title":"function fpgaGetNumMetrics","text":"
                                                                                    Enumerates number of metrics. 
                                                                                    fpga_result fpgaGetNumMetrics (\n    fpga_handle handle,\n    uint64_t * num_metrics\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened fpga resource 
                                                                                      •  
                                                                                    • num_metrics Number of metrics are discovered in fpga resource
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_NOT_FOUND if the Metrics are not discovered 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/metrics.h
                                                                                    "},{"location":"opae-code/metrics_8h_source/","title":"File metrics.h","text":"
                                                                                    File List > docs > sw > include > opae > metrics.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_METRICS_H__\n#define __FPGA_METRICS_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaGetNumMetrics(fpga_handle handle,\n                uint64_t *num_metrics);\n\nfpga_result fpgaGetMetricsInfo(fpga_handle handle,\n                fpga_metric_info *metric_info,\n                uint64_t *num_metrics);\n\nfpga_result fpgaGetMetricsByIndex(fpga_handle handle,\n                uint64_t *metric_num,\n                uint64_t num_metric_indexes,\n                fpga_metric *metrics);\n\nfpga_result fpgaGetMetricsByName(fpga_handle handle,\n                char **metrics_names,\n                uint64_t num_metric_names,\n                fpga_metric *metrics);\n\n\nfpga_result fpgaGetMetricsThresholdInfo(fpga_handle handle,\n                struct metric_threshold *metric_thresholds,\n                uint32_t *num_thresholds);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_METRICS_H__\n
                                                                                    "},{"location":"opae-code/mmio_8h/","title":"File mmio.h","text":"
                                                                                    FileList > docs > sw > include > opae > mmio.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Functions for mapping and accessing MMIO space. More...
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/mmio_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaMapMMIO (fpga_handle handle, uint32_t mmio_num, uint64_t ** mmio_ptr) Map MMIO space. fpga_result fpgaReadMMIO32 (fpga_handle handle, uint32_t mmio_num, uint64_t offset, uint32_t * value) Read 32 bit value from MMIO space. fpga_result fpgaReadMMIO64 (fpga_handle handle, uint32_t mmio_num, uint64_t offset, uint64_t * value) Read 64 bit value from MMIO space. fpga_result fpgaUnmapMMIO (fpga_handle handle, uint32_t mmio_num) Unmap MMIO space. fpga_result fpgaWriteMMIO32 (fpga_handle handle, uint32_t mmio_num, uint64_t offset, uint32_t value) Write 32 bit value to MMIO space. fpga_result fpgaWriteMMIO512 (fpga_handle handle, uint32_t mmio_num, uint64_t offset, const void * value) Write 512 bit value to MMIO space. fpga_result fpgaWriteMMIO64 (fpga_handle handle, uint32_t mmio_num, uint64_t offset, uint64_t value) Write 64 bit value to MMIO space."},{"location":"opae-code/mmio_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    Most FPGA accelerators provide access to control registers through memory-mappable address spaces, commonly referred to as \"MMIO spaces\". This file provides functions to map, unmap, read, and write MMIO spaces.
                                                                                     
                                                                                    Note that an accelerator may have multiple MMIO spaces, denoted by the mmio_num argument of the APIs below. The meaning and properties of each MMIO space are up to the accelerator designer. 
                                                                                    "},{"location":"opae-code/mmio_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/mmio_8h/#function-fpgamapmmio","title":"function fpgaMapMMIO","text":"
                                                                                    Map MMIO space. 
                                                                                    fpga_result fpgaMapMMIO (\n    fpga_handle handle,\n    uint32_t mmio_num,\n    uint64_t ** mmio_ptr\n) \n
                                                                                     
                                                                                    This function will return a pointer to the specified MMIO space of the target object in process virtual memory, if supported by the target. Some MMIO spaces may be restricted to privileged processes, depending on the used handle and type.
                                                                                     
                                                                                    After mapping the respective MMIO space, you can access it through direct pointer operations (observing supported access sizes and alignments of the target platform and accelerator).
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    Some targets (such as the ASE simulator) do not support memory-mapping of IO register spaces and will not return a pointer to an actually mapped space. Instead, they will return FPGA_NOT_SUPPORTED. Usually, these platforms still allow the application to issue MMIO operations using fpgaReadMMIO32(), fpgaWriteMMIO32(), fpgeReadMMIO64(), and fpgaWriteMMIO64().
                                                                                     
                                                                                    If the caller passes in NULL for mmio_ptr, no mapping will be performed, and no virtual address will be returned, though the call will return FPGA_OK. This implies that all accesses will be performed through fpgaReadMMIO32(), fpgaWriteMMIO32(), fpgeReadMMIO64(), and fpgaWriteMMIO64(). This is the only supported case for ASE.
                                                                                     
                                                                                    The number of available MMIO spaces can be retrieved through the num_mmio property (fpgaPropertyGetNumMMIO()).
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened resource 
                                                                                      •  
                                                                                    • mmio_num Number of MMIO space to access 
                                                                                      •  
                                                                                    • mmio_ptr Pointer to memory where a pointer to the MMIO space will be returned. May be NULL, in which case no pointer is returned. Returned address may be NULL if underlying platform does not support memory mapping for register access. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. FPGA_NO_ACCESS if the process' permissions are not sufficient to map the requested MMIO space. FPGA_NOT_SUPPORTED if platform does not support memory mapped IO. 
                                                                                    "},{"location":"opae-code/mmio_8h/#function-fpgareadmmio32","title":"function fpgaReadMMIO32","text":"
                                                                                    Read 32 bit value from MMIO space. 
                                                                                    fpga_result fpgaReadMMIO32 (\n    fpga_handle handle,\n    uint32_t mmio_num,\n    uint64_t offset,\n    uint32_t * value\n) \n
                                                                                     
                                                                                    This function will read from MMIO space of the target object at a specified offset.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource 
                                                                                      •  
                                                                                    • mmio_num Number of MMIO space to access 
                                                                                      •  
                                                                                    • offset Byte offset into MMIO space 
                                                                                      •  
                                                                                    • value Pointer to memory where read value is returned (32 bit) 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                                                                    "},{"location":"opae-code/mmio_8h/#function-fpgareadmmio64","title":"function fpgaReadMMIO64","text":"
                                                                                    Read 64 bit value from MMIO space. 
                                                                                    fpga_result fpgaReadMMIO64 (\n    fpga_handle handle,\n    uint32_t mmio_num,\n    uint64_t offset,\n    uint64_t * value\n) \n
                                                                                     
                                                                                    This function will read from MMIO space of the target object at a specified offset.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource 
                                                                                      •  
                                                                                    • mmio_num Number of MMIO space to access 
                                                                                      •  
                                                                                    • offset Byte offset into MMIO space 
                                                                                      •  
                                                                                    • value Pointer to memory where read value is returned (64 bit) 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                                                                    "},{"location":"opae-code/mmio_8h/#function-fpgaunmapmmio","title":"function fpgaUnmapMMIO","text":"
                                                                                    Unmap MMIO space. 
                                                                                    fpga_result fpgaUnmapMMIO (\n    fpga_handle handle,\n    uint32_t mmio_num\n) \n
                                                                                     
                                                                                    This function will unmap a previously mapped MMIO space of the target object, rendering any pointers to it invalid.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This call is only supported by hardware targets, not by ASE simulation.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened resource 
                                                                                      •  
                                                                                    • mmio_num Number of MMIO space to access 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                                                                    "},{"location":"opae-code/mmio_8h/#function-fpgawritemmio32","title":"function fpgaWriteMMIO32","text":"
                                                                                    Write 32 bit value to MMIO space. 
                                                                                    fpga_result fpgaWriteMMIO32 (\n    fpga_handle handle,\n    uint32_t mmio_num,\n    uint64_t offset,\n    uint32_t value\n) \n
                                                                                     
                                                                                    This function will write to MMIO space of the target object at a specified offset.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource 
                                                                                      •  
                                                                                    • mmio_num Number of MMIO space to access 
                                                                                      •  
                                                                                    • offset Byte offset into MMIO space 
                                                                                      •  
                                                                                    • value Value to write (32 bit) 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                                                                    "},{"location":"opae-code/mmio_8h/#function-fpgawritemmio512","title":"function fpgaWriteMMIO512","text":"
                                                                                    Write 512 bit value to MMIO space. 
                                                                                    fpga_result fpgaWriteMMIO512 (\n    fpga_handle handle,\n    uint32_t mmio_num,\n    uint64_t offset,\n    const void * value\n) \n
                                                                                     
                                                                                    512 bit MMIO writes may not be supported on all platforms.
                                                                                     
                                                                                    This function will write to MMIO space of the target object at a specified offset.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource 
                                                                                      •  
                                                                                    • mmio_num Number of MMIO space to access 
                                                                                      •  
                                                                                    • offset Byte offset into MMIO space 
                                                                                      •  
                                                                                    • value Pointer to memory holding value to write (512 bits) 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                                                                    "},{"location":"opae-code/mmio_8h/#function-fpgawritemmio64","title":"function fpgaWriteMMIO64","text":"
                                                                                    Write 64 bit value to MMIO space. 
                                                                                    fpga_result fpgaWriteMMIO64 (\n    fpga_handle handle,\n    uint32_t mmio_num,\n    uint64_t offset,\n    uint64_t value\n) \n
                                                                                     
                                                                                    This function will write to MMIO space of the target object at a specified offset.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource 
                                                                                      •  
                                                                                    • mmio_num Number of MMIO space to access 
                                                                                      •  
                                                                                    • offset Byte offset into MMIO space 
                                                                                      •  
                                                                                    • value Value to write (64 bit) 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/mmio.h
                                                                                    "},{"location":"opae-code/mmio_8h_source/","title":"File mmio.h","text":"
                                                                                    File List > docs > sw > include > opae > mmio.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_MMIO_H__\n#define __FPGA_MMIO_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaWriteMMIO64(fpga_handle handle,\n                uint32_t mmio_num, uint64_t offset,\n                uint64_t value);\n\nfpga_result fpgaReadMMIO64(fpga_handle handle,\n               uint32_t mmio_num,\n               uint64_t offset, uint64_t *value);\n\nfpga_result fpgaWriteMMIO32(fpga_handle handle,\n                uint32_t mmio_num, uint64_t offset,\n                uint32_t value);\n\nfpga_result fpgaReadMMIO32(fpga_handle handle,\n               uint32_t mmio_num,\n               uint64_t offset, uint32_t *value);\n\nfpga_result fpgaWriteMMIO512(fpga_handle handle,\n                uint32_t mmio_num, uint64_t offset,\n                const void *value);\n\nfpga_result fpgaMapMMIO(fpga_handle handle,\n            uint32_t mmio_num, uint64_t **mmio_ptr);\n\nfpga_result fpgaUnmapMMIO(fpga_handle handle,\n              uint32_t mmio_num);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_MMIO_H__\n
                                                                                    "},{"location":"opae-code/properties_8h/","title":"File properties.h","text":"
                                                                                    FileList > docs > sw > include > opae > properties.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Functions for examining and manipulating fpga_properties objects.More...
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/properties_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaClearProperties (fpga_properties prop) Clear a fpga_properties object. fpga_result fpgaCloneProperties (fpga_properties src, fpga_properties * dst) Clone a fpga_properties object. fpga_result fpgaDestroyProperties (fpga_properties * prop) Destroy a fpga_properties object. fpga_result fpgaGetProperties (fpga_token token, fpga_properties * prop) Create a fpga_properties object. fpga_result fpgaGetPropertiesFromHandle (fpga_handle handle, fpga_properties * prop) Create a fpga_properties object. fpga_result fpgaPropertiesGetAcceleratorState (const fpga_properties prop, fpga_accelerator_state * state) Get the state of a accelerator resource property. fpga_result fpgaPropertiesGetBBSID (const fpga_properties prop, uint64_t * bbs_id) Get the BBS ID of an FPGA resource property. fpga_result fpgaPropertiesGetBBSVersion (const fpga_properties prop, fpga_version * bbs_version) Get the BBS Version of an FPGA resource property. fpga_result fpgaPropertiesGetBus (const fpga_properties prop, uint8_t * bus) Get the PCI bus number of a resource. fpga_result fpgaPropertiesGetCapabilities (const fpga_properties prop, uint64_t * capabilities) Get the capabilities FPGA resource property. fpga_result fpgaPropertiesGetDevice (const fpga_properties prop, uint8_t * device) Get the PCI device number of a resource. fpga_result fpgaPropertiesGetDeviceID (const fpga_properties prop, uint16_t * device_id) Get the device id of the resource. fpga_result fpgaPropertiesGetFunction (const fpga_properties prop, uint8_t * function) Get the PCI function number of a resource. fpga_result fpgaPropertiesGetGUID (const fpga_properties prop, fpga_guid * guid) Get the GUID of a resource. fpga_result fpgaPropertiesGetInterface (const fpga_properties prop, fpga_interface * interface) Get the OPAE plugin interface implemented by a resource. fpga_result fpgaPropertiesGetLocalMemorySize (const fpga_properties prop, uint64_t * lms) Get the local memory size of an FPGA resource property. fpga_result fpgaPropertiesGetModel (const fpga_properties prop, char * model) Get the model of an FPGA resource property. fpga_result fpgaPropertiesGetNumErrors (const fpga_properties prop, uint32_t * num_errors) Get the number of errors that can be reported by a resource. fpga_result fpgaPropertiesGetNumInterrupts (const fpga_properties prop, uint32_t * num_interrupts) Get the number of interrupts. fpga_result fpgaPropertiesGetNumMMIO (const fpga_properties prop, uint32_t * mmio_spaces) Get the number of mmio spaces. fpga_result fpgaPropertiesGetNumSlots (const fpga_properties prop, uint32_t * num_slots) Get the number of slots of an FPGA resource property. fpga_result fpgaPropertiesGetObjectID (const fpga_properties prop, uint64_t * object_id) Get the object ID of a resource. fpga_result fpgaPropertiesGetObjectType (const fpga_properties prop, fpga_objtype * objtype) Get the object type of a resource. fpga_result fpgaPropertiesGetParent (const fpga_properties prop, fpga_token * parent) Get the token of the parent object. fpga_result fpgaPropertiesGetSegment (const fpga_properties prop, uint16_t * segment) Get the PCI segment number of a resource. fpga_result fpgaPropertiesGetSocketID (const fpga_properties prop, uint8_t * socket_id) Get the socket id of a resource. fpga_result fpgaPropertiesGetSubsystemDeviceID (const fpga_properties prop, uint16_t * subsystem_device_id) Get the subsystem device id of an FPGA resource property. fpga_result fpgaPropertiesGetSubsystemVendorID (const fpga_properties prop, uint16_t * subsystem_vendor_id) Get the subsystem vendor id of an FPGA resource property. fpga_result fpgaPropertiesGetVendorID (const fpga_properties prop, uint16_t * vendor_id) Get the vendor id of an FPGA resource property. fpga_result fpgaPropertiesSetAcceleratorState (fpga_properties prop, fpga_accelerator_state state) Set the state of an accelerator resource property. fpga_result fpgaPropertiesSetBBSID (fpga_properties prop, uint64_t bbs_id) Set the BBS ID of an FPGA resource property. fpga_result fpgaPropertiesSetBBSVersion (fpga_properties prop, fpga_version version) Set the BBS Version of an FPGA resource property. fpga_result fpgaPropertiesSetBus (fpga_properties prop, uint8_t bus) Set the PCI bus number of a resource. fpga_result fpgaPropertiesSetCapabilities (fpga_properties prop, uint64_t capabilities) Set the capabilities of an FPGA resource property. fpga_result fpgaPropertiesSetDevice (fpga_properties prop, uint8_t device) Set the PCI device number of a resource. fpga_result fpgaPropertiesSetDeviceID (fpga_properties prop, uint16_t device_id) Set the device id of the resource. fpga_result fpgaPropertiesSetFunction (fpga_properties prop, uint8_t function) Set the PCI function number of a resource. fpga_result fpgaPropertiesSetGUID (fpga_properties prop, fpga_guid guid) Set the GUID of a resource. fpga_result fpgaPropertiesSetInterface (const fpga_properties prop, fpga_interface interface) Set the OPAE plugin interface implemented by a resource. fpga_result fpgaPropertiesSetLocalMemorySize (fpga_properties prop, uint64_t lms) Set the local memory size of an FPGA resource property. fpga_result fpgaPropertiesSetModel (fpga_properties prop, char * model) Set the model of an FPGA resource property. fpga_result fpgaPropertiesSetNumErrors (const fpga_properties prop, uint32_t num_errors) Set the number of error registers. fpga_result fpgaPropertiesSetNumInterrupts (fpga_properties prop, uint32_t num_interrupts) Set the number of interrupts. fpga_result fpgaPropertiesSetNumMMIO (fpga_properties prop, uint32_t mmio_spaces) Set the number of mmio spaces. fpga_result fpgaPropertiesSetNumSlots (fpga_properties prop, uint32_t num_slots) Set the number of slots of an FPGA resource property. fpga_result fpgaPropertiesSetObjectID (const fpga_properties prop, uint64_t object_id) Set the object ID of a resource. fpga_result fpgaPropertiesSetObjectType (fpga_properties prop, fpga_objtype objtype) Set the object type of a resource. fpga_result fpgaPropertiesSetParent (fpga_properties prop, fpga_token parent) Set the token of the parent object. fpga_result fpgaPropertiesSetSegment (fpga_properties prop, uint16_t segment) Set the PCI segment number of a resource. fpga_result fpgaPropertiesSetSocketID (fpga_properties prop, uint8_t socket_id) Set the socket id of the resource. fpga_result fpgaPropertiesSetSubsystemDeviceID (fpga_properties prop, uint16_t subsystem_device_id) Set the subsystem device id of an FPGA resource property. fpga_result fpgaPropertiesSetSubsystemVendorID (fpga_properties prop, uint16_t subsystem_vendor_id) Set the subsystem vendor id of an FPGA resource property. fpga_result fpgaPropertiesSetVendorID (fpga_properties prop, uint16_t vendor_id) Set the vendor id of an FPGA resource property. fpga_result fpgaUpdateProperties (fpga_token token, fpga_properties prop) Update a fpga_properties object."},{"location":"opae-code/properties_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    In OPAE, fpga_properties objects are used both for obtaining information about resources and for selectively enumerating resources based on their properties. This file provides accessor functions (get/set) to allow reading and writing individual items of an fpga_properties object. Generally, not all object types supported by OPAE carry all properties. If you call a property accessor method on a fpga_properties object that does not support this particular property, it will return FPGA_INVALID_PARAM.
                                                                                    "},{"location":"opae-code/properties_8h/#accessor-return-values","title":"Accessor Return Values","text":"
                                                                                    In addition to the return values specified in the documentation below, all accessor functions return FPGA_OK on success, FPGA_INVALID_PARAM if you pass NULL or invalid parameters (i.e. non-initialized properties objects), FPGA_EXCEPTION if an internal exception occurred trying to access the properties object, FPGA_NOT_FOUND if the requested property is not part of the supplied properties object. 
                                                                                    "},{"location":"opae-code/properties_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/properties_8h/#function-fpgaclearproperties","title":"function fpgaClearProperties","text":"
                                                                                    Clear a fpga_properties object. 
                                                                                    fpga_result fpgaClearProperties (\n    fpga_properties prop\n) \n
                                                                                     
                                                                                    Sets all fields of the properties object pointed at by 'prop' to 'don't care', which implies that the fpga_properties object would match all FPGA resources if used for an fpgaEnumerate() query. The matching criteria can be further refined by using fpgaSet* functions on the properties object.
                                                                                     
                                                                                    Instead of creating a new fpga_properties object every time, this function can be used to re-use fpga_properties objects from previous queries.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop fpga_properties object to clear 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if prop is not a valid object. FPGA_EXCEPTION if an * internal exception occured when trying to access prop. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgacloneproperties","title":"function fpgaCloneProperties","text":"
                                                                                    Clone a fpga_properties object. 
                                                                                    fpga_result fpgaCloneProperties (\n    fpga_properties src,\n    fpga_properties * dst\n) \n
                                                                                     
                                                                                    Creates a copy of an fpga_properties object.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This call creates a new properties object and allocates memory for it. Both the 'src' and the newly created 'dst' objects will eventually need to be destroyed using fpgaDestroyProperties().
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • src fpga_properties object to copy 
                                                                                      •  
                                                                                    • dst New fpga_properties object cloned from 'src' 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if src is not a valid object, or if dst is NULL. FPGA_NO_MEMORY if there was not enough memory to allocate an fpga_properties object for dst. FPGA_EXCEPTION if an internal exception occurred either accessing src or updating dst. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgadestroyproperties","title":"function fpgaDestroyProperties","text":"
                                                                                    Destroy a fpga_properties object. 
                                                                                    fpga_result fpgaDestroyProperties (\n    fpga_properties * prop\n) \n
                                                                                     
                                                                                    Destroys an existing fpga_properties object that the caller has previously created using fpgaGetProperties() or fpgaCloneProperties().
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    fpgaDestroyProperties() requires the address of an fpga_properties object, similar to fpgaGetPropertiesFromHandle(), fpgaGetProperties(), and fpgaCloneProperties(). Passing any other value results in undefined behavior.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Pointer to the fpga_properties object to destroy 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM is prop is not a valid object. FPGA_EXCEPTION if an internal exception occurrred while trying to access prop. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgagetproperties","title":"function fpgaGetProperties","text":"
                                                                                    Create a fpga_properties object. 
                                                                                    fpga_result fpgaGetProperties (\n    fpga_token token,\n    fpga_properties * prop\n) \n
                                                                                     
                                                                                    Initializes the memory pointed at by prop to represent a properties object, and populates it with the properties of the resource referred to by token. Individual properties can then be queried using fpgaPropertiesGet*() accessor functions.
                                                                                     
                                                                                    If token is NULL, an \"empty\" properties object is created to be used as a filter for fpgaEnumerate(). All individual fields are set to dont care`, which implies that the fpga_properties object would match all FPGA resources if used for an fpgaEnumerate() query. The matching criteria can be further refined by using fpgaSet* functions on the properties object, or the object can be populated with the actual properties of a resource by using fpgaUpdateProperties().
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    fpgaGetProperties() will allocate memory for the created properties object returned in prop. It is the responsibility of the caller to free this memory after use by calling fpgaDestroyProperties().
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • token Token to get properties for. Can be NULL, which will create an empty properties object to be used as a filter for fpgaEnumerate(). 
                                                                                      •  
                                                                                    • prop Pointer to a variable of type fpga_properties 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_NO_MEMORY if no memory could be allocated to create the fpga_properties object. FPGA_EXCEPTION if an exception happend while initializing the fpga_properties object. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgagetpropertiesfromhandle","title":"function fpgaGetPropertiesFromHandle","text":"
                                                                                    Create a fpga_properties object. 
                                                                                    fpga_result fpgaGetPropertiesFromHandle (\n    fpga_handle handle,\n    fpga_properties * prop\n) \n
                                                                                     
                                                                                    Initializes the memory pointed at by prop to represent a properties object, and populates it with the properties of the resource referred to by handle. Individual properties can then be queried using fpgaPropertiesGet*() accessor functions.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    fpgaGetPropertiesFromHandle() will allocate memory for the created properties object returned in prop. It is the responsibility of the caller to free this memory after use by calling fpgaDestroyProperties().
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Open handle to get properties for. 
                                                                                      •  
                                                                                    • prop Pointer to a variable of type fpga_properties 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_NO_MEMORY if no memory could be allocated to create the fpga_properties object. FPGA_EXCEPTION if an exception happend while initializing the fpga_properties object. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetacceleratorstate","title":"function fpgaPropertiesGetAcceleratorState","text":"
                                                                                    Get the state of a accelerator resource property. 
                                                                                    fpga_result fpgaPropertiesGetAcceleratorState (\n    const fpga_properties prop,\n    fpga_accelerator_state * state\n) \n
                                                                                     
                                                                                    Returns the accelerator state of a accelerator.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query - must be of type FPGA_ACCELERATOR 
                                                                                      •  
                                                                                    • state Pointer to a accelerator state variable of the accelerator 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_ACCELERATOR. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetbbsid","title":"function fpgaPropertiesGetBBSID","text":"
                                                                                    Get the BBS ID of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesGetBBSID (\n    const fpga_properties prop,\n    uint64_t * bbs_id\n) \n
                                                                                     
                                                                                    Returns the blue bitstream id of an FPGA.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • bbs_id Pointer to a bbs id variable of the FPGA 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetbbsversion","title":"function fpgaPropertiesGetBBSVersion","text":"
                                                                                    Get the BBS Version of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesGetBBSVersion (\n    const fpga_properties prop,\n    fpga_version * bbs_version\n) \n
                                                                                     
                                                                                    Returns the blue bitstream version of an FPGA.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • bbs_version Pointer to a bbs version variable of the FPGA 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetbus","title":"function fpgaPropertiesGetBus","text":"
                                                                                    Get the PCI bus number of a resource. 
                                                                                    fpga_result fpgaPropertiesGetBus (\n    const fpga_properties prop,\n    uint8_t * bus\n) \n
                                                                                     
                                                                                    Returns the bus number the queried resource.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • bus Pointer to a PCI bus variable of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetcapabilities","title":"function fpgaPropertiesGetCapabilities","text":"
                                                                                    Get the capabilities FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesGetCapabilities (\n    const fpga_properties prop,\n    uint64_t * capabilities\n) \n
                                                                                     
                                                                                    Returns the capabilities of an FPGA. Capabilities is a bitfield value
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • capabilities Pointer to a capabilities variable of the FPGA 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This API is not currently supported. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetdevice","title":"function fpgaPropertiesGetDevice","text":"
                                                                                    Get the PCI device number of a resource. 
                                                                                    fpga_result fpgaPropertiesGetDevice (\n    const fpga_properties prop,\n    uint8_t * device\n) \n
                                                                                     
                                                                                    Returns the device number the queried resource.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • device Pointer to a PCI device variable of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetdeviceid","title":"function fpgaPropertiesGetDeviceID","text":"
                                                                                    Get the device id of the resource. 
                                                                                    fpga_result fpgaPropertiesGetDeviceID (\n    const fpga_properties prop,\n    uint16_t * device_id\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • device_id Pointer to a device id variable of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetfunction","title":"function fpgaPropertiesGetFunction","text":"
                                                                                    Get the PCI function number of a resource. 
                                                                                    fpga_result fpgaPropertiesGetFunction (\n    const fpga_properties prop,\n    uint8_t * function\n) \n
                                                                                     
                                                                                    Returns the function number the queried resource.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • function Pointer to PCI function variable of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetguid","title":"function fpgaPropertiesGetGUID","text":"
                                                                                    Get the GUID of a resource. 
                                                                                    fpga_result fpgaPropertiesGetGUID (\n    const fpga_properties prop,\n    fpga_guid * guid\n) \n
                                                                                     
                                                                                    Returns the GUID of an FPGA or accelerator object.
                                                                                     
                                                                                    For an accelerator, the GUID uniquely identifies a specific accelerator context type, i.e. different accelerators will have different GUIDs. For an FPGA, the GUID is used to identify a certain instance of an FPGA, e.g. to determine whether a given bitstream would be compatible.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • guid Pointer to a GUID of the slot variable 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetinterface","title":"function fpgaPropertiesGetInterface","text":"
                                                                                    Get the OPAE plugin interface implemented by a resource. 
                                                                                    fpga_result fpgaPropertiesGetInterface (\n    const fpga_properties prop,\n    fpga_interface * interface\n) \n
                                                                                     
                                                                                    Returns the plugin interface enumerator.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • interface Pointer to an fpga_interface location to store the interface in 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetlocalmemorysize","title":"function fpgaPropertiesGetLocalMemorySize","text":"
                                                                                    Get the local memory size of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesGetLocalMemorySize (\n    const fpga_properties prop,\n    uint64_t * lms\n) \n
                                                                                     
                                                                                    Returns the local memory size of an FPGA.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • lms Pointer to a memory size variable of the FPGA 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This API is not currently supported. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetmodel","title":"function fpgaPropertiesGetModel","text":"
                                                                                    Get the model of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesGetModel (\n    const fpga_properties prop,\n    char * model\n) \n
                                                                                     
                                                                                    Returns the model of an FPGA.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • model Model of the FPGA resource (string of minimum FPGA_MODEL_LENGTH length 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This API is not currently supported. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetnumerrors","title":"function fpgaPropertiesGetNumErrors","text":"
                                                                                    Get the number of errors that can be reported by a resource. 
                                                                                    fpga_result fpgaPropertiesGetNumErrors (\n    const fpga_properties prop,\n    uint32_t * num_errors\n) \n
                                                                                     
                                                                                    Returns the number of error registers understood by a resource.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • num_errors Pointer to a 32 bit memory location to store the number of supported errors in 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetnuminterrupts","title":"function fpgaPropertiesGetNumInterrupts","text":"
                                                                                    Get the number of interrupts. 
                                                                                    fpga_result fpgaPropertiesGetNumInterrupts (\n    const fpga_properties prop,\n    uint32_t * num_interrupts\n) \n
                                                                                     
                                                                                    Returns the number of interrupts of an accelerator properties structure.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query - must be of type FPGA_ACCELERATOR 
                                                                                      •  
                                                                                    • num_interrupts Pointer to a variable for number of interrupts 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_ACCELERATOR. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetnummmio","title":"function fpgaPropertiesGetNumMMIO","text":"
                                                                                    Get the number of mmio spaces. 
                                                                                    fpga_result fpgaPropertiesGetNumMMIO (\n    const fpga_properties prop,\n    uint32_t * mmio_spaces\n) \n
                                                                                     
                                                                                    Returns the number of mmio spaces of an AFU properties structure.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query - must be of type FPGA_ACCELERATOR 
                                                                                      •  
                                                                                    • mmio_spaces Pointer to a variable for number of mmio spaces 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_ACCELERATOR. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetnumslots","title":"function fpgaPropertiesGetNumSlots","text":"
                                                                                    Get the number of slots of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesGetNumSlots (\n    const fpga_properties prop,\n    uint32_t * num_slots\n) \n
                                                                                     
                                                                                    Returns the number of slots present in an FPGA.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • num_slots Pointer to number of slots variable of the FPGA 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetobjectid","title":"function fpgaPropertiesGetObjectID","text":"
                                                                                    Get the object ID of a resource. 
                                                                                    fpga_result fpgaPropertiesGetObjectID (\n    const fpga_properties prop,\n    uint64_t * object_id\n) \n
                                                                                     
                                                                                    Returns the object ID of a resource. The object ID is a 64 bit identifier that is unique within a single node or system. It represents a similar concept as the token, but can be used across processes (e.g. passed on the command line).
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • object_id Pointer to a 64bit memory location to store the object ID in 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetobjecttype","title":"function fpgaPropertiesGetObjectType","text":"
                                                                                    Get the object type of a resource. 
                                                                                    fpga_result fpgaPropertiesGetObjectType (\n    const fpga_properties prop,\n    fpga_objtype * objtype\n) \n
                                                                                     
                                                                                    Returns the object type of the queried resource.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • objtype Pointer to an object type variable of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetparent","title":"function fpgaPropertiesGetParent","text":"
                                                                                    Get the token of the parent object. 
                                                                                    fpga_result fpgaPropertiesGetParent (\n    const fpga_properties prop,\n    fpga_token * parent\n) \n
                                                                                     
                                                                                    Returns the token of the parent of the queried resource in '*parent'.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • parent Pointer to a token variable of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_NOT_FOUND if resource does not have a parent (e.g. an FPGA_DEVICE resource does not have parents). Also see \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetsegment","title":"function fpgaPropertiesGetSegment","text":"
                                                                                    Get the PCI segment number of a resource. 
                                                                                    fpga_result fpgaPropertiesGetSegment (\n    const fpga_properties prop,\n    uint16_t * segment\n) \n
                                                                                     
                                                                                    Returns the segment number of the queried resource.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • segment Pointer to a PCI segment variable of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetsocketid","title":"function fpgaPropertiesGetSocketID","text":"
                                                                                    Get the socket id of a resource. 
                                                                                    fpga_result fpgaPropertiesGetSocketID (\n    const fpga_properties prop,\n    uint8_t * socket_id\n) \n
                                                                                     
                                                                                    Returns the socket id of the queried resource.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • socket_id Pointer to a socket id variable of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetsubsystemdeviceid","title":"function fpgaPropertiesGetSubsystemDeviceID","text":"
                                                                                    Get the subsystem device id of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesGetSubsystemDeviceID (\n    const fpga_properties prop,\n    uint16_t * subsystem_device_id\n) \n
                                                                                     
                                                                                    Returns the subsystem device id of an FPGA.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • subsystem_device_id Pointer to a device id variable of the FPGA 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetsubsystemvendorid","title":"function fpgaPropertiesGetSubsystemVendorID","text":"
                                                                                    Get the subsystem vendor id of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesGetSubsystemVendorID (\n    const fpga_properties prop,\n    uint16_t * subsystem_vendor_id\n) \n
                                                                                     
                                                                                    Returns the subsystem vendor id of an FPGA.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • subsystem_vendor_id Pointer to a vendor id variable of the FPGA 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiesgetvendorid","title":"function fpgaPropertiesGetVendorID","text":"
                                                                                    Get the vendor id of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesGetVendorID (\n    const fpga_properties prop,\n    uint16_t * vendor_id\n) \n
                                                                                     
                                                                                    Returns the vendor id of an FPGA.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • vendor_id Pointer to a vendor id variable of the FPGA 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This API is not currently supported. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetacceleratorstate","title":"function fpgaPropertiesSetAcceleratorState","text":"
                                                                                    Set the state of an accelerator resource property. 
                                                                                    fpga_result fpgaPropertiesSetAcceleratorState (\n    fpga_properties prop,\n    fpga_accelerator_state state\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify - must be of type FPGA_ACCELERATOR 
                                                                                      •  
                                                                                    • state accelerator state of the accelerator resource 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_ACCELERATOR. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetbbsid","title":"function fpgaPropertiesSetBBSID","text":"
                                                                                    Set the BBS ID of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesSetBBSID (\n    fpga_properties prop,\n    uint64_t bbs_id\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • bbs_id Blue bitstream id of the FPGA resource 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetbbsversion","title":"function fpgaPropertiesSetBBSVersion","text":"
                                                                                    Set the BBS Version of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesSetBBSVersion (\n    fpga_properties prop,\n    fpga_version version\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • version Blue bitstream version of the FPGA resource 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetbus","title":"function fpgaPropertiesSetBus","text":"
                                                                                    Set the PCI bus number of a resource. 
                                                                                    fpga_result fpgaPropertiesSetBus (\n    fpga_properties prop,\n    uint8_t bus\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify 
                                                                                      •  
                                                                                    • bus PCI bus number of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetcapabilities","title":"function fpgaPropertiesSetCapabilities","text":"
                                                                                    Set the capabilities of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesSetCapabilities (\n    fpga_properties prop,\n    uint64_t capabilities\n) \n
                                                                                     
                                                                                    Capabilities is a bitfield value
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • capabilities Capabilities of the FPGA resource 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This API is not currently supported. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetdevice","title":"function fpgaPropertiesSetDevice","text":"
                                                                                    Set the PCI device number of a resource. 
                                                                                    fpga_result fpgaPropertiesSetDevice (\n    fpga_properties prop,\n    uint8_t device\n) \n
                                                                                     
                                                                                    Enforces the limitation on the number of devices as specified in the PCI spec.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify 
                                                                                      •  
                                                                                    • device PCI device number of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetdeviceid","title":"function fpgaPropertiesSetDeviceID","text":"
                                                                                    Set the device id of the resource. 
                                                                                    fpga_result fpgaPropertiesSetDeviceID (\n    fpga_properties prop,\n    uint16_t device_id\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify 
                                                                                      •  
                                                                                    • device_id Device id of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetfunction","title":"function fpgaPropertiesSetFunction","text":"
                                                                                    Set the PCI function number of a resource. 
                                                                                    fpga_result fpgaPropertiesSetFunction (\n    fpga_properties prop,\n    uint8_t function\n) \n
                                                                                     
                                                                                    Enforces the limitation on the number of functions as specified in the PCI spec.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify 
                                                                                      •  
                                                                                    • function PCI function number of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetguid","title":"function fpgaPropertiesSetGUID","text":"
                                                                                    Set the GUID of a resource. 
                                                                                    fpga_result fpgaPropertiesSetGUID (\n    fpga_properties prop,\n    fpga_guid guid\n) \n
                                                                                     
                                                                                    Sets the GUID of an FPGA or accelerator object.
                                                                                     
                                                                                    For an accelerator, the GUID uniquely identifies a specific accelerator context type, i.e. different accelerators will have different GUIDs. For an FPGA, the GUID is used to identify a certain instance of an FPGA, e.g. to determine whether a given bitstream would be compatible.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify 
                                                                                      •  
                                                                                    • guid Pointer to a GUID of the slot variable 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetinterface","title":"function fpgaPropertiesSetInterface","text":"
                                                                                    Set the OPAE plugin interface implemented by a resource. 
                                                                                    fpga_result fpgaPropertiesSetInterface (\n    const fpga_properties prop,\n    fpga_interface interface\n) \n
                                                                                     
                                                                                    Set the plugin interface enumerator.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • interface The interface enumerator to set 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetlocalmemorysize","title":"function fpgaPropertiesSetLocalMemorySize","text":"
                                                                                    Set the local memory size of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesSetLocalMemorySize (\n    fpga_properties prop,\n    uint64_t lms\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • lms Local memory size of the FPGA resource 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This API is not currently supported. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetmodel","title":"function fpgaPropertiesSetModel","text":"
                                                                                    Set the model of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesSetModel (\n    fpga_properties prop,\n    char * model\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • model Model of the FPGA resource (string of maximum FPGA_MODEL_LENGTH length 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This API is not currently supported. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetnumerrors","title":"function fpgaPropertiesSetNumErrors","text":"
                                                                                    Set the number of error registers. 
                                                                                    fpga_result fpgaPropertiesSetNumErrors (\n    const fpga_properties prop,\n    uint32_t num_errors\n) \n
                                                                                     
                                                                                    Set the number of error registers understood by a resource to enumerate.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • num_errors Number of errors 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetnuminterrupts","title":"function fpgaPropertiesSetNumInterrupts","text":"
                                                                                    Set the number of interrupts. 
                                                                                    fpga_result fpgaPropertiesSetNumInterrupts (\n    fpga_properties prop,\n    uint32_t num_interrupts\n) \n
                                                                                     
                                                                                    Sets the number of interrupts of an accelerator properties structure.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify - must be of type FPGA_ACCELERATOR 
                                                                                      •  
                                                                                    • num_interrupts Number of interrupts of the accelerator 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_ACCELERATOR. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetnummmio","title":"function fpgaPropertiesSetNumMMIO","text":"
                                                                                    Set the number of mmio spaces. 
                                                                                    fpga_result fpgaPropertiesSetNumMMIO (\n    fpga_properties prop,\n    uint32_t mmio_spaces\n) \n
                                                                                     
                                                                                    Sets the number of mmio spaces of an AFU properties structure.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify - must be of type FPGA_ACCELERATOR 
                                                                                      •  
                                                                                    • mmio_spaces Number of MMIO spaces of the accelerator 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_ACCELERATOR. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetnumslots","title":"function fpgaPropertiesSetNumSlots","text":"
                                                                                    Set the number of slots of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesSetNumSlots (\n    fpga_properties prop,\n    uint32_t num_slots\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • num_slots Number of slots of the FPGA 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetobjectid","title":"function fpgaPropertiesSetObjectID","text":"
                                                                                    Set the object ID of a resource. 
                                                                                    fpga_result fpgaPropertiesSetObjectID (\n    const fpga_properties prop,\n    uint64_t object_id\n) \n
                                                                                     
                                                                                    Sets the object ID of a resource. The object ID is a 64 bit identifier that is unique within a single node or system. It represents a similar concept as the token, but can be used across processes (e.g. passed on the command line).
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to query 
                                                                                      •  
                                                                                    • object_id A 64bit value to use as the object ID 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetobjecttype","title":"function fpgaPropertiesSetObjectType","text":"
                                                                                    Set the object type of a resource. 
                                                                                    fpga_result fpgaPropertiesSetObjectType (\n    fpga_properties prop,\n    fpga_objtype objtype\n) \n
                                                                                     
                                                                                    Sets the object type of the resource. * Currently supported object types are FPGA_DEVICE and FPGA_ACCELERATOR.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify 
                                                                                      •  
                                                                                    • objtype Object type of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetparent","title":"function fpgaPropertiesSetParent","text":"
                                                                                    Set the token of the parent object. 
                                                                                    fpga_result fpgaPropertiesSetParent (\n    fpga_properties prop,\n    fpga_token parent\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify 
                                                                                      •  
                                                                                    • parent Pointer to a token variable of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetsegment","title":"function fpgaPropertiesSetSegment","text":"
                                                                                    Set the PCI segment number of a resource. 
                                                                                    fpga_result fpgaPropertiesSetSegment (\n    fpga_properties prop,\n    uint16_t segment\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify 
                                                                                      •  
                                                                                    • segment PCI segment number of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetsocketid","title":"function fpgaPropertiesSetSocketID","text":"
                                                                                    Set the socket id of the resource. 
                                                                                    fpga_result fpgaPropertiesSetSocketID (\n    fpga_properties prop,\n    uint8_t socket_id\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify 
                                                                                      •  
                                                                                    • socket_id Socket id of the resource 'prop' is associated with 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    See \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetsubsystemdeviceid","title":"function fpgaPropertiesSetSubsystemDeviceID","text":"
                                                                                    Set the subsystem device id of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesSetSubsystemDeviceID (\n    fpga_properties prop,\n    uint16_t subsystem_device_id\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify 
                                                                                      •  
                                                                                    • subsystem_device_id Subsystem Device id of the FPGA resource 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetsubsystemvendorid","title":"function fpgaPropertiesSetSubsystemVendorID","text":"
                                                                                    Set the subsystem vendor id of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesSetSubsystemVendorID (\n    fpga_properties prop,\n    uint16_t subsystem_vendor_id\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify 
                                                                                      •  
                                                                                    • subsystem_vendor_id Subsystem Vendor id of the FPGA resource 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. See also \"Accessor Return Values\" in properties.h. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgapropertiessetvendorid","title":"function fpgaPropertiesSetVendorID","text":"
                                                                                    Set the vendor id of an FPGA resource property. 
                                                                                    fpga_result fpgaPropertiesSetVendorID (\n    fpga_properties prop,\n    uint16_t vendor_id\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • prop Properties object to modify - must be of type FPGA_DEVICE 
                                                                                      •  
                                                                                    • vendor_id Vendor id of the FPGA resource 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if object type is not FPGA_DEVICE. See also \"Accessor Return Values\" in properties.h.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    This API is not currently supported. 
                                                                                    "},{"location":"opae-code/properties_8h/#function-fpgaupdateproperties","title":"function fpgaUpdateProperties","text":"
                                                                                    Update a fpga_properties object. 
                                                                                    fpga_result fpgaUpdateProperties (\n    fpga_token token,\n    fpga_properties prop\n) \n
                                                                                     
                                                                                    Populates the properties object 'prop' with properties of the resource referred to by 'token'. Unlike fpgaGetProperties(), this call will not create a new properties object or allocate memory for it, but use a previously created properties object.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • token Token to retrieve properties for 
                                                                                      •  
                                                                                    • prop fpga_properties object to update 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if token or prop are not valid objects. FPGA_NOT_FOUND if the resource referred to by token was not found. FPGA_NO_DRIVER if not driver is loaded. FPGA_EXCEPTION if an internal exception occured when trying to update prop. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/properties.h
                                                                                    "},{"location":"opae-code/properties_8h_source/","title":"File properties.h","text":"
                                                                                    File List > docs > sw > include > opae > properties.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017-2021, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_PROPERTIES_H__\n#define __FPGA_PROPERTIES_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaGetPropertiesFromHandle(fpga_handle handle, fpga_properties *prop);\n\nfpga_result fpgaGetProperties(fpga_token token, fpga_properties *prop);\n\nfpga_result fpgaUpdateProperties(fpga_token token, fpga_properties prop);\n\nfpga_result fpgaClearProperties(fpga_properties prop);\n\nfpga_result fpgaCloneProperties(fpga_properties src, fpga_properties *dst);\n\nfpga_result fpgaDestroyProperties(fpga_properties *prop);\n\nfpga_result fpgaPropertiesGetParent(const fpga_properties prop,\n                    fpga_token *parent);\n\nfpga_result fpgaPropertiesSetParent(fpga_properties prop,\n                    fpga_token parent);\nfpga_result fpgaPropertiesGetObjectType(const fpga_properties prop,\n                    fpga_objtype *objtype);\n\nfpga_result fpgaPropertiesSetObjectType(fpga_properties prop,\n                    fpga_objtype objtype);\nfpga_result fpgaPropertiesGetSegment(const fpga_properties prop, uint16_t *segment);\n\nfpga_result fpgaPropertiesSetSegment(fpga_properties prop, uint16_t segment);\n\nfpga_result fpgaPropertiesGetBus(const fpga_properties prop, uint8_t *bus);\n\nfpga_result fpgaPropertiesSetBus(fpga_properties prop, uint8_t bus);\n\nfpga_result fpgaPropertiesGetDevice(const fpga_properties prop,\n                    uint8_t *device);\n\nfpga_result fpgaPropertiesSetDevice(fpga_properties prop,\n                    uint8_t device);\n\nfpga_result fpgaPropertiesGetFunction(const fpga_properties prop,\n                      uint8_t *function);\n\nfpga_result fpgaPropertiesSetFunction(fpga_properties prop,\n                      uint8_t function);\n\nfpga_result fpgaPropertiesGetSocketID(const fpga_properties prop,\n                      uint8_t *socket_id);\n\nfpga_result fpgaPropertiesSetSocketID(fpga_properties prop,\n                      uint8_t socket_id);\n\nfpga_result fpgaPropertiesGetDeviceID(const fpga_properties prop,\n                      uint16_t *device_id);\n\nfpga_result fpgaPropertiesSetDeviceID(fpga_properties prop,\n                      uint16_t device_id);\n\nfpga_result fpgaPropertiesGetNumSlots(const fpga_properties prop,\n                      uint32_t *num_slots);\n\nfpga_result fpgaPropertiesSetNumSlots(fpga_properties prop,\n                      uint32_t num_slots);\n\nfpga_result fpgaPropertiesGetBBSID(const fpga_properties prop,\n                   uint64_t *bbs_id);\n\n\nfpga_result fpgaPropertiesSetBBSID(fpga_properties prop,\n                   uint64_t bbs_id);\n\n\nfpga_result fpgaPropertiesGetBBSVersion(const fpga_properties prop,\n                    fpga_version *bbs_version);\n\nfpga_result fpgaPropertiesSetBBSVersion(fpga_properties prop,\n                    fpga_version version);\n\n\nfpga_result fpgaPropertiesGetVendorID(const fpga_properties prop,\n                      uint16_t *vendor_id);\n\n\nfpga_result fpgaPropertiesSetVendorID(fpga_properties prop,\n                      uint16_t vendor_id);\n\nfpga_result fpgaPropertiesGetModel(const fpga_properties prop,\n                   char *model);\n\n\nfpga_result fpgaPropertiesSetModel(fpga_properties prop,\n                   char *model);\n\n\nfpga_result fpgaPropertiesGetLocalMemorySize(const fpga_properties prop,\n                         uint64_t *lms);\n\n\nfpga_result fpgaPropertiesSetLocalMemorySize(fpga_properties prop,\n                         uint64_t lms);\n\nfpga_result fpgaPropertiesGetCapabilities(const fpga_properties prop,\n                      uint64_t *capabilities);\n\n\nfpga_result fpgaPropertiesSetCapabilities(fpga_properties prop,\n                      uint64_t capabilities);\n\nfpga_result fpgaPropertiesGetGUID(const fpga_properties prop,\n                  fpga_guid *guid);\n\nfpga_result fpgaPropertiesSetGUID(fpga_properties prop, fpga_guid guid);\n\nfpga_result fpgaPropertiesGetNumMMIO(const fpga_properties prop,\n                     uint32_t *mmio_spaces);\n\nfpga_result fpgaPropertiesSetNumMMIO(fpga_properties prop,\n                     uint32_t mmio_spaces);\n\nfpga_result fpgaPropertiesGetNumInterrupts(const fpga_properties prop,\n                       uint32_t *num_interrupts);\n\nfpga_result fpgaPropertiesSetNumInterrupts(fpga_properties prop,\n                       uint32_t num_interrupts);\n\nfpga_result fpgaPropertiesGetAcceleratorState(const fpga_properties prop,\n                          fpga_accelerator_state *state);\n\n\nfpga_result fpgaPropertiesSetAcceleratorState(fpga_properties prop,\n                          fpga_accelerator_state state);\n\nfpga_result fpgaPropertiesGetObjectID(const fpga_properties prop,\n                        uint64_t *object_id);\n\n\nfpga_result fpgaPropertiesSetObjectID(const fpga_properties prop,\n                        uint64_t object_id);\n\n\nfpga_result fpgaPropertiesGetNumErrors(const fpga_properties prop,\n                       uint32_t *num_errors);\n\n\nfpga_result fpgaPropertiesSetNumErrors(const fpga_properties prop,\n                       uint32_t num_errors);\n\nfpga_result fpgaPropertiesGetInterface(const fpga_properties prop,\n                       fpga_interface *interface);\n\nfpga_result fpgaPropertiesSetInterface(const fpga_properties prop,\n                       fpga_interface interface);\n\nfpga_result fpgaPropertiesGetSubsystemVendorID(const fpga_properties prop,\n                           uint16_t *subsystem_vendor_id);\n\n\nfpga_result fpgaPropertiesSetSubsystemVendorID(fpga_properties prop,\n                           uint16_t subsystem_vendor_id);\n\nfpga_result fpgaPropertiesGetSubsystemDeviceID(const fpga_properties prop,\n                           uint16_t *subsystem_device_id);\n\n\nfpga_result fpgaPropertiesSetSubsystemDeviceID(fpga_properties prop,\n                           uint16_t subsystem_device_id);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_PROPERTIES_H__\n
                                                                                    "},{"location":"opae-code/sysobject_8h/","title":"File sysobject.h","text":"
                                                                                    FileList > docs > sw > include > opae > sysobject.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Functions to read/write from system objects. More...
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/sysobject_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaDestroyObject (fpga_object * obj) Free memory used for the fpga_object data structure. fpga_result fpgaHandleGetObject (fpga_handle handle, const char * name, fpga_object * object, int flags) Create an fpga_object data structure from a handle. fpga_result fpgaObjectGetObject (fpga_object parent, const char * name, fpga_object * object, int flags) Create an fpga_object data structure from a parent object. fpga_result fpgaObjectGetObjectAt (fpga_object parent, size_t idx, fpga_object * object) Create an fpga_object data structure from a parent object using a given index. fpga_result fpgaObjectGetSize (fpga_object obj, uint32_t * value, int flags) Retrieve the size of the object. fpga_result fpgaObjectGetType (fpga_object obj, enum fpga_sysobject_type * type) Get the sysobject type (container or attribute) fpga_result fpgaObjectRead (fpga_object obj, uint8_t * buffer, size_t offset, size_t len, int flags) Read bytes from an FPGA object. fpga_result fpgaObjectRead64 (fpga_object obj, uint64_t * value, int flags) Read a 64-bit value from an FPGA object. fpga_result fpgaObjectWrite64 (fpga_object obj, uint64_t value, int flags) Write 64-bit value to an FPGA object. fpga_result fpgaTokenGetObject (fpga_token token, const char * name, fpga_object * object, int flags) Create an fpga_object data structures."},{"location":"opae-code/sysobject_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    On Linux systems with the OPAE kernel driver, this is used to access sysfs nodes created by the driver. 
                                                                                    "},{"location":"opae-code/sysobject_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/sysobject_8h/#function-fpgadestroyobject","title":"function fpgaDestroyObject","text":"
                                                                                    Free memory used for the fpga_object data structure. 
                                                                                    fpga_result fpgaDestroyObject (\n    fpga_object * obj\n) \n
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    fpgaDestroyObject() requires the address of an fpga_object as created by fpgaTokenGetObject(), fpgaHandleGetObject(), or fpgaObjectGetObject(). Passing any other value results in undefind behavior.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • obj Pointer to the fpga_object instance to destroy
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success, FPGA_INVALID_PARAM if the object is NULL, FPGA_EXCEPTION if an internal error is encountered. 
                                                                                    "},{"location":"opae-code/sysobject_8h/#function-fpgahandlegetobject","title":"function fpgaHandleGetObject","text":"
                                                                                    Create an fpga_object data structure from a handle. 
                                                                                    fpga_result fpgaHandleGetObject (\n    fpga_handle handle,\n    const char * name,\n    fpga_object * object,\n    int flags\n) \n
                                                                                     
                                                                                    An fpga_object is a handle to an FPGA resource which can be an attribute, register, or container. This object has read/write access..
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle identifying a resource (accelerator or device) 
                                                                                      •  
                                                                                    • name A key identifying an object belonging to a resource. 
                                                                                      •  
                                                                                    • object Pointer to memory to store the object in 
                                                                                      •  
                                                                                    • flags Control behavior of object identification and creation FPGA_OBJECT_GLOB is used to indicate that the name should be treated as a globbing expression. FPGA_OBJECT_RECURSE_ONE indicates that subobjects be created for objects one level down from the object identified by name. FPGA_OBJECT_RECURSE_ALL indicates that subobjects be created for all objects below the current object identified by name.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_NOT_FOUND if an object cannot be found with the given key. FPGA_NOT_SUPPORTED if this function is not supported by the current implementation of this API.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    Names that begin with '.' or '/' or contain '..' are not allowed and result in FPGA_INVALID_PARAM being returned 
                                                                                    "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectgetobject","title":"function fpgaObjectGetObject","text":"
                                                                                    Create an fpga_object data structure from a parent object. 
                                                                                    fpga_result fpgaObjectGetObject (\n    fpga_object parent,\n    const char * name,\n    fpga_object * object,\n    int flags\n) \n
                                                                                     
                                                                                    An fpga_object is a handle to an FPGA resource which can be an attribute, register, or container. If the parent object was created with a handle, then the new object will inherit the handle allowing it to have read-write access to the object data.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • parent A parent container fpga_object. 
                                                                                      •  
                                                                                    • name A key identifying a sub-object of the parent container. 
                                                                                      •  
                                                                                    • object Pointer to memory to store the object in. 
                                                                                      •  
                                                                                    • flags Control behavior of object identification and creation. FPGA_OBJECT_GLOB is used to indicate that the name should be treated as a globbing expression. FPGA_OBJECT_RECURSE_ONE indicates that subobjects be created for objects one level down from the object identified by name. FPGA_OBJECT_RECURSE_ALL indicates that subobjects be created for all objects below the current object identified by name.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid - this includes a parent object that is not a container object. FPGA_NOT_FOUND if an object cannot be found with the given key. FPGA_NOT_SUPPORTED if this function is not supported by the current implementation of this API.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    Names that begin with '.' or '/' or contain '..' are not allowed and result in FPGA_INVALID_PARAM being returned 
                                                                                    "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectgetobjectat","title":"function fpgaObjectGetObjectAt","text":"
                                                                                    Create an fpga_object data structure from a parent object using a given index. 
                                                                                    fpga_result fpgaObjectGetObjectAt (\n    fpga_object parent,\n    size_t idx,\n    fpga_object * object\n) \n
                                                                                     
                                                                                    An fpga_object is a handle to an FPGA resource which can be an attribute, register, or container. If the parent object was created with a handle, then the new object will inherit the handle allowing it to have read-write access to the object data.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • parent A parent container 'fpga_object' 
                                                                                      •  
                                                                                    • idx A positive index less than the size reported by the parent. 
                                                                                      •  
                                                                                    • object Pointer to memory to store the object in.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid - this includes a parent object that is not a container object. FPGA_NOT_FOUND if an object cannot be found with the given key. FPGA_NOT_SUPPORTED if this function is not supported by the current implementation of this API. 
                                                                                    "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectgetsize","title":"function fpgaObjectGetSize","text":"
                                                                                    Retrieve the size of the object. 
                                                                                    fpga_result fpgaObjectGetSize (\n    fpga_object obj,\n    uint32_t * value,\n    int flags\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • obj An fpga_object instance. 
                                                                                      •  
                                                                                    • value Pointer to variable to store size in. 
                                                                                      •  
                                                                                    • flags Flags that control how the object is read If FPGA_OBJECT_SYNC is used then object will update its buffered copy before retrieving the size.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of supplied parameters is invalid. FPGA_EXCEPTION if error occurred. 
                                                                                    "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectgettype","title":"function fpgaObjectGetType","text":"
                                                                                    Get the sysobject type (container or attribute) 
                                                                                    fpga_result fpgaObjectGetType (\n    fpga_object obj,\n    enum fpga_sysobject_type * type\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • obj An fpga_object instance 
                                                                                      •  
                                                                                    • type The type of object (FPGA_OBJECT_CONTAINER or FPGA_OBJECT_ATTRIBUTE)
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success, FPGA_INVALID_PARAM if any of the supplied parameters are null or invalid 
                                                                                    "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectread","title":"function fpgaObjectRead","text":"
                                                                                    Read bytes from an FPGA object. 
                                                                                    fpga_result fpgaObjectRead (\n    fpga_object obj,\n    uint8_t * buffer,\n    size_t offset,\n    size_t len,\n    int flags\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • obj An fpga_object instance. 
                                                                                      •  
                                                                                    • buffer Pointer to a buffer to read bytes into. 
                                                                                      •  
                                                                                    • offset Byte offset relative to objects internal buffer where to begin reading bytes from. 
                                                                                      •  
                                                                                    • len The length, in bytes, to read from the object. 
                                                                                      •  
                                                                                    • flags Flags that control how object is read If FPGA_OBJECT_SYNC is used then object will update its buffered copy before retrieving the data.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success, FPGA_INVALID_PARAM if any of the supplied parameters is invalid 
                                                                                    "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectread64","title":"function fpgaObjectRead64","text":"
                                                                                    Read a 64-bit value from an FPGA object. 
                                                                                    fpga_result fpgaObjectRead64 (\n    fpga_object obj,\n    uint64_t * value,\n    int flags\n) \n
                                                                                     
                                                                                    The value is assumed to be in string format and will be parsed. See flags below for changing that behavior.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • obj An fpga_object instance 
                                                                                      •  
                                                                                    • value Pointer to a 64-bit variable to store the value in 
                                                                                      •  
                                                                                    • flags Flags that control how the object is read If FPGA_OBJECT_SYNC is used then object will update its buffered copy before retrieving the data. If FPGA_OBJECT_RAW is used, then the data will be read as raw bytes into the uint64_t pointer variable.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success, FPGA_INVALID_PARAM if any of the supplied parameters is invalid 
                                                                                    "},{"location":"opae-code/sysobject_8h/#function-fpgaobjectwrite64","title":"function fpgaObjectWrite64","text":"
                                                                                    Write 64-bit value to an FPGA object. 
                                                                                    fpga_result fpgaObjectWrite64 (\n    fpga_object obj,\n    uint64_t value,\n    int flags\n) \n
                                                                                     
                                                                                    The value will be converted to string before writing. See flags below for changing that behavior.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • obj An fpga_object instance. 
                                                                                      •  
                                                                                    • value The value to write to the object 
                                                                                      •  
                                                                                    • flags Flags that control how the object is written If FPGA_OBJECT_RAW is used, then the value will be written as raw bytes.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success, FPGA_INVALID_PARAM if any of the supplied parameters is invalid
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    The object must have been created using a handle to a resource. 
                                                                                    "},{"location":"opae-code/sysobject_8h/#function-fpgatokengetobject","title":"function fpgaTokenGetObject","text":"
                                                                                    Create an fpga_object data structures. 
                                                                                    fpga_result fpgaTokenGetObject (\n    fpga_token token,\n    const char * name,\n    fpga_object * object,\n    int flags\n) \n
                                                                                     
                                                                                    An fpga_object is a handle to an FPGA resource which can be an attribute, register or a container. This object is read-only.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • token Token identifying a resource (accelerator or device) 
                                                                                      •  
                                                                                    • name A key identifying an object belonging to a resource. 
                                                                                      •  
                                                                                    • object Pointer to memory to store the object in 
                                                                                      •  
                                                                                    • flags Control behavior of object identification and creation. FPGA_OBJECT_GLOB is used to indicate that the name should be treated as a globbing expression. FPGA_OBJECT_RECURSE_ONE indicates that subobjects be created for objects one level down from the object identified by name. FPGA_OBJECT_RECURSE_ALL indicates that subobjects be created for all objects below the current object identified by name.
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if any of the supplied parameters is invalid. FPGA_NOT_FOUND if an object cannot be found with the given key. FPGA_NOT_SUPPORTED if this function is not supported by the current implementation of this API.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    Names that begin with '.' or '/' or contain '..' are not allowed and result in FPGA_INVALID_PARAM being returned 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/sysobject.h
                                                                                    "},{"location":"opae-code/sysobject_8h_source/","title":"File sysobject.h","text":"
                                                                                    File List > docs > sw > include > opae > sysobject.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017-2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_SYSOBJECT_H__\n#define __FPGA_SYSOBJECT_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaTokenGetObject(fpga_token token, const char *name,\n                   fpga_object *object, int flags);\n\nfpga_result fpgaHandleGetObject(fpga_handle handle, const char *name,\n                fpga_object *object, int flags);\n\nfpga_result fpgaObjectGetObject(fpga_object parent, const char *name,\n                fpga_object *object, int flags);\n\nfpga_result fpgaObjectGetObjectAt(fpga_object parent, size_t idx,\n                  fpga_object *object);\nfpga_result fpgaObjectGetType(fpga_object obj, enum fpga_sysobject_type *type);\n\nfpga_result fpgaDestroyObject(fpga_object *obj);\n\nfpga_result fpgaObjectGetSize(fpga_object obj, uint32_t *value, int flags);\n\nfpga_result fpgaObjectRead(fpga_object obj, uint8_t *buffer, size_t offset,\n               size_t len, int flags);\n\nfpga_result fpgaObjectRead64(fpga_object obj, uint64_t *value, int flags);\n\nfpga_result fpgaObjectWrite64(fpga_object obj, uint64_t value, int flags);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif /* !__FPGA_SYSOBJECT_H__ */\n
                                                                                    "},{"location":"opae-code/types_8h/","title":"File types.h","text":"
                                                                                    FileList > docs > sw > include > opae > types.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Type definitions for FPGA API. More...
                                                                                     
                                                                                       
                                                                                    • #include <stdint.h>
                                                                                      •  
                                                                                    • #include <stddef.h>
                                                                                      •  
                                                                                    • #include <stdbool.h>
                                                                                      •  
                                                                                    • #include <opae/types_enum.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/types_8h/#classes","title":"Classes","text":"Type Name struct _fpga_token_header Internal token type header. struct fpga_error_info struct fpga_metric Metric struct. struct fpga_metric_info Metric info struct. struct fpga_version Semantic version. struct metric_threshold struct threshold Threshold struct."},{"location":"opae-code/types_8h/#public-types","title":"Public Types","text":"Type Name typedef void * fpga_event_handle Handle to an event object. typedef uint8_t fpga_guid Globally unique identifier (GUID) typedef void * fpga_handle Handle to an FPGA resource. typedef struct fpga_metric fpga_metric Metric struct. typedef struct fpga_metric_info fpga_metric_info Metric info struct. typedef void * fpga_object Object pertaining to an FPGA resource as identified by a unique name. typedef void * fpga_properties Object for expressing FPGA resource properties. typedef void * fpga_token Token for referencing FPGA resources. typedef struct _fpga_token_header fpga_token_header Internal token type header. typedef struct metric_threshold metric_threshold union metric_value Metric value union. typedef struct threshold threshold Threshold struct."},{"location":"opae-code/types_8h/#macros","title":"Macros","text":"Type Name define FPGA_ERROR_NAME_MAX  64Information about an error register. define FPGA_METRIC_STR_SIZE  256FPGA Metric string size. define fpga_is_parent_child (__parent_hdr, __child_hdr) Determine token parent/child relationship."},{"location":"opae-code/types_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    OPAE uses the three opaque types fpga_properties, fpga_token, and fpga_handle to create a hierarchy of objects that can be used to enumerate, reference, acquire, and query FPGA resources. This object model is designed to be extensible to account for different FPGA architectures and platforms.
                                                                                    "},{"location":"opae-code/types_8h/#initialization","title":"Initialization","text":"
                                                                                    OPAEs management of the opaque types fpga_properties, fpga_token, and fpga_handle relies on the proper initialization of variables of these types. In other words, before doing anything with a variable of one of these opaque types, you need to first initialize them.
                                                                                     
                                                                                    The respective functions that initialize opaque types are:
                                                                                     
                                                                                       
                                                                                    • fpgaGetProperties() and fpgaCloneProperties() for fpga_properties
                                                                                      •  
                                                                                    • fpgaEnumerate() and fpgaCloneToken() for fpga_token
                                                                                      •  
                                                                                    • fpgaOpen() for fpga_handle
                                                                                      •  
                                                                                     
                                                                                    This should intuitively make sense - fpgaGetProperties() creates fpga_properties objects, fpgaEnumerate() creates fpga_token objects, fpgaOpen() creates fpga_handle objects, and fpgaCloneProperties() and fpgaCloneToken() clone (create) fpga_properties and fpga_token objects, respectively.
                                                                                     
                                                                                    Since these opaque types are interpreted as pointers (they are typedef'd to a void *), passing an uninitialized opaque type into any function except the respective initailzation function will result in undefined behaviour, because OPAE will try to follow an invalid pointer. Undefined behaviour in this case may include an unexpected error code, or an application crash. 
                                                                                    "},{"location":"opae-code/types_8h/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/types_8h/#typedef-fpga_event_handle","title":"typedef fpga_event_handle","text":"
                                                                                    Handle to an event object. 
                                                                                    typedef void* fpga_event_handle;\n
                                                                                     
                                                                                    OPAE provides an interface to asynchronous events that can be generated by different FPGA resources. The event API provides functions to register for these events; associated with every event a process has registered for is an fpga_event_handle, which encapsulates the OS-specific data structure for event objects.
                                                                                     
                                                                                    After use, fpga_event_handle objects should be destroyed using fpgaDestroyEventHandle() to free backing memory used by the fpga_event_handle object. 
                                                                                    "},{"location":"opae-code/types_8h/#typedef-fpga_guid","title":"typedef fpga_guid","text":"
                                                                                    Globally unique identifier (GUID) 
                                                                                    typedef uint8_t fpga_guid[16];\n
                                                                                     
                                                                                    GUIDs are used widely within OPAE for helping identify FPGA resources. For example, every FPGA resource has a guid property, which can be (and in the case of FPGA_ACCELERATOR resource primarily is) used for enumerating a resource of a specific type.
                                                                                     
                                                                                    fpga_guid is compatible with libuuid's uuid_t, so users can use libuuid functions like uuid_parse() to create and work with GUIDs. 
                                                                                    "},{"location":"opae-code/types_8h/#typedef-fpga_handle","title":"typedef fpga_handle","text":"
                                                                                    Handle to an FPGA resource. 
                                                                                    typedef void* fpga_handle;\n
                                                                                     
                                                                                    A valid fpga_handle object, as populated by fpgaOpen(), denotes ownership of an FPGA resource. Note that ownership can be exclusive or shared, depending on the flags used in fpgaOpen(). Ownership can be released by calling fpgaClose(), which will render the underlying handle invalid.
                                                                                     
                                                                                    Many OPAE C API functions require a valid token (which is synonymous with ownership of the resource). 
                                                                                    "},{"location":"opae-code/types_8h/#typedef-fpga_metric","title":"typedef fpga_metric","text":"
                                                                                    typedef struct fpga_metric fpga_metric;\n
                                                                                    "},{"location":"opae-code/types_8h/#typedef-fpga_metric_info","title":"typedef fpga_metric_info","text":"
                                                                                    typedef struct fpga_metric_info fpga_metric_info;\n
                                                                                    "},{"location":"opae-code/types_8h/#typedef-fpga_object","title":"typedef fpga_object","text":"
                                                                                    Object pertaining to an FPGA resource as identified by a unique name. 
                                                                                    typedef void* fpga_object;\n
                                                                                     
                                                                                    An fpga_object represents either a device attribute or a container of attributes. Similar to filesystems, a '/' may be used to seperate objects in an object hierarchy. Once on object is acquired, it may be used to read or write data in a resource attribute or to query sub-objects if the object is a container object. The data in an object is buffered and will be kept around until the object is destroyed. Additionally, the data in an attribute can by synchronized from the owning resource using the FPGA_OBJECT_SYNC flag during read operations. The name identifying the object is unique with respect to the resource that owns it. A parent resource may be identified by an fpga_token object, by an fpga_handle object, or another fpga_object object. If a handle object is used when opening the object, then the object is opened with read-write access. Otherwise, the object is read-only. 
                                                                                    "},{"location":"opae-code/types_8h/#typedef-fpga_properties","title":"typedef fpga_properties","text":"
                                                                                    Object for expressing FPGA resource properties. 
                                                                                    typedef void* fpga_properties;\n
                                                                                     
                                                                                    fpga_properties objects encapsulate all enumerable information about an FPGA resources. They can be used for two purposes: selective enumeration (discovery) and querying information about existing resources.
                                                                                     
                                                                                    For selective enumeration, usually an empty fpga_properties object is created (using fpgaGetProperties()) and then populated with the desired criteria for enumeration. An array of fpga_properties can then be passed to fpgaEnumerate(), which will return a list of fpga_token objects matching these criteria.
                                                                                     
                                                                                    For querying properties of existing FPGA resources, fpgaGetProperties() can also take an fpga_token and will return an fpga_properties object populated with information about the resource referenced by that token.
                                                                                     
                                                                                    After use, fpga_properties objects should be destroyed using fpga_destroyProperties() to free backing memory used by the fpga_properties object. 
                                                                                    "},{"location":"opae-code/types_8h/#typedef-fpga_token","title":"typedef fpga_token","text":"
                                                                                    Token for referencing FPGA resources. 
                                                                                    typedef void* fpga_token;\n
                                                                                     
                                                                                    An fpga_token serves as a reference to a specific FPGA resource present in the system. Holding an fpga_token does not constitute ownership of the FPGA resource - it merely allows the user to query further information about a resource, or to use fpgaOpen() to acquire ownership.
                                                                                     
                                                                                    fpga_tokens are usually returned by fpgaEnumerate() or fpgaPropertiesGetParent(), and used by fpgaOpen() to acquire ownership and yield a handle to the resource. Some API calls also take fpga_tokens as arguments if they don't require ownership of the resource in question. 
                                                                                    "},{"location":"opae-code/types_8h/#typedef-fpga_token_header","title":"typedef fpga_token_header","text":"
                                                                                    Internal token type header. 
                                                                                    typedef struct _fpga_token_header fpga_token_header;\n
                                                                                     
                                                                                    Each plugin (dfl: libxfpga.so, vfio: libopae-v.so) implements its own proprietary token type. This header must appear at offset zero within that structure.
                                                                                     
                                                                                    eg, see lib/plugins/xfpga/types_int.h:struct _fpga_token and lib/plugins/vfio/opae_vfio.h:struct _vfio_token. 
                                                                                    "},{"location":"opae-code/types_8h/#typedef-metric_threshold","title":"typedef metric_threshold","text":"
                                                                                    typedef struct metric_threshold metric_threshold;\n
                                                                                    "},{"location":"opae-code/types_8h/#union-metric_value","title":"union metric_value","text":""},{"location":"opae-code/types_8h/#typedef-threshold","title":"typedef threshold","text":"
                                                                                    typedef struct threshold threshold;\n
                                                                                    "},{"location":"opae-code/types_8h/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/types_8h/#define-fpga_error_name_max","title":"define FPGA_ERROR_NAME_MAX","text":"
                                                                                    Information about an error register. 
                                                                                    #define FPGA_ERROR_NAME_MAX 64\n
                                                                                     
                                                                                    This data structure captures information about an error register exposed by an accelerator resource. The error API provides functions to retrieve these information structures from a particular resource. 
                                                                                    "},{"location":"opae-code/types_8h/#define-fpga_metric_str_size","title":"define FPGA_METRIC_STR_SIZE","text":"
                                                                                    #define FPGA_METRIC_STR_SIZE 256\n
                                                                                    "},{"location":"opae-code/types_8h/#define-fpga_is_parent_child","title":"define fpga_is_parent_child","text":"
                                                                                    Determine token parent/child relationship. 
                                                                                    #define fpga_is_parent_child (\n    __parent_hdr,\n    __child_hdr\n) (((__parent_hdr)->objtype == FPGA_DEVICE ) && \\\n ((__child_hdr)->objtype == FPGA_ACCELERATOR ) && \\\n ((__parent_hdr)->segment == (__child_hdr)->segment) && \\\n ((__parent_hdr)->bus == (__child_hdr)->bus) && \\\n ((__parent_hdr)->device == (__child_hdr)->device))\n
                                                                                     
                                                                                    Given pointers to two fpga_token_header structs, determine whether the first is the parent of the second. A parent will have objtype == FPGA_DEVICE. A child will have objtype == FPGA_ACCELERATOR. The PCIe address of the two headers will match in all but the function fields. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/types.h
                                                                                    "},{"location":"opae-code/types_8h_source/","title":"File types.h","text":"
                                                                                    File List > docs > sw > include > opae > types.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018-2022, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_TYPES_H__\n#define __FPGA_TYPES_H__\n\n#include <stdint.h>\n#include <stddef.h>\n#include <stdbool.h>\n#include <opae/types_enum.h>\n\ntypedef void *fpga_properties;\n\ntypedef void *fpga_token;\n\ntypedef void *fpga_handle;\n\ntypedef uint8_t fpga_guid[16];\n\ntypedef struct {\n    uint8_t major;        \n    uint8_t minor;        \n    uint16_t patch;       \n} fpga_version;\n\ntypedef void *fpga_event_handle;\n\n#define FPGA_ERROR_NAME_MAX 64\nstruct fpga_error_info {\n    char name[FPGA_ERROR_NAME_MAX];   \n    bool can_clear;                   \n};\n\ntypedef void *fpga_object;\n\n#define FPGA_METRIC_STR_SIZE   256\ntypedef union {\n    uint64_t   ivalue;  // Metric integer value\n    double     dvalue;  // Metric double value\n    float      fvalue;  // Metric float value\n    bool       bvalue;  // Metric bool value\n} metric_value;\n\n\ntypedef struct fpga_metric_info {\n    uint64_t metric_num;                         // Metric index num\n    fpga_guid metric_guid;                       // Metric guid\n    char qualifier_name[FPGA_METRIC_STR_SIZE];   // Metric full name\n    char group_name[FPGA_METRIC_STR_SIZE];       // Metric group name\n    char metric_name[FPGA_METRIC_STR_SIZE];      // Metric name\n    char metric_units[FPGA_METRIC_STR_SIZE];     // Metric units\n    enum fpga_metric_datatype metric_datatype;   // Metric data type\n    enum fpga_metric_type metric_type;           // Metric group type\n} fpga_metric_info;\n\ntypedef struct fpga_metric {\n    uint64_t metric_num;    // Metric index num\n    metric_value value;     // Metric value\n    bool isvalid;           // Metric value is valid\n} fpga_metric;\n\n\ntypedef struct threshold {\n    char threshold_name[FPGA_METRIC_STR_SIZE]; // Threshold name\n    uint32_t is_valid;                         // Threshold is valid\n    double value;                              // Threshold value\n} threshold;\n\ntypedef struct metric_threshold {\n    char metric_name[FPGA_METRIC_STR_SIZE];        // Metric Threshold name\n    threshold upper_nr_threshold;                  // Upper Non-Recoverable Threshold\n    threshold upper_c_threshold;                   // Upper Critical Threshold\n    threshold upper_nc_threshold;                  // Upper Non-Critical Threshold\n    threshold lower_nr_threshold;                  // Lower Non-Recoverable Threshold\n    threshold lower_c_threshold;                   // Lower Critical Threshold\n    threshold lower_nc_threshold;                  // Lower Non-Critical Threshold\n    threshold hysteresis;                          // Hysteresis\n} metric_threshold;\n\ntypedef struct _fpga_token_header {\n    uint64_t magic;\n    uint16_t vendor_id;\n    uint16_t device_id;\n    uint16_t segment;\n    uint8_t bus;\n    uint8_t device;\n    uint8_t function;\n    fpga_interface interface;\n    fpga_objtype objtype;\n    uint64_t object_id;\n    fpga_guid guid;\n    uint16_t subsystem_vendor_id;\n    uint16_t subsystem_device_id;\n} fpga_token_header;\n\n#define fpga_is_parent_child(__parent_hdr, __child_hdr) \\\n(((__parent_hdr)->objtype == FPGA_DEVICE) && \\\n ((__child_hdr)->objtype == FPGA_ACCELERATOR) && \\\n ((__parent_hdr)->segment == (__child_hdr)->segment) && \\\n ((__parent_hdr)->bus == (__child_hdr)->bus) && \\\n ((__parent_hdr)->device == (__child_hdr)->device))\n\n#endif // __FPGA_TYPES_H__\n
                                                                                    "},{"location":"opae-code/types__enum_8h/","title":"File types_enum.h","text":"
                                                                                    FileList > docs > sw > include > opae > types_enum.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Definitions of enumerated types for the OPAE C API. More...
                                                                                    "},{"location":"opae-code/types__enum_8h/#public-types","title":"Public Types","text":"Type Name enum fpga_accelerator_state accelerator state enum fpga_buffer_flags Buffer flags. enum fpga_event_type FPGA events. enum fpga_interface OPAE plugin interface. enum fpga_metric_datatype Metrics data type. enum fpga_metric_type fpga metrics types opae defines power,thermal, performance counter and afu metric types enum fpga_objtype OPAE FPGA resources (objects) enum fpga_open_flags Open flags. enum fpga_reconf_flags Reconfiguration flags. enum fpga_result OPAE C API function return codes. enum fpga_sysobject_flags enum fpga_sysobject_type"},{"location":"opae-code/types__enum_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    This file defines return and error codes, event and object types, states, and flags as used or reported by OPAE C API functions. 
                                                                                    "},{"location":"opae-code/types__enum_8h/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/types__enum_8h/#enum-fpga_accelerator_state","title":"enum fpga_accelerator_state","text":"
                                                                                    enum fpga_accelerator_state {\n    FPGA_ACCELERATOR_ASSIGNED = 0,\n    FPGA_ACCELERATOR_UNASSIGNED\n};\n
                                                                                    "},{"location":"opae-code/types__enum_8h/#enum-fpga_buffer_flags","title":"enum fpga_buffer_flags","text":"
                                                                                    Buffer flags. 
                                                                                    enum fpga_buffer_flags {\n    FPGA_BUF_PREALLOCATED = (1u << 0),\n    FPGA_BUF_QUIET = (1u << 1),\n    FPGA_BUF_READ_ONLY = (1u << 2)\n};\n
                                                                                     
                                                                                    These flags can be passed to the fpgaPrepareBuffer() function. 
                                                                                    "},{"location":"opae-code/types__enum_8h/#enum-fpga_event_type","title":"enum fpga_event_type","text":"
                                                                                    FPGA events. 
                                                                                    enum fpga_event_type {\n    FPGA_EVENT_INTERRUPT = 0,\n    FPGA_EVENT_ERROR,\n    FPGA_EVENT_POWER_THERMAL\n};\n
                                                                                     
                                                                                    OPAE currently defines the following event types that applications can register for. Note that not all FPGA resources and target platforms may support all event types. 
                                                                                    "},{"location":"opae-code/types__enum_8h/#enum-fpga_interface","title":"enum fpga_interface","text":"
                                                                                    OPAE plugin interface. 
                                                                                    enum fpga_interface {\n    FPGA_IFC_DFL = 0,\n    FPGA_IFC_VFIO,\n    FPGA_IFC_SIM_DFL,\n    FPGA_IFC_SIM_VFIO,\n    FPGA_IFC_UIO\n};\n
                                                                                     
                                                                                    These are the supported plugin interfaces. 
                                                                                    "},{"location":"opae-code/types__enum_8h/#enum-fpga_metric_datatype","title":"enum fpga_metric_datatype","text":"
                                                                                    enum fpga_metric_datatype {\n    FPGA_METRIC_DATATYPE_INT,\n    FPGA_METRIC_DATATYPE_FLOAT,\n    FPGA_METRIC_DATATYPE_DOUBLE,\n    FPGA_METRIC_DATATYPE_BOOL,\n    FPGA_METRIC_DATATYPE_UNKNOWN\n};\n
                                                                                    "},{"location":"opae-code/types__enum_8h/#enum-fpga_metric_type","title":"enum fpga_metric_type","text":"
                                                                                    enum fpga_metric_type {\n    FPGA_METRIC_TYPE_POWER,\n    FPGA_METRIC_TYPE_THERMAL,\n    FPGA_METRIC_TYPE_PERFORMANCE_CTR,\n    FPGA_METRIC_TYPE_AFU,\n    FPGA_METRIC_TYPE_UNKNOWN\n};\n
                                                                                    "},{"location":"opae-code/types__enum_8h/#enum-fpga_objtype","title":"enum fpga_objtype","text":"
                                                                                    OPAE FPGA resources (objects) 
                                                                                    enum fpga_objtype {\n    FPGA_DEVICE = 0,\n    FPGA_ACCELERATOR\n};\n
                                                                                     
                                                                                    These are the FPGA resources currently supported by the OPAE object model. 
                                                                                    "},{"location":"opae-code/types__enum_8h/#enum-fpga_open_flags","title":"enum fpga_open_flags","text":"
                                                                                    Open flags. 
                                                                                    enum fpga_open_flags {\n    FPGA_OPEN_SHARED = (1u << 0)\n};\n
                                                                                     
                                                                                    These flags can be passed to the fpgaOpen() function. 
                                                                                    "},{"location":"opae-code/types__enum_8h/#enum-fpga_reconf_flags","title":"enum fpga_reconf_flags","text":"
                                                                                    Reconfiguration flags. 
                                                                                    enum fpga_reconf_flags {\n    FPGA_RECONF_FORCE = (1u << 0),\n    FPGA_RECONF_SKIP_USRCLK = (1u << 1)\n};\n
                                                                                     
                                                                                    These flags can be passed to the fpgaReconfigureSlot() function. 
                                                                                    "},{"location":"opae-code/types__enum_8h/#enum-fpga_result","title":"enum fpga_result","text":"
                                                                                    OPAE C API function return codes. 
                                                                                    enum fpga_result {\n    FPGA_OK = 0,\n    FPGA_INVALID_PARAM,\n    FPGA_BUSY,\n    FPGA_EXCEPTION,\n    FPGA_NOT_FOUND,\n    FPGA_NO_MEMORY,\n    FPGA_NOT_SUPPORTED,\n    FPGA_NO_DRIVER,\n    FPGA_NO_DAEMON,\n    FPGA_NO_ACCESS,\n    FPGA_RECONF_ERROR\n};\n
                                                                                     
                                                                                    Every public API function exported by the OPAE C library will return one of these codes. Usually, FPGA_OK denotes successful completion of the requested operation, while any return code other than FPGA_OK indicates an error or other deviation from the expected behavior. Users of the OPAE C API should always check the return codes of the APIs they call, and not use output parameters of functions that did not execute successfully.
                                                                                     
                                                                                    The fpgaErrStr() function converts error codes into printable messages.
                                                                                     
                                                                                    OPAE also has a logging mechanism that allows a developer to get more information about why a particular call failed with a specific message. If enabled, any function that returns an error code different from FPGA_OK will also print out a message with further details. This mechanism can be enabled by setting the environment variable LIBOPAE_LOG to 1 before running the respective application. 
                                                                                    "},{"location":"opae-code/types__enum_8h/#enum-fpga_sysobject_flags","title":"enum fpga_sysobject_flags","text":"
                                                                                    enum fpga_sysobject_flags {\n    FPGA_OBJECT_SYNC = (1u << 0),\n    FPGA_OBJECT_GLOB = (1u << 1),\n    FPGA_OBJECT_RAW =\n        (1u << 2),\n    FPGA_OBJECT_RECURSE_ONE =\n        (1u\n         << 3),\n    FPGA_OBJECT_RECURSE_ALL =\n        (1u\n         << 4)\n};\n
                                                                                    "},{"location":"opae-code/types__enum_8h/#enum-fpga_sysobject_type","title":"enum fpga_sysobject_type","text":"
                                                                                    enum fpga_sysobject_type {\n    FPGA_OBJECT_CONTAINER =\n        (1u << 0),\n    FPGA_OBJECT_ATTRIBUTE =\n        (1u << 1)\n};\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/types_enum.h
                                                                                    "},{"location":"opae-code/types__enum_8h_source/","title":"File types_enum.h","text":"
                                                                                    File List > docs > sw > include > opae > types_enum.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017-2022, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_TYPES_ENUM_H__\n#define __FPGA_TYPES_ENUM_H__\n\ntypedef enum {\n    FPGA_OK = 0,         \n    FPGA_INVALID_PARAM,  \n    FPGA_BUSY,           \n    FPGA_EXCEPTION,      \n    FPGA_NOT_FOUND,      \n    FPGA_NO_MEMORY,      \n    FPGA_NOT_SUPPORTED,  \n    FPGA_NO_DRIVER,      \n    FPGA_NO_DAEMON,      \n    FPGA_NO_ACCESS,      \n    FPGA_RECONF_ERROR    \n} fpga_result;\n\ntypedef enum {\n    FPGA_EVENT_INTERRUPT = 0,   \n    FPGA_EVENT_ERROR,           \n    FPGA_EVENT_POWER_THERMAL    \n} fpga_event_type;\n\n/* TODO: consider adding lifecycle events in the future\n * to help with orchestration.  Need a complete specification\n * before including them in the API.  Proposed events:\n *  FPGA_EVENT_APPEAR\n *  FPGA_EVENT_DISAPPEAR\n *  FPGA_EVENT_CHANGE\n */\n\ntypedef enum {\n    FPGA_ACCELERATOR_ASSIGNED = 0,\n    FPGA_ACCELERATOR_UNASSIGNED\n} fpga_accelerator_state;\n\ntypedef enum {\n    FPGA_DEVICE = 0,\n    FPGA_ACCELERATOR\n} fpga_objtype;\n\ntypedef enum {\n    FPGA_IFC_DFL = 0,\n    FPGA_IFC_VFIO,\n    FPGA_IFC_SIM_DFL,\n    FPGA_IFC_SIM_VFIO,\n    FPGA_IFC_UIO,\n} fpga_interface;\n\nenum fpga_buffer_flags {\n    FPGA_BUF_PREALLOCATED = (1u << 0), \n    FPGA_BUF_QUIET = (1u << 1),        \n    FPGA_BUF_READ_ONLY = (1u << 2)     \n};\n\nenum fpga_open_flags {\n    FPGA_OPEN_SHARED = (1u << 0)\n};\n\nenum fpga_reconf_flags {\n    FPGA_RECONF_FORCE = (1u << 0),\n    FPGA_RECONF_SKIP_USRCLK = (1u << 1)\n};\n\nenum fpga_sysobject_flags {\n    FPGA_OBJECT_SYNC = (1u << 0), \n    FPGA_OBJECT_GLOB = (1u << 1), \n    FPGA_OBJECT_RAW =\n        (1u << 2), \n    FPGA_OBJECT_RECURSE_ONE =\n        (1u\n         << 3), \n    FPGA_OBJECT_RECURSE_ALL =\n        (1u\n         << 4) \n};\n\nenum fpga_sysobject_type {\n    FPGA_OBJECT_CONTAINER =\n        (1u << 0), \n    FPGA_OBJECT_ATTRIBUTE =\n        (1u << 1) \n};\n\nenum fpga_metric_type {\n    FPGA_METRIC_TYPE_POWER,             // Metric power\n    FPGA_METRIC_TYPE_THERMAL,           // Metric Thermal\n    FPGA_METRIC_TYPE_PERFORMANCE_CTR,   // Metric Performance counter\n    FPGA_METRIC_TYPE_AFU,               // Metric AFU\n    FPGA_METRIC_TYPE_UNKNOWN            // Unknown\n};\n\nenum fpga_metric_datatype {\n    FPGA_METRIC_DATATYPE_INT,       // Metric datatype integer\n    FPGA_METRIC_DATATYPE_FLOAT,     // Metric datatype float\n    FPGA_METRIC_DATATYPE_DOUBLE,    // Metric datatype double\n    FPGA_METRIC_DATATYPE_BOOL,      // Metric datatype bool\n    FPGA_METRIC_DATATYPE_UNKNOWN    // Metric datatype unknown\n};\n\n#endif // __FPGA_TYPES_ENUM_H__\n
                                                                                    "},{"location":"opae-code/uio_8h/","title":"File uio.h","text":"
                                                                                    FileList > docs > sw > include > opae > uio.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    APIs to manage a PCIe device via UIO. More...
                                                                                     
                                                                                       
                                                                                    • #include <stdio.h>
                                                                                      •  
                                                                                    • #include <stdint.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/uio_8h/#classes","title":"Classes","text":"Type Name struct opae_uio OPAE UIO device abstraction. struct opae_uio_device_region MMIO region info."},{"location":"opae-code/uio_8h/#public-functions","title":"Public Functions","text":"Type Name void opae_uio_close (struct opae_uio * u) Release and close a UIO device. int opae_uio_open (struct opae_uio * u, const char * dfl_device) Open and populate a UIO device. int opae_uio_region_get (struct opae_uio * u, uint32_t index, uint8_t ** ptr, size_t * size) Query device MMIO region."},{"location":"opae-code/uio_8h/#macros","title":"Macros","text":"Type Name define OPAE_UIO_PATH_MAX  256"},{"location":"opae-code/uio_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    Presents a simple interface for interacting with a DFL device that is bound to its UIO driver. See https://kernel.org/doc/html/v4.14/driver-api/uio-howto.html for a description of UIO.
                                                                                     
                                                                                    Provides APIs for opening/closing the device and for querying info about the MMIO regions of the device. 
                                                                                    "},{"location":"opae-code/uio_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/uio_8h/#function-opae_uio_close","title":"function opae_uio_close","text":"
                                                                                    Release and close a UIO device. 
                                                                                    void opae_uio_close (\n    struct opae_uio * u\n) \n
                                                                                     
                                                                                    The given device pointer must have been previously initialized by opae_uio_open. Releases all data structures.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • u Storage for the device info. May be stack-resident.
                                                                                      •  
                                                                                     
                                                                                    Example  struct opae_uio u;
                                                                                     
                                                                                    if (opae_uio_open(&u, \"dfl_dev.10\")) {   // handle error } else {   // interact with the device   ...   // free the device   opae_uio_close(&u); }
                                                                                     
                                                                                    Example  $ sudo opaeuio -r dfl_dev.10
                                                                                    "},{"location":"opae-code/uio_8h/#function-opae_uio_open","title":"function opae_uio_open","text":"
                                                                                    Open and populate a UIO device. 
                                                                                    int opae_uio_open (\n    struct opae_uio * u,\n    const char * dfl_device\n) \n
                                                                                     
                                                                                    Opens the Device Feature List device corresponding to the device name given in dfl_device, eg \"dfl_dev.10\". The device must be bound to the dfl-uio-pdev driver prior to opening it. The data structures corresponding to the MMIO regions are initialized.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • u Storage for the device. May be stack-resident. 
                                                                                      •  
                                                                                    • dfl_device The name of the desired DFL device. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error. Zero on success.
                                                                                     
                                                                                    Example  $ sudo opaeuio -i -u lab -g lab dfl_dev.10
                                                                                     
                                                                                    Example  struct opae_uio u;
                                                                                     
                                                                                    if (opae_uio_open(&u, \"dfl_dev.10\")) {   // handle error }
                                                                                    "},{"location":"opae-code/uio_8h/#function-opae_uio_region_get","title":"function opae_uio_region_get","text":"
                                                                                    Query device MMIO region. 
                                                                                    int opae_uio_region_get (\n    struct opae_uio * u,\n    uint32_t index,\n    uint8_t ** ptr,\n    size_t * size\n) \n
                                                                                     
                                                                                    Retrieves info describing the MMIO region with the given region index. The device structure u must have been previously opened by a call to opae_uio_open.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • u The open OPAE UIO device. 
                                                                                      •  
                                                                                    • index The zero-based index of the desired MMIO region. 
                                                                                      •  
                                                                                    • ptr Optional pointer to receive the virtual address for the region. Pass NULL to ignore. 
                                                                                      •  
                                                                                    • size Optional pointer to receive the size of the MMIO region. Pass NULL to ignore. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error (including index out-of-range). Zero on success.
                                                                                     
                                                                                    Example  struct opae_uio u;
                                                                                     
                                                                                    uint8_t *virt = NULL; size_t size = 0;
                                                                                     
                                                                                    if (opae_uio_open(&u, \"dfl_dev.10\")) {   // handle error } else {   opae_uio_region_get(&u, 0, &virt, &size); }
                                                                                    "},{"location":"opae-code/uio_8h/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/uio_8h/#define-opae_uio_path_max","title":"define OPAE_UIO_PATH_MAX","text":"
                                                                                    #define OPAE_UIO_PATH_MAX 256\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/uio.h
                                                                                    "},{"location":"opae-code/uio_8h_source/","title":"File uio.h","text":"
                                                                                    File List > docs > sw > include > opae > uio.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2020, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __OPAE_UIO_H__\n#define __OPAE_UIO_H__\n\n#include <stdio.h>\n#include <stdint.h>\n\n#define OPAE_UIO_PATH_MAX 256\n\nstruct opae_uio_device_region {\n    uint32_t region_index;\n    uint8_t *region_ptr;\n    size_t region_page_offset;\n    size_t region_size;\n    struct opae_uio_device_region *next;\n};\n\nstruct opae_uio {\n    char device_path[OPAE_UIO_PATH_MAX];\n    int device_fd;\n    struct opae_uio_device_region *regions;\n};\n\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nint opae_uio_open(struct opae_uio *u,\n          const char *dfl_device);\n\nint opae_uio_region_get(struct opae_uio *u,\n            uint32_t index,\n            uint8_t **ptr,\n            size_t *size);\n\nvoid opae_uio_close(struct opae_uio *u);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __OPAE_UIO_H__\n
                                                                                    "},{"location":"opae-code/umsg_8h/","title":"File umsg.h","text":"
                                                                                    FileList > docs > sw > include > opae > umsg.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    FPGA UMsg API. 
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/umsg_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaGetNumUmsg (fpga_handle handle, uint64_t * value) Get number of Umsgs. fpga_result fpgaGetUmsgPtr (fpga_handle handle, uint64_t ** umsg_ptr) Access UMsg memory directly. fpga_result fpgaSetUmsgAttributes (fpga_handle handle, uint64_t value) Sets Umsg hint. fpga_result fpgaTriggerUmsg (fpga_handle handle, uint64_t value) Trigger Umsg."},{"location":"opae-code/umsg_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/umsg_8h/#function-fpgagetnumumsg","title":"function fpgaGetNumUmsg","text":"
                                                                                    Get number of Umsgs. 
                                                                                    fpga_result fpgaGetNumUmsg (\n    fpga_handle handle,\n    uint64_t * value\n) \n
                                                                                     
                                                                                    Retuns number of umsg supported by AFU.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource 
                                                                                      •  
                                                                                    • value Returns number of UMsgs 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if input parameter combination is not valid. FPGA_EXCEPTION if input parameter fpga handle is not valid. 
                                                                                    "},{"location":"opae-code/umsg_8h/#function-fpgagetumsgptr","title":"function fpgaGetUmsgPtr","text":"
                                                                                    Access UMsg memory directly. 
                                                                                    fpga_result fpgaGetUmsgPtr (\n    fpga_handle handle,\n    uint64_t ** umsg_ptr\n) \n
                                                                                     
                                                                                    This function will return a pointer to the memory allocated for low latency accelerator notifications (UMsgs).
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource 
                                                                                      •  
                                                                                    • umsg_ptr Pointer to memory where a pointer to the virtual address space will be returned 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if input parameter combination is not valid. FPGA_EXCEPTION if input parameter fpga handle is not valid. FPGA_NO_MEMORY if memory allocation fails or system doesn't configure huge pages. 
                                                                                    "},{"location":"opae-code/umsg_8h/#function-fpgasetumsgattributes","title":"function fpgaSetUmsgAttributes","text":"
                                                                                    Sets Umsg hint. 
                                                                                    fpga_result fpgaSetUmsgAttributes (\n    fpga_handle handle,\n    uint64_t value\n) \n
                                                                                     
                                                                                    Writes usmg hint bit.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource 
                                                                                      •  
                                                                                    • value Value to use for UMsg hint, Umsg hit is N wide bitvector where N = number of Umsgs. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if input parameter combination is not valid. FPGA_EXCEPTION if input parameter fpga handle is not valid. 
                                                                                    "},{"location":"opae-code/umsg_8h/#function-fpgatriggerumsg","title":"function fpgaTriggerUmsg","text":"
                                                                                    Trigger Umsg. 
                                                                                    fpga_result fpgaTriggerUmsg (\n    fpga_handle handle,\n    uint64_t value\n) \n
                                                                                     
                                                                                    Writes a 64-bit value to trigger low-latency accelerator notification mechanism (UMsgs).
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource 
                                                                                      •  
                                                                                    • value Value to use for UMsg 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if input parameter combination is not valid. FPGA_EXCEPTION if input parameter fpga handle is not valid. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/umsg.h
                                                                                    "},{"location":"opae-code/umsg_8h_source/","title":"File umsg.h","text":"
                                                                                    File List > docs > sw > include > opae > umsg.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_UMSG_H__\n#define __FPGA_UMSG_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaGetNumUmsg(fpga_handle handle, uint64_t *value);\n\nfpga_result fpgaSetUmsgAttributes(fpga_handle handle,\n                  uint64_t value);\n\nfpga_result fpgaTriggerUmsg(fpga_handle handle, uint64_t value);\n\nfpga_result fpgaGetUmsgPtr(fpga_handle handle, uint64_t **umsg_ptr);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_UMSG_H__\n
                                                                                    "},{"location":"opae-code/userclk_8h/","title":"File userclk.h","text":"
                                                                                    FileList > docs > sw > include > opae > userclk.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Functions for setting and get afu user clock. 
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/userclk_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaGetUserClock (fpga_handle handle, uint64_t * high_clk, uint64_t * low_clk, int flags) Get afu user clock high and low. fpga_result fpgaSetUserClock (fpga_handle handle, uint64_t high_clk, uint64_t low_clk, int flags) set afu user clock high and low"},{"location":"opae-code/userclk_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/userclk_8h/#function-fpgagetuserclock","title":"function fpgaGetUserClock","text":"
                                                                                    Get afu user clock high and low. 
                                                                                    fpga_result fpgaGetUserClock (\n    fpga_handle handle,\n    uint64_t * high_clk,\n    uint64_t * low_clk,\n    int flags\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource. 
                                                                                      •  
                                                                                    • high_clk AFU High user clock frequency in MHz. 
                                                                                      •  
                                                                                    • low_clk AFU Low user clock frequency in MHz. 
                                                                                      •  
                                                                                    • flags Flags Reserved.
                                                                                      •  
                                                                                     
                                                                                    .*
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if invalid parameters were provided, or if the parameter combination is not valid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                                                                    "},{"location":"opae-code/userclk_8h/#function-fpgasetuserclock","title":"function fpgaSetUserClock","text":"
                                                                                    set afu user clock high and low 
                                                                                    fpga_result fpgaSetUserClock (\n    fpga_handle handle,\n    uint64_t high_clk,\n    uint64_t low_clk,\n    int flags\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • handle Handle to previously opened accelerator resource. 
                                                                                      •  
                                                                                    • high_clk AFU High user clock frequency in MHz. 
                                                                                      •  
                                                                                    • low_clk AFU Low user clock frequency in MHz. 
                                                                                      •  
                                                                                    • flags Flags Reserved.
                                                                                      •  
                                                                                     
                                                                                    .*
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_OK on success. FPGA_INVALID_PARAM if invalid parameters were provided, or if the parameter combination is not valid. FPGA_EXCEPTION if an internal exception occurred while trying to access the handle. 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/userclk.h
                                                                                    "},{"location":"opae-code/userclk_8h_source/","title":"File userclk.h","text":"
                                                                                    File List > docs > sw > include > opae > userclk.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2018, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_AFU_USER_CLOCK_H__\n#define __FPGA_AFU_USER_CLOCK_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaSetUserClock(fpga_handle handle,\n                uint64_t high_clk, uint64_t low_clk, int flags);\n\nfpga_result fpgaGetUserClock(fpga_handle handle,\n                uint64_t *high_clk, uint64_t *low_clk, int flags);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_AFU_USER_CLOCK_H__\n
                                                                                    "},{"location":"opae-code/utils_8h/","title":"File utils.h","text":"
                                                                                    FileList > docs > sw > include > opae > utils.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    Utility functions and macros for the FPGA API. 
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    • #include <stdio.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/utils_8h/#public-functions","title":"Public Functions","text":"Type Name const char * fpgaErrStr (fpga_result e) Return human-readable error message."},{"location":"opae-code/utils_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/utils_8h/#function-fpgaerrstr","title":"function fpgaErrStr","text":"
                                                                                    Return human-readable error message. 
                                                                                    const char * fpgaErrStr (\n    fpga_result e\n) \n
                                                                                     
                                                                                    Returns a pointer to a human-readable error message corresponding to the provided fpga_error error code.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • e Error code (as returned by another FPGA API function 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Pointer to a descriptive error message string 
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/utils.h
                                                                                    "},{"location":"opae-code/utils_8h_source/","title":"File utils.h","text":"
                                                                                    File List > docs > sw > include > opae > utils.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_UTILS_H__\n#define __FPGA_UTILS_H__\n\n#include <opae/types.h>\n#include <stdio.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nconst char *fpgaErrStr(fpga_result e);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_UTILS_H__\n
                                                                                    "},{"location":"opae-code/version_8h/","title":"File version.h","text":"
                                                                                    FileList > docs > sw > include > opae > version.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                       
                                                                                    • #include <opae/types.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/version_8h/#public-functions","title":"Public Functions","text":"Type Name fpga_result fpgaGetOPAECBuildString (char * build_str, size_t len) Get build information about the OPAE library as a string. fpga_result fpgaGetOPAECVersion (fpga_version * version) Get version information about the OPAE library. fpga_result fpgaGetOPAECVersionString (char * version_str, size_t len) Get version information about the OPAE library as a string."},{"location":"opae-code/version_8h/#macros","title":"Macros","text":"Type Name define FPGA_BUILD_STR_MAX  41 define FPGA_VERSION_STR_MAX  10"},{"location":"opae-code/version_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/version_8h/#function-fpgagetopaecbuildstring","title":"function fpgaGetOPAECBuildString","text":"
                                                                                    Get build information about the OPAE library as a string. 
                                                                                    fpga_result fpgaGetOPAECBuildString (\n    char * build_str,\n    size_t len\n) \n
                                                                                     
                                                                                    Retrieve the build identifier of the OPAE library.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • build_str String to copy build information into 
                                                                                      •  
                                                                                    • len Length of build_str 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if build_str is NULL, FPGA_EXCEPTION if the version string cannot be copied into build_str, FPGA_OK otherwise. 
                                                                                    "},{"location":"opae-code/version_8h/#function-fpgagetopaecversion","title":"function fpgaGetOPAECVersion","text":"
                                                                                    Get version information about the OPAE library. 
                                                                                    fpga_result fpgaGetOPAECVersion (\n    fpga_version * version\n) \n
                                                                                     
                                                                                    Retrieve major version, minor version, and revision information about the OPAE library.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • version FPGA version 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if any of the output parameters is NULL, FPGA_OK otherwise. 
                                                                                    "},{"location":"opae-code/version_8h/#function-fpgagetopaecversionstring","title":"function fpgaGetOPAECVersionString","text":"
                                                                                    Get version information about the OPAE library as a string. 
                                                                                    fpga_result fpgaGetOPAECVersionString (\n    char * version_str,\n    size_t len\n) \n
                                                                                     
                                                                                    Retrieve major version, minor version, and revision information about the OPAE library, encoded in a human-readable string (e.g. \"1.0.0\").
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • version_str String to copy version information into 
                                                                                      •  
                                                                                    • len Length of version_str 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    FPGA_INVALID_PARAM if version_str is NULL, FPGA_EXCEPTION if the version string cannot be copied into version_str, FPGA_OK otherwise. 
                                                                                    "},{"location":"opae-code/version_8h/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/version_8h/#define-fpga_build_str_max","title":"define FPGA_BUILD_STR_MAX","text":"
                                                                                    #define FPGA_BUILD_STR_MAX 41\n
                                                                                    "},{"location":"opae-code/version_8h/#define-fpga_version_str_max","title":"define FPGA_VERSION_STR_MAX","text":"
                                                                                    #define FPGA_VERSION_STR_MAX 10\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/version.h
                                                                                    "},{"location":"opae-code/version_8h_source/","title":"File version.h","text":"
                                                                                    File List > docs > sw > include > opae > version.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __FPGA_VERSION_H__\n#define __FPGA_VERSION_H__\n\n#include <opae/types.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nfpga_result fpgaGetOPAECVersion(fpga_version *version);\n\nfpga_result fpgaGetOPAECVersionString(char *version_str, size_t len);\n#define FPGA_VERSION_STR_MAX 10\n\nfpga_result fpgaGetOPAECBuildString(char *build_str, size_t len);\n#define FPGA_BUILD_STR_MAX 41\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __FPGA_VERSION_H__\n
                                                                                    "},{"location":"opae-code/vfio_8h/","title":"File vfio.h","text":"
                                                                                    FileList > docs > sw > include > opae > vfio.h
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    APIs to manage a PCIe device via vfio-pci. More...
                                                                                     
                                                                                       
                                                                                    • #include <stdio.h>
                                                                                      •  
                                                                                    • #include <stdint.h>
                                                                                      •  
                                                                                    • #include <pthread.h>
                                                                                      •  
                                                                                    • #include <linux/vfio.h>
                                                                                      •  
                                                                                    • #include <opae/mem_alloc.h>
                                                                                      •  
                                                                                    • #include <opae/hash_map.h>
                                                                                      •  
                                                                                    "},{"location":"opae-code/vfio_8h/#classes","title":"Classes","text":"Type Name struct opae_vfio OPAE VFIO device abstraction. struct opae_vfio_buffer System DMA buffer. struct opae_vfio_device VFIO device. struct opae_vfio_device_irq Interrupt info. struct opae_vfio_device_region MMIO region info. struct opae_vfio_group VFIO group. struct opae_vfio_iova_range IO Virtual Address Range. struct opae_vfio_sparse_info MMIO sparse-mappable region info."},{"location":"opae-code/vfio_8h/#public-types","title":"Public Types","text":"Type Name enum opae_vfio_buffer_flags Flags for opae_vfio_buffer_allocate_ex() ."},{"location":"opae-code/vfio_8h/#public-functions","title":"Public Functions","text":"Type Name int opae_vfio_buffer_allocate (struct opae_vfio * v, size_t * size, uint8_t ** buf, uint64_t * iova) Allocate and map system buffer. int opae_vfio_buffer_allocate_ex (struct opae_vfio * v, size_t * size, uint8_t ** buf, uint64_t * iova, int flags) Allocate and map system buffer (extended w/ flags) int opae_vfio_buffer_free (struct opae_vfio * v, uint8_t * buf) Unmap and free a system buffer. struct opae_vfio_buffer * opae_vfio_buffer_info (struct opae_vfio * v, uint8_t * vaddr) Extract the internal data structure pointer for the given vaddr. void opae_vfio_close (struct opae_vfio * v) Release and close a VFIO device. int opae_vfio_irq_disable (struct opae_vfio * v, uint32_t index, uint32_t subindex) Disable an IRQ. int opae_vfio_irq_enable (struct opae_vfio * v, uint32_t index, uint32_t subindex, int event_fd) Enable an IRQ. int opae_vfio_irq_mask (struct opae_vfio * v, uint32_t index, uint32_t subindex) Mask an IRQ. int opae_vfio_irq_unmask (struct opae_vfio * v, uint32_t index, uint32_t subindex) Unmask an IRQ. int opae_vfio_open (struct opae_vfio * v, const char * pciaddr) Open and populate a VFIO device. int opae_vfio_region_get (struct opae_vfio * v, uint32_t index, uint8_t ** ptr, size_t * size) Query device MMIO region. int opae_vfio_secure_open (struct opae_vfio * v, const char * pciaddr, const char * token) Open and populate a VFIO device."},{"location":"opae-code/vfio_8h/#detailed-description","title":"Detailed Description","text":"
                                                                                    Presents a simple interface for interacting with a PCIe device that is bound to the vfio-pci driver. See https://kernel.org/doc/Documentation/vfio.txt for a description of vfio-pci.
                                                                                     
                                                                                    Provides APIs for opening/closing the device, querying info about the MMIO regions of the device, and allocating/mapping and freeing/unmapping DMA buffers. 
                                                                                    "},{"location":"opae-code/vfio_8h/#public-types-documentation","title":"Public Types Documentation","text":""},{"location":"opae-code/vfio_8h/#enum-opae_vfio_buffer_flags","title":"enum opae_vfio_buffer_flags","text":"
                                                                                    enum opae_vfio_buffer_flags {\n    OPAE_VFIO_BUF_PREALLOCATED = 1\n};\n
                                                                                    "},{"location":"opae-code/vfio_8h/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/vfio_8h/#function-opae_vfio_buffer_allocate","title":"function opae_vfio_buffer_allocate","text":"
                                                                                    Allocate and map system buffer. 
                                                                                    int opae_vfio_buffer_allocate (\n    struct opae_vfio * v,\n    size_t * size,\n    uint8_t ** buf,\n    uint64_t * iova\n) \n
                                                                                     
                                                                                    Allocate, map, and retrieve info for a system buffer capable of DMA. Saves an entry in the v->cont_buffers list. If the buffer is not explicitly freed by opae_vfio_buffer_free, it will be freed during opae_vfio_close.
                                                                                     
                                                                                    mmap is used for the allocation. If the size is greater than 2MB, then the allocation request is fulfilled by a 1GB huge page. Else, if the size is greater than 4096, then the request is fulfilled by a 2MB huge page. Else, the request is fulfilled by the non-huge page pool.
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    Allocations from the huge page pool require that huge pages be configured on the system. Huge pages may be configured on the kernel boot command prompt. Example default_hugepagesz=1G hugepagesz=1G hugepages=2 hugepagesz=2M hugepages=8
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    Huge pages may also be configured at runtime. Example sudo sh -c 'echo 8 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages' sudo sh -c 'echo 2 > /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages'
                                                                                     
                                                                                    Note:
                                                                                     
                                                                                    Be sure that the IOMMU is also enabled using the follow kernel boot command: intel_iommu=on
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • v The open OPAE VFIO device. 
                                                                                      •  
                                                                                    • size A pointer to the requested size. The size may be rounded to the next page size prior to return from the function. 
                                                                                      •  
                                                                                    • buf Optional pointer to receive the virtual address for the buffer. Pass NULL to ignore. 
                                                                                      •  
                                                                                    • iova Optional pointer to receive the IOVA address for the buffer. Pass NULL to ignore. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error. Zero on success.
                                                                                     
                                                                                    Example  opae_vfio v;
                                                                                     
                                                                                    size_t sz; uint8_t *buf_2m_virt = NULL; uint8_t *buf_1g_virt = NULL; uint64_t buf_2m_iova = 0; uint64_t buf_1g_iova = 0;
                                                                                     
                                                                                    if (opae_vfio_open(&v, \"0000:00:00.0\")) {   // handle error } else {   sz = 2 * 1024 * 1024;   if (opae_vfio_buffer_allocate(&v,                                 &sz,                                 &buf_2m_virt,                                 &buf_2m_iova)) {     // handle allocation error   }
                                                                                     
                                                                                    sz = 1024 * 1024 * 1024;   if (opae_vfio_buffer_allocate(&v,                                 &sz,                                 &buf_1g_virt,                                 &buf_1g_iova)) {     // handle allocation error   } }
                                                                                    "},{"location":"opae-code/vfio_8h/#function-opae_vfio_buffer_allocate_ex","title":"function opae_vfio_buffer_allocate_ex","text":"
                                                                                    Allocate and map system buffer (extended w/ flags) 
                                                                                    int opae_vfio_buffer_allocate_ex (\n    struct opae_vfio * v,\n    size_t * size,\n    uint8_t ** buf,\n    uint64_t * iova,\n    int flags\n) \n
                                                                                     
                                                                                    Allocate, map, and retrieve info for a system buffer capable of DMA. Saves an entry in the v->cont_buffers list. If the buffer is not explicitly freed by opae_vfio_buffer_free, it will be freed during opae_vfio_close, unless OPAE_VFIO_BUF_PREALLOCATED is used in which case the buffer is not freed by this library.
                                                                                     
                                                                                    When not using OPAE_VFIO_BUF_PREALLOCATED, mmap is used for the allocation. If the size is greater than 2MB, then the allocation request is fulfilled by a 1GB huge page. Else, if the size is greater than 4096, then the request is fulfilled by a 2MB huge page. Else, the request is fulfilled by the non-huge page pool.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • v The open OPAE VFIO device. 
                                                                                      •  
                                                                                    • size A pointer to the requested size. The size may be rounded to the next page size prior to return from the function. 
                                                                                      •  
                                                                                    • buf Optional pointer to receive the virtual address for the buffer/input buffer pointer when using OPAE_VFIO_BUF_PREALLOCATED. Pass NULL to ignore. 
                                                                                      •  
                                                                                    • iova Optional pointer to receive the IOVA address for the buffer. Pass NULL to ignore. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error. Zero on success.
                                                                                     
                                                                                    Example  opae_vfio v;
                                                                                     
                                                                                    size_t sz = MY_BUF_SIZE; uint8_t *prealloc_virt = NULL; uint64_t iova = 0;
                                                                                     
                                                                                    prealloc_virt = allocate_my_buffer(sz);
                                                                                     
                                                                                    if (opae_vfio_open(&v, \"0000:00:00.0\")) {   // handle error } else {   if (opae_vfio_buffer_allocate_ex(&v,                                    &sz,                                    &prealloc_virt,                                    &iova,                                    OPAE_VFIO_BUF_PREALLOCATED)) {     // handle allocation error   } }
                                                                                    "},{"location":"opae-code/vfio_8h/#function-opae_vfio_buffer_free","title":"function opae_vfio_buffer_free","text":"
                                                                                    Unmap and free a system buffer. 
                                                                                    int opae_vfio_buffer_free (\n    struct opae_vfio * v,\n    uint8_t * buf\n) \n
                                                                                     
                                                                                    The buffer corresponding to buf must have been created by a previous call to opae_vfio_buffer_allocate.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • v The open OPAE VFIO device. 
                                                                                      •  
                                                                                    • buf The virtual address corresponding to the buffer to be freed. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error. Zero on success.
                                                                                     
                                                                                    Example  size_t sz; uint8_t *buf_2m_virt = NULL; uint64_t buf_2m_iova = 0;
                                                                                     
                                                                                    sz = 2 * 1024 * 1024; if (opae_vfio_buffer_allocate(&v,                               &sz,                               &buf_2m_virt,                               &buf_2m_iova)) {   // handle allocation error } else {   // use the buffer
                                                                                     
                                                                                    if (opae_vfio_buffer_free(&v, buf_2m_virt)) {     // handle free error   } }
                                                                                    "},{"location":"opae-code/vfio_8h/#function-opae_vfio_buffer_info","title":"function opae_vfio_buffer_info","text":"
                                                                                    Extract the internal data structure pointer for the given vaddr. 
                                                                                    struct opae_vfio_buffer * opae_vfio_buffer_info (\n    struct opae_vfio * v,\n    uint8_t * vaddr\n) \n
                                                                                     
                                                                                    The virtual address vaddr must correspond to a buffer previously allocated by opae_vfio_buffer_allocate() or opae_vfio_buffer_allocate_ex().
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • v The open OPAE VFIO device. 
                                                                                      •  
                                                                                    • vaddr The user virtual address of the desired buffer info struct. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    NULL on lookup error. 
                                                                                    "},{"location":"opae-code/vfio_8h/#function-opae_vfio_close","title":"function opae_vfio_close","text":"
                                                                                    Release and close a VFIO device. 
                                                                                    void opae_vfio_close (\n    struct opae_vfio * v\n) \n
                                                                                     
                                                                                    The given device pointer must have been previously initialized by opae_vfio_open. Releases all data structures, including any DMA buffer allocations that have not be explicitly freed by opae_vfio_buffer_free.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • v Storage for the device info. May be stack-resident.
                                                                                      •  
                                                                                     
                                                                                    Example  opae_vfio v;
                                                                                     
                                                                                    if (opae_vfio_open(&v, \"0000:00:00.0\")) {   // handle error } else {   // interact with the device   ...   // free the device   opae_vfio_close(&v); }
                                                                                     
                                                                                    Example  $ sudo opaevfio -r 0000:00:00.0
                                                                                    "},{"location":"opae-code/vfio_8h/#function-opae_vfio_irq_disable","title":"function opae_vfio_irq_disable","text":"
                                                                                    Disable an IRQ. 
                                                                                    int opae_vfio_irq_disable (\n    struct opae_vfio * v,\n    uint32_t index,\n    uint32_t subindex\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • v The open OPAE VFIO device. 
                                                                                      •  
                                                                                    • index The IRQ category. For MSI-X, use VFIO_PCI_MSIX_IRQ_INDEX. 
                                                                                      •  
                                                                                    • subindex The IRQ to disable. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error. Zero on success. 
                                                                                    "},{"location":"opae-code/vfio_8h/#function-opae_vfio_irq_enable","title":"function opae_vfio_irq_enable","text":"
                                                                                    Enable an IRQ. 
                                                                                    int opae_vfio_irq_enable (\n    struct opae_vfio * v,\n    uint32_t index,\n    uint32_t subindex,\n    int event_fd\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • v The open OPAE VFIO device. 
                                                                                      •  
                                                                                    • index The IRQ category. For MSI-X, use VFIO_PCI_MSIX_IRQ_INDEX. 
                                                                                      •  
                                                                                    • subindex The IRQ to enable. 
                                                                                      •  
                                                                                    • event_fd The file descriptor, created by eventfd(). Interrupts will result in this event_fd being signaled. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error. Zero on success. 
                                                                                    "},{"location":"opae-code/vfio_8h/#function-opae_vfio_irq_mask","title":"function opae_vfio_irq_mask","text":"
                                                                                    Mask an IRQ. 
                                                                                    int opae_vfio_irq_mask (\n    struct opae_vfio * v,\n    uint32_t index,\n    uint32_t subindex\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • v The open OPAE VFIO device. 
                                                                                      •  
                                                                                    • index The IRQ category. For MSI-X, use VFIO_PCI_MSIX_IRQ_INDEX. 
                                                                                      •  
                                                                                    • subindex The IRQ to mask. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error. Zero on success. 
                                                                                    "},{"location":"opae-code/vfio_8h/#function-opae_vfio_irq_unmask","title":"function opae_vfio_irq_unmask","text":"
                                                                                    Unmask an IRQ. 
                                                                                    int opae_vfio_irq_unmask (\n    struct opae_vfio * v,\n    uint32_t index,\n    uint32_t subindex\n) \n
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • v The open OPAE VFIO device. 
                                                                                      •  
                                                                                    • index The IRQ category. For MSI-X, use VFIO_PCI_MSIX_IRQ_INDEX. 
                                                                                      •  
                                                                                    • subindex The IRQ to unmask. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error. Zero on success. 
                                                                                    "},{"location":"opae-code/vfio_8h/#function-opae_vfio_open","title":"function opae_vfio_open","text":"
                                                                                    Open and populate a VFIO device. 
                                                                                    int opae_vfio_open (\n    struct opae_vfio * v,\n    const char * pciaddr\n) \n
                                                                                     
                                                                                    Opens the PCIe device corresponding to the address given in pciaddr. The device must be bound to the vfio-pci driver prior to opening it. The data structures corresponding to IOVA space, MMIO regions, and DMA buffers are initialized.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • v Storage for the device info. May be stack-resident. 
                                                                                      •  
                                                                                    • pciaddr The PCIe address of the requested device. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error. Zero on success.
                                                                                     
                                                                                    Example  $ sudo opaevfio -i 0000:00:00.0 -u user -g group
                                                                                     
                                                                                    Example  opae_vfio v;
                                                                                     
                                                                                    if (opae_vfio_open(&v, \"0000:00:00.0\")) {   // handle error }
                                                                                    "},{"location":"opae-code/vfio_8h/#function-opae_vfio_region_get","title":"function opae_vfio_region_get","text":"
                                                                                    Query device MMIO region. 
                                                                                    int opae_vfio_region_get (\n    struct opae_vfio * v,\n    uint32_t index,\n    uint8_t ** ptr,\n    size_t * size\n) \n
                                                                                     
                                                                                    Retrieves info describing the MMIO region with the given region index. The device structure v must have been previously opened by a call to opae_vfio_open.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • v The open OPAE VFIO device. 
                                                                                      •  
                                                                                    • index The zero-based index of the desired MMIO region. 
                                                                                      •  
                                                                                    • ptr Optional pointer to receive the virtual address for the region. Pass NULL to ignore. 
                                                                                      •  
                                                                                    • size Optional pointer to receive the size of the MMIO region. Pass NULL to ignore. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error (including index out-of-range). Zero on success.
                                                                                     
                                                                                    Example  opae_vfio v;
                                                                                     
                                                                                    uint8_t *fme_virt = NULL; uint8_t *port_virt = NULL; size_t fme_size = 0; size_t port_size = 0;
                                                                                     
                                                                                    if (opae_vfio_open(&v, \"0000:00:00.0\")) {   // handle error } else {   opae_vfio_region_get(&v, 0, &fme_virt, &fme_size);   opae_vfio_region_get(&v, 2, &port_virt, &port_size); }
                                                                                    "},{"location":"opae-code/vfio_8h/#function-opae_vfio_secure_open","title":"function opae_vfio_secure_open","text":"
                                                                                    Open and populate a VFIO device. 
                                                                                    int opae_vfio_secure_open (\n    struct opae_vfio * v,\n    const char * pciaddr,\n    const char * token\n) \n
                                                                                     
                                                                                    Opens the PCIe device corresponding to the address given in pciaddr, using the VF token (GUID) given in token. The device must be bound to the vfio-pci driver prior to opening it. The data structures corresponding to IOVA space, MMIO regions, and DMA buffers are initialized.
                                                                                     
                                                                                    Parameters:
                                                                                     
                                                                                       
                                                                                    • v Storage for the device info. May be stack-resident. 
                                                                                      •  
                                                                                    • pciaddr The PCIe address of the requested device. 
                                                                                      •  
                                                                                    • token The GUID representing the VF token. 
                                                                                      •  
                                                                                     
                                                                                    Returns:
                                                                                     
                                                                                    Non-zero on error. Zero on success.
                                                                                     
                                                                                    Example  $ sudo opaevfio -i 0000:00:00.0 -u user -g group
                                                                                     
                                                                                    Example  opae_vfio v;
                                                                                     
                                                                                    if (opae_vfio_secure_open(&v, \"0000:00:00.0\",                           \"00f5ad6b-2edd-422e-9d1e-34124c686fec\")) {   // handle error }
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/include/opae/vfio.h
                                                                                    "},{"location":"opae-code/vfio_8h_source/","title":"File vfio.h","text":"
                                                                                    File List > docs > sw > include > opae > vfio.h
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2020-2023, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifndef __OPAE_VFIO_H__\n#define __OPAE_VFIO_H__\n\n#include <stdio.h>\n#include <stdint.h>\n#include <pthread.h>\n\n#include <linux/vfio.h>\n#include <opae/mem_alloc.h>\n#include <opae/hash_map.h>\n\nstruct opae_vfio_iova_range {\n    uint64_t start;             \n    uint64_t end;               \n    struct opae_vfio_iova_range *next;  \n};\n\nstruct opae_vfio_group {\n    char *group_device; \n    int group_fd;       \n};\n\nstruct opae_vfio_sparse_info {\n    uint32_t index;             \n    uint32_t offset;            \n    uint32_t size;              \n    uint8_t *ptr;               \n    struct opae_vfio_sparse_info *next; \n};\n\nstruct opae_vfio_device_region {\n    uint32_t region_index;              \n    uint8_t *region_ptr;                \n    size_t region_size;             \n    struct opae_vfio_sparse_info *region_sparse;    \n    struct opae_vfio_device_region *next;       \n};\n\nstruct opae_vfio_device_irq {\n    uint32_t flags;             \n    uint32_t index;             \n    uint32_t count;             \n    int32_t *event_fds;         \n    int32_t *masks;             \n    struct opae_vfio_device_irq *next;  \n};\n\nstruct opae_vfio_device {\n    int device_fd;                  \n    uint64_t device_config_offset;          \n    uint32_t device_num_regions;            \n    struct opae_vfio_device_region *regions;    \n    uint32_t device_num_irqs;           \n    struct opae_vfio_device_irq *irqs;      \n};\n\nstruct opae_vfio_buffer {\n    uint8_t *buffer_ptr;        \n    size_t buffer_size;     \n    uint64_t buffer_iova;       \n    int flags;          \n};\n\nstruct opae_vfio {\n    pthread_mutex_t lock;               \n    char *cont_device;              \n    char *cont_pciaddr;             \n    int cont_fd;                    \n    struct opae_vfio_iova_range *cont_ranges;   \n    struct mem_alloc iova_alloc;            \n    struct opae_vfio_group group;           \n    struct opae_vfio_device device;         \n    opae_hash_map cont_buffers;     \n};\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nint opae_vfio_open(struct opae_vfio *v,\n           const char *pciaddr);\n\nint opae_vfio_secure_open(struct opae_vfio *v,\n              const char *pciaddr,\n              const char *token);\n\nint opae_vfio_region_get(struct opae_vfio *v,\n             uint32_t index,\n             uint8_t **ptr,\n             size_t *size);\n\nint opae_vfio_buffer_allocate(struct opae_vfio *v,\n                  size_t *size,\n                  uint8_t **buf,\n                  uint64_t *iova);\n\nenum opae_vfio_buffer_flags {\n    OPAE_VFIO_BUF_PREALLOCATED = 1, \n};\n\nint opae_vfio_buffer_allocate_ex(struct opae_vfio *v,\n                 size_t *size,\n                 uint8_t **buf,\n                 uint64_t *iova,\n                 int flags);\n\nstruct opae_vfio_buffer *opae_vfio_buffer_info(struct opae_vfio *v,\n                           uint8_t *vaddr);\n\nint opae_vfio_buffer_free(struct opae_vfio *v,\n              uint8_t *buf);\n\nint opae_vfio_irq_enable(struct opae_vfio *v,\n             uint32_t index,\n             uint32_t subindex,\n             int event_fd);\n\nint opae_vfio_irq_unmask(struct opae_vfio *v,\n             uint32_t index,\n             uint32_t subindex);\n\nint opae_vfio_irq_mask(struct opae_vfio *v,\n               uint32_t index,\n               uint32_t subindex);\n\nint opae_vfio_irq_disable(struct opae_vfio *v,\n              uint32_t index,\n              uint32_t subindex);\n\nvoid opae_vfio_close(struct opae_vfio *v);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif // __cplusplus\n\n#endif // __OPAE_VFIO_H__\n
                                                                                    "},{"location":"opae-code/dir_9a6968a8846ef48cff617fcd6355d7b4/","title":"Dir docs/sw/samples","text":"
                                                                                    FileList > docs > sw > samples
                                                                                    "},{"location":"opae-code/dir_9a6968a8846ef48cff617fcd6355d7b4/#directories","title":"Directories","text":"Type Name dir hello_events dir hello_fpga 
                                                                                    The documentation for this class was generated from the following file docs/sw/samples/
                                                                                    "},{"location":"opae-code/dir_d66a8e4b979fa79493bebe26e2602d2b/","title":"Dir docs/sw/samples/hello_events","text":"
                                                                                    FileList > docs > sw > samples > hello_events
                                                                                    "},{"location":"opae-code/dir_d66a8e4b979fa79493bebe26e2602d2b/#files","title":"Files","text":"Type Name file hello_events.c A code sample of using OPAE event API. 
                                                                                    The documentation for this class was generated from the following file docs/sw/samples/hello_events/
                                                                                    "},{"location":"opae-code/hello__events_8c/","title":"File hello_events.c","text":"
                                                                                    FileList > docs > sw > samples > hello_events > hello_events.c
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    A code sample of using OPAE event API. More...
                                                                                     
                                                                                       
                                                                                    • #include <stdio.h>
                                                                                      •  
                                                                                    • #include <stdlib.h>
                                                                                      •  
                                                                                    • #include <string.h>
                                                                                      •  
                                                                                    • #include <unistd.h>
                                                                                      •  
                                                                                    • #include <getopt.h>
                                                                                      •  
                                                                                    • #include <poll.h>
                                                                                      •  
                                                                                    • #include <errno.h>
                                                                                      •  
                                                                                    • #include <sys/stat.h>
                                                                                      •  
                                                                                    • #include <pthread.h>
                                                                                      •  
                                                                                    • #include <opae/fpga.h>
                                                                                      •  
                                                                                    • #include <argsfilter.h>
                                                                                      •  
                                                                                    • #include \"mock/opae_std.h\"
                                                                                      •  
                                                                                    "},{"location":"opae-code/hello__events_8c/#classes","title":"Classes","text":"Type Name struct ras_inject_error"},{"location":"opae-code/hello__events_8c/#public-functions","title":"Public Functions","text":"Type Name void * error_thread (void * arg)  fpga_result find_fpga (fpga_properties device_filter, fpga_token * fpga, uint32_t * num_matches)  void help (void)  fpga_result inject_ras_fatal_error (fpga_token fme_token, uint8_t err)  int main (int argc, char * argv)  fpga_result parse_args (int argc, char * argv)  void print_err (const char * s, fpga_result res)  int usleep (unsigned)"},{"location":"opae-code/hello__events_8c/#macros","title":"Macros","text":"Type Name define FME_SYSFS_INJECT_ERROR  \"errors/inject_errors\" define GETOPT_STRING  \"hv\" define ON_ERR_GOTO (res, label, desc)"},{"location":"opae-code/hello__events_8c/#detailed-description","title":"Detailed Description","text":"
                                                                                    This sample starts two processes. One process injects an artificial fatal error to sysfs; while the other tries to asynchronously capture and handle the event. This sample code exercises all major functions of the event API, including creating and destroying event handles, register and unregister events, polling on event file descriptor, and getting the OS object associated with an event. For a full discussion of OPAE event API, refer to event.h. 
                                                                                    "},{"location":"opae-code/hello__events_8c/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/hello__events_8c/#function-error_thread","title":"function error_thread","text":"
                                                                                    void * error_thread (\n    void * arg\n) \n
                                                                                    "},{"location":"opae-code/hello__events_8c/#function-find_fpga","title":"function find_fpga","text":"
                                                                                    fpga_result find_fpga (\n    fpga_properties device_filter,\n    fpga_token * fpga,\n    uint32_t * num_matches\n) \n
                                                                                    "},{"location":"opae-code/hello__events_8c/#function-help","title":"function help","text":"
                                                                                    void help (\n    void\n) \n
                                                                                    "},{"location":"opae-code/hello__events_8c/#function-inject_ras_fatal_error","title":"function inject_ras_fatal_error","text":"
                                                                                    fpga_result inject_ras_fatal_error (\n    fpga_token fme_token,\n    uint8_t err\n) \n
                                                                                    "},{"location":"opae-code/hello__events_8c/#function-main","title":"function main","text":"
                                                                                    int main (\n    int argc,\n    char * argv\n) \n
                                                                                    "},{"location":"opae-code/hello__events_8c/#function-parse_args","title":"function parse_args","text":"
                                                                                    fpga_result parse_args (\n    int argc,\n    char * argv\n) \n
                                                                                    "},{"location":"opae-code/hello__events_8c/#function-print_err","title":"function print_err","text":"
                                                                                    void print_err (\n    const char * s,\n    fpga_result res\n) \n
                                                                                    "},{"location":"opae-code/hello__events_8c/#function-usleep","title":"function usleep","text":"
                                                                                    int usleep (\n    unsigned\n) \n
                                                                                    "},{"location":"opae-code/hello__events_8c/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/hello__events_8c/#define-fme_sysfs_inject_error","title":"define FME_SYSFS_INJECT_ERROR","text":"
                                                                                    #define FME_SYSFS_INJECT_ERROR \"errors/inject_errors\"\n
                                                                                    "},{"location":"opae-code/hello__events_8c/#define-getopt_string","title":"define GETOPT_STRING","text":"
                                                                                    #define GETOPT_STRING \"hv\"\n
                                                                                    "},{"location":"opae-code/hello__events_8c/#define-on_err_goto","title":"define ON_ERR_GOTO","text":"
                                                                                    #define ON_ERR_GOTO (\n    res,\n    label,\n    desc\n) do {                                       \\\n        if ((res) != FPGA_OK ) {            \\ print_err ((desc), (res));  \\\n            goto label;                \\\n        }                                  \\\n    } while (0)\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/samples/hello_events/hello_events.c
                                                                                    "},{"location":"opae-code/hello__events_8c_source/","title":"File hello_events.c","text":"
                                                                                    File List > docs > sw > samples > hello_events > hello_events.c
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017-2021, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifdef HAVE_CONFIG_H\n#include <config.h>\n#endif // HAVE_CONFIG_H\n#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n#include <unistd.h>\n#include <getopt.h>\n#include <poll.h>\n#include <errno.h>\n#include <sys/stat.h>\n#include <pthread.h>\n\n#include <opae/fpga.h>\n#include <argsfilter.h>\n#include \"mock/opae_std.h\"\n\nint usleep(unsigned);\n\n#define FME_SYSFS_INJECT_ERROR \"errors/inject_errors\"\n\n#define ON_ERR_GOTO(res, label, desc)              \\\n    do {                                       \\\n        if ((res) != FPGA_OK) {            \\\n            print_err((desc), (res));  \\\n            goto label;                \\\n        }                                  \\\n    } while (0)\n\nvoid print_err(const char *s, fpga_result res)\n{\n    fprintf(stderr, \"Error %s: %s\\n\", s, fpgaErrStr(res));\n}\n\n// RAS Error Inject CSR\nstruct ras_inject_error {\n    union {\n        uint64_t csr;\n        struct {\n            /* Catastrophic  error */\n            uint64_t  catastrophicr_error : 1;\n            /* Fatal error */\n            uint64_t  fatal_error : 1;\n            /* Non-fatal error */\n            uint64_t  nonfatal_error : 1;\n            /* Reserved */\n            uint64_t  rsvd : 61;\n        };\n    };\n};\n\nfpga_result inject_ras_fatal_error(fpga_token fme_token, uint8_t err)\n{\n    fpga_result             res1       = FPGA_OK;\n    fpga_result             res2       = FPGA_OK;\n    fpga_handle             fme_handle = NULL;\n    struct ras_inject_error inj_error  = { {0} };\n    fpga_object             inj_err_object;\n\n    res1 = fpgaOpen(fme_token, &fme_handle, FPGA_OPEN_SHARED);\n    if (res1 != FPGA_OK) {\n        OPAE_ERR(\"Failed to open FPGA\");\n        return res1;\n    }\n\n    res1 = fpgaHandleGetObject(fme_handle, FME_SYSFS_INJECT_ERROR, &inj_err_object, 0);\n    ON_ERR_GOTO(res1, out_close, \"Failed to get Handle Object\");\n\n    // Inject fatal error\n    inj_error.fatal_error = err;\n\n    res1 = fpgaObjectWrite64(inj_err_object, inj_error.csr, 0);\n    ON_ERR_GOTO(res1, out_destroy_obj, \"Failed to Read Object\");\n\nout_destroy_obj:\n    res2 = fpgaDestroyObject(&inj_err_object);\n    ON_ERR_GOTO(res2, out_close, \"Failed to Destroy Object\");\nout_close:\n    res2 = fpgaClose(fme_handle);\n    if (res2 != FPGA_OK) {\n        OPAE_ERR(\"Failed to close FPGA\");\n    }\n\n    return res1 != FPGA_OK ? res1 : res2;\n}\n\nvoid *error_thread(void *arg)\n{\n    fpga_token token = (fpga_token) arg;\n    fpga_result res;\n\n    usleep(5000000);\n    printf(\"injecting error\\n\");\n    res = inject_ras_fatal_error(token, 1);\n    if (res != FPGA_OK)\n        print_err(\"setting inject error register\", res);\n\n    usleep(5000000);\n    printf(\"clearing error\\n\");\n    res = inject_ras_fatal_error(token, 0);\n    if (res != FPGA_OK)\n        print_err(\"clearing inject error register\", res);\n\n    return NULL;\n}\n\nvoid help(void)\n{\n    printf(\"\\n\"\n           \"hello_events\\n\"\n           \"OPAE Events API sample\\n\"\n           \"\\n\"\n           \"Usage:\\n\"\n           \"    hello_events [-hv] [-S <segment>] [-B <bus>] [-D <device>] [-F <function>] [PCI_ADDR]\\n\"\n           \"\\n\"\n           \"                -h,--help           Print this help\\n\"\n           \"                -v,--version        Print version info and exit\\n\"\n           \"\\n\");\n}\n\n/*\n * Parse command line arguments\n */\n#define GETOPT_STRING \"hv\"\nfpga_result parse_args(int argc, char *argv[])\n{\n    struct option longopts[] = {\n        { \"help\",    no_argument, NULL, 'h' },\n        { \"version\", no_argument, NULL, 'v' },\n        { NULL,      0,           NULL, 0   }\n    };\n\n    int getopt_ret;\n    int option_index;\n\n    while (-1 != (getopt_ret = getopt_long(argc, argv, GETOPT_STRING, longopts, &option_index))) {\n        const char *tmp_optarg = optarg;\n        /* checks to see if optarg is null and if not goes to value of optarg */\n        if ((optarg) && ('=' == *tmp_optarg)) {\n            ++tmp_optarg;\n        }\n\n    switch (getopt_ret) {\n    case 'h': /* help */\n        help();\n        return -1;\n\n    case 'v': /* version */\n        printf(\"hello_events %s %s%s\\n\",\n               OPAE_VERSION,\n               OPAE_GIT_COMMIT_HASH,\n               OPAE_GIT_SRC_TREE_DIRTY ? \"*\":\"\");\n        return -1;\n\n    default: /* invalid option */\n        fprintf(stderr, \"Invalid cmdline options\\n\");\n        return FPGA_EXCEPTION;\n    }\n    }\n\n    return FPGA_OK;\n}\n\nfpga_result find_fpga(fpga_properties device_filter,\n              fpga_token *fpga,\n              uint32_t *num_matches)\n{\n    fpga_properties filter = NULL;\n    fpga_result res        = FPGA_OK;\n    fpga_result dres       = FPGA_OK;\n\n    res = fpgaCloneProperties(device_filter, &filter);\n    ON_ERR_GOTO(res, out, \"cloning properties object\");\n\n    res = fpgaPropertiesSetObjectType(filter, FPGA_DEVICE);\n    ON_ERR_GOTO(res, out_destroy, \"setting interface ID\");\n\n    res = fpgaEnumerate(&filter, 1, fpga, 1, num_matches);\n    ON_ERR_GOTO(res, out_destroy, \"enumerating FPGAs\");\n\nout_destroy:\n    dres = fpgaDestroyProperties(&filter);\n    ON_ERR_GOTO(dres, out, \"destroying properties object\");\nout:\n    return (res == FPGA_OK) ? dres : res;\n}\n\nint main(int argc, char *argv[])\n{\n    fpga_token         fpga_device_token = NULL;\n    fpga_handle        fpga_device_handle = NULL;\n    uint32_t           num_matches = 1;\n    fpga_result        res1 = FPGA_OK;\n    fpga_result        res2 = FPGA_OK;\n    fpga_event_handle  eh;\n    uint64_t           count = 0;\n    int                res;\n    struct pollfd      pfd;\n    int                timeout = 10000;\n    int                poll_ret = 0;\n    ssize_t            bytes_read = 0;\n    pthread_t          errthr;\n    fpga_properties    device_filter = NULL;\n\n    res1 = fpgaGetProperties(NULL, &device_filter);\n    if (res1) {\n        print_err(\"failed to alloc properties\", res1);\n        return res1;\n    }\n\n    if (opae_set_properties_from_args(device_filter,\n                      &res1,\n                      &argc,\n                      argv)) {\n        print_err(\"failed arg parse\", res1);\n        res1 = FPGA_EXCEPTION;\n        goto out_exit;\n    } else if (res1) {\n        print_err(\"failed to set properties\", res1);\n        goto out_exit;\n    }\n\n    res1 = parse_args(argc, argv);\n    if ((int)res1 < 0)\n        goto out_exit;\n    ON_ERR_GOTO(res1, out_exit, \"parsing arguments\");\n\n    res1 = find_fpga(device_filter, &fpga_device_token, &num_matches);\n    ON_ERR_GOTO(res1, out_exit, \"finding FPGA accelerator\");\n\n    if (num_matches < 1) {\n        fprintf(stderr, \"No matches for address provided.\\n\");\n        res1 = FPGA_NOT_FOUND;\n        goto out_exit;\n    }\n\n    if (num_matches > 1) {\n        fprintf(stderr, \"Found more than one suitable slot.\\n\");\n    }\n\n    res1 = fpgaOpen(fpga_device_token, &fpga_device_handle, FPGA_OPEN_SHARED);\n    ON_ERR_GOTO(res1, out_destroy_tok, \"opening accelerator\");\n\n    res1 = fpgaCreateEventHandle(&eh);\n    ON_ERR_GOTO(res1, out_close, \"creating event handle\");\n\n    res1 = fpgaRegisterEvent(fpga_device_handle, FPGA_EVENT_ERROR, eh, 0);\n    ON_ERR_GOTO(res1, out_destroy_eh, \"registering an FME event\");\n\n    printf(\"Waiting for interrupts now...\\n\");\n\n    res = pthread_create(&errthr, NULL, error_thread, fpga_device_token);\n    if (res) {\n        printf(\"Failed to create error_thread.\\n\");\n        res1 = FPGA_EXCEPTION;\n        goto out_destroy_eh;\n    }\n\n\n    res1 = fpgaGetOSObjectFromEventHandle(eh, &pfd.fd);\n    ON_ERR_GOTO(res1, out_join, \"getting file descriptor\");\n\n    pfd.events = POLLIN;\n    poll_ret = poll(&pfd, 1, timeout);\n\n    if (poll_ret < 0) {\n        printf(\"Poll error errno = %s\\n\", strerror(errno));\n        res1 = FPGA_EXCEPTION;\n        goto out_join;\n    } else if (poll_ret == 0) {\n         printf(\"Poll timeout occurred\\n\");\n         res1 = FPGA_EXCEPTION;\n         goto out_join;\n    } else {\n         printf(\"FME Interrupt occurred\\n\");\n         bytes_read = opae_read(pfd.fd, &count, sizeof(count));\n         if (bytes_read <= 0)\n             printf(\"WARNING: error reading from poll fd: %s\\n\",\n                     bytes_read < 0 ? strerror(errno) : \"zero bytes read\");\n    }\n\n    res1 = fpgaUnregisterEvent(fpga_device_handle, FPGA_EVENT_ERROR, eh);\n    ON_ERR_GOTO(res1, out_join, \"unregistering an FME event\");\n\n    printf(\"Successfully tested Register/Unregister for FME events!\\n\");\n\nout_join:\n    pthread_join(errthr, NULL);\n\nout_destroy_eh:\n    res2 = fpgaDestroyEventHandle(&eh);\n    ON_ERR_GOTO(res2, out_close, \"deleting event handle\");\n\nout_close:\n    res2 = fpgaClose(fpga_device_handle);\n    ON_ERR_GOTO(res2, out_destroy_tok, \"closing accelerator\");\n\nout_destroy_tok:\n    res2 = fpgaDestroyToken(&fpga_device_token);\n    ON_ERR_GOTO(res2, out_exit, \"destroying token\");\n\nout_exit:\n    fpgaDestroyProperties(&device_filter);\n    return res1 != FPGA_OK ? res1 : res2;\n}\n
                                                                                    "},{"location":"opae-code/dir_a3c160366dc832de1042e5d4d49ef034/","title":"Dir docs/sw/samples/hello_fpga","text":"
                                                                                    FileList > docs > sw > samples > hello_fpga
                                                                                    "},{"location":"opae-code/dir_a3c160366dc832de1042e5d4d49ef034/#files","title":"Files","text":"Type Name file hello_fpga.c A code sample illustrates the basic usage of the OPAE C API. 
                                                                                    The documentation for this class was generated from the following file docs/sw/samples/hello_fpga/
                                                                                    "},{"location":"opae-code/hello__fpga_8c/","title":"File hello_fpga.c","text":"
                                                                                    FileList > docs > sw > samples > hello_fpga > hello_fpga.c
                                                                                     
                                                                                    Go to the source code of this file.
                                                                                     
                                                                                    A code sample illustrates the basic usage of the OPAE C API. More...
                                                                                     
                                                                                       
                                                                                    • #include <stdio.h>
                                                                                      •  
                                                                                    • #include <stdlib.h>
                                                                                      •  
                                                                                    • #include <string.h>
                                                                                      •  
                                                                                    • #include <stdint.h>
                                                                                      •  
                                                                                    • #include <getopt.h>
                                                                                      •  
                                                                                    • #include <unistd.h>
                                                                                      •  
                                                                                    • #include <uuid/uuid.h>
                                                                                      •  
                                                                                    • #include <opae/fpga.h>
                                                                                      •  
                                                                                    • #include <argsfilter.h>
                                                                                      •  
                                                                                    • #include \"mock/opae_std.h\"
                                                                                      •  
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#classes","title":"Classes","text":"Type Name struct cache_line struct config"},{"location":"opae-code/hello__fpga_8c/#public-attributes","title":"Public Attributes","text":"Type Name struct config config   = = { .open_flags = 0, .run_n3000 = 0 }"},{"location":"opae-code/hello__fpga_8c/#public-functions","title":"Public Functions","text":"Type Name fpga_result find_fpga (fpga_properties device_filter, fpga_guid afu_guid, fpga_token * accelerator_token, uint32_t * num_matches_accelerators)  fpga_result find_nlb_n3000 (fpga_handle accelerator_handle, uint64_t * afu_baddr)  void help (void)  int main (int argc, char * argv)  fpga_result parse_args (int argc, char * argv)  void print_err (const char * s, fpga_result res)  bool probe_for_ase (void)  int usleep (unsigned)"},{"location":"opae-code/hello__fpga_8c/#macros","title":"Macros","text":"Type Name define CACHELINE_ALIGNED_ADDR (p) ((p) >> LOG2_CL) define CL (x) ((x) * 64) define CSR_AFU_DSM_BASEL  0x0110 define CSR_CFG  0x0140 define CSR_CTL  0x0138 define CSR_DST_ADDR  0x0128 define CSR_NUM_LINES  0x0130 define CSR_SRC_ADDR  0x0120 define CSR_STATUS1  0x0168 define DSM_STATUS_TEST_COMPLETE  0x40 define FPGA_NLB0_UUID_H  0xd8424dc4a4a3c413 define FPGA_NLB0_UUID_L  0xf89e433683f9040b define GETOPT_STRING  \"hscv\" define LOG2_CL  6 define LPBK1_BUFFER_ALLOCATION_SIZE MB(2) define LPBK1_BUFFER_SIZE MB(1) define LPBK1_DSM_SIZE MB(2) define MB (x) ((x) * 1024 * 1024) define N3000_AFUID  \"9AEFFE5F-8457-0612-C000-C9660D824272\" define NLB0_AFUID  \"D8424DC4-A4A3-C413-F89E-433683F9040B\" define ON_ERR_GOTO (res, label, desc)  define TEST_TIMEOUT  30000"},{"location":"opae-code/hello__fpga_8c/#detailed-description","title":"Detailed Description","text":"
                                                                                    The sample is a host application that demonstrates the basic steps of interacting with FPGA using the OPAE library. These steps include:
                                                                                     
                                                                                       
                                                                                    • FPGA enumeration
                                                                                      •  
                                                                                    • Resource acquiring and releasing
                                                                                      •  
                                                                                    • Managing shared memory buffer
                                                                                      •  
                                                                                    • MMIO read and write
                                                                                      •  
                                                                                     
                                                                                    The sample also demonstrates OPAE's object model, such as tokens, handles, and properties.
                                                                                     
                                                                                    The sample requires a native loopback mode (NLB) test image to be loaded on the FPGA. Refer to Quick Start Guide for full instructions on building, configuring, and running this code sample. 
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#public-attributes-documentation","title":"Public Attributes Documentation","text":""},{"location":"opae-code/hello__fpga_8c/#variable-config","title":"variable config","text":"
                                                                                    struct config config;\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#public-functions-documentation","title":"Public Functions Documentation","text":""},{"location":"opae-code/hello__fpga_8c/#function-find_fpga","title":"function find_fpga","text":"
                                                                                    fpga_result find_fpga (\n    fpga_properties device_filter,\n    fpga_guid afu_guid,\n    fpga_token * accelerator_token,\n    uint32_t * num_matches_accelerators\n) \n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#function-find_nlb_n3000","title":"function find_nlb_n3000","text":"
                                                                                    fpga_result find_nlb_n3000 (\n    fpga_handle accelerator_handle,\n    uint64_t * afu_baddr\n) \n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#function-help","title":"function help","text":"
                                                                                    void help (\n    void\n) \n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#function-main","title":"function main","text":"
                                                                                    int main (\n    int argc,\n    char * argv\n) \n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#function-parse_args","title":"function parse_args","text":"
                                                                                    fpga_result parse_args (\n    int argc,\n    char * argv\n) \n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#function-print_err","title":"function print_err","text":"
                                                                                    void print_err (\n    const char * s,\n    fpga_result res\n) \n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#function-probe_for_ase","title":"function probe_for_ase","text":"
                                                                                    bool probe_for_ase (\n    void\n) \n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#function-usleep","title":"function usleep","text":"
                                                                                    int usleep (\n    unsigned\n) \n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#macro-definition-documentation","title":"Macro Definition Documentation","text":""},{"location":"opae-code/hello__fpga_8c/#define-cacheline_aligned_addr","title":"define CACHELINE_ALIGNED_ADDR","text":"
                                                                                    #define CACHELINE_ALIGNED_ADDR (\n    p\n) ((p) >> LOG2_CL )\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-cl","title":"define CL","text":"
                                                                                    #define CL (\n    x\n) ((x) * 64)\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-csr_afu_dsm_basel","title":"define CSR_AFU_DSM_BASEL","text":"
                                                                                    #define CSR_AFU_DSM_BASEL 0x0110\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-csr_cfg","title":"define CSR_CFG","text":"
                                                                                    #define CSR_CFG 0x0140\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-csr_ctl","title":"define CSR_CTL","text":"
                                                                                    #define CSR_CTL 0x0138\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-csr_dst_addr","title":"define CSR_DST_ADDR","text":"
                                                                                    #define CSR_DST_ADDR 0x0128\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-csr_num_lines","title":"define CSR_NUM_LINES","text":"
                                                                                    #define CSR_NUM_LINES 0x0130\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-csr_src_addr","title":"define CSR_SRC_ADDR","text":"
                                                                                    #define CSR_SRC_ADDR 0x0120\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-csr_status1","title":"define CSR_STATUS1","text":"
                                                                                    #define CSR_STATUS1 0x0168\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-dsm_status_test_complete","title":"define DSM_STATUS_TEST_COMPLETE","text":"
                                                                                    #define DSM_STATUS_TEST_COMPLETE 0x40\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-fpga_nlb0_uuid_h","title":"define FPGA_NLB0_UUID_H","text":"
                                                                                    #define FPGA_NLB0_UUID_H 0xd8424dc4a4a3c413\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-fpga_nlb0_uuid_l","title":"define FPGA_NLB0_UUID_L","text":"
                                                                                    #define FPGA_NLB0_UUID_L 0xf89e433683f9040b\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-getopt_string","title":"define GETOPT_STRING","text":"
                                                                                    #define GETOPT_STRING \"hscv\"\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-log2_cl","title":"define LOG2_CL","text":"
                                                                                    #define LOG2_CL 6\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-lpbk1_buffer_allocation_size","title":"define LPBK1_BUFFER_ALLOCATION_SIZE","text":"
                                                                                    #define LPBK1_BUFFER_ALLOCATION_SIZE MB (2)\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-lpbk1_buffer_size","title":"define LPBK1_BUFFER_SIZE","text":"
                                                                                    #define LPBK1_BUFFER_SIZE MB (1)\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-lpbk1_dsm_size","title":"define LPBK1_DSM_SIZE","text":"
                                                                                    #define LPBK1_DSM_SIZE MB (2)\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-mb","title":"define MB","text":"
                                                                                    #define MB (\n    x\n) ((x) * 1024 * 1024)\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-n3000_afuid","title":"define N3000_AFUID","text":"
                                                                                    #define N3000_AFUID \"9AEFFE5F-8457-0612-C000-C9660D824272\"\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-nlb0_afuid","title":"define NLB0_AFUID","text":"
                                                                                    #define NLB0_AFUID \"D8424DC4-A4A3-C413-F89E-433683F9040B\"\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-on_err_goto","title":"define ON_ERR_GOTO","text":"
                                                                                    #define ON_ERR_GOTO (\n    res,\n    label,\n    desc\n) do {                                       \\\n        if ((res) != FPGA_OK ) {            \\ print_err ((desc), (res));  \\\n            goto label;                \\\n        }                                  \\\n    } while (0)\n
                                                                                    "},{"location":"opae-code/hello__fpga_8c/#define-test_timeout","title":"define TEST_TIMEOUT","text":"
                                                                                    #define TEST_TIMEOUT 30000\n
                                                                                     
                                                                                    The documentation for this class was generated from the following file docs/sw/samples/hello_fpga/hello_fpga.c
                                                                                    "},{"location":"opae-code/hello__fpga_8c_source/","title":"File hello_fpga.c","text":"
                                                                                    File List > docs > sw > samples > hello_fpga > hello_fpga.c
                                                                                     
                                                                                    Go to the documentation of this file. 
                                                                                     
                                                                                    // Copyright(c) 2017-2022, Intel Corporation\n//\n// Redistribution  and  use  in source  and  binary  forms,  with  or  without\n// modification, are permitted provided that the following conditions are met:\n//\n// * Redistributions of  source code  must retain the  above copyright notice,\n//   this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright notice,\n//   this list of conditions and the following disclaimer in the documentation\n//   and/or other materials provided with the distribution.\n// * Neither the name  of Intel Corporation  nor the names of its contributors\n//   may be used to  endorse or promote  products derived  from this  software\n//   without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE\n// IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n// ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE\n// LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR\n// CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF\n// SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS\n// INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN\n// CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)\n// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE\n// POSSIBILITY OF SUCH DAMAGE.\n\n#ifdef HAVE_CONFIG_H\n#include <config.h>\n#endif // HAVE_CONFIG_H\n#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n#include <stdint.h>\n#include <getopt.h>\n#include <unistd.h>\n\n#include <uuid/uuid.h>\n#include <opae/fpga.h>\n#include <argsfilter.h>\n#include \"mock/opae_std.h\"\n\nint usleep(unsigned);\n\n#ifndef TEST_TIMEOUT\n#define TEST_TIMEOUT 30000\n#endif // TEST_TIMEOUT\n\n#ifndef CL\n# define CL(x)                       ((x) * 64)\n#endif // CL\n#ifndef LOG2_CL\n# define LOG2_CL                     6\n#endif // LOG2_CL\n#ifndef MB\n# define MB(x)                       ((x) * 1024 * 1024)\n#endif // MB\n\n#define CACHELINE_ALIGNED_ADDR(p) ((p) >> LOG2_CL)\n\n#define LPBK1_BUFFER_SIZE            MB(1)\n#define LPBK1_BUFFER_ALLOCATION_SIZE MB(2)\n#define LPBK1_DSM_SIZE               MB(2)\n#define CSR_SRC_ADDR                 0x0120\n#define CSR_DST_ADDR                 0x0128\n#define CSR_CTL                      0x0138\n#define CSR_STATUS1                  0x0168\n#define CSR_CFG                      0x0140\n#define CSR_NUM_LINES                0x0130\n#define DSM_STATUS_TEST_COMPLETE     0x40\n#define CSR_AFU_DSM_BASEL            0x0110\n\n/* NLB0 AFU_ID */\n#define NLB0_AFUID \"D8424DC4-A4A3-C413-F89E-433683F9040B\"\n\n/* NLB0 AFU_ID for N3000 */\n#define N3000_AFUID \"9AEFFE5F-8457-0612-C000-C9660D824272\"\n#define FPGA_NLB0_UUID_H 0xd8424dc4a4a3c413\n#define FPGA_NLB0_UUID_L 0xf89e433683f9040b\n\n\n/*\n * macro to check return codes, print error message, and goto cleanup label\n * NOTE: this changes the program flow (uses goto)!\n */\n#define ON_ERR_GOTO(res, label, desc)              \\\n    do {                                       \\\n        if ((res) != FPGA_OK) {            \\\n            print_err((desc), (res));  \\\n            goto label;                \\\n        }                                  \\\n    } while (0)\n\n/* Type definitions */\ntypedef struct {\n    uint32_t uint[16];\n} cache_line;\n\nvoid print_err(const char *s, fpga_result res)\n{\n    fprintf(stderr, \"Error %s: %s\\n\", s, fpgaErrStr(res));\n}\n\n/*\n * Global configuration of bus, set during parse_args()\n * */\nstruct config {\n    int open_flags;\n    int run_n3000;\n}\n\nconfig = {\n    .open_flags = 0,\n    .run_n3000 = 0\n};\n\nvoid help(void)\n{\n    printf(\"\\n\"\n           \"hello_fpga\\n\"\n           \"OPAE Native Loopback 0 (NLB0) sample\\n\"\n           \"\\n\"\n           \"Usage:\\n\"\n           \"        hello_fpga [-schv] [-S <segment>] [-B <bus>] [-D <device>] [-F <function>] [PCI_ADDR]\\n\"\n           \"\\n\"\n           \"                -s,--shared         Open accelerator in shared mode\\n\"\n           \"                -c,--n3000          Assume N3000 MMIO layout\\n\"\n           \"                -h,--help           Print this help\\n\"\n           \"                -v,--version        Print version info and exit\\n\"\n           \"\\n\");\n}\n\n#define GETOPT_STRING \"hscv\"\nfpga_result parse_args(int argc, char *argv[])\n{\n    struct option longopts[] = {\n        { \"help\",    no_argument, NULL, 'h' },\n        { \"shared\",  no_argument, NULL, 's' },\n        { \"n3000\",   no_argument, NULL, 'c' },\n        { \"version\", no_argument, NULL, 'v' },\n        { NULL,      0,           NULL,  0  }\n    };\n\n    int getopt_ret;\n    int option_index;\n\n    char version[32];\n    char build[32];\n\n    while (-1 != (getopt_ret = getopt_long(argc, argv, GETOPT_STRING,\n                        longopts, &option_index))) {\n        const char *tmp_optarg = optarg;\n        /* Checks to see if optarg is null and if not it goes to value of optarg */\n        if ((optarg) && ('=' == *tmp_optarg)) {\n            ++tmp_optarg;\n        }\n\n        switch (getopt_ret) {\n        case 'h':\n            help();\n            return -1;\n        case 's':\n            config.open_flags |= FPGA_OPEN_SHARED;\n            break;\n        case 'c':\n            config.run_n3000 = 1;\n            break;\n        case 'v':\n            fpgaGetOPAECVersionString(version, sizeof(version));\n            fpgaGetOPAECBuildString(build, sizeof(build));\n            printf(\"hello_fpga %s %s\\n\",\n                   version, build);\n            return -1;\n\n        default: /* invalid option */\n            fprintf(stderr, \"Invalid cmdline option \\n\");\n            return FPGA_EXCEPTION;\n        }\n    }\n\n    return FPGA_OK;\n}\n\nfpga_result find_fpga(fpga_properties device_filter,\n              fpga_guid afu_guid,\n              fpga_token *accelerator_token,\n              uint32_t *num_matches_accelerators)\n{\n    fpga_properties filter = NULL;\n    fpga_result res1;\n    fpga_result res2 = FPGA_OK;\n\n    res1 = fpgaCloneProperties(device_filter, &filter);\n    ON_ERR_GOTO(res1, out, \"cloning properties object\");\n\n    res1 = fpgaPropertiesSetObjectType(filter, FPGA_ACCELERATOR);\n    ON_ERR_GOTO(res1, out_destroy, \"setting object type\");\n\n    res1 = fpgaPropertiesSetGUID(filter, afu_guid);\n    ON_ERR_GOTO(res1, out_destroy, \"setting GUID\");\n\n    res1 = fpgaEnumerate(&filter, 1, accelerator_token, 1, num_matches_accelerators);\n    ON_ERR_GOTO(res1, out_destroy, \"enumerating accelerators\");\n\nout_destroy:\n    res2 = fpgaDestroyProperties(&filter);\n    ON_ERR_GOTO(res2, out, \"destroying properties object\");\nout:\n    return res1 != FPGA_OK ? res1 : res2;\n}\n\n/* Is the FPGA simulated with ASE? */\nbool probe_for_ase(void)\n{\n    fpga_result r = FPGA_OK;\n    uint16_t device_id = 0;\n    fpga_properties filter = NULL;\n    uint32_t num_matches = 1;\n    fpga_token fme_token;\n\n    /* Connect to the FPGA management engine */\n    fpgaGetProperties(NULL, &filter);\n    fpgaPropertiesSetObjectType(filter, FPGA_DEVICE);\n\n    /* Connecting to one is sufficient to find ASE */\n    fpgaEnumerate(&filter, 1, &fme_token, 1, &num_matches);\n    if (0 != num_matches) {\n        /* Retrieve the device ID of the FME */\n        fpgaDestroyProperties(&filter);\n        fpgaGetProperties(fme_token, &filter);\n        r = fpgaPropertiesGetDeviceID(filter, &device_id);\n        fpgaDestroyToken(&fme_token);\n    }\n    fpgaDestroyProperties(&filter);\n\n    /* ASE's device ID is 0xa5e */\n    return ((FPGA_OK == r) && (0xa5e == device_id));\n}\n\nfpga_result find_nlb_n3000(fpga_handle accelerator_handle,\n               uint64_t *afu_baddr)\n{\n\n    fpga_result res1 = FPGA_OK;\n    int end_of_list = 0;\n    int nlb0_found = 0;\n    uint64_t header = 0;\n    uint64_t uuid_hi = 0;\n    uint64_t uuid_lo = 0;\n    uint64_t next_offset = 0;\n    uint64_t nlb0_offset = 0;\n\n    /* find NLB0 in AFU */\n    do {\n        // Read the next feature header\n        res1 = fpgaReadMMIO64(accelerator_handle, 0, nlb0_offset, &header);\n        ON_ERR_GOTO(res1, out_exit, \"fpgaReadMMIO64\");\n        res1 = fpgaReadMMIO64(accelerator_handle, 0, nlb0_offset+8, &uuid_lo);\n        ON_ERR_GOTO(res1, out_exit, \"fpgaReadMMIO64\");\n        res1 = fpgaReadMMIO64(accelerator_handle, 0, nlb0_offset+16, &uuid_hi);\n        ON_ERR_GOTO(res1, out_exit, \"fpgaReadMMIO64\");\n        // printf(\"%zx: %zx %zx %zx\\n\", nlb0_offset, header, uuid_lo, uuid_hi);\n        if ((((header >> 60) & 0xf) == 0x1) &&\n            (uuid_lo == FPGA_NLB0_UUID_L) && (uuid_hi == FPGA_NLB0_UUID_H)) {\n            nlb0_found = 1;\n            break;\n        }\n        // End of the list flag\n        end_of_list = (header >> 40) & 1;\n        // Move to the next feature header\n        next_offset = (header >> 16) & 0xffffff;\n        if ((next_offset == 0xffff) || (next_offset == 0)) {\n            nlb0_found = 0;\n            break;\n        }\n        nlb0_offset = nlb0_offset + next_offset;\n    } while (!end_of_list);\n\n    if (!nlb0_found) {\n        printf(\"AFU NLB0 not found\\n\");\n        return FPGA_EXCEPTION;\n    }\n\n    printf(\"AFU NLB0 found @ %zx\\n\", nlb0_offset);\n    *afu_baddr = nlb0_offset;\n    return FPGA_OK;\n\nout_exit:\n    return FPGA_EXCEPTION;\n}\n\nint main(int argc, char *argv[])\n{\n    fpga_token         accelerator_token;\n    fpga_handle        accelerator_handle;\n    fpga_guid          guid;\n    uint32_t           num_matches_accelerators = 0;\n    uint32_t           use_ase;\n\n    volatile uint64_t *dsm_ptr    = NULL;\n    volatile uint64_t *status_ptr = NULL;\n    volatile uint64_t *input_ptr  = NULL;\n    volatile uint64_t *output_ptr = NULL;\n\n    uint64_t           dsm_wsid;\n    uint64_t           input_wsid;\n    uint64_t           output_wsid;\n    uint32_t           i;\n    uint32_t           timeout;\n    fpga_result        res1 = FPGA_OK;\n    fpga_result        res2 = FPGA_OK;\n    fpga_properties    device_filter = NULL;\n\n    res1 = fpgaGetProperties(NULL, &device_filter);\n    if (res1 != FPGA_OK) {\n        print_err(\"failed to allocate properties.\\n\", res1);\n        return 1;\n    }\n\n    if (opae_set_properties_from_args(device_filter,\n                      &res1,\n                      &argc,\n                      argv)) {\n        print_err(\"failed arg parse.\\n\", res1);\n        res1 = FPGA_EXCEPTION;\n        goto out_exit;\n    } else if (res1) {\n        print_err(\"failed to set properties.\\n\", res1);\n        goto out_exit;\n    }\n\n    res1 = parse_args(argc, argv);\n    if ((int)res1 < 0)\n        goto out_exit;\n    ON_ERR_GOTO(res1, out_exit, \"parsing arguments\");\n\n    if (config.run_n3000) {\n        if (uuid_parse(N3000_AFUID, guid) < 0)\n            res1 = FPGA_EXCEPTION;\n    } else {\n        if (uuid_parse(NLB0_AFUID, guid) < 0)\n            res1 = FPGA_EXCEPTION;\n    }\n\n    ON_ERR_GOTO(res1, out_exit, \"parsing guid\");\n\n    use_ase = probe_for_ase();\n    if (use_ase) {\n        printf(\"Running in ASE mode\\n\");\n    }\n\n    /* Look for accelerator with NLB0_AFUID */\n    res1 = find_fpga(device_filter,\n             guid,\n             &accelerator_token,\n             &num_matches_accelerators);\n    ON_ERR_GOTO(res1, out_exit, \"finding FPGA accelerator\");\n\n    if (num_matches_accelerators == 0) {\n        res1 = FPGA_NOT_FOUND;\n    }\n    ON_ERR_GOTO(res1, out_exit, \"no matching accelerator\");\n\n    if (num_matches_accelerators > 1) {\n        printf(\"Found more than one suitable accelerator. \");\n    }\n\n\n    /* Open accelerator and map MMIO */\n    res1 = fpgaOpen(accelerator_token, &accelerator_handle, config.open_flags);\n    ON_ERR_GOTO(res1, out_destroy_tok, \"opening accelerator\");\n\n    res1 = fpgaMapMMIO(accelerator_handle, 0, NULL);\n    ON_ERR_GOTO(res1, out_close, \"mapping MMIO space\");\n\n\n    /* Allocate buffers */\n    res1 = fpgaPrepareBuffer(accelerator_handle, LPBK1_DSM_SIZE,\n                (void **)&dsm_ptr, &dsm_wsid, 0);\n    ON_ERR_GOTO(res1, out_close, \"allocating DSM buffer\");\n\n    res1 = fpgaPrepareBuffer(accelerator_handle, LPBK1_BUFFER_ALLOCATION_SIZE,\n               (void **)&input_ptr, &input_wsid, 0);\n    ON_ERR_GOTO(res1, out_free_dsm, \"allocating input buffer\");\n\n    res1 = fpgaPrepareBuffer(accelerator_handle, LPBK1_BUFFER_ALLOCATION_SIZE,\n               (void **)&output_ptr, &output_wsid, 0);\n    ON_ERR_GOTO(res1, out_free_input, \"allocating output buffer\");\n\n    printf(\"Running Test\\n\");\n\n    /* Initialize buffers */\n    memset((void *)dsm_ptr,    0,    LPBK1_DSM_SIZE);\n    memset((void *)input_ptr,  0xAF, LPBK1_BUFFER_SIZE);\n    memset((void *)output_ptr, 0xBE, LPBK1_BUFFER_SIZE);\n\n    cache_line *cl_ptr = (cache_line *)input_ptr;\n    for (i = 0; i < LPBK1_BUFFER_SIZE / CL(1); ++i) {\n        cl_ptr[i].uint[15] = i+1; /* set the last uint in every cacheline */\n    }\n\n\n    /* Reset accelerator */\n    res1 = fpgaReset(accelerator_handle);\n    ON_ERR_GOTO(res1, out_free_output, \"resetting accelerator\");\n\n    uint64_t nlb_base_addr = 0;\n\n    if (config.run_n3000) {\n        res1 = find_nlb_n3000(accelerator_handle, &nlb_base_addr);\n        ON_ERR_GOTO(res1, out_free_output, \"finding nlb in AFU\");\n    }\n\n    /* Program DMA addresses */\n    uint64_t iova = 0;\n    res1 = fpgaGetIOAddress(accelerator_handle, dsm_wsid, &iova);\n    ON_ERR_GOTO(res1, out_free_output, \"getting DSM IOVA\");\n\n    res1 = fpgaWriteMMIO64(accelerator_handle, 0, nlb_base_addr + CSR_AFU_DSM_BASEL, iova);\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_AFU_DSM_BASEL\");\n\n    res1 = fpgaWriteMMIO32(accelerator_handle, 0, nlb_base_addr + CSR_CTL, 0);\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_CFG\");\n    res1 = fpgaWriteMMIO32(accelerator_handle, 0, nlb_base_addr + CSR_CTL, 1);\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_CFG\");\n\n    res1 = fpgaGetIOAddress(accelerator_handle, input_wsid, &iova);\n    ON_ERR_GOTO(res1, out_free_output, \"getting input IOVA\");\n    res1 = fpgaWriteMMIO64(accelerator_handle, 0, nlb_base_addr + CSR_SRC_ADDR, CACHELINE_ALIGNED_ADDR(iova));\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_SRC_ADDR\");\n\n    res1 = fpgaGetIOAddress(accelerator_handle, output_wsid, &iova);\n    ON_ERR_GOTO(res1, out_free_output, \"getting output IOVA\");\n    res1 = fpgaWriteMMIO64(accelerator_handle, 0, nlb_base_addr + CSR_DST_ADDR, CACHELINE_ALIGNED_ADDR(iova));\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_DST_ADDR\");\n\n    res1 = fpgaWriteMMIO32(accelerator_handle, 0, nlb_base_addr + CSR_NUM_LINES, LPBK1_BUFFER_SIZE / CL(1));\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_NUM_LINES\");\n    res1 = fpgaWriteMMIO32(accelerator_handle, 0, nlb_base_addr + CSR_CFG, 0x42000);\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_CFG\");\n\n    status_ptr = dsm_ptr + DSM_STATUS_TEST_COMPLETE/sizeof(uint64_t);\n\n\n    /* Start the test */\n    res1 = fpgaWriteMMIO32(accelerator_handle, 0, nlb_base_addr + CSR_CTL, 3);\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_CFG\");\n\n    /* Wait for test completion */\n    timeout = TEST_TIMEOUT;\n    while (0 == ((*status_ptr) & 0x1)) {\n        usleep(100);\n        if (!use_ase && (--timeout == 0)) {\n            res1 = FPGA_EXCEPTION;\n            ON_ERR_GOTO(res1, out_free_output, \"test timed out\");\n        }\n    }\n\n    /* Stop the device */\n    res1 = fpgaWriteMMIO32(accelerator_handle, 0, nlb_base_addr + CSR_CTL, 7);\n    ON_ERR_GOTO(res1, out_free_output, \"writing CSR_CFG\");\n\n    /* Wait for the AFU's read/write traffic to complete */\n    uint32_t afu_traffic_trips = 0;\n    while (afu_traffic_trips < 100) {\n        /*\n         * CSR_STATUS1 holds two 32 bit values: num pending reads and writes.\n         * Wait for it to be 0.\n         */\n        uint64_t s1;\n        res1 = fpgaReadMMIO64(accelerator_handle, 0, nlb_base_addr + CSR_STATUS1, &s1);\n        ON_ERR_GOTO(res1, out_free_output, \"reading CSR_STATUS1\");\n        if (s1 == 0) {\n            break;\n        }\n\n        afu_traffic_trips += 1;\n        usleep(1000);\n    }\n\n    /* Check output buffer contents */\n    for (i = 0; i < LPBK1_BUFFER_SIZE; i++) {\n        if (((uint8_t *)output_ptr)[i] != ((uint8_t *)input_ptr)[i]) {\n            fprintf(stderr, \"Output does NOT match input \"\n                \"at offset %i!\\n\", i);\n            break;\n        }\n    }\n\n    printf(\"Done Running Test\\n\");\n\n\n    /* Release buffers */\nout_free_output:\n    res2 = fpgaReleaseBuffer(accelerator_handle, output_wsid);\n    ON_ERR_GOTO(res2, out_free_input, \"releasing output buffer\");\nout_free_input:\n    res2 = fpgaReleaseBuffer(accelerator_handle, input_wsid);\n    ON_ERR_GOTO(res2, out_free_dsm, \"releasing input buffer\");\nout_free_dsm:\n    res2 = fpgaReleaseBuffer(accelerator_handle, dsm_wsid);\n    ON_ERR_GOTO(res2, out_unmap, \"releasing DSM buffer\");\n\n    /* Unmap MMIO space */\nout_unmap:\n    res2 = fpgaUnmapMMIO(accelerator_handle, 0);\n    ON_ERR_GOTO(res2, out_close, \"unmapping MMIO space\");\n\n    /* Release accelerator */\nout_close:\n    res2 = fpgaClose(accelerator_handle);\n    ON_ERR_GOTO(res2, out_destroy_tok, \"closing accelerator\");\n\n    /* Destroy token */\nout_destroy_tok:\n    res2 = fpgaDestroyToken(&accelerator_token);\n    ON_ERR_GOTO(res2, out_exit, \"destroying token\");\n\nout_exit:\n    fpgaDestroyProperties(&device_filter);\n    return res1 != FPGA_OK ? res1 : res2;\n}\n
                                                                                    "},{"location":"opae-code/namespaces/","title":"Namespace List","text":"
                                                                                    Here is a list of all namespaces with brief descriptions:
                                                                                     
                                                                                       
                                                                                    • namespace opae 
                                                                                         
                                                                                      • namespace fpga 
                                                                                           
                                                                                        • namespace types 
                                                                                             
                                                                                          • namespace detail 
                                                                                            •  
                                                                                           
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    • namespace std 
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/","title":"Class Index","text":""},{"location":"opae-code/classes/#b","title":"b","text":"
                                                                                       
                                                                                    • busy (opae::fpga::types)
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#c","title":"c","text":"
                                                                                       
                                                                                    • cache_line
                                                                                      •  
                                                                                    • config
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#e","title":"e","text":"
                                                                                       
                                                                                    • error (opae::fpga::types)
                                                                                      •  
                                                                                    • event (opae::fpga::types)
                                                                                      •  
                                                                                    • except (opae::fpga::types)
                                                                                      •  
                                                                                    • exception (opae::fpga::types)
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#f","title":"f","text":"
                                                                                       
                                                                                    • fpga_error_info
                                                                                      •  
                                                                                    • fpga_metric
                                                                                      •  
                                                                                    • fpga_metric_info
                                                                                      •  
                                                                                    • fpga_version
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#g","title":"g","text":"
                                                                                       
                                                                                    • guid_t (opae::fpga::types)
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#h","title":"h","text":"
                                                                                       
                                                                                    • handle (opae::fpga::types)
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#i","title":"i","text":"
                                                                                       
                                                                                    • invalid_param (opae::fpga::types)
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#m","title":"m","text":"
                                                                                       
                                                                                    • mem_alloc
                                                                                      •  
                                                                                    • mem_link
                                                                                      •  
                                                                                    • metric_threshold
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#n","title":"n","text":"
                                                                                       
                                                                                    • no_access (opae::fpga::types)
                                                                                      •  
                                                                                    • no_daemon (opae::fpga::types)
                                                                                      •  
                                                                                    • no_driver (opae::fpga::types)
                                                                                      •  
                                                                                    • no_memory (opae::fpga::types)
                                                                                      •  
                                                                                    • not_found (opae::fpga::types)
                                                                                      •  
                                                                                    • not_supported (opae::fpga::types)
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#o","title":"o","text":"
                                                                                       
                                                                                    • opae_uio
                                                                                      •  
                                                                                    • opae_uio_device_region
                                                                                      •  
                                                                                    • opae_vfio
                                                                                      •  
                                                                                    • opae_vfio_buffer
                                                                                      •  
                                                                                    • opae_vfio_device
                                                                                      •  
                                                                                    • opae_vfio_device_irq
                                                                                      •  
                                                                                    • opae_vfio_device_region
                                                                                      •  
                                                                                    • opae_vfio_group
                                                                                      •  
                                                                                    • opae_vfio_iova_range
                                                                                      •  
                                                                                    • opae_vfio_sparse_info
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#p","title":"p","text":"
                                                                                       
                                                                                    • properties (opae::fpga::types)
                                                                                      •  
                                                                                    • pvalue (opae::fpga::types)
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#r","title":"r","text":"
                                                                                       
                                                                                    • reconf_error (opae::fpga::types)
                                                                                      •  
                                                                                    • ras_inject_error
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#s","title":"s","text":"
                                                                                       
                                                                                    • shared_buffer (opae::fpga::types)
                                                                                      •  
                                                                                    • src_location (opae::fpga::types)
                                                                                      •  
                                                                                    • sysobject (opae::fpga::types)
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#t","title":"t","text":"
                                                                                       
                                                                                    • token (opae::fpga::types)
                                                                                      •  
                                                                                    • type_t (opae::fpga::types::event)
                                                                                      •  
                                                                                    • threshold
                                                                                      •  
                                                                                    "},{"location":"opae-code/classes/#v","title":"v","text":"
                                                                                       
                                                                                    • version (opae::fpga::types)
                                                                                      •  
                                                                                     
                                                                                    ## \\
                                                                                     
                                                                                       
                                                                                    • _fpga_token_header
                                                                                      •  
                                                                                    • _opae_hash_map
                                                                                      •  
                                                                                    • _opae_hash_map_item
                                                                                      •  
                                                                                    "},{"location":"opae-code/hierarchy/","title":"Class Hierarchy","text":"
                                                                                    This inheritance list is sorted roughly, but not completely, alphabetically:
                                                                                     
                                                                                       
                                                                                    • class opae::fpga::types::error An error object represents an error register for a resource. 
                                                                                      •  
                                                                                    • class opae::fpga::types::event Wraps fpga event routines in OPAE C. 
                                                                                      •  
                                                                                    • class opae::fpga::types::handle An allocated accelerator resource. 
                                                                                      •  
                                                                                    • class opae::fpga::types::properties Wraps an OPAE fpga_properties object. 
                                                                                      •  
                                                                                    • class opae::fpga::types::shared_buffer Host/AFU shared memory blocks. 
                                                                                      •  
                                                                                    • class opae::fpga::types::src_location Identify a particular line in a source file. 
                                                                                      •  
                                                                                    • class opae::fpga::types::sysobject Wraps the OPAE fpga_object primitive. 
                                                                                      •  
                                                                                    • class opae::fpga::types::token Wraps the OPAE fpga_token primitive. 
                                                                                      •  
                                                                                    • class opae::fpga::types::version 
                                                                                      •  
                                                                                    • struct _fpga_token_header Internal token type header. 
                                                                                      •  
                                                                                    • struct _opae_hash_map Hash map object. 
                                                                                      •  
                                                                                    • struct _opae_hash_map_item List link item. 
                                                                                      •  
                                                                                    • struct cache_line 
                                                                                      •  
                                                                                    • struct config 
                                                                                      •  
                                                                                    • struct fpga_error_info 
                                                                                      •  
                                                                                    • struct fpga_metric Metric struct. 
                                                                                      •  
                                                                                    • struct fpga_metric_info Metric info struct. 
                                                                                      •  
                                                                                    • struct fpga_version Semantic version. 
                                                                                      •  
                                                                                    • struct mem_alloc 
                                                                                      •  
                                                                                    • struct mem_link Provides an API for allocating/freeing a logical address space. 
                                                                                      •  
                                                                                    • struct metric_threshold 
                                                                                      •  
                                                                                    • struct opae::fpga::types::event::type_t C++ struct that is interchangeable with fpga_event_type enum. 
                                                                                      •  
                                                                                    • struct opae::fpga::types::guid_t Representation of the guid member of a properties object. 
                                                                                      •  
                                                                                    • struct opae::fpga::types::pvalue Wraps OPAE properties defined in the OPAE C API by associating an fpga_properties reference with the getters and setters defined for a property.
                                                                                      •  
                                                                                    • struct opae_uio OPAE UIO device abstraction. 
                                                                                      •  
                                                                                    • struct opae_uio_device_region MMIO region info. 
                                                                                      •  
                                                                                    • struct opae_vfio OPAE VFIO device abstraction. 
                                                                                      •  
                                                                                    • struct opae_vfio_buffer System DMA buffer. 
                                                                                      •  
                                                                                    • struct opae_vfio_device VFIO device. 
                                                                                      •  
                                                                                    • struct opae_vfio_device_irq Interrupt info. 
                                                                                      •  
                                                                                    • struct opae_vfio_device_region MMIO region info. 
                                                                                      •  
                                                                                    • struct opae_vfio_group VFIO group. 
                                                                                      •  
                                                                                    • struct opae_vfio_iova_range IO Virtual Address Range. 
                                                                                      •  
                                                                                    • struct opae_vfio_sparse_info MMIO sparse-mappable region info. 
                                                                                      •  
                                                                                    • struct ras_inject_error 
                                                                                      •  
                                                                                    • struct threshold Threshold struct. 
                                                                                      •  
                                                                                    • class std::exception 
                                                                                         
                                                                                      • class opae::fpga::types::except Generic OPAE exception. 
                                                                                           
                                                                                        • class opae::fpga::types::busy busy exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::exception exception exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::invalid_param invalid_param exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_access no_access exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_daemon no_daemon exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_driver no_driver exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_memory no_memory exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_found not_found exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_supported not_supported exception
                                                                                          •  
                                                                                        • class opae::fpga::types::reconf_error reconf_error exception
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                      • class opae::fpga::types::except Generic OPAE exception. 
                                                                                           
                                                                                        • class opae::fpga::types::busy busy exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::exception exception exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::invalid_param invalid_param exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_access no_access exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_daemon no_daemon exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_driver no_driver exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_memory no_memory exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_found not_found exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_supported not_supported exception
                                                                                          •  
                                                                                        • class opae::fpga::types::reconf_error reconf_error exception
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                      • class opae::fpga::types::except Generic OPAE exception. 
                                                                                           
                                                                                        • class opae::fpga::types::busy busy exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::exception exception exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::invalid_param invalid_param exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_access no_access exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_daemon no_daemon exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_driver no_driver exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_memory no_memory exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_found not_found exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_supported not_supported exception
                                                                                          •  
                                                                                        • class opae::fpga::types::reconf_error reconf_error exception
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                      • class opae::fpga::types::except Generic OPAE exception. 
                                                                                           
                                                                                        • class opae::fpga::types::busy busy exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::exception exception exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::invalid_param invalid_param exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_access no_access exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_daemon no_daemon exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_driver no_driver exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_memory no_memory exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_found not_found exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_supported not_supported exception
                                                                                          •  
                                                                                        • class opae::fpga::types::reconf_error reconf_error exception
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                      • class opae::fpga::types::except Generic OPAE exception. 
                                                                                           
                                                                                        • class opae::fpga::types::busy busy exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::exception exception exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::invalid_param invalid_param exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_access no_access exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_daemon no_daemon exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_driver no_driver exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_memory no_memory exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_found not_found exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_supported not_supported exception
                                                                                          •  
                                                                                        • class opae::fpga::types::reconf_error reconf_error exception
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                      • class opae::fpga::types::except Generic OPAE exception. 
                                                                                           
                                                                                        • class opae::fpga::types::busy busy exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::exception exception exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::invalid_param invalid_param exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_access no_access exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_daemon no_daemon exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_driver no_driver exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_memory no_memory exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_found not_found exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_supported not_supported exception
                                                                                          •  
                                                                                        • class opae::fpga::types::reconf_error reconf_error exception
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                      • class opae::fpga::types::except Generic OPAE exception. 
                                                                                           
                                                                                        • class opae::fpga::types::busy busy exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::exception exception exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::invalid_param invalid_param exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_access no_access exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_daemon no_daemon exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_driver no_driver exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_memory no_memory exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_found not_found exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_supported not_supported exception
                                                                                          •  
                                                                                        • class opae::fpga::types::reconf_error reconf_error exception
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                      • class opae::fpga::types::except Generic OPAE exception. 
                                                                                           
                                                                                        • class opae::fpga::types::busy busy exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::exception exception exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::invalid_param invalid_param exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_access no_access exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_daemon no_daemon exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_driver no_driver exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_memory no_memory exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_found not_found exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_supported not_supported exception
                                                                                          •  
                                                                                        • class opae::fpga::types::reconf_error reconf_error exception
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                      • class opae::fpga::types::except Generic OPAE exception. 
                                                                                           
                                                                                        • class opae::fpga::types::busy busy exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::exception exception exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::invalid_param invalid_param exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_access no_access exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_daemon no_daemon exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_driver no_driver exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_memory no_memory exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_found not_found exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_supported not_supported exception
                                                                                          •  
                                                                                        • class opae::fpga::types::reconf_error reconf_error exception
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                      • class opae::fpga::types::except Generic OPAE exception. 
                                                                                           
                                                                                        • class opae::fpga::types::busy busy exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::exception exception exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::invalid_param invalid_param exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_access no_access exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_daemon no_daemon exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_driver no_driver exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_memory no_memory exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_found not_found exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_supported not_supported exception
                                                                                          •  
                                                                                        • class opae::fpga::types::reconf_error reconf_error exception
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                      • class opae::fpga::types::except Generic OPAE exception. 
                                                                                           
                                                                                        • class opae::fpga::types::busy busy exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::exception exception exception 
                                                                                          •  
                                                                                        • class opae::fpga::types::invalid_param invalid_param exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_access no_access exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_daemon no_daemon exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_driver no_driver exception
                                                                                          •  
                                                                                        • class opae::fpga::types::no_memory no_memory exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_found not_found exception
                                                                                          •  
                                                                                        • class opae::fpga::types::not_supported not_supported exception
                                                                                          •  
                                                                                        • class opae::fpga::types::reconf_error reconf_error exception
                                                                                          •  
                                                                                         
                                                                                        •  
                                                                                       
                                                                                      •  
                                                                                    "},{"location":"opae-code/modules/","title":"Modules","text":"
                                                                                    Here is a list of all modules:
                                                                                    "},{"location":"opae-code/todo/","title":"Todo List","text":""},{"location":"opae-code/todo/#global-fpgaregisterevent-fpga_handle-handle-fpga_event_type-event_type-fpga_event_handle-event_handle-uint32_t-flags","title":"Global fpgaRegisterEvent  (fpga_handle handle, fpga_event_type event_type, fpga_event_handle event_handle, uint32_t flags)","text":"
                                                                                    define if calling fpgaRegisterEvent multiple times with the same event_handle is an error condition or if it is silently ignored.  define if calling fpgaUnregisterEvent multiple times with the same event_handle is an error condition or if it is silently ignored.
                                                                                    "},{"location":"opae-code/pages/","title":"Class List","text":"
                                                                                    Here are the classes, structs, unions and interfaces with brief descriptions:
                                                                                    "},{"location":"opae-code/class_members/","title":"Class Members","text":""},{"location":"opae-code/class_members/#a","title":"a","text":"
                                                                                       
                                                                                    • allocated (mem_alloc)
                                                                                      •  
                                                                                    • address (mem_link)
                                                                                      •  
                                                                                    • accelerator_state (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • allocate (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • attach (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • as_string (opae::fpga::types::version)
                                                                                      •  
                                                                                    • as_struct (opae::fpga::types::version)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#b","title":"b","text":"
                                                                                       
                                                                                    • bus (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • buckets (_opae_hash_map)
                                                                                      •  
                                                                                    • busy (opae::fpga::types::busy)
                                                                                      •  
                                                                                    • buf_ (opae::fpga::types::except)
                                                                                      •  
                                                                                    • bbs_id (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • bbs_version (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • bytes (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • build (opae::fpga::types::version)
                                                                                      •  
                                                                                    • buffer_iova (opae_vfio_buffer)
                                                                                      •  
                                                                                    • buffer_ptr (opae_vfio_buffer)
                                                                                      •  
                                                                                    • buffer_size (opae_vfio_buffer)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#c","title":"c","text":"
                                                                                       
                                                                                    • cleanup_context (_opae_hash_map)
                                                                                      •  
                                                                                    • can_clear (fpga_error_info, opae::fpga::types::error)
                                                                                      •  
                                                                                    • c_type (opae::fpga::types::error, opae::fpga::types::guid_t, opae::fpga::types::handle, opae::fpga::types::properties, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject, opae::fpga::types::token)
                                                                                      •  
                                                                                    • close (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • capabilities (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • copy_ (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • copy_t (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • compare (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • cont_buffers (opae_vfio)
                                                                                      •  
                                                                                    • cont_device (opae_vfio)
                                                                                      •  
                                                                                    • cont_fd (opae_vfio)
                                                                                      •  
                                                                                    • cont_pciaddr (opae_vfio)
                                                                                      •  
                                                                                    • cont_ranges (opae_vfio)
                                                                                      •  
                                                                                    • count (opae_vfio_device_irq)
                                                                                      •  
                                                                                    • catastrophicr_error (ras_inject_error)
                                                                                      •  
                                                                                    • csr (ras_inject_error)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#d","title":"d","text":"
                                                                                       
                                                                                    • device (_fpga_token_header, opae::fpga::types::properties, opae_vfio)
                                                                                      •  
                                                                                    • device_id (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • data_ (opae::fpga::types::guid_t)
                                                                                      •  
                                                                                    • device_fd (opae_uio, opae_vfio_device)
                                                                                      •  
                                                                                    • device_path (opae_uio)
                                                                                      •  
                                                                                    • device_config_offset (opae_vfio_device)
                                                                                      •  
                                                                                    • device_num_irqs (opae_vfio_device)
                                                                                      •  
                                                                                    • device_num_regions (opae_vfio_device)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#e","title":"e","text":"
                                                                                       
                                                                                    • error (opae::fpga::types::error, opae::fpga::types::event::type_t)
                                                                                      •  
                                                                                    • error_info_ (opae::fpga::types::error)
                                                                                      •  
                                                                                    • error_num_ (opae::fpga::types::error)
                                                                                      •  
                                                                                    • event (opae::fpga::types::event)
                                                                                      •  
                                                                                    • event_handle_ (opae::fpga::types::event)
                                                                                      •  
                                                                                    • except (opae::fpga::types::except)
                                                                                      •  
                                                                                    • exception (opae::fpga::types::exception)
                                                                                      •  
                                                                                    • enumerate (opae::fpga::types::token)
                                                                                      •  
                                                                                    • event_fds (opae_vfio_device_irq)
                                                                                      •  
                                                                                    • end (opae_vfio_iova_range)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#f","title":"f","text":"
                                                                                       
                                                                                    • function (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • flags (_opae_hash_map, opae_vfio_buffer, opae_vfio_device_irq)
                                                                                      •  
                                                                                    • free (mem_alloc)
                                                                                      •  
                                                                                    • fill (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • file (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    • file_ (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    • fn (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    • fn_ (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    • fatal_error (ras_inject_error)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#g","title":"g","text":"
                                                                                       
                                                                                    • guid (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • group_name (fpga_metric_info)
                                                                                      •  
                                                                                    • get (opae::fpga::types::error, opae::fpga::types::event, opae::fpga::types::properties, opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • guid_t (opae::fpga::types::guid_t)
                                                                                      •  
                                                                                    • get_token (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • get_ (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • get_value (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • getter_t (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • get_parent (opae::fpga::types::token)
                                                                                      •  
                                                                                    • group (opae_vfio)
                                                                                      •  
                                                                                    • group_device (opae_vfio_group)
                                                                                      •  
                                                                                    • group_fd (opae_vfio_group)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#h","title":"h","text":"
                                                                                       
                                                                                    • hash_seed (_opae_hash_map)
                                                                                      •  
                                                                                    • hysteresis (metric_threshold)
                                                                                      •  
                                                                                    • handle_ (opae::fpga::types::event, opae::fpga::types::handle, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • handle (opae::fpga::types::handle)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#i","title":"i","text":"
                                                                                       
                                                                                    • interface (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • isvalid (fpga_metric)
                                                                                      •  
                                                                                    • interrupt (opae::fpga::types::event::type_t)
                                                                                      •  
                                                                                    • invalidate (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • is_set (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • is_set_ (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • invalid_param (opae::fpga::types::invalid_param)
                                                                                      •  
                                                                                    • io_address (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • io_address_ (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • iova_alloc (opae_vfio)
                                                                                      •  
                                                                                    • irqs (opae_vfio_device)
                                                                                      •  
                                                                                    • index (opae_vfio_device_irq, opae_vfio_sparse_info)
                                                                                      •  
                                                                                    • is_valid (threshold)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#k","title":"k","text":"
                                                                                       
                                                                                    • key_cleanup (_opae_hash_map)
                                                                                      •  
                                                                                    • key_compare (_opae_hash_map)
                                                                                      •  
                                                                                    • key_hash (_opae_hash_map)
                                                                                      •  
                                                                                    • key (_opae_hash_map_item)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#l","title":"l","text":"
                                                                                       
                                                                                    • lower_c_threshold (metric_threshold)
                                                                                      •  
                                                                                    • lower_nc_threshold (metric_threshold)
                                                                                      •  
                                                                                    • lower_nr_threshold (metric_threshold)
                                                                                      •  
                                                                                    • loc_ (opae::fpga::types::except)
                                                                                      •  
                                                                                    • local_memory_size (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • len_ (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • line (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    • line_ (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    • lock (opae_vfio)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#m","title":"m","text":"
                                                                                       
                                                                                    • magic (_fpga_token_header)
                                                                                      •  
                                                                                    • metric_num (fpga_metric, fpga_metric_info)
                                                                                      •  
                                                                                    • metric_datatype (fpga_metric_info)
                                                                                      •  
                                                                                    • metric_guid (fpga_metric_info)
                                                                                      •  
                                                                                    • metric_name (fpga_metric_info, metric_threshold)
                                                                                      •  
                                                                                    • metric_type (fpga_metric_info)
                                                                                      •  
                                                                                    • metric_units (fpga_metric_info)
                                                                                      •  
                                                                                    • major (fpga_version)
                                                                                      •  
                                                                                    • minor (fpga_version)
                                                                                      •  
                                                                                    • MAX_EXCEPT (opae::fpga::types::except)
                                                                                      •  
                                                                                    • msg_ (opae::fpga::types::except)
                                                                                      •  
                                                                                    • mmio_ptr (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • model (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • masks (opae_vfio_device_irq)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#n","title":"n","text":"
                                                                                       
                                                                                    • num_buckets (_opae_hash_map)
                                                                                      •  
                                                                                    • next (_opae_hash_map_item, mem_link, opae_uio_device_region, opae_vfio_device_irq, opae_vfio_device_region, opae_vfio_iova_range, opae_vfio_sparse_info)
                                                                                      •  
                                                                                    • name (fpga_error_info, opae::fpga::types::error)
                                                                                      •  
                                                                                    • no_access (opae::fpga::types::no_access)
                                                                                      •  
                                                                                    • no_daemon (opae::fpga::types::no_daemon)
                                                                                      •  
                                                                                    • no_driver (opae::fpga::types::no_driver)
                                                                                      •  
                                                                                    • no_memory (opae::fpga::types::no_memory)
                                                                                      •  
                                                                                    • not_found (opae::fpga::types::not_found)
                                                                                      •  
                                                                                    • not_supported (opae::fpga::types::not_supported)
                                                                                      •  
                                                                                    • none (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • num_errors (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • num_interrupts (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • num_mmio (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • num_slots (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • nonfatal_error (ras_inject_error)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#o","title":"o","text":"
                                                                                       
                                                                                    • object_id (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • objtype (_fpga_token_header)
                                                                                      •  
                                                                                    • open_flags (config)
                                                                                      •  
                                                                                    • operator= (opae::fpga::types::error, opae::fpga::types::guid_t, opae::fpga::types::handle, opae::fpga::types::properties, opae::fpga::types::pvalue, opae::fpga::types::shared_buffer, opae::fpga::types::src_location, opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • operator fpga_event_type (opae::fpga::types::event::type_t)
                                                                                      •  
                                                                                    • operator fpga_event_handle (opae::fpga::types::event)
                                                                                      •  
                                                                                    • os_object (opae::fpga::types::event)
                                                                                      •  
                                                                                    • os_object_ (opae::fpga::types::event)
                                                                                      •  
                                                                                    • operator fpga_result (opae::fpga::types::except)
                                                                                      •  
                                                                                    • operator uint8_t * (opae::fpga::types::guid_t)
                                                                                      •  
                                                                                    • operator== (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • open (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • operator fpga_handle (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • operator fpga_properties (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • operator copy_t (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • owner (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • operator fpga_object (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • operator fpga_token (opae::fpga::types::token)
                                                                                      •  
                                                                                    • offset (opae_vfio_sparse_info)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#p","title":"p","text":"
                                                                                       
                                                                                    • patch (fpga_version)
                                                                                      •  
                                                                                    • prev (mem_link)
                                                                                      •  
                                                                                    • ptr_t (opae::fpga::types::error, opae::fpga::types::event, opae::fpga::types::handle, opae::fpga::types::properties, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject, opae::fpga::types::token)
                                                                                      •  
                                                                                    • power_thermal (opae::fpga::types::event::type_t)
                                                                                      •  
                                                                                    • parse (opae::fpga::types::guid_t)
                                                                                      •  
                                                                                    • props_ (opae::fpga::types::guid_t, opae::fpga::types::properties, opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • parent (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • properties (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • pvalue (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • ptr (opae_vfio_sparse_info)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#q","title":"q","text":"
                                                                                       
                                                                                    • qualifier_name (fpga_metric_info)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#r","title":"r","text":"
                                                                                       
                                                                                    • run_n3000 (config)
                                                                                      •  
                                                                                    • read_value (opae::fpga::types::error)
                                                                                      •  
                                                                                    • register_event (opae::fpga::types::event)
                                                                                      •  
                                                                                    • res_ (opae::fpga::types::except)
                                                                                      •  
                                                                                    • read_csr32 (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • read_csr64 (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • reconfigure (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • reset (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • reconf_error (opae::fpga::types::reconf_error)
                                                                                      •  
                                                                                    • read (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • release (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • read64 (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • regions (opae_uio, opae_vfio_device)
                                                                                      •  
                                                                                    • region_index (opae_uio_device_region, opae_vfio_device_region)
                                                                                      •  
                                                                                    • region_page_offset (opae_uio_device_region)
                                                                                      •  
                                                                                    • region_ptr (opae_uio_device_region, opae_vfio_device_region)
                                                                                      •  
                                                                                    • region_size (opae_uio_device_region, opae_vfio_device_region)
                                                                                      •  
                                                                                    • region_sparse (opae_vfio_device_region)
                                                                                      •  
                                                                                    • rsvd (ras_inject_error)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#s","title":"s","text":"
                                                                                       
                                                                                    • segment (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • subsystem_device_id (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • subsystem_vendor_id (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • size (mem_link, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject, opae_vfio_sparse_info)
                                                                                      •  
                                                                                    • socket_id (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • set_ (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • setter_t (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • shared_buffer (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • size_t (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • src_location (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    • sysobject (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • sysobject_ (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • start (opae_vfio_iova_range)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#t","title":"t","text":"
                                                                                       
                                                                                    • token_ (opae::fpga::types::error, opae::fpga::types::handle, opae::fpga::types::sysobject, opae::fpga::types::token)
                                                                                      •  
                                                                                    • type_ (opae::fpga::types::event::type_t, opae::fpga::types::event)
                                                                                      •  
                                                                                    • type_t (opae::fpga::types::event::type_t)
                                                                                      •  
                                                                                    • type (opae::fpga::types::properties, opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • token (opae::fpga::types::token)
                                                                                      •  
                                                                                    • threshold_name (threshold)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#u","title":"u","text":"
                                                                                       
                                                                                    • uint (cache_line)
                                                                                      •  
                                                                                    • upper_c_threshold (metric_threshold)
                                                                                      •  
                                                                                    • upper_nc_threshold (metric_threshold)
                                                                                      •  
                                                                                    • upper_nr_threshold (metric_threshold)
                                                                                      •  
                                                                                    • update (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#v","title":"v","text":"
                                                                                       
                                                                                    • vendor_id (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • value_cleanup (_opae_hash_map)
                                                                                      •  
                                                                                    • value (_opae_hash_map_item, fpga_metric, threshold)
                                                                                      •  
                                                                                    • virt_ (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#w","title":"w","text":"
                                                                                       
                                                                                    • what (opae::fpga::types::except)
                                                                                      •  
                                                                                    • write_csr32 (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • write_csr512 (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • write_csr64 (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • write (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • wsid (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • wsid_ (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • write64 (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#_1","title":"~","text":"
                                                                                       
                                                                                    • ~error (opae::fpga::types::error)
                                                                                      •  
                                                                                    • ~event (opae::fpga::types::event)
                                                                                      •  
                                                                                    • ~handle (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • ~properties (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • ~shared_buffer (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • ~sysobject (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • ~token (opae::fpga::types::token)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_members/#_2","title":"@","text":"
                                                                                       
                                                                                    • @1 (ras_inject_error)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/","title":"Class Member Functions","text":""},{"location":"opae-code/class_member_functions/#a","title":"a","text":"
                                                                                       
                                                                                    • allocate (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • attach (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • as_string (opae::fpga::types::version)
                                                                                      •  
                                                                                    • as_struct (opae::fpga::types::version)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#b","title":"b","text":"
                                                                                       
                                                                                    • busy (opae::fpga::types::busy)
                                                                                      •  
                                                                                    • bytes (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • build (opae::fpga::types::version)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#c","title":"c","text":"
                                                                                       
                                                                                    • c_type (opae::fpga::types::error, opae::fpga::types::guid_t, opae::fpga::types::handle, opae::fpga::types::properties, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject, opae::fpga::types::token)
                                                                                      •  
                                                                                    • can_clear (opae::fpga::types::error)
                                                                                      •  
                                                                                    • close (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • compare (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#e","title":"e","text":"
                                                                                       
                                                                                    • error (opae::fpga::types::error)
                                                                                      •  
                                                                                    • event (opae::fpga::types::event)
                                                                                      •  
                                                                                    • except (opae::fpga::types::except)
                                                                                      •  
                                                                                    • exception (opae::fpga::types::exception)
                                                                                      •  
                                                                                    • enumerate (opae::fpga::types::token)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#f","title":"f","text":"
                                                                                       
                                                                                    • fill (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • file (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    • fn (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#g","title":"g","text":"
                                                                                       
                                                                                    • get (opae::fpga::types::error, opae::fpga::types::event, opae::fpga::types::properties, opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • guid_t (opae::fpga::types::guid_t)
                                                                                      •  
                                                                                    • get_token (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • get_value (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • get_parent (opae::fpga::types::token)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#h","title":"h","text":"
                                                                                       
                                                                                    • handle (opae::fpga::types::handle)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#i","title":"i","text":"
                                                                                       
                                                                                    • invalidate (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • is_set (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • invalid_param (opae::fpga::types::invalid_param)
                                                                                      •  
                                                                                    • io_address (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#l","title":"l","text":"
                                                                                       
                                                                                    • line (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#m","title":"m","text":"
                                                                                       
                                                                                    • mmio_ptr (opae::fpga::types::handle)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#n","title":"n","text":"
                                                                                       
                                                                                    • name (opae::fpga::types::error)
                                                                                      •  
                                                                                    • no_access (opae::fpga::types::no_access)
                                                                                      •  
                                                                                    • no_daemon (opae::fpga::types::no_daemon)
                                                                                      •  
                                                                                    • no_driver (opae::fpga::types::no_driver)
                                                                                      •  
                                                                                    • no_memory (opae::fpga::types::no_memory)
                                                                                      •  
                                                                                    • not_found (opae::fpga::types::not_found)
                                                                                      •  
                                                                                    • not_supported (opae::fpga::types::not_supported)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#o","title":"o","text":"
                                                                                       
                                                                                    • operator= (opae::fpga::types::error, opae::fpga::types::guid_t, opae::fpga::types::handle, opae::fpga::types::properties, opae::fpga::types::pvalue, opae::fpga::types::shared_buffer, opae::fpga::types::src_location, opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • operator fpga_event_type (opae::fpga::types::event::type_t)
                                                                                      •  
                                                                                    • operator fpga_event_handle (opae::fpga::types::event)
                                                                                      •  
                                                                                    • os_object (opae::fpga::types::event)
                                                                                      •  
                                                                                    • operator fpga_result (opae::fpga::types::except)
                                                                                      •  
                                                                                    • operator uint8_t * (opae::fpga::types::guid_t)
                                                                                      •  
                                                                                    • operator== (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • open (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • operator fpga_handle (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • operator fpga_properties (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • operator copy_t (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • owner (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • operator fpga_object (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • operator fpga_token (opae::fpga::types::token)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#p","title":"p","text":"
                                                                                       
                                                                                    • parse (opae::fpga::types::guid_t)
                                                                                      •  
                                                                                    • properties (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • pvalue (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#r","title":"r","text":"
                                                                                       
                                                                                    • read_value (opae::fpga::types::error)
                                                                                      •  
                                                                                    • register_event (opae::fpga::types::event)
                                                                                      •  
                                                                                    • read_csr32 (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • read_csr64 (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • reconfigure (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • reset (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • reconf_error (opae::fpga::types::reconf_error)
                                                                                      •  
                                                                                    • read (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • release (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • read64 (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#s","title":"s","text":"
                                                                                       
                                                                                    • shared_buffer (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • size (opae::fpga::types::shared_buffer, opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • src_location (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    • sysobject (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#t","title":"t","text":"
                                                                                       
                                                                                    • type_t (opae::fpga::types::event::type_t)
                                                                                      •  
                                                                                    • type (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • token (opae::fpga::types::token)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#u","title":"u","text":"
                                                                                       
                                                                                    • update (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#w","title":"w","text":"
                                                                                       
                                                                                    • what (opae::fpga::types::except)
                                                                                      •  
                                                                                    • write_csr32 (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • write_csr512 (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • write_csr64 (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • write (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • wsid (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • write64 (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_functions/#_1","title":"~","text":"
                                                                                       
                                                                                    • ~error (opae::fpga::types::error)
                                                                                      •  
                                                                                    • ~event (opae::fpga::types::event)
                                                                                      •  
                                                                                    • ~handle (opae::fpga::types::handle)
                                                                                      •  
                                                                                    • ~properties (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • ~shared_buffer (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • ~sysobject (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • ~token (opae::fpga::types::token)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/","title":"Class Member Variables","text":""},{"location":"opae-code/class_member_variables/#a","title":"a","text":"
                                                                                       
                                                                                    • allocated (mem_alloc)
                                                                                      •  
                                                                                    • address (mem_link)
                                                                                      •  
                                                                                    • accelerator_state (opae::fpga::types::properties)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#b","title":"b","text":"
                                                                                       
                                                                                    • bus (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • buckets (_opae_hash_map)
                                                                                      •  
                                                                                    • buf_ (opae::fpga::types::except)
                                                                                      •  
                                                                                    • bbs_id (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • bbs_version (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • buffer_iova (opae_vfio_buffer)
                                                                                      •  
                                                                                    • buffer_ptr (opae_vfio_buffer)
                                                                                      •  
                                                                                    • buffer_size (opae_vfio_buffer)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#c","title":"c","text":"
                                                                                       
                                                                                    • cleanup_context (_opae_hash_map)
                                                                                      •  
                                                                                    • can_clear (fpga_error_info)
                                                                                      •  
                                                                                    • capabilities (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • copy_ (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • cont_buffers (opae_vfio)
                                                                                      •  
                                                                                    • cont_device (opae_vfio)
                                                                                      •  
                                                                                    • cont_fd (opae_vfio)
                                                                                      •  
                                                                                    • cont_pciaddr (opae_vfio)
                                                                                      •  
                                                                                    • cont_ranges (opae_vfio)
                                                                                      •  
                                                                                    • count (opae_vfio_device_irq)
                                                                                      •  
                                                                                    • catastrophicr_error (ras_inject_error)
                                                                                      •  
                                                                                    • csr (ras_inject_error)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#d","title":"d","text":"
                                                                                       
                                                                                    • device (_fpga_token_header, opae::fpga::types::properties, opae_vfio)
                                                                                      •  
                                                                                    • device_id (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • data_ (opae::fpga::types::guid_t)
                                                                                      •  
                                                                                    • device_fd (opae_uio, opae_vfio_device)
                                                                                      •  
                                                                                    • device_path (opae_uio)
                                                                                      •  
                                                                                    • device_config_offset (opae_vfio_device)
                                                                                      •  
                                                                                    • device_num_irqs (opae_vfio_device)
                                                                                      •  
                                                                                    • device_num_regions (opae_vfio_device)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#e","title":"e","text":"
                                                                                       
                                                                                    • error_info_ (opae::fpga::types::error)
                                                                                      •  
                                                                                    • error_num_ (opae::fpga::types::error)
                                                                                      •  
                                                                                    • event_handle_ (opae::fpga::types::event)
                                                                                      •  
                                                                                    • error (opae::fpga::types::event::type_t)
                                                                                      •  
                                                                                    • event_fds (opae_vfio_device_irq)
                                                                                      •  
                                                                                    • end (opae_vfio_iova_range)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#f","title":"f","text":"
                                                                                       
                                                                                    • function (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • flags (_opae_hash_map, opae_vfio_buffer, opae_vfio_device_irq)
                                                                                      •  
                                                                                    • free (mem_alloc)
                                                                                      •  
                                                                                    • file_ (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    • fn_ (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    • fatal_error (ras_inject_error)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#g","title":"g","text":"
                                                                                       
                                                                                    • guid (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • group_name (fpga_metric_info)
                                                                                      •  
                                                                                    • get_ (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • group (opae_vfio)
                                                                                      •  
                                                                                    • group_device (opae_vfio_group)
                                                                                      •  
                                                                                    • group_fd (opae_vfio_group)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#h","title":"h","text":"
                                                                                       
                                                                                    • hash_seed (_opae_hash_map)
                                                                                      •  
                                                                                    • hysteresis (metric_threshold)
                                                                                      •  
                                                                                    • handle_ (opae::fpga::types::event, opae::fpga::types::handle, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#i","title":"i","text":"
                                                                                       
                                                                                    • interface (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • isvalid (fpga_metric)
                                                                                      •  
                                                                                    • interrupt (opae::fpga::types::event::type_t)
                                                                                      •  
                                                                                    • is_set_ (opae::fpga::types::guid_t, opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • io_address_ (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • iova_alloc (opae_vfio)
                                                                                      •  
                                                                                    • irqs (opae_vfio_device)
                                                                                      •  
                                                                                    • index (opae_vfio_device_irq, opae_vfio_sparse_info)
                                                                                      •  
                                                                                    • is_valid (threshold)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#k","title":"k","text":"
                                                                                       
                                                                                    • key_cleanup (_opae_hash_map)
                                                                                      •  
                                                                                    • key_compare (_opae_hash_map)
                                                                                      •  
                                                                                    • key_hash (_opae_hash_map)
                                                                                      •  
                                                                                    • key (_opae_hash_map_item)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#l","title":"l","text":"
                                                                                       
                                                                                    • lower_c_threshold (metric_threshold)
                                                                                      •  
                                                                                    • lower_nc_threshold (metric_threshold)
                                                                                      •  
                                                                                    • lower_nr_threshold (metric_threshold)
                                                                                      •  
                                                                                    • loc_ (opae::fpga::types::except)
                                                                                      •  
                                                                                    • local_memory_size (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • len_ (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    • line_ (opae::fpga::types::src_location)
                                                                                      •  
                                                                                    • lock (opae_vfio)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#m","title":"m","text":"
                                                                                       
                                                                                    • magic (_fpga_token_header)
                                                                                      •  
                                                                                    • metric_num (fpga_metric, fpga_metric_info)
                                                                                      •  
                                                                                    • metric_datatype (fpga_metric_info)
                                                                                      •  
                                                                                    • metric_guid (fpga_metric_info)
                                                                                      •  
                                                                                    • metric_name (fpga_metric_info, metric_threshold)
                                                                                      •  
                                                                                    • metric_type (fpga_metric_info)
                                                                                      •  
                                                                                    • metric_units (fpga_metric_info)
                                                                                      •  
                                                                                    • major (fpga_version)
                                                                                      •  
                                                                                    • minor (fpga_version)
                                                                                      •  
                                                                                    • MAX_EXCEPT (opae::fpga::types::except)
                                                                                      •  
                                                                                    • msg_ (opae::fpga::types::except)
                                                                                      •  
                                                                                    • model (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • masks (opae_vfio_device_irq)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#n","title":"n","text":"
                                                                                       
                                                                                    • num_buckets (_opae_hash_map)
                                                                                      •  
                                                                                    • next (_opae_hash_map_item, mem_link, opae_uio_device_region, opae_vfio_device_irq, opae_vfio_device_region, opae_vfio_iova_range, opae_vfio_sparse_info)
                                                                                      •  
                                                                                    • name (fpga_error_info)
                                                                                      •  
                                                                                    • none (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • num_errors (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • num_interrupts (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • num_mmio (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • num_slots (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • nonfatal_error (ras_inject_error)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#o","title":"o","text":"
                                                                                       
                                                                                    • object_id (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • objtype (_fpga_token_header)
                                                                                      •  
                                                                                    • open_flags (config)
                                                                                      •  
                                                                                    • os_object_ (opae::fpga::types::event)
                                                                                      •  
                                                                                    • offset (opae_vfio_sparse_info)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#p","title":"p","text":"
                                                                                       
                                                                                    • patch (fpga_version)
                                                                                      •  
                                                                                    • prev (mem_link)
                                                                                      •  
                                                                                    • power_thermal (opae::fpga::types::event::type_t)
                                                                                      •  
                                                                                    • props_ (opae::fpga::types::guid_t, opae::fpga::types::properties, opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • parent (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • ptr (opae_vfio_sparse_info)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#q","title":"q","text":"
                                                                                       
                                                                                    • qualifier_name (fpga_metric_info)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#r","title":"r","text":"
                                                                                       
                                                                                    • run_n3000 (config)
                                                                                      •  
                                                                                    • res_ (opae::fpga::types::except)
                                                                                      •  
                                                                                    • regions (opae_uio, opae_vfio_device)
                                                                                      •  
                                                                                    • region_index (opae_uio_device_region, opae_vfio_device_region)
                                                                                      •  
                                                                                    • region_page_offset (opae_uio_device_region)
                                                                                      •  
                                                                                    • region_ptr (opae_uio_device_region, opae_vfio_device_region)
                                                                                      •  
                                                                                    • region_size (opae_uio_device_region, opae_vfio_device_region)
                                                                                      •  
                                                                                    • region_sparse (opae_vfio_device_region)
                                                                                      •  
                                                                                    • rsvd (ras_inject_error)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#s","title":"s","text":"
                                                                                       
                                                                                    • segment (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • subsystem_device_id (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • subsystem_vendor_id (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • size (mem_link, opae_vfio_sparse_info)
                                                                                      •  
                                                                                    • socket_id (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • set_ (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • sysobject_ (opae::fpga::types::sysobject)
                                                                                      •  
                                                                                    • start (opae_vfio_iova_range)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#t","title":"t","text":"
                                                                                       
                                                                                    • token_ (opae::fpga::types::error, opae::fpga::types::handle, opae::fpga::types::sysobject, opae::fpga::types::token)
                                                                                      •  
                                                                                    • type_ (opae::fpga::types::event::type_t, opae::fpga::types::event)
                                                                                      •  
                                                                                    • type (opae::fpga::types::properties)
                                                                                      •  
                                                                                    • threshold_name (threshold)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#u","title":"u","text":"
                                                                                       
                                                                                    • uint (cache_line)
                                                                                      •  
                                                                                    • upper_c_threshold (metric_threshold)
                                                                                      •  
                                                                                    • upper_nc_threshold (metric_threshold)
                                                                                      •  
                                                                                    • upper_nr_threshold (metric_threshold)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#v","title":"v","text":"
                                                                                       
                                                                                    • vendor_id (_fpga_token_header, opae::fpga::types::properties)
                                                                                      •  
                                                                                    • value_cleanup (_opae_hash_map)
                                                                                      •  
                                                                                    • value (_opae_hash_map_item, fpga_metric, threshold)
                                                                                      •  
                                                                                    • virt_ (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#w","title":"w","text":"
                                                                                       
                                                                                    • wsid_ (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_variables/#_1","title":"@","text":"
                                                                                       
                                                                                    • @1 (ras_inject_error)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_typedefs/","title":"Class Member Typedefs","text":""},{"location":"opae-code/class_member_typedefs/#c","title":"c","text":"
                                                                                       
                                                                                    • copy_t (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_typedefs/#g","title":"g","text":"
                                                                                       
                                                                                    • getter_t (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_typedefs/#p","title":"p","text":"
                                                                                       
                                                                                    • ptr_t (opae::fpga::types::error, opae::fpga::types::event, opae::fpga::types::handle, opae::fpga::types::properties, opae::fpga::types::shared_buffer, opae::fpga::types::sysobject, opae::fpga::types::token)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_typedefs/#s","title":"s","text":"
                                                                                       
                                                                                    • setter_t (opae::fpga::types::pvalue)
                                                                                      •  
                                                                                    • size_t (opae::fpga::types::shared_buffer)
                                                                                      •  
                                                                                    "},{"location":"opae-code/class_member_enums/","title":"Class Member Enums","text":""},{"location":"opae-code/namespace_members/","title":"Namespace Members","text":""},{"location":"opae-code/namespace_members/#a","title":"a","text":"
                                                                                       
                                                                                    • assert_fpga_ok (opae::fpga::types::detail)
                                                                                      •  
                                                                                    "},{"location":"opae-code/namespace_members/#e","title":"e","text":"
                                                                                       
                                                                                    • exception_fn (opae::fpga::types::detail)
                                                                                      •  
                                                                                    "},{"location":"opae-code/namespace_members/#i","title":"i","text":"
                                                                                       
                                                                                    • is_ok (opae::fpga::types::detail)
                                                                                      •  
                                                                                    "},{"location":"opae-code/namespace_members/#o","title":"o","text":"
                                                                                       
                                                                                    • opae_exceptions (opae::fpga::types::detail)
                                                                                      •  
                                                                                    "},{"location":"opae-code/namespace_member_functions/","title":"Namespace Member Functions","text":""},{"location":"opae-code/namespace_member_functions/#a","title":"a","text":"
                                                                                       
                                                                                    • assert_fpga_ok (opae::fpga::types::detail)
                                                                                      •  
                                                                                    "},{"location":"opae-code/namespace_member_functions/#i","title":"i","text":"
                                                                                       
                                                                                    • is_ok (opae::fpga::types::detail)
                                                                                      •  
                                                                                    "},{"location":"opae-code/namespace_member_variables/","title":"Namespace Member Variables","text":""},{"location":"opae-code/namespace_member_variables/#o","title":"o","text":"
                                                                                       
                                                                                    • opae_exceptions (opae::fpga::types::detail)
                                                                                      •  
                                                                                    "},{"location":"opae-code/namespace_member_typedefs/","title":"Namespace Member Typedefs","text":""},{"location":"opae-code/namespace_member_typedefs/#e","title":"e","text":"
                                                                                       
                                                                                    • exception_fn (opae::fpga::types::detail)
                                                                                      •  
                                                                                    "},{"location":"opae-code/namespace_member_enums/","title":"Namespace Member Enums","text":""},{"location":"opae-code/functions/","title":"Functions","text":""},{"location":"opae-code/functions/#e","title":"e","text":"
                                                                                       
                                                                                    • error_thread (hello_events.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/functions/#f","title":"f","text":"
                                                                                       
                                                                                    • fpgaClose (access.h)
                                                                                      •  
                                                                                    • fpgaOpen (access.h)
                                                                                      •  
                                                                                    • fpgaReset (access.h)
                                                                                      •  
                                                                                    • fpgaGetIOAddress (buffer.h)
                                                                                      •  
                                                                                    • fpgaPrepareBuffer (buffer.h)
                                                                                      •  
                                                                                    • fpgaReleaseBuffer (buffer.h)
                                                                                      •  
                                                                                    • fpgaCloneToken (enum.h)
                                                                                      •  
                                                                                    • fpgaDestroyToken (enum.h)
                                                                                      •  
                                                                                    • fpgaEnumerate (enum.h)
                                                                                      •  
                                                                                    • fpgaClearAllErrors (error.h)
                                                                                      •  
                                                                                    • fpgaClearError (error.h)
                                                                                      •  
                                                                                    • fpgaGetErrorInfo (error.h)
                                                                                      •  
                                                                                    • fpgaReadError (error.h)
                                                                                      •  
                                                                                    • fpgaCreateEventHandle (event.h)
                                                                                      •  
                                                                                    • fpgaDestroyEventHandle (event.h)
                                                                                      •  
                                                                                    • fpgaGetOSObjectFromEventHandle (event.h)
                                                                                      •  
                                                                                    • fpgaRegisterEvent (event.h)
                                                                                      •  
                                                                                    • fpgaUnregisterEvent (event.h)
                                                                                      •  
                                                                                    • fpgaFinalize (init.h)
                                                                                      •  
                                                                                    • fpgaInitialize (init.h)
                                                                                      •  
                                                                                    • fpgaAssignPortToInterface (manage.h)
                                                                                      •  
                                                                                    • fpgaAssignToInterface (manage.h)
                                                                                      •  
                                                                                    • fpgaReconfigureSlot (manage.h)
                                                                                      •  
                                                                                    • fpgaReleaseFromInterface (manage.h)
                                                                                      •  
                                                                                    • fpgaGetMetricsByIndex (metrics.h)
                                                                                      •  
                                                                                    • fpgaGetMetricsByName (metrics.h)
                                                                                      •  
                                                                                    • fpgaGetMetricsInfo (metrics.h)
                                                                                      •  
                                                                                    • fpgaGetMetricsThresholdInfo (metrics.h)
                                                                                      •  
                                                                                    • fpgaGetNumMetrics (metrics.h)
                                                                                      •  
                                                                                    • fpgaMapMMIO (mmio.h)
                                                                                      •  
                                                                                    • fpgaReadMMIO32 (mmio.h)
                                                                                      •  
                                                                                    • fpgaReadMMIO64 (mmio.h)
                                                                                      •  
                                                                                    • fpgaUnmapMMIO (mmio.h)
                                                                                      •  
                                                                                    • fpgaWriteMMIO32 (mmio.h)
                                                                                      •  
                                                                                    • fpgaWriteMMIO512 (mmio.h)
                                                                                      •  
                                                                                    • fpgaWriteMMIO64 (mmio.h)
                                                                                      •  
                                                                                    • fpgaClearProperties (properties.h)
                                                                                      •  
                                                                                    • fpgaCloneProperties (properties.h)
                                                                                      •  
                                                                                    • fpgaDestroyProperties (properties.h)
                                                                                      •  
                                                                                    • fpgaGetProperties (properties.h)
                                                                                      •  
                                                                                    • fpgaGetPropertiesFromHandle (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetAcceleratorState (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetBBSID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetBBSVersion (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetBus (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetCapabilities (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetDevice (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetDeviceID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetFunction (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetGUID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetInterface (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetLocalMemorySize (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetModel (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetNumErrors (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetNumInterrupts (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetNumMMIO (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetNumSlots (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetObjectID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetObjectType (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetParent (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetSegment (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetSocketID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetSubsystemDeviceID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetSubsystemVendorID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesGetVendorID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetAcceleratorState (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetBBSID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetBBSVersion (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetBus (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetCapabilities (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetDevice (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetDeviceID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetFunction (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetGUID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetInterface (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetLocalMemorySize (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetModel (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetNumErrors (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetNumInterrupts (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetNumMMIO (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetNumSlots (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetObjectID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetObjectType (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetParent (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetSegment (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetSocketID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetSubsystemDeviceID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetSubsystemVendorID (properties.h)
                                                                                      •  
                                                                                    • fpgaPropertiesSetVendorID (properties.h)
                                                                                      •  
                                                                                    • fpgaUpdateProperties (properties.h)
                                                                                      •  
                                                                                    • fpgaDestroyObject (sysobject.h)
                                                                                      •  
                                                                                    • fpgaHandleGetObject (sysobject.h)
                                                                                      •  
                                                                                    • fpgaObjectGetObject (sysobject.h)
                                                                                      •  
                                                                                    • fpgaObjectGetObjectAt (sysobject.h)
                                                                                      •  
                                                                                    • fpgaObjectGetSize (sysobject.h)
                                                                                      •  
                                                                                    • fpgaObjectGetType (sysobject.h)
                                                                                      •  
                                                                                    • fpgaObjectRead (sysobject.h)
                                                                                      •  
                                                                                    • fpgaObjectRead64 (sysobject.h)
                                                                                      •  
                                                                                    • fpgaObjectWrite64 (sysobject.h)
                                                                                      •  
                                                                                    • fpgaTokenGetObject (sysobject.h)
                                                                                      •  
                                                                                    • fpgaGetNumUmsg (umsg.h)
                                                                                      •  
                                                                                    • fpgaGetUmsgPtr (umsg.h)
                                                                                      •  
                                                                                    • fpgaSetUmsgAttributes (umsg.h)
                                                                                      •  
                                                                                    • fpgaTriggerUmsg (umsg.h)
                                                                                      •  
                                                                                    • fpgaGetUserClock (userclk.h)
                                                                                      •  
                                                                                    • fpgaSetUserClock (userclk.h)
                                                                                      •  
                                                                                    • fpgaErrStr (utils.h)
                                                                                      •  
                                                                                    • fpgaGetOPAECBuildString (version.h)
                                                                                      •  
                                                                                    • fpgaGetOPAECVersion (version.h)
                                                                                      •  
                                                                                    • fpgaGetOPAECVersionString (version.h)
                                                                                      •  
                                                                                    • find_fpga (hello_events.c, hello_fpga.c)
                                                                                      •  
                                                                                    • find_nlb_n3000 (hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/functions/#h","title":"h","text":"
                                                                                       
                                                                                    • help (hello_events.c, hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/functions/#i","title":"i","text":"
                                                                                       
                                                                                    • inject_ras_fatal_error (hello_events.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/functions/#m","title":"m","text":"
                                                                                       
                                                                                    • mem_alloc_add_free (mem_alloc.h)
                                                                                      •  
                                                                                    • mem_alloc_destroy (mem_alloc.h)
                                                                                      •  
                                                                                    • mem_alloc_get (mem_alloc.h)
                                                                                      •  
                                                                                    • mem_alloc_init (mem_alloc.h)
                                                                                      •  
                                                                                    • mem_alloc_put (mem_alloc.h)
                                                                                      •  
                                                                                    • main (hello_events.c, hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/functions/#o","title":"o","text":"
                                                                                       
                                                                                    • opae_hash_map_add (hash_map.h)
                                                                                      •  
                                                                                    • opae_hash_map_destroy (hash_map.h)
                                                                                      •  
                                                                                    • opae_hash_map_find (hash_map.h)
                                                                                      •  
                                                                                    • opae_hash_map_init (hash_map.h)
                                                                                      •  
                                                                                    • opae_hash_map_is_empty (hash_map.h)
                                                                                      •  
                                                                                    • opae_hash_map_remove (hash_map.h)
                                                                                      •  
                                                                                    • opae_u64_key_compare (hash_map.h)
                                                                                      •  
                                                                                    • opae_u64_key_hash (hash_map.h)
                                                                                      •  
                                                                                    • opae_print (log.h)
                                                                                      •  
                                                                                    • opae_uio_close (uio.h)
                                                                                      •  
                                                                                    • opae_uio_open (uio.h)
                                                                                      •  
                                                                                    • opae_uio_region_get (uio.h)
                                                                                      •  
                                                                                    • opae_vfio_buffer_allocate (vfio.h)
                                                                                      •  
                                                                                    • opae_vfio_buffer_allocate_ex (vfio.h)
                                                                                      •  
                                                                                    • opae_vfio_buffer_free (vfio.h)
                                                                                      •  
                                                                                    • opae_vfio_buffer_info (vfio.h)
                                                                                      •  
                                                                                    • opae_vfio_close (vfio.h)
                                                                                      •  
                                                                                    • opae_vfio_irq_disable (vfio.h)
                                                                                      •  
                                                                                    • opae_vfio_irq_enable (vfio.h)
                                                                                      •  
                                                                                    • opae_vfio_irq_mask (vfio.h)
                                                                                      •  
                                                                                    • opae_vfio_irq_unmask (vfio.h)
                                                                                      •  
                                                                                    • opae_vfio_open (vfio.h)
                                                                                      •  
                                                                                    • opae_vfio_region_get (vfio.h)
                                                                                      •  
                                                                                    • opae_vfio_secure_open (vfio.h)
                                                                                      •  
                                                                                    "},{"location":"opae-code/functions/#p","title":"p","text":"
                                                                                       
                                                                                    • parse_args (hello_events.c, hello_fpga.c)
                                                                                      •  
                                                                                    • print_err (hello_events.c, hello_fpga.c)
                                                                                      •  
                                                                                    • probe_for_ase (hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/functions/#u","title":"u","text":"
                                                                                       
                                                                                    • usleep (hello_events.c, hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/macros/","title":"Macros","text":""},{"location":"opae-code/macros/#a","title":"a","text":"
                                                                                       
                                                                                    • ASSERT_FPGA_OK (except.h)
                                                                                      •  
                                                                                    "},{"location":"opae-code/macros/#c","title":"c","text":"
                                                                                       
                                                                                    • CACHELINE_ALIGNED_ADDR (hello_fpga.c)
                                                                                      •  
                                                                                    • CL (hello_fpga.c)
                                                                                      •  
                                                                                    • CSR_AFU_DSM_BASEL (hello_fpga.c)
                                                                                      •  
                                                                                    • CSR_CFG (hello_fpga.c)
                                                                                      •  
                                                                                    • CSR_CTL (hello_fpga.c)
                                                                                      •  
                                                                                    • CSR_DST_ADDR (hello_fpga.c)
                                                                                      •  
                                                                                    • CSR_NUM_LINES (hello_fpga.c)
                                                                                      •  
                                                                                    • CSR_SRC_ADDR (hello_fpga.c)
                                                                                      •  
                                                                                    • CSR_STATUS1 (hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/macros/#d","title":"d","text":"
                                                                                       
                                                                                    • DSM_STATUS_TEST_COMPLETE (hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/macros/#f","title":"f","text":"
                                                                                       
                                                                                    • FPGA_ERROR_NAME_MAX (types.h)
                                                                                      •  
                                                                                    • FPGA_METRIC_STR_SIZE (types.h)
                                                                                      •  
                                                                                    • fpga_is_parent_child (types.h)
                                                                                      •  
                                                                                    • FPGA_BUILD_STR_MAX (version.h)
                                                                                      •  
                                                                                    • FPGA_VERSION_STR_MAX (version.h)
                                                                                      •  
                                                                                    • FME_SYSFS_INJECT_ERROR (hello_events.c)
                                                                                      •  
                                                                                    • FPGA_NLB0_UUID_H (hello_fpga.c)
                                                                                      •  
                                                                                    • FPGA_NLB0_UUID_L (hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/macros/#g","title":"g","text":"
                                                                                       
                                                                                    • GETOPT_STRING (hello_events.c, hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/macros/#l","title":"l","text":"
                                                                                       
                                                                                    • LOG2_CL (hello_fpga.c)
                                                                                      •  
                                                                                    • LPBK1_BUFFER_ALLOCATION_SIZE (hello_fpga.c)
                                                                                      •  
                                                                                    • LPBK1_BUFFER_SIZE (hello_fpga.c)
                                                                                      •  
                                                                                    • LPBK1_DSM_SIZE (hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/macros/#m","title":"m","text":"
                                                                                       
                                                                                    • MB (hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/macros/#n","title":"n","text":"
                                                                                       
                                                                                    • N3000_AFUID (hello_fpga.c)
                                                                                      •  
                                                                                    • NLB0_AFUID (hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/macros/#o","title":"o","text":"
                                                                                       
                                                                                    • OPAECXX_HERE (except.h)
                                                                                      •  
                                                                                    • OPAE_DBG (log.h)
                                                                                      •  
                                                                                    • OPAE_DEFAULT_LOGLEVEL (log.h)
                                                                                      •  
                                                                                    • OPAE_ERR (log.h)
                                                                                      •  
                                                                                    • OPAE_MSG (log.h)
                                                                                      •  
                                                                                    • OPAE_UIO_PATH_MAX (uio.h)
                                                                                      •  
                                                                                    • ON_ERR_GOTO (hello_events.c, hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/macros/#t","title":"t","text":"
                                                                                       
                                                                                    • TEST_TIMEOUT (hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/macros/#_","title":"_","text":"
                                                                                       
                                                                                    • __SHORT_FILE__ (log.h)
                                                                                      •  
                                                                                    "},{"location":"opae-code/variables/","title":"Variables","text":""},{"location":"opae-code/variables/#c","title":"c","text":"
                                                                                       
                                                                                    • config (hello_fpga.c)
                                                                                      •  
                                                                                    "},{"location":"opae-code/variables/#f","title":"f","text":"
                                                                                       
                                                                                    • fpga_event_handle (types.h)
                                                                                      •  
                                                                                    • fpga_guid (types.h)
                                                                                      •  
                                                                                    • fpga_handle (types.h)
                                                                                      •  
                                                                                    • fpga_metric (types.h)
                                                                                      •  
                                                                                    • fpga_metric_info (types.h)
                                                                                      •  
                                                                                    • fpga_object (types.h)
                                                                                      •  
                                                                                    • fpga_properties (types.h)
                                                                                      •  
                                                                                    • fpga_token (types.h)
                                                                                      •  
                                                                                    • fpga_token_header (types.h)
                                                                                      •  
                                                                                    • fpga_accelerator_state (types_enum.h)
                                                                                      •  
                                                                                    • fpga_buffer_flags (types_enum.h)
                                                                                      •  
                                                                                    • fpga_event_type (types_enum.h)
                                                                                      •  
                                                                                    • fpga_interface (types_enum.h)
                                                                                      •  
                                                                                    • fpga_metric_datatype (types_enum.h)
                                                                                      •  
                                                                                    • fpga_metric_type (types_enum.h)
                                                                                      •  
                                                                                    • fpga_objtype (types_enum.h)
                                                                                      •  
                                                                                    • fpga_open_flags (types_enum.h)
                                                                                      •  
                                                                                    • fpga_reconf_flags (types_enum.h)
                                                                                      •  
                                                                                    • fpga_result (types_enum.h)
                                                                                      •  
                                                                                    • fpga_sysobject_flags (types_enum.h)
                                                                                      •  
                                                                                    • fpga_sysobject_type (types_enum.h)
                                                                                      •  
                                                                                    "},{"location":"opae-code/variables/#m","title":"m","text":"
                                                                                       
                                                                                    • metric_threshold (types.h)
                                                                                      •  
                                                                                    "},{"location":"opae-code/variables/#o","title":"o","text":"
                                                                                       
                                                                                    • opae_hash_map (hash_map.h)
                                                                                      •  
                                                                                    • opae_hash_map_flags (hash_map.h)
                                                                                      •  
                                                                                    • opae_hash_map_item (hash_map.h)
                                                                                      •  
                                                                                    • opae_loglevel (log.h)
                                                                                      •  
                                                                                    • opae_vfio_buffer_flags (vfio.h)
                                                                                      •  
                                                                                    "},{"location":"opae-code/variables/#t","title":"t","text":"
                                                                                       
                                                                                    • threshold (types.h)
                                                                                      •  
                                                                                    "},{"location":"opae-code/variables/#_","title":"_","text":"
                                                                                       
                                                                                    • _opae_hash_map_flags (hash_map.h)
                                                                                      •  
                                                                                    "},{"location":"opae-code/links/","title":"Links","text":"
                                                                                    * Related Pages     * Todo List   * Modules   * Class List     * struct _fpga_token_header     * struct _opae_hash_map     * struct _opae_hash_map_item     * struct cache_line     * struct config     * struct fpga_error_info     * struct fpga_metric     * struct fpga_metric_info     * struct fpga_version     * struct mem_alloc     * struct mem_link     * struct metric_threshold     * union metric_value     * namespace opae       * namespace opae::fpga         * namespace opae::fpga::types           * class opae::fpga::types::busy           * namespace opae::fpga::types::detail           * class opae::fpga::types::error           * class opae::fpga::types::event             * struct opae::fpga::types::event::type_t           * class opae::fpga::types::except           * class opae::fpga::types::exception           * struct opae::fpga::types::guid_t           * class opae::fpga::types::handle           * class opae::fpga::types::invalid_param           * class opae::fpga::types::no_access           * class opae::fpga::types::no_daemon           * class opae::fpga::types::no_driver           * class opae::fpga::types::no_memory           * class opae::fpga::types::not_found           * class opae::fpga::types::not_supported           * class opae::fpga::types::properties           * struct opae::fpga::types::pvalue           * class opae::fpga::types::reconf_error           * class opae::fpga::types::shared_buffer           * class opae::fpga::types::src_location           * class opae::fpga::types::sysobject           * class opae::fpga::types::token           * class opae::fpga::types::version     * struct opae_uio     * struct opae_uio_device_region     * struct opae_vfio     * struct opae_vfio_buffer     * struct opae_vfio_device     * struct opae_vfio_device_irq     * struct opae_vfio_device_region     * struct opae_vfio_group     * struct opae_vfio_iova_range     * struct opae_vfio_sparse_info     * struct ras_inject_error     * namespace std     * struct threshold   * Namespace ListNamespace List   * Namespace Members   * Namespace Member Functions   * Namespace Member Variables   * Namespace Member Typedefs   * Namespace Member Enumerations   * Class Index   * Class Hierarchy   * Class Members   * Class Member Functions   * Class Member Variables   * Class Member Typedefs   * Class Member Enumerations   * Files     * docs       * docs/sw         * docs/sw/include           * docs/sw/include/opae             * access.h             * access.h source             * buffer.h             * buffer.h source             * docs/sw/include/opae/cxx               * core.h               * core.h source               * docs/sw/include/opae/cxx/core                 * errors.h                 * errors.h source                 * events.h                 * events.h source                 * except.h                 * except.h source                 * handle.h                 * handle.h source                 * properties.h                 * properties.h source                 * pvalue.h                 * pvalue.h source                 * shared_buffer.h                 * shared_buffer.h source                 * sysobject.h                 * sysobject.h source                 * token.h                 * token.h source                 * version.h                 * version.h source             * enum.h             * enum.h source             * error.h             * error.h source             * event.h             * event.h source             * fpga.h             * fpga.h source             * hash_map.h             * hash_map.h source             * init.h             * init.h source             * log.h             * log.h source             * manage.h             * manage.h source             * mem_alloc.h             * mem_alloc.h source             * metrics.h             * metrics.h source             * mmio.h             * mmio.h source             * properties.h             * properties.h source             * sysobject.h             * sysobject.h source             * types.h             * types.h source             * types_enum.h             * types_enum.h source             * uio.h             * uio.h source             * umsg.h             * umsg.h source             * userclk.h             * userclk.h source             * utils.h             * utils.h source             * version.h             * version.h source             * vfio.h             * vfio.h source         * docs/sw/samples           * docs/sw/samples/hello_events             * hello_events.c             * hello_events.c source           * docs/sw/samples/hello_fpga             * hello_fpga.c             * hello_fpga.c source   * File Variables   * File Functions   * File Macros
                                                                                    "}]}
\ No newline at end of file
diff --git a/ofs-2023.3-2/sitemap.xml.gz b/ofs-2023.3-2/sitemap.xml.gz
index ce8416f90c1c5f37ac9a14b3d3c3b84ed1716892..e06b4757c62712739dfd83e62af35af63dfcb656 100644
GIT binary patch
delta 13
Ucmb=gXP58h;7E_!H<7&p035FbPXGV_

delta 13
Ucmb=gXP58h;Fx}R??m